def make_fixture(class_name, logger, job_num, *args, **kwargs): """Provide factory function for creating Fixture instances.""" fixturelib = FixtureLib() if class_name in _BUILDERS: builder = _BUILDERS[class_name]() return builder.build_fixture(logger, job_num, fixturelib, *args, **kwargs) if class_name not in _FIXTURES: raise ValueError("Unknown fixture class '%s'" % class_name) # Special case MongoDFixture or _MongosFixture for now since we only add one option. # If there's more logic, we should add a builder class for them. if class_name in ["MongoDFixture", "_MongoSFixture"]: return _FIXTURES[class_name](logger, job_num, fixturelib, *args, add_feature_flags=bool( config.ENABLED_FEATURE_FLAGS), **kwargs) return _FIXTURES[class_name](logger, job_num, fixturelib, *args, **kwargs)
def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: str) -> Set[str]: """Get a list of commands in a given API version by calling listCommands.""" assert mongod_or_mongos in ("mongod", "mongos") logging.info("Calling listCommands on %s", mongod_or_mongos) dbpath = TemporaryDirectory() fixturelib = FixtureLib() mongod_executable = os.path.join(install_dir, "mongod") mongos_executable = os.path.join(install_dir, "mongos") if mongod_or_mongos == "mongod": logger = loggers.new_fixture_logger("MongoDFixture", 0) logger.parent = LOGGER fixture: interface.Fixture = fixturelib.make_fixture( "MongoDFixture", logger, 0, dbpath_prefix=dbpath.name, mongod_executable=mongod_executable, mongod_options={"set_parameters": {}}) else: logger = loggers.new_fixture_logger("ShardedClusterFixture", 0) logger.parent = LOGGER fixture = fixturelib.make_fixture( "ShardedClusterFixture", logger, 0, dbpath_prefix=dbpath.name, mongos_executable=mongos_executable, mongod_executable=mongod_executable, mongod_options={"set_parameters": {}}) fixture.setup() fixture.await_ready() try: client = MongoClient(fixture.get_driver_connection_url()) reply = client.admin.command('listCommands') commands = { name for name, info in reply['commands'].items() if api_version in info['apiVersions'] } logging.info("Found %s commands in API Version %s on %s", len(commands), api_version, mongod_or_mongos) return commands finally: fixture.teardown()
def make_fixture(class_name, logger, job_num, *args, **kwargs): """Provide factory function for creating Fixture instances.""" fixturelib = FixtureLib() if class_name not in _FIXTURES: raise ValueError("Unknown fixture class '%s'" % class_name) return _FIXTURES[class_name](logger, job_num, fixturelib, *args, **kwargs)
def setUp(self): self.logger = logging.getLogger("job_unittest") fixturelib = FixtureLib() self.__noop_fixture = _fixtures.NoOpFixture(logger=self.logger, job_num=0, fixturelib=fixturelib) self.__noop_fixture.setup = mock.Mock() self.__noop_fixture.teardown = mock.Mock() test_report = mock.Mock() test_report.find_test_info().status = "pass" self.__job_object = job.Job(job_num=0, logger=self.logger, fixture=self.__noop_fixture, hooks=[], report=test_report, archival=None, suite_options=None, test_queue_logger=self.logger)
def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements logger, job_num, test_id=None, executable=None, connection_string=None, filename=None, test_filename=None, process_kwargs=None, **kwargs): """Return a Process instance that starts a mongo shell. The shell is started with the given connection string and arguments constructed from 'kwargs'. :param filename: the file run by the mongo shell :param test_filename: The test file - it's usually `filename`, but may be different for tests that use JS runner files, which in turn run real JS files. """ executable = utils.default_if_none( utils.default_if_none(executable, config.MONGO_EXECUTABLE), config.DEFAULT_MONGO_EXECUTABLE) args = [executable] eval_sb = [] # String builder. global_vars = kwargs.pop("global_vars", {}).copy() def basename(filepath): return os.path.splitext(os.path.basename(filepath))[0] if test_filename is not None: test_name = basename(test_filename) elif filename is not None: test_name = basename(filename) else: test_name = None shortcut_opts = { "backupOnRestartDir": (config.BACKUP_ON_RESTART_DIR, None), "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True), "mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""), "noJournal": (config.NO_JOURNAL, False), "storageEngine": (config.STORAGE_ENGINE, ""), "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""), "testName": (test_name, ""), "transportLayer": (config.TRANSPORT_LAYER, ""), "wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""), "wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""), "wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""), # Evergreen variables. "evergreenDebugSymbolsUrl": (config.DEBUG_SYMBOLS_URL, ""), } test_data = global_vars.get("TestData", {}).copy() for opt_name in shortcut_opts: (opt_value, opt_default) = shortcut_opts[opt_name] if opt_value is not None: test_data[opt_name] = opt_value elif opt_name not in test_data: # Only use 'opt_default' if the property wasn't set in the YAML configuration. test_data[opt_name] = opt_default global_vars["TestData"] = test_data if config.EVERGREEN_TASK_ID is not None: test_data["inEvergreen"] = True # Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since # they are dictionaries, they will be converted to JavaScript objects when passed to the shell # by the _format_shell_vars() function. mongod_set_parameters = test_data.get("setParameters", {}).copy() mongos_set_parameters = test_data.get("setParametersMongos", {}).copy() mongocryptd_set_parameters = test_data.get("setParametersMongocryptd", {}).copy() # Propagate additional setParameters to mongod processes spawned by the mongo shell. Command # line options to resmoke.py override the YAML configuration. if config.MONGOD_SET_PARAMETERS is not None: mongod_set_parameters.update( utils.load_yaml(config.MONGOD_SET_PARAMETERS)) # Propagate additional setParameters to mongos processes spawned by the mongo shell. Command # line options to resmoke.py override the YAML configuration. if config.MONGOS_SET_PARAMETERS is not None: mongos_set_parameters.update( utils.load_yaml(config.MONGOS_SET_PARAMETERS)) # Propagate additional setParameters to mongocryptd processes spawned by the mongo shell. # Command line options to resmoke.py override the YAML configuration. if config.MONGOCRYPTD_SET_PARAMETERS is not None: mongocryptd_set_parameters.update( utils.load_yaml(config.MONGOCRYPTD_SET_PARAMETERS)) fixturelib = FixtureLib() mongod_launcher = standalone.MongodLauncher(fixturelib) # If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its # value to a default. mongod_set_parameters.setdefault( "logComponentVerbosity", mongod_launcher.get_default_log_component_verbosity_for_mongod()) # If the 'enableFlowControl' setParameter for mongod was not already specified, we set its value # to a default. if config.FLOW_CONTROL is not None: mongod_set_parameters.setdefault("enableFlowControl", config.FLOW_CONTROL == "on") # Set the default value for minimum resharding operation duration to 5 seconds. mongod_set_parameters.setdefault( "reshardingMinimumOperationDurationMillis", 5000) mongos_launcher = shardedcluster.MongosLauncher(fixturelib) # If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its # value to a default. mongos_set_parameters.setdefault( "logComponentVerbosity", mongos_launcher.default_mongos_log_component_verbosity()) test_data["setParameters"] = mongod_set_parameters test_data["setParametersMongos"] = mongos_set_parameters test_data["setParametersMongocryptd"] = mongocryptd_set_parameters test_data["undoRecorderPath"] = config.UNDO_RECORDER_PATH # There's a periodic background thread that checks for and aborts expired transactions. # "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring # and being aborted by the background thread. It defaults to 60 seconds, which is too short to # be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen # execution timeout. if "transactionLifetimeLimitSeconds" not in test_data: test_data["transactionLifetimeLimitSeconds"] = 24 * 60 * 60 if "eval_prepend" in kwargs: eval_sb.append(str(kwargs.pop("eval_prepend"))) # If nodb is specified, pass the connection string through TestData so it can be used inside the # test, then delete it so it isn't given as an argument to the mongo shell. if "nodb" in kwargs and connection_string is not None: test_data["connectionString"] = connection_string connection_string = None for var_name in global_vars: _format_shell_vars(eval_sb, [var_name], global_vars[var_name]) if "eval" in kwargs: eval_sb.append(str(kwargs.pop("eval"))) # Load this file to allow a callback to validate collections before shutting down mongod. eval_sb.append( "load('jstests/libs/override_methods/validate_collections_on_shutdown.js');" ) # Load a callback to check UUID consistency before shutting down a ShardingTest. eval_sb.append( "load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');" ) # Load a callback to check index consistency before shutting down a ShardingTest. eval_sb.append( "load('jstests/libs/override_methods/check_indexes_consistent_across_cluster.js');" ) # Load a callback to check that all orphans are deleted before shutting down a ShardingTest. eval_sb.append( "load('jstests/libs/override_methods/check_orphans_are_deleted.js');") # Load this file to retry operations that fail due to in-progress background operations. eval_sb.append( "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');" ) eval_sb.append( "(function() { Timestamp.prototype.toString = function() { throw new Error(\"Cannot toString timestamps. Consider using timestampCmp() for comparison or tojson(<variable>) for output.\"); } })();" ) eval_str = "; ".join(eval_sb) args.append("--eval") args.append(eval_str) if connection_string is not None: # The --host and --port options are ignored by the mongo shell when an explicit connection # string is specified. We remove these options to avoid any ambiguity with what server the # logged mongo shell invocation will connect to. if "port" in kwargs: kwargs.pop("port") if "host" in kwargs: kwargs.pop("host") # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) if connection_string is not None: args.append(connection_string) # Have the mongo shell run the specified file. if filename is not None: args.append(filename) _set_keyfile_permissions(test_data) process_kwargs = utils.default_if_none(process_kwargs, {}) process_kwargs["job_num"] = job_num process_kwargs["test_id"] = test_id return make_process(logger, args, **process_kwargs)
class TestMergeMongoOptionDicts(unittest.TestCase): def setUp(self) -> None: self.under_test = FixtureLib() def test_merge_empty(self): # pylint: disable=no-self-use original = { "dbpath": "value0", self.under_test.SET_PARAMETERS_KEY: { "param1": "value1", "param2": "value2", } } override = {} merged = self.under_test.merge_mongo_option_dicts( copy.deepcopy(original), override) self.assertDictEqual(merged, original) def test_merge_non_params(self): # pylint: disable=no-self-use non_param1_key = "non_param1" non_param2_key = "non_param2" original = { non_param1_key: "value0", non_param2_key: { "nested_param1": "value0", }, self.under_test.SET_PARAMETERS_KEY: { "param1": "value1", } } override = { non_param1_key: "value1", non_param2_key: "value1", } self.under_test.merge_mongo_option_dicts(original, override) expected = { non_param1_key: "value1", non_param2_key: "value1", self.under_test.SET_PARAMETERS_KEY: { "param1": "value1", } } self.assertEqual(original, expected) def test_merge_params(self): # pylint: disable=no-self-use original = { "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: { "param1": "value", "param2": { "param3": "value", }, } } override = { self.under_test.SET_PARAMETERS_KEY: { "param2": { "param3": { "param4": "value" } } } } self.under_test.merge_mongo_option_dicts(original, override) expected = { "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: { "param1": "value", "param2": { "param3": { "param4": "value" } } } } self.assertDictEqual(original, expected)
def setUp(self) -> None: self.under_test = FixtureLib()
def __init__(self, should_raise=False): logger = logging.getLogger("fixture_unittests") fixturelib = FixtureLib() interface.Fixture.__init__(self, logger, 99, fixturelib) self._should_raise = should_raise