def make_fixture(class_name, logger, job_num, *args, **kwargs):
"""Provide factory function for creating Fixture instances."""
fixturelib = FixtureLib()
if class_name in _BUILDERS:
builder = _BUILDERS[class_name]()
return builder.build_fixture(logger, job_num, fixturelib, *args,
**kwargs)
if class_name not in _FIXTURES:
raise ValueError("Unknown fixture class '%s'" % class_name)
# Special case MongoDFixture or _MongosFixture for now since we only add one option.
# If there's more logic, we should add a builder class for them.
if class_name in ["MongoDFixture", "_MongoSFixture"]:
return _FIXTURES[class_name](logger,
job_num,
fixturelib,
*args,
add_feature_flags=bool(
config.ENABLED_FEATURE_FLAGS),
**kwargs)
return _FIXTURES[class_name](logger, job_num, fixturelib, *args, **kwargs)
def list_commands_for_api(api_version: str, mongod_or_mongos: str,
install_dir: str) -> Set[str]:
"""Get a list of commands in a given API version by calling listCommands."""
assert mongod_or_mongos in ("mongod", "mongos")
logging.info("Calling listCommands on %s", mongod_or_mongos)
dbpath = TemporaryDirectory()
fixturelib = FixtureLib()
mongod_executable = os.path.join(install_dir, "mongod")
mongos_executable = os.path.join(install_dir, "mongos")
if mongod_or_mongos == "mongod":
logger = loggers.new_fixture_logger("MongoDFixture", 0)
logger.parent = LOGGER
fixture: interface.Fixture = fixturelib.make_fixture(
"MongoDFixture",
logger,
0,
dbpath_prefix=dbpath.name,
mongod_executable=mongod_executable,
mongod_options={"set_parameters": {}})
else:
logger = loggers.new_fixture_logger("ShardedClusterFixture", 0)
logger.parent = LOGGER
fixture = fixturelib.make_fixture(
"ShardedClusterFixture",
logger,
0,
dbpath_prefix=dbpath.name,
mongos_executable=mongos_executable,
mongod_executable=mongod_executable,
mongod_options={"set_parameters": {}})
fixture.setup()
fixture.await_ready()
try:
client = MongoClient(fixture.get_driver_connection_url())
reply = client.admin.command('listCommands')
commands = {
name
for name, info in reply['commands'].items()
if api_version in info['apiVersions']
}
logging.info("Found %s commands in API Version %s on %s",
len(commands), api_version, mongod_or_mongos)
return commands
finally:
fixture.teardown()
def make_fixture(class_name, logger, job_num, *args, **kwargs):
"""Provide factory function for creating Fixture instances."""
fixturelib = FixtureLib()
if class_name not in _FIXTURES:
raise ValueError("Unknown fixture class '%s'" % class_name)
return _FIXTURES[class_name](logger, job_num, fixturelib, *args, **kwargs)
def setUp(self):
self.logger = logging.getLogger("job_unittest")
fixturelib = FixtureLib()
self.__noop_fixture = _fixtures.NoOpFixture(logger=self.logger, job_num=0,
fixturelib=fixturelib)
self.__noop_fixture.setup = mock.Mock()
self.__noop_fixture.teardown = mock.Mock()
test_report = mock.Mock()
test_report.find_test_info().status = "pass"
self.__job_object = job.Job(job_num=0, logger=self.logger, fixture=self.__noop_fixture,
hooks=[], report=test_report, archival=None, suite_options=None,
test_queue_logger=self.logger)
def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements
logger,
job_num,
test_id=None,
executable=None,
connection_string=None,
filename=None,
test_filename=None,
process_kwargs=None,
**kwargs):
"""Return a Process instance that starts a mongo shell.
The shell is started with the given connection string and arguments constructed from 'kwargs'.
:param filename: the file run by the mongo shell
:param test_filename: The test file - it's usually `filename`, but may be different for
tests that use JS runner files, which in turn run real JS files.
"""
executable = utils.default_if_none(
utils.default_if_none(executable, config.MONGO_EXECUTABLE),
config.DEFAULT_MONGO_EXECUTABLE)
args = [executable]
eval_sb = [] # String builder.
global_vars = kwargs.pop("global_vars", {}).copy()
def basename(filepath):
return os.path.splitext(os.path.basename(filepath))[0]
if test_filename is not None:
test_name = basename(test_filename)
elif filename is not None:
test_name = basename(filename)
else:
test_name = None
shortcut_opts = {
"backupOnRestartDir": (config.BACKUP_ON_RESTART_DIR, None),
"enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True),
"mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""),
"noJournal": (config.NO_JOURNAL, False),
"storageEngine": (config.STORAGE_ENGINE, ""),
"storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""),
"testName": (test_name, ""),
"transportLayer": (config.TRANSPORT_LAYER, ""),
"wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""),
"wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""),
"wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""),
# Evergreen variables.
"evergreenDebugSymbolsUrl": (config.DEBUG_SYMBOLS_URL, ""),
}
test_data = global_vars.get("TestData", {}).copy()
for opt_name in shortcut_opts:
(opt_value, opt_default) = shortcut_opts[opt_name]
if opt_value is not None:
test_data[opt_name] = opt_value
elif opt_name not in test_data:
# Only use 'opt_default' if the property wasn't set in the YAML configuration.
test_data[opt_name] = opt_default
global_vars["TestData"] = test_data
if config.EVERGREEN_TASK_ID is not None:
test_data["inEvergreen"] = True
# Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since
# they are dictionaries, they will be converted to JavaScript objects when passed to the shell
# by the _format_shell_vars() function.
mongod_set_parameters = test_data.get("setParameters", {}).copy()
mongos_set_parameters = test_data.get("setParametersMongos", {}).copy()
mongocryptd_set_parameters = test_data.get("setParametersMongocryptd",
{}).copy()
# Propagate additional setParameters to mongod processes spawned by the mongo shell. Command
# line options to resmoke.py override the YAML configuration.
if config.MONGOD_SET_PARAMETERS is not None:
mongod_set_parameters.update(
utils.load_yaml(config.MONGOD_SET_PARAMETERS))
# Propagate additional setParameters to mongos processes spawned by the mongo shell. Command
# line options to resmoke.py override the YAML configuration.
if config.MONGOS_SET_PARAMETERS is not None:
mongos_set_parameters.update(
utils.load_yaml(config.MONGOS_SET_PARAMETERS))
# Propagate additional setParameters to mongocryptd processes spawned by the mongo shell.
# Command line options to resmoke.py override the YAML configuration.
if config.MONGOCRYPTD_SET_PARAMETERS is not None:
mongocryptd_set_parameters.update(
utils.load_yaml(config.MONGOCRYPTD_SET_PARAMETERS))
fixturelib = FixtureLib()
mongod_launcher = standalone.MongodLauncher(fixturelib)
# If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its
# value to a default.
mongod_set_parameters.setdefault(
"logComponentVerbosity",
mongod_launcher.get_default_log_component_verbosity_for_mongod())
# If the 'enableFlowControl' setParameter for mongod was not already specified, we set its value
# to a default.
if config.FLOW_CONTROL is not None:
mongod_set_parameters.setdefault("enableFlowControl",
config.FLOW_CONTROL == "on")
# Set the default value for minimum resharding operation duration to 5 seconds.
mongod_set_parameters.setdefault(
"reshardingMinimumOperationDurationMillis", 5000)
mongos_launcher = shardedcluster.MongosLauncher(fixturelib)
# If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its
# value to a default.
mongos_set_parameters.setdefault(
"logComponentVerbosity",
mongos_launcher.default_mongos_log_component_verbosity())
test_data["setParameters"] = mongod_set_parameters
test_data["setParametersMongos"] = mongos_set_parameters
test_data["setParametersMongocryptd"] = mongocryptd_set_parameters
test_data["undoRecorderPath"] = config.UNDO_RECORDER_PATH
# There's a periodic background thread that checks for and aborts expired transactions.
# "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring
# and being aborted by the background thread. It defaults to 60 seconds, which is too short to
# be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen
# execution timeout.
if "transactionLifetimeLimitSeconds" not in test_data:
test_data["transactionLifetimeLimitSeconds"] = 24 * 60 * 60
if "eval_prepend" in kwargs:
eval_sb.append(str(kwargs.pop("eval_prepend")))
# If nodb is specified, pass the connection string through TestData so it can be used inside the
# test, then delete it so it isn't given as an argument to the mongo shell.
if "nodb" in kwargs and connection_string is not None:
test_data["connectionString"] = connection_string
connection_string = None
for var_name in global_vars:
_format_shell_vars(eval_sb, [var_name], global_vars[var_name])
if "eval" in kwargs:
eval_sb.append(str(kwargs.pop("eval")))
# Load this file to allow a callback to validate collections before shutting down mongod.
eval_sb.append(
"load('jstests/libs/override_methods/validate_collections_on_shutdown.js');"
)
# Load a callback to check UUID consistency before shutting down a ShardingTest.
eval_sb.append(
"load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');"
)
# Load a callback to check index consistency before shutting down a ShardingTest.
eval_sb.append(
"load('jstests/libs/override_methods/check_indexes_consistent_across_cluster.js');"
)
# Load a callback to check that all orphans are deleted before shutting down a ShardingTest.
eval_sb.append(
"load('jstests/libs/override_methods/check_orphans_are_deleted.js');")
# Load this file to retry operations that fail due to in-progress background operations.
eval_sb.append(
"load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');"
)
eval_sb.append(
"(function() { Timestamp.prototype.toString = function() { throw new Error(\"Cannot toString timestamps. Consider using timestampCmp() for comparison or tojson(<variable>) for output.\"); } })();"
)
eval_str = "; ".join(eval_sb)
args.append("--eval")
args.append(eval_str)
if connection_string is not None:
# The --host and --port options are ignored by the mongo shell when an explicit connection
# string is specified. We remove these options to avoid any ambiguity with what server the
# logged mongo shell invocation will connect to.
if "port" in kwargs:
kwargs.pop("port")
if "host" in kwargs:
kwargs.pop("host")
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
if connection_string is not None:
args.append(connection_string)
# Have the mongo shell run the specified file.
if filename is not None:
args.append(filename)
_set_keyfile_permissions(test_data)
process_kwargs = utils.default_if_none(process_kwargs, {})
process_kwargs["job_num"] = job_num
process_kwargs["test_id"] = test_id
return make_process(logger, args, **process_kwargs)
class TestMergeMongoOptionDicts(unittest.TestCase):
def setUp(self) -> None:
self.under_test = FixtureLib()
def test_merge_empty(self): # pylint: disable=no-self-use
original = {
"dbpath": "value0",
self.under_test.SET_PARAMETERS_KEY: {
"param1": "value1",
"param2": "value2",
}
}
override = {}
merged = self.under_test.merge_mongo_option_dicts(
copy.deepcopy(original), override)
self.assertDictEqual(merged, original)
def test_merge_non_params(self): # pylint: disable=no-self-use
non_param1_key = "non_param1"
non_param2_key = "non_param2"
original = {
non_param1_key: "value0",
non_param2_key: {
"nested_param1": "value0",
},
self.under_test.SET_PARAMETERS_KEY: {
"param1": "value1",
}
}
override = {
non_param1_key: "value1",
non_param2_key: "value1",
}
self.under_test.merge_mongo_option_dicts(original, override)
expected = {
non_param1_key: "value1",
non_param2_key: "value1",
self.under_test.SET_PARAMETERS_KEY: {
"param1": "value1",
}
}
self.assertEqual(original, expected)
def test_merge_params(self): # pylint: disable=no-self-use
original = {
"dbpath": "value",
self.under_test.SET_PARAMETERS_KEY: {
"param1": "value",
"param2": {
"param3": "value",
},
}
}
override = {
self.under_test.SET_PARAMETERS_KEY: {
"param2": {
"param3": {
"param4": "value"
}
}
}
}
self.under_test.merge_mongo_option_dicts(original, override)
expected = {
"dbpath": "value",
self.under_test.SET_PARAMETERS_KEY: {
"param1": "value",
"param2": {
"param3": {
"param4": "value"
}
}
}
}
self.assertDictEqual(original, expected)
def setUp(self) -> None:
self.under_test = FixtureLib()
def __init__(self, should_raise=False):
logger = logging.getLogger("fixture_unittests")
fixturelib = FixtureLib()
interface.Fixture.__init__(self, logger, 99, fixturelib)
self._should_raise = should_raise
