def mongos_program(logger, executable=None, process_kwargs=None, **kwargs):
"""Return a Process instance that starts a mongos with arguments constructed from 'kwargs'."""
executable = utils.default_if_none(executable,
config.DEFAULT_MONGOS_EXECUTABLE)
args = [executable]
# Apply the --setParameter command line argument. Command line options to resmoke.py override
# the YAML configuration.
suite_set_parameters = kwargs.pop("set_parameters", {})
if config.MONGOS_SET_PARAMETERS is not None:
suite_set_parameters.update(
utils.load_yaml(config.MONGOS_SET_PARAMETERS))
# Set default log verbosity levels if none were specified.
if "logComponentVerbosity" not in suite_set_parameters:
suite_set_parameters[
"logComponentVerbosity"] = default_mongos_log_component_verbosity(
)
_add_testing_set_parameters(suite_set_parameters)
_apply_set_parameters(args, suite_set_parameters)
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
_set_keyfile_permissions(kwargs)
process_kwargs = utils.default_if_none(process_kwargs, {})
return make_process(logger, args, **process_kwargs)
def _mongos_program(logger,
job_num,
test_id=None,
executable=None,
process_kwargs=None,
**kwargs):
"""Return a Process instance that starts a mongos with arguments constructed from 'kwargs'."""
executable = utils.default_if_none(executable,
config.DEFAULT_MONGOS_EXECUTABLE)
# Apply the --setParameter command line argument. Command line options to resmoke.py override
# the YAML configuration.
suite_set_parameters = kwargs.setdefault("set_parameters", {})
if config.MONGOS_SET_PARAMETERS is not None:
suite_set_parameters.update(
utils.load_yaml(config.MONGOS_SET_PARAMETERS))
# Set default log verbosity levels if none were specified.
if "logComponentVerbosity" not in suite_set_parameters:
suite_set_parameters[
"logComponentVerbosity"] = default_mongos_log_component_verbosity(
)
standalone.add_testing_set_parameters(suite_set_parameters)
return core.programs.mongos_program(logger, job_num, test_id, executable,
process_kwargs, **kwargs)
def fuzz_set_parameters(seed, user_provided_params):
"""Randomly generate mongod configurations and wiredTigerConnectionString."""
rng = random.Random(seed)
ret = {}
params = [generate_flow_control_parameters(rng), generate_independent_parameters(rng)]
for dct in params:
for key, value in dct.items():
ret[key] = value
for key, value in utils.load_yaml(user_provided_params).items():
ret[key] = value
return utils.dump_yaml(ret), generate_eviction_configs(rng)
def get_set_param_errors(process_params):
agg_set_params = collections.defaultdict(list)
for set_param in process_params:
for key, value in utils.load_yaml(set_param).items():
agg_set_params[key] += [value]
errors = []
for key, values in agg_set_params.items():
if len(values) == 1:
continue
for left, _ in enumerate(values):
for right in range(left + 1, len(values)):
if values[left] != values[right]:
errors.append(
f"setParameter has multiple distinct values. Key: {key} Values: {values}"
)
return errors
def __init__(self, filename=None, yaml_string=None, raw_dict=None):
"""Init from a yaml file, from a yaml string, or default-construct."""
super(HistoryDict, self).__init__()
if filename is not None and yaml_string is not None:
raise ValueError(
"Cannot construct HistoryDict from both yaml object and file.")
self._history_store = defaultdict(list)
self._value_store = dict()
self._global_time = 0
raw_dict = default_if_none(raw_dict, {})
if filename is not None:
raw_dict = load_yaml_file(filename)
elif yaml_string is not None:
raw_dict = load_yaml(yaml_string)
else:
return # Just default-construct.
schema_version = raw_dict["SchemaVersion"]
if schema_version != SCHEMA_VERSION:
raise ValueError(
f"Invalid schema version. Expected {SCHEMA_VERSION} but found {schema_version}."
)
history_dict = raw_dict["History"]
for key in history_dict:
for raw_access in history_dict[key]:
access = Access.from_dict(raw_access)
self._history_store[key].append(access)
self._global_time = max(access.time, self._global_time)
last_val = self._retrieve_last_value(key)
if last_val is not TOMBSTONE:
self._value_store[key] = last_val
# The next recorded global time should be 1 higher than the last.
self._global_time += 1
def _merge_set_params(param_list):
ret = {}
for set_param in param_list:
ret.update(utils.load_yaml(set_param))
return utils.dump_yaml(ret)
def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements
logger,
job_num,
test_id=None,
executable=None,
connection_string=None,
filename=None,
test_filename=None,
process_kwargs=None,
**kwargs):
"""Return a Process instance that starts a mongo shell.
The shell is started with the given connection string and arguments constructed from 'kwargs'.
:param filename: the file run by the mongo shell
:param test_filename: The test file - it's usually `filename`, but may be different for
tests that use JS runner files, which in turn run real JS files.
"""
executable = utils.default_if_none(
utils.default_if_none(executable, config.MONGO_EXECUTABLE),
config.DEFAULT_MONGO_EXECUTABLE)
args = [executable]
eval_sb = [] # String builder.
global_vars = kwargs.pop("global_vars", {}).copy()
def basename(filepath):
return os.path.splitext(os.path.basename(filepath))[0]
if test_filename is not None:
test_name = basename(test_filename)
elif filename is not None:
test_name = basename(filename)
else:
test_name = None
shortcut_opts = {
"backupOnRestartDir": (config.BACKUP_ON_RESTART_DIR, None),
"enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True),
"mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""),
"noJournal": (config.NO_JOURNAL, False),
"storageEngine": (config.STORAGE_ENGINE, ""),
"storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""),
"testName": (test_name, ""),
"transportLayer": (config.TRANSPORT_LAYER, ""),
"wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""),
"wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""),
"wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""),
# Evergreen variables.
"evergreenDebugSymbolsUrl": (config.DEBUG_SYMBOLS_URL, ""),
}
test_data = global_vars.get("TestData", {}).copy()
for opt_name in shortcut_opts:
(opt_value, opt_default) = shortcut_opts[opt_name]
if opt_value is not None:
test_data[opt_name] = opt_value
elif opt_name not in test_data:
# Only use 'opt_default' if the property wasn't set in the YAML configuration.
test_data[opt_name] = opt_default
global_vars["TestData"] = test_data
if config.EVERGREEN_TASK_ID is not None:
test_data["inEvergreen"] = True
# Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since
# they are dictionaries, they will be converted to JavaScript objects when passed to the shell
# by the _format_shell_vars() function.
mongod_set_parameters = test_data.get("setParameters", {}).copy()
mongos_set_parameters = test_data.get("setParametersMongos", {}).copy()
mongocryptd_set_parameters = test_data.get("setParametersMongocryptd",
{}).copy()
# Propagate additional setParameters to mongod processes spawned by the mongo shell. Command
# line options to resmoke.py override the YAML configuration.
if config.MONGOD_SET_PARAMETERS is not None:
mongod_set_parameters.update(
utils.load_yaml(config.MONGOD_SET_PARAMETERS))
# Propagate additional setParameters to mongos processes spawned by the mongo shell. Command
# line options to resmoke.py override the YAML configuration.
if config.MONGOS_SET_PARAMETERS is not None:
mongos_set_parameters.update(
utils.load_yaml(config.MONGOS_SET_PARAMETERS))
# Propagate additional setParameters to mongocryptd processes spawned by the mongo shell.
# Command line options to resmoke.py override the YAML configuration.
if config.MONGOCRYPTD_SET_PARAMETERS is not None:
mongocryptd_set_parameters.update(
utils.load_yaml(config.MONGOCRYPTD_SET_PARAMETERS))
fixturelib = FixtureLib()
mongod_launcher = standalone.MongodLauncher(fixturelib)
# If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its
# value to a default.
mongod_set_parameters.setdefault(
"logComponentVerbosity",
mongod_launcher.get_default_log_component_verbosity_for_mongod())
# If the 'enableFlowControl' setParameter for mongod was not already specified, we set its value
# to a default.
if config.FLOW_CONTROL is not None:
mongod_set_parameters.setdefault("enableFlowControl",
config.FLOW_CONTROL == "on")
# Set the default value for minimum resharding operation duration to 5 seconds.
mongod_set_parameters.setdefault(
"reshardingMinimumOperationDurationMillis", 5000)
mongos_launcher = shardedcluster.MongosLauncher(fixturelib)
# If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its
# value to a default.
mongos_set_parameters.setdefault(
"logComponentVerbosity",
mongos_launcher.default_mongos_log_component_verbosity())
test_data["setParameters"] = mongod_set_parameters
test_data["setParametersMongos"] = mongos_set_parameters
test_data["setParametersMongocryptd"] = mongocryptd_set_parameters
test_data["undoRecorderPath"] = config.UNDO_RECORDER_PATH
# There's a periodic background thread that checks for and aborts expired transactions.
# "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring
# and being aborted by the background thread. It defaults to 60 seconds, which is too short to
# be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen
# execution timeout.
if "transactionLifetimeLimitSeconds" not in test_data:
test_data["transactionLifetimeLimitSeconds"] = 24 * 60 * 60
if "eval_prepend" in kwargs:
eval_sb.append(str(kwargs.pop("eval_prepend")))
# If nodb is specified, pass the connection string through TestData so it can be used inside the
# test, then delete it so it isn't given as an argument to the mongo shell.
if "nodb" in kwargs and connection_string is not None:
test_data["connectionString"] = connection_string
connection_string = None
for var_name in global_vars:
_format_shell_vars(eval_sb, [var_name], global_vars[var_name])
if "eval" in kwargs:
eval_sb.append(str(kwargs.pop("eval")))
# Load this file to allow a callback to validate collections before shutting down mongod.
eval_sb.append(
"load('jstests/libs/override_methods/validate_collections_on_shutdown.js');"
)
# Load a callback to check UUID consistency before shutting down a ShardingTest.
eval_sb.append(
"load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');"
)
# Load a callback to check index consistency before shutting down a ShardingTest.
eval_sb.append(
"load('jstests/libs/override_methods/check_indexes_consistent_across_cluster.js');"
)
# Load a callback to check that all orphans are deleted before shutting down a ShardingTest.
eval_sb.append(
"load('jstests/libs/override_methods/check_orphans_are_deleted.js');")
# Load this file to retry operations that fail due to in-progress background operations.
eval_sb.append(
"load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');"
)
eval_sb.append(
"(function() { Timestamp.prototype.toString = function() { throw new Error(\"Cannot toString timestamps. Consider using timestampCmp() for comparison or tojson(<variable>) for output.\"); } })();"
)
eval_str = "; ".join(eval_sb)
args.append("--eval")
args.append(eval_str)
if connection_string is not None:
# The --host and --port options are ignored by the mongo shell when an explicit connection
# string is specified. We remove these options to avoid any ambiguity with what server the
# logged mongo shell invocation will connect to.
if "port" in kwargs:
kwargs.pop("port")
if "host" in kwargs:
kwargs.pop("host")
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
if connection_string is not None:
args.append(connection_string)
# Have the mongo shell run the specified file.
if filename is not None:
args.append(filename)
_set_keyfile_permissions(test_data)
process_kwargs = utils.default_if_none(process_kwargs, {})
process_kwargs["job_num"] = job_num
process_kwargs["test_id"] = test_id
return make_process(logger, args, **process_kwargs)
def mongod_program( # pylint: disable=too-many-branches,too-many-statements
logger,
executable=None,
process_kwargs=None,
**kwargs):
"""Return a Process instance that starts mongod arguments constructed from 'kwargs'."""
executable = utils.default_if_none(executable,
config.DEFAULT_MONGOD_EXECUTABLE)
args = [executable]
# Apply the --setParameter command line argument. Command line options to resmoke.py override
# the YAML configuration.
suite_set_parameters = kwargs.pop("set_parameters", {})
if config.MONGOD_SET_PARAMETERS is not None:
suite_set_parameters.update(
utils.load_yaml(config.MONGOD_SET_PARAMETERS))
# Set default log verbosity levels if none were specified.
if "logComponentVerbosity" not in suite_set_parameters:
suite_set_parameters[
"logComponentVerbosity"] = get_default_log_component_verbosity_for_mongod(
executable)
# minNumChunksForSessionsCollection controls the minimum number of chunks the balancer will
# enforce for the sessions collection. If the actual number of chunks is less, the balancer will
# issue split commands to create more chunks. As a result, the balancer will also end up moving
# chunks for the sessions collection to balance the chunks across shards. Unless the suite is
# explicitly prepared to handle these background migrations, set the parameter to 1.
if "configsvr" in kwargs and "minNumChunksForSessionsCollection" not in suite_set_parameters:
suite_set_parameters["minNumChunksForSessionsCollection"] = 1
# orphanCleanupDelaySecs controls an artificial delay before cleaning up an orphaned chunk
# that has migrated off of a shard, meant to allow most dependent queries on secondaries to
# complete first. It defaults to 900, or 15 minutes, which is prohibitively long for tests.
# Setting it in the .yml file overrides this.
if "shardsvr" in kwargs and "orphanCleanupDelaySecs" not in suite_set_parameters:
suite_set_parameters["orphanCleanupDelaySecs"] = 1
# The LogicalSessionCache does automatic background refreshes in the server. This is
# race-y for tests, since tests trigger their own immediate refreshes instead. Turn off
# background refreshing for tests. Set in the .yml file to override this.
if "disableLogicalSessionCacheRefresh" not in suite_set_parameters:
suite_set_parameters["disableLogicalSessionCacheRefresh"] = True
# Set coordinateCommitReturnImmediatelyAfterPersistingDecision to false so that tests do
# not need to rely on causal consistency or explicity wait for the transaction to finish
# committing.
if executable != LAST_STABLE_MONGOD_BINARY and \
"coordinateCommitReturnImmediatelyAfterPersistingDecision" not in suite_set_parameters:
suite_set_parameters[
"coordinateCommitReturnImmediatelyAfterPersistingDecision"] = False
# There's a periodic background thread that checks for and aborts expired transactions.
# "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring
# and being aborted by the background thread. It defaults to 60 seconds, which is too short to
# be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen
# execution timeout.
if "transactionLifetimeLimitSeconds" not in suite_set_parameters:
suite_set_parameters["transactionLifetimeLimitSeconds"] = 24 * 60 * 60
# Hybrid index builds drain writes received during the build process in batches of 1000 writes
# by default. Not all tests perform enough writes to exercise the code path where multiple
# batches are applied, which means certain bugs are harder to encounter. Set this level lower
# so there are more opportunities to drain writes in multiple batches.
if "maxIndexBuildDrainBatchSize" not in suite_set_parameters:
suite_set_parameters["maxIndexBuildDrainBatchSize"] = 10
# The periodic no-op writer writes an oplog entry of type='n' once every 10 seconds. This has
# the potential to mask issues such as SERVER-31609 because it allows the operationTime of
# cluster to advance even if the client is blocked for other reasons. We should disable the
# periodic no-op writer. Set in the .yml file to override this.
if "replSet" in kwargs and "writePeriodicNoops" not in suite_set_parameters:
suite_set_parameters["writePeriodicNoops"] = False
# The default time for stepdown and quiesce mode in response to SIGTERM is 15 seconds. Reduce
# this to 100ms for faster shutdown. On branches 4.4 and earlier, there is no quiesce mode, but
# the default time for stepdown is 10 seconds.
# TODO(SERVER-47797): Remove reference to waitForStepDownOnNonCommandShutdown.
if ("replSet" in kwargs and "waitForStepDownOnNonCommandShutdown"
not in suite_set_parameters
and "shutdownTimeoutMillisForSignaledShutdown"
not in suite_set_parameters):
if executable == LAST_STABLE_MONGOD_BINARY:
suite_set_parameters["waitForStepDownOnNonCommandShutdown"] = False
else:
suite_set_parameters[
"shutdownTimeoutMillisForSignaledShutdown"] = 100
if "enableFlowControl" not in suite_set_parameters and config.FLOW_CONTROL is not None:
suite_set_parameters["enableFlowControl"] = (
config.FLOW_CONTROL == "on")
if ("failpoint.flowControlTicketOverride" not in suite_set_parameters
and config.FLOW_CONTROL_TICKETS is not None):
suite_set_parameters["failpoint.flowControlTicketOverride"] = {
"mode": "alwaysOn",
"data": {
"numTickets": config.FLOW_CONTROL_TICKETS
}
}
# TODO(SERVER-48645): Only keep the else block once v4.4 is not longer the last stable version
if executable == LAST_STABLE_MONGOD_BINARY:
suite_set_parameters.setdefault("enableTestCommands", True)
else:
_add_testing_set_parameters(suite_set_parameters)
_apply_set_parameters(args, suite_set_parameters)
shortcut_opts = {
"enableMajorityReadConcern": config.MAJORITY_READ_CONCERN,
"nojournal": config.NO_JOURNAL,
"serviceExecutor": config.SERVICE_EXECUTOR,
"storageEngine": config.STORAGE_ENGINE,
"transportLayer": config.TRANSPORT_LAYER,
"wiredTigerCollectionConfigString": config.WT_COLL_CONFIG,
"wiredTigerEngineConfigString": config.WT_ENGINE_CONFIG,
"wiredTigerIndexConfigString": config.WT_INDEX_CONFIG,
}
if config.STORAGE_ENGINE == "inMemory":
shortcut_opts["inMemorySizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE
elif config.STORAGE_ENGINE == "rocksdb":
shortcut_opts["rocksdbCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE
elif config.STORAGE_ENGINE == "wiredTiger" or config.STORAGE_ENGINE is None:
shortcut_opts[
"wiredTigerCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE
# These options are just flags, so they should not take a value.
opts_without_vals = ("nojournal", "logappend")
# Have the --nojournal command line argument to resmoke.py unset the journal option.
if shortcut_opts["nojournal"] and "journal" in kwargs:
del kwargs["journal"]
# Ensure that config servers run with journaling enabled.
if "configsvr" in kwargs:
shortcut_opts["nojournal"] = False
kwargs["journal"] = ""
# Command line options override the YAML configuration.
for opt_name in shortcut_opts:
opt_value = shortcut_opts[opt_name]
if opt_name in opts_without_vals:
# Options that are specified as --flag on the command line are represented by a boolean
# value where True indicates that the flag should be included in 'kwargs'.
if opt_value:
kwargs[opt_name] = ""
else:
# Options that are specified as --key=value on the command line are represented by a
# value where None indicates that the key-value pair shouldn't be included in 'kwargs'.
if opt_value is not None:
kwargs[opt_name] = opt_value
# Override the storage engine specified on the command line with "wiredTiger" if running a
# config server replica set.
if "replSet" in kwargs and "configsvr" in kwargs:
kwargs["storageEngine"] = "wiredTiger"
# Apply the rest of the command line arguments.
_apply_kwargs(args, kwargs)
_set_keyfile_permissions(kwargs)
process_kwargs = utils.default_if_none(process_kwargs, {})
return make_process(logger, args, **process_kwargs)
