def mongos_program(logger, executable=None, process_kwargs=None, **kwargs): """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" executable = utils.default_if_none(executable, config.DEFAULT_MONGOS_EXECUTABLE) args = [executable] # Apply the --setParameter command line argument. Command line options to resmoke.py override # the YAML configuration. suite_set_parameters = kwargs.pop("set_parameters", {}) if config.MONGOS_SET_PARAMETERS is not None: suite_set_parameters.update( utils.load_yaml(config.MONGOS_SET_PARAMETERS)) # Set default log verbosity levels if none were specified. if "logComponentVerbosity" not in suite_set_parameters: suite_set_parameters[ "logComponentVerbosity"] = default_mongos_log_component_verbosity( ) _add_testing_set_parameters(suite_set_parameters) _apply_set_parameters(args, suite_set_parameters) # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) _set_keyfile_permissions(kwargs) process_kwargs = utils.default_if_none(process_kwargs, {}) return make_process(logger, args, **process_kwargs)
def _mongos_program(logger, job_num, test_id=None, executable=None, process_kwargs=None, **kwargs): """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" executable = utils.default_if_none(executable, config.DEFAULT_MONGOS_EXECUTABLE) # Apply the --setParameter command line argument. Command line options to resmoke.py override # the YAML configuration. suite_set_parameters = kwargs.setdefault("set_parameters", {}) if config.MONGOS_SET_PARAMETERS is not None: suite_set_parameters.update( utils.load_yaml(config.MONGOS_SET_PARAMETERS)) # Set default log verbosity levels if none were specified. if "logComponentVerbosity" not in suite_set_parameters: suite_set_parameters[ "logComponentVerbosity"] = default_mongos_log_component_verbosity( ) standalone.add_testing_set_parameters(suite_set_parameters) return core.programs.mongos_program(logger, job_num, test_id, executable, process_kwargs, **kwargs)
def fuzz_set_parameters(seed, user_provided_params): """Randomly generate mongod configurations and wiredTigerConnectionString.""" rng = random.Random(seed) ret = {} params = [generate_flow_control_parameters(rng), generate_independent_parameters(rng)] for dct in params: for key, value in dct.items(): ret[key] = value for key, value in utils.load_yaml(user_provided_params).items(): ret[key] = value return utils.dump_yaml(ret), generate_eviction_configs(rng)
def get_set_param_errors(process_params): agg_set_params = collections.defaultdict(list) for set_param in process_params: for key, value in utils.load_yaml(set_param).items(): agg_set_params[key] += [value] errors = [] for key, values in agg_set_params.items(): if len(values) == 1: continue for left, _ in enumerate(values): for right in range(left + 1, len(values)): if values[left] != values[right]: errors.append( f"setParameter has multiple distinct values. Key: {key} Values: {values}" ) return errors
def __init__(self, filename=None, yaml_string=None, raw_dict=None): """Init from a yaml file, from a yaml string, or default-construct.""" super(HistoryDict, self).__init__() if filename is not None and yaml_string is not None: raise ValueError( "Cannot construct HistoryDict from both yaml object and file.") self._history_store = defaultdict(list) self._value_store = dict() self._global_time = 0 raw_dict = default_if_none(raw_dict, {}) if filename is not None: raw_dict = load_yaml_file(filename) elif yaml_string is not None: raw_dict = load_yaml(yaml_string) else: return # Just default-construct. schema_version = raw_dict["SchemaVersion"] if schema_version != SCHEMA_VERSION: raise ValueError( f"Invalid schema version. Expected {SCHEMA_VERSION} but found {schema_version}." ) history_dict = raw_dict["History"] for key in history_dict: for raw_access in history_dict[key]: access = Access.from_dict(raw_access) self._history_store[key].append(access) self._global_time = max(access.time, self._global_time) last_val = self._retrieve_last_value(key) if last_val is not TOMBSTONE: self._value_store[key] = last_val # The next recorded global time should be 1 higher than the last. self._global_time += 1
def _merge_set_params(param_list): ret = {} for set_param in param_list: ret.update(utils.load_yaml(set_param)) return utils.dump_yaml(ret)
def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches,too-many-locals,too-many-statements logger, job_num, test_id=None, executable=None, connection_string=None, filename=None, test_filename=None, process_kwargs=None, **kwargs): """Return a Process instance that starts a mongo shell. The shell is started with the given connection string and arguments constructed from 'kwargs'. :param filename: the file run by the mongo shell :param test_filename: The test file - it's usually `filename`, but may be different for tests that use JS runner files, which in turn run real JS files. """ executable = utils.default_if_none( utils.default_if_none(executable, config.MONGO_EXECUTABLE), config.DEFAULT_MONGO_EXECUTABLE) args = [executable] eval_sb = [] # String builder. global_vars = kwargs.pop("global_vars", {}).copy() def basename(filepath): return os.path.splitext(os.path.basename(filepath))[0] if test_filename is not None: test_name = basename(test_filename) elif filename is not None: test_name = basename(filename) else: test_name = None shortcut_opts = { "backupOnRestartDir": (config.BACKUP_ON_RESTART_DIR, None), "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True), "mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""), "noJournal": (config.NO_JOURNAL, False), "storageEngine": (config.STORAGE_ENGINE, ""), "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""), "testName": (test_name, ""), "transportLayer": (config.TRANSPORT_LAYER, ""), "wiredTigerCollectionConfigString": (config.WT_COLL_CONFIG, ""), "wiredTigerEngineConfigString": (config.WT_ENGINE_CONFIG, ""), "wiredTigerIndexConfigString": (config.WT_INDEX_CONFIG, ""), # Evergreen variables. "evergreenDebugSymbolsUrl": (config.DEBUG_SYMBOLS_URL, ""), } test_data = global_vars.get("TestData", {}).copy() for opt_name in shortcut_opts: (opt_value, opt_default) = shortcut_opts[opt_name] if opt_value is not None: test_data[opt_name] = opt_value elif opt_name not in test_data: # Only use 'opt_default' if the property wasn't set in the YAML configuration. test_data[opt_name] = opt_default global_vars["TestData"] = test_data if config.EVERGREEN_TASK_ID is not None: test_data["inEvergreen"] = True # Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since # they are dictionaries, they will be converted to JavaScript objects when passed to the shell # by the _format_shell_vars() function. mongod_set_parameters = test_data.get("setParameters", {}).copy() mongos_set_parameters = test_data.get("setParametersMongos", {}).copy() mongocryptd_set_parameters = test_data.get("setParametersMongocryptd", {}).copy() # Propagate additional setParameters to mongod processes spawned by the mongo shell. Command # line options to resmoke.py override the YAML configuration. if config.MONGOD_SET_PARAMETERS is not None: mongod_set_parameters.update( utils.load_yaml(config.MONGOD_SET_PARAMETERS)) # Propagate additional setParameters to mongos processes spawned by the mongo shell. Command # line options to resmoke.py override the YAML configuration. if config.MONGOS_SET_PARAMETERS is not None: mongos_set_parameters.update( utils.load_yaml(config.MONGOS_SET_PARAMETERS)) # Propagate additional setParameters to mongocryptd processes spawned by the mongo shell. # Command line options to resmoke.py override the YAML configuration. if config.MONGOCRYPTD_SET_PARAMETERS is not None: mongocryptd_set_parameters.update( utils.load_yaml(config.MONGOCRYPTD_SET_PARAMETERS)) fixturelib = FixtureLib() mongod_launcher = standalone.MongodLauncher(fixturelib) # If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its # value to a default. mongod_set_parameters.setdefault( "logComponentVerbosity", mongod_launcher.get_default_log_component_verbosity_for_mongod()) # If the 'enableFlowControl' setParameter for mongod was not already specified, we set its value # to a default. if config.FLOW_CONTROL is not None: mongod_set_parameters.setdefault("enableFlowControl", config.FLOW_CONTROL == "on") # Set the default value for minimum resharding operation duration to 5 seconds. mongod_set_parameters.setdefault( "reshardingMinimumOperationDurationMillis", 5000) mongos_launcher = shardedcluster.MongosLauncher(fixturelib) # If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its # value to a default. mongos_set_parameters.setdefault( "logComponentVerbosity", mongos_launcher.default_mongos_log_component_verbosity()) test_data["setParameters"] = mongod_set_parameters test_data["setParametersMongos"] = mongos_set_parameters test_data["setParametersMongocryptd"] = mongocryptd_set_parameters test_data["undoRecorderPath"] = config.UNDO_RECORDER_PATH # There's a periodic background thread that checks for and aborts expired transactions. # "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring # and being aborted by the background thread. It defaults to 60 seconds, which is too short to # be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen # execution timeout. if "transactionLifetimeLimitSeconds" not in test_data: test_data["transactionLifetimeLimitSeconds"] = 24 * 60 * 60 if "eval_prepend" in kwargs: eval_sb.append(str(kwargs.pop("eval_prepend"))) # If nodb is specified, pass the connection string through TestData so it can be used inside the # test, then delete it so it isn't given as an argument to the mongo shell. if "nodb" in kwargs and connection_string is not None: test_data["connectionString"] = connection_string connection_string = None for var_name in global_vars: _format_shell_vars(eval_sb, [var_name], global_vars[var_name]) if "eval" in kwargs: eval_sb.append(str(kwargs.pop("eval"))) # Load this file to allow a callback to validate collections before shutting down mongod. eval_sb.append( "load('jstests/libs/override_methods/validate_collections_on_shutdown.js');" ) # Load a callback to check UUID consistency before shutting down a ShardingTest. eval_sb.append( "load('jstests/libs/override_methods/check_uuids_consistent_across_cluster.js');" ) # Load a callback to check index consistency before shutting down a ShardingTest. eval_sb.append( "load('jstests/libs/override_methods/check_indexes_consistent_across_cluster.js');" ) # Load a callback to check that all orphans are deleted before shutting down a ShardingTest. eval_sb.append( "load('jstests/libs/override_methods/check_orphans_are_deleted.js');") # Load this file to retry operations that fail due to in-progress background operations. eval_sb.append( "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');" ) eval_sb.append( "(function() { Timestamp.prototype.toString = function() { throw new Error(\"Cannot toString timestamps. Consider using timestampCmp() for comparison or tojson(<variable>) for output.\"); } })();" ) eval_str = "; ".join(eval_sb) args.append("--eval") args.append(eval_str) if connection_string is not None: # The --host and --port options are ignored by the mongo shell when an explicit connection # string is specified. We remove these options to avoid any ambiguity with what server the # logged mongo shell invocation will connect to. if "port" in kwargs: kwargs.pop("port") if "host" in kwargs: kwargs.pop("host") # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) if connection_string is not None: args.append(connection_string) # Have the mongo shell run the specified file. if filename is not None: args.append(filename) _set_keyfile_permissions(test_data) process_kwargs = utils.default_if_none(process_kwargs, {}) process_kwargs["job_num"] = job_num process_kwargs["test_id"] = test_id return make_process(logger, args, **process_kwargs)
def mongod_program( # pylint: disable=too-many-branches,too-many-statements logger, executable=None, process_kwargs=None, **kwargs): """Return a Process instance that starts mongod arguments constructed from 'kwargs'.""" executable = utils.default_if_none(executable, config.DEFAULT_MONGOD_EXECUTABLE) args = [executable] # Apply the --setParameter command line argument. Command line options to resmoke.py override # the YAML configuration. suite_set_parameters = kwargs.pop("set_parameters", {}) if config.MONGOD_SET_PARAMETERS is not None: suite_set_parameters.update( utils.load_yaml(config.MONGOD_SET_PARAMETERS)) # Set default log verbosity levels if none were specified. if "logComponentVerbosity" not in suite_set_parameters: suite_set_parameters[ "logComponentVerbosity"] = get_default_log_component_verbosity_for_mongod( executable) # minNumChunksForSessionsCollection controls the minimum number of chunks the balancer will # enforce for the sessions collection. If the actual number of chunks is less, the balancer will # issue split commands to create more chunks. As a result, the balancer will also end up moving # chunks for the sessions collection to balance the chunks across shards. Unless the suite is # explicitly prepared to handle these background migrations, set the parameter to 1. if "configsvr" in kwargs and "minNumChunksForSessionsCollection" not in suite_set_parameters: suite_set_parameters["minNumChunksForSessionsCollection"] = 1 # orphanCleanupDelaySecs controls an artificial delay before cleaning up an orphaned chunk # that has migrated off of a shard, meant to allow most dependent queries on secondaries to # complete first. It defaults to 900, or 15 minutes, which is prohibitively long for tests. # Setting it in the .yml file overrides this. if "shardsvr" in kwargs and "orphanCleanupDelaySecs" not in suite_set_parameters: suite_set_parameters["orphanCleanupDelaySecs"] = 1 # The LogicalSessionCache does automatic background refreshes in the server. This is # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off # background refreshing for tests. Set in the .yml file to override this. if "disableLogicalSessionCacheRefresh" not in suite_set_parameters: suite_set_parameters["disableLogicalSessionCacheRefresh"] = True # Set coordinateCommitReturnImmediatelyAfterPersistingDecision to false so that tests do # not need to rely on causal consistency or explicity wait for the transaction to finish # committing. if executable != LAST_STABLE_MONGOD_BINARY and \ "coordinateCommitReturnImmediatelyAfterPersistingDecision" not in suite_set_parameters: suite_set_parameters[ "coordinateCommitReturnImmediatelyAfterPersistingDecision"] = False # There's a periodic background thread that checks for and aborts expired transactions. # "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring # and being aborted by the background thread. It defaults to 60 seconds, which is too short to # be reliable for our tests. Setting it to 24 hours, so that it is longer than the Evergreen # execution timeout. if "transactionLifetimeLimitSeconds" not in suite_set_parameters: suite_set_parameters["transactionLifetimeLimitSeconds"] = 24 * 60 * 60 # Hybrid index builds drain writes received during the build process in batches of 1000 writes # by default. Not all tests perform enough writes to exercise the code path where multiple # batches are applied, which means certain bugs are harder to encounter. Set this level lower # so there are more opportunities to drain writes in multiple batches. if "maxIndexBuildDrainBatchSize" not in suite_set_parameters: suite_set_parameters["maxIndexBuildDrainBatchSize"] = 10 # The periodic no-op writer writes an oplog entry of type='n' once every 10 seconds. This has # the potential to mask issues such as SERVER-31609 because it allows the operationTime of # cluster to advance even if the client is blocked for other reasons. We should disable the # periodic no-op writer. Set in the .yml file to override this. if "replSet" in kwargs and "writePeriodicNoops" not in suite_set_parameters: suite_set_parameters["writePeriodicNoops"] = False # The default time for stepdown and quiesce mode in response to SIGTERM is 15 seconds. Reduce # this to 100ms for faster shutdown. On branches 4.4 and earlier, there is no quiesce mode, but # the default time for stepdown is 10 seconds. # TODO(SERVER-47797): Remove reference to waitForStepDownOnNonCommandShutdown. if ("replSet" in kwargs and "waitForStepDownOnNonCommandShutdown" not in suite_set_parameters and "shutdownTimeoutMillisForSignaledShutdown" not in suite_set_parameters): if executable == LAST_STABLE_MONGOD_BINARY: suite_set_parameters["waitForStepDownOnNonCommandShutdown"] = False else: suite_set_parameters[ "shutdownTimeoutMillisForSignaledShutdown"] = 100 if "enableFlowControl" not in suite_set_parameters and config.FLOW_CONTROL is not None: suite_set_parameters["enableFlowControl"] = ( config.FLOW_CONTROL == "on") if ("failpoint.flowControlTicketOverride" not in suite_set_parameters and config.FLOW_CONTROL_TICKETS is not None): suite_set_parameters["failpoint.flowControlTicketOverride"] = { "mode": "alwaysOn", "data": { "numTickets": config.FLOW_CONTROL_TICKETS } } # TODO(SERVER-48645): Only keep the else block once v4.4 is not longer the last stable version if executable == LAST_STABLE_MONGOD_BINARY: suite_set_parameters.setdefault("enableTestCommands", True) else: _add_testing_set_parameters(suite_set_parameters) _apply_set_parameters(args, suite_set_parameters) shortcut_opts = { "enableMajorityReadConcern": config.MAJORITY_READ_CONCERN, "nojournal": config.NO_JOURNAL, "serviceExecutor": config.SERVICE_EXECUTOR, "storageEngine": config.STORAGE_ENGINE, "transportLayer": config.TRANSPORT_LAYER, "wiredTigerCollectionConfigString": config.WT_COLL_CONFIG, "wiredTigerEngineConfigString": config.WT_ENGINE_CONFIG, "wiredTigerIndexConfigString": config.WT_INDEX_CONFIG, } if config.STORAGE_ENGINE == "inMemory": shortcut_opts["inMemorySizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE elif config.STORAGE_ENGINE == "rocksdb": shortcut_opts["rocksdbCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE elif config.STORAGE_ENGINE == "wiredTiger" or config.STORAGE_ENGINE is None: shortcut_opts[ "wiredTigerCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE # These options are just flags, so they should not take a value. opts_without_vals = ("nojournal", "logappend") # Have the --nojournal command line argument to resmoke.py unset the journal option. if shortcut_opts["nojournal"] and "journal" in kwargs: del kwargs["journal"] # Ensure that config servers run with journaling enabled. if "configsvr" in kwargs: shortcut_opts["nojournal"] = False kwargs["journal"] = "" # Command line options override the YAML configuration. for opt_name in shortcut_opts: opt_value = shortcut_opts[opt_name] if opt_name in opts_without_vals: # Options that are specified as --flag on the command line are represented by a boolean # value where True indicates that the flag should be included in 'kwargs'. if opt_value: kwargs[opt_name] = "" else: # Options that are specified as --key=value on the command line are represented by a # value where None indicates that the key-value pair shouldn't be included in 'kwargs'. if opt_value is not None: kwargs[opt_name] = opt_value # Override the storage engine specified on the command line with "wiredTiger" if running a # config server replica set. if "replSet" in kwargs and "configsvr" in kwargs: kwargs["storageEngine"] = "wiredTiger" # Apply the rest of the command line arguments. _apply_kwargs(args, kwargs) _set_keyfile_permissions(kwargs) process_kwargs = utils.default_if_none(process_kwargs, {}) return make_process(logger, args, **process_kwargs)