def after_test(self, test, test_report):
"""Run mongo shell to call replSetTest.awaitReplication()."""
start_time = time.time()
client_conn = self.fixture.get_driver_connection_url()
js_cmds = """
conn = '{}';
try {{
rst = new ReplSetTest(conn);
rst.awaitReplication();
}} catch (e) {{
if (e.code === ErrorCodes.InterruptedAtShutdown || e.code === ErrorCodes.ShutdownInProgress) {{
jsTestLog("Ignoring shutdown error: " + tojson(e));
}}
throw e;
}}"""
shell_options = {"nodb": "", "eval": js_cmds.format(client_conn)}
shell_proc = core.programs.mongo_shell_program(self.hook_logger,
**shell_options)
shell_proc.start()
return_code = shell_proc.wait()
if return_code:
raise errors.ServerFailure(
"Awaiting replication failed for {}".format(client_conn))
self.hook_logger.info("WaitForReplication took %0.4f seconds",
time.time() - start_time)
def after_test(self, test, test_report):
"""Run mongo shell to call replSetTest.awaitReplication()."""
start_time = time.time()
client_conn = self.fixture.get_driver_connection_url()
js_cmds = """
conn = '{}';
try {{
rst = new ReplSetTest(conn);
rst.awaitReplication();
}} catch (e) {{
jsTestLog("WaitForReplication got error: " + tojson(e));
if (!e.message.includes('The server is in quiesce mode and will shut down')) {{
throw e;
}}
jsTestLog("Ignoring shutdown error in quiesce mode");
}}"""
shell_options = {"nodb": "", "eval": js_cmds.format(client_conn)}
shell_proc = core.programs.mongo_shell_program(self.hook_logger,
self.fixture.job_num,
**shell_options)
shell_proc.start()
return_code = shell_proc.wait()
if return_code:
raise errors.ServerFailure(
"Awaiting replication failed for {}".format(client_conn))
self.hook_logger.info("WaitForReplication took %0.4f seconds",
time.time() - start_time)
def build_fixture(self, logger, job_num, fixturelib, *args, **kwargs):
"""Build a sharded cluster."""
mixed_bin_versions = kwargs.get("mixed_bin_versions", config.MIXED_BIN_VERSIONS)
old_bin_version = kwargs.pop("old_bin_version", config.MULTIVERSION_BIN_VERSION)
is_multiversion = mixed_bin_versions is not None
num_shards = kwargs.pop("num_shards", 1)
num_shards_option = config.NUM_SHARDS
num_shards = num_shards if not num_shards_option else num_shards_option
kwargs["num_shards"] = num_shards
num_rs_nodes_per_shard = kwargs.pop("num_rs_nodes_per_shard", 1)
num_rs_nodes_per_shard_option = config.NUM_REPLSET_NODES
num_rs_nodes_per_shard = num_rs_nodes_per_shard if not num_rs_nodes_per_shard_option else num_rs_nodes_per_shard_option
kwargs["num_rs_nodes_per_shard"] = num_rs_nodes_per_shard
num_mongos = kwargs.pop("num_mongos", 1)
kwargs["num_mongos"] = num_mongos
mongos_executable = default_if_none(
kwargs.get("mongos_executable"), config.MONGOS_EXECUTABLE,
config.DEFAULT_MONGOS_EXECUTABLE)
if is_multiversion:
len_versions = len(mixed_bin_versions)
num_mongods = num_shards * num_rs_nodes_per_shard
if len_versions != num_mongods:
msg = ("The number of binary versions specified: {} do not match the number of"
" nodes in the sharded cluster: {}.").format(len_versions, num_mongods)
raise errors.ServerFailure(msg)
from buildscripts.resmokelib import multiversionconstants
mongos_executable = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_MONGOS_BINARY,
config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_MONGOS_BINARY
}[old_bin_version]
kwargs["mongos_executable"] = mongos_executable
sharded_cluster = _FIXTURES[self.REGISTERED_NAME](logger, job_num, fixturelib, *args,
**kwargs)
config_svr = self._new_configsvr(sharded_cluster, is_multiversion)
sharded_cluster.install_configsvr(config_svr)
for rs_shard_index in range(num_shards):
rs_shard = self._new_rs_shard(sharded_cluster, mixed_bin_versions, rs_shard_index,
num_rs_nodes_per_shard)
sharded_cluster.install_rs_shard(rs_shard)
for mongos_index in range(num_mongos):
mongos = self._new_mongos(sharded_cluster, mongos_executable, mongos_index, num_mongos)
sharded_cluster.install_mongos(mongos)
return sharded_cluster
def after_test(self, test, test_report):
"""Run monger shell to call replSetTest.awaitReplication()."""
start_time = time.time()
client_conn = self.fixture.get_driver_connection_url()
js_cmds = "conn = '{}'; rst = new ReplSetTest(conn); rst.awaitReplication();".format(
client_conn)
shell_options = {"nodb": "", "eval": js_cmds}
shell_proc = core.programs.monger_shell_program(self.hook_logger, **shell_options)
shell_proc.start()
return_code = shell_proc.wait()
if return_code:
raise errors.ServerFailure("Awaiting replication failed for {}".format(client_conn))
self.hook_logger.info("WaitForReplication took %0.4f seconds", time.time() - start_time)
def _do_teardown(self, mode=None):
"""Shut down the replica sets."""
self.logger.info("Stopping all replica sets...")
running_at_start = self.is_running()
if not running_at_start:
self.logger.warning("All replica sets were expected to be running, but weren't.")
teardown_handler = interface.FixtureTeardownHandler(self.logger)
for replica_set in self.replica_sets:
teardown_handler.teardown(replica_set, "replica_set", mode=mode)
if teardown_handler.was_successful():
self.logger.info("Successfully stopped all replica sets.")
else:
self.logger.error("Stopping the fixture failed.")
raise errors.ServerFailure(teardown_handler.get_error_message())
def _validate_multiversion_options(
kwargs: Dict[str, Any],
mixed_bin_versions: Optional[List[str]]) -> None:
"""Error out if the number of binary versions does not match the number of nodes in sharded cluster.
:param kwargs: sharded cluster fixture kwargs
:param mixed_bin_versions: the list of bin versions
"""
if mixed_bin_versions is not None:
len_versions = len(mixed_bin_versions)
num_mongods = kwargs["num_shards"] * kwargs[
"num_rs_nodes_per_shard"]
if len_versions != num_mongods:
msg = (
"The number of binary versions specified: {} do not match the number of"
" nodes in the sharded cluster: {}.").format(
len_versions, num_mongods)
raise errors.ServerFailure(msg)
def wait(self, timeout=None):
"""Wait until process has terminated and all output has been consumed by the logger pipes."""
return_code = self._process.wait(timeout)
if self._recorder is not None:
self.logger.info('Saving the UndoDB recording; it may take a few minutes...')
recorder_return = self._recorder.wait(timeout)
if recorder_return != 0:
raise errors.ServerFailure(
"UndoDB live-record did not terminate correctly. This is likely a bug with UndoDB. "
"Please record the logs and notify the #server-testing Slack channel")
if self._stdout_pipe:
self._stdout_pipe.wait_until_finished()
if self._stderr_pipe:
self._stderr_pipe.wait_until_finished()
return return_code
def after_test(self, test, test_report): # noqa: D205,D400
"""Instruct the background thread to stop running the dbhash check now that 'test' has
finished running.
"""
if self._background_job is None:
return
self.logger.info("Pausing the background thread.")
self._background_job.pause()
if self._background_job.exc_info is not None:
if isinstance(self._background_job.exc_info[1], errors.TestFailure):
# If the mongo shell process running the JavaScript file exited with a non-zero
# return code, then we raise an errors.ServerFailure exception to cause resmoke.py's
# test execution to stop.
raise errors.ServerFailure(self._background_job.exc_info[1].args[0])
else:
self.logger.error("Encountered an error inside the background thread.",
exc_info=self._background_job.exc_info)
raise self._background_job.exc_info[1]
def _validate_multiversion_options(
kwargs: Dict[str, Any],
mixed_bin_versions: Optional[List[str]]) -> None:
"""Error out if the number of binary versions does not match the number of nodes in replica set.
:param kwargs: sharded cluster fixture kwargs
:param mixed_bin_versions: the list of bin versions
"""
if mixed_bin_versions is not None:
num_versions = len(mixed_bin_versions)
replset_config_options = kwargs.get("replset_config_options", {})
is_config_svr = "configsvr" in replset_config_options and replset_config_options[
"configsvr"]
if num_versions != kwargs["num_nodes"] and not is_config_svr:
msg = (
"The number of binary versions specified: {} do not match the number of"
" nodes in the replica set: {}.").format(
num_versions, kwargs["num_nodes"])
raise errors.ServerFailure(msg)
def get_last_optime(client):
"""Get the latest optime.
This function is derived from _getLastOpTime() in ReplSetTest.
"""
repl_set_status = client.admin.command({"replSetGetStatus": 1})
conn_status = [m for m in repl_set_status["members"] if "self" in m][0]
optime = conn_status["optime"]
optime_is_empty = False
if isinstance(optime, bson.Timestamp): # PV0
optime_is_empty = (optime == bson.Timestamp(0, 0))
else: # PV1
optime_is_empty = (optime["ts"] == bson.Timestamp(0, 0)
and optime["t"] == -1)
if optime_is_empty:
raise errors.ServerFailure(
"Uninitialized opTime being reported by {addr[0]}:{addr[1]}: {repl_set_status}"
.format(addr=client.address, repl_set_status=repl_set_status))
return optime
def after_test(self, test, test_report):
"""After test execution."""
try:
JSHook.after_test(self, test, test_report)
except errors.TestFailure as err:
raise errors.ServerFailure(err.args[0])
def _do_teardown(self):
if self._should_raise:
raise errors.ServerFailure(self.ERROR_MESSAGE)
def build_fixture(self, logger, job_num, fixturelib, *args, **kwargs): # pylint: disable=too-many-locals
"""Build a replica set."""
# We hijack the mixed_bin_versions passed to the fixture.
mixed_bin_versions = kwargs.pop("mixed_bin_versions", config.MIXED_BIN_VERSIONS)
old_bin_version = kwargs.pop("old_bin_version", config.MULTIVERSION_BIN_VERSION)
# We also hijack the num_nodes because we need it here.
num_nodes = kwargs.pop("num_nodes", 2)
num_replset_nodes = config.NUM_REPLSET_NODES
num_nodes = num_replset_nodes if num_replset_nodes else num_nodes
kwargs["num_nodes"] = num_nodes
replset_config_options = kwargs.get("replset_config_options", {})
mongod_executable = default_if_none(
kwargs.get("mongod_executable"), config.MONGOD_EXECUTABLE,
config.DEFAULT_MONGOD_EXECUTABLE)
kwargs["mongod_executable"] = mongod_executable
latest_mongod = mongod_executable
from buildscripts.resmokelib import multiversionconstants
fcv = multiversionconstants.LATEST_FCV
executables = {BinVersionEnum.NEW: latest_mongod}
classes = {BinVersionEnum.NEW: self.latest_class}
# Default to NEW for all bin versions; may be overridden below.
mongod_binary_versions = [BinVersionEnum.NEW for _ in range(num_nodes)]
is_multiversion = mixed_bin_versions is not None
if is_multiversion:
old_shell_version = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_MONGO_BINARY,
config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY
}[old_bin_version]
old_mongod_version = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_MONGOD_BINARY,
config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_MONGOD_BINARY
}[old_bin_version]
executables[BinVersionEnum.OLD] = old_mongod_version
classes[BinVersionEnum.OLD] = f"{self.latest_class}{self.multiversion_class_suffix}"
load_version(version_path_suffix=self.multiversion_class_suffix,
shell_path=old_shell_version)
is_config_svr = "configsvr" in replset_config_options and replset_config_options[
"configsvr"]
mongod_binary_versions = [x for x in mixed_bin_versions]
num_versions = len(mixed_bin_versions)
fcv = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_FCV, config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_FCV
}[old_bin_version]
if num_versions != num_nodes and not is_config_svr:
msg = ("The number of binary versions specified: {} do not match the number of"
" nodes in the replica set: {}.").format(num_versions, num_nodes)
raise errors.ServerFailure(msg)
replset = _FIXTURES[self.REGISTERED_NAME](logger, job_num, fixturelib, *args, **kwargs)
replset.set_fcv(fcv)
for node_index in range(replset.num_nodes):
node = self._new_mongod(replset, node_index, executables, classes,
mongod_binary_versions[node_index], is_multiversion)
replset.install_mongod(node)
if replset.start_initial_sync_node:
if not replset.initial_sync_node:
replset.initial_sync_node_idx = replset.num_nodes
# TODO: This adds the linear chain and steady state param now, is that ok?
replset.initial_sync_node = self._new_mongod(replset, replset.initial_sync_node_idx,
executables, classes,
BinVersionEnum.NEW, is_multiversion)
return replset
def build_fixture(self, logger, job_num, fixturelib, *args, **kwargs): # pylint: disable=too-many-locals
"""Build a replica set."""
# We hijack the mixed_bin_versions passed to the fixture.
mixed_bin_versions = kwargs.pop("mixed_bin_versions",
config.MIXED_BIN_VERSIONS)
old_bin_version = kwargs.pop("old_bin_version",
config.MULTIVERSION_BIN_VERSION)
# We also hijack the num_nodes because we need it here.
num_nodes = kwargs.pop("num_nodes", 2)
num_replset_nodes = config.NUM_REPLSET_NODES
num_nodes = num_replset_nodes if num_replset_nodes else num_nodes
kwargs["num_nodes"] = num_nodes
replset_config_options = kwargs.get("replset_config_options", {})
mongod_executable = default_if_none(kwargs.get("mongod_executable"),
config.MONGOD_EXECUTABLE,
config.DEFAULT_MONGOD_EXECUTABLE)
kwargs["mongod_executable"] = mongod_executable
num_nodes = kwargs["num_nodes"]
latest_mongod = mongod_executable
latest_class = "MongoDFixture"
executables = []
classes = []
fcv = None
multiversion_class_suffix = "_" + old_bin_version
shell_version = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_MONGO_BINARY,
config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_MONGO_BINARY
}[old_bin_version]
mongod_version = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_MONGOD_BINARY,
config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_MONGOD_BINARY
}[old_bin_version]
if mixed_bin_versions is None:
executables = [latest_mongod for x in range(num_nodes)]
classes = [latest_class for x in range(num_nodes)]
else:
is_config_svr = "configsvr" in replset_config_options and replset_config_options[
"configsvr"]
load_version(version_path_suffix=multiversion_class_suffix,
shell_path=shell_version)
if not is_config_svr:
executables = [
latest_mongod if (x == "new") else mongod_version
for x in mixed_bin_versions
]
classes = [
latest_class if (x == "new") else
f"{latest_class}{multiversion_class_suffix}"
for x in mixed_bin_versions
]
if is_config_svr:
# Our documented recommended path for upgrading shards lets us assume that config
# server nodes will always be fully upgraded before shard nodes.
executables = [latest_mongod, latest_mongod]
classes = [latest_class, latest_class]
num_versions = len(mixed_bin_versions)
fcv = {
config.MultiversionOptions.LAST_LTS:
multiversionconstants.LAST_LTS_FCV,
config.MultiversionOptions.LAST_CONTINUOUS:
multiversionconstants.LAST_CONTINUOUS_FCV
}[old_bin_version]
if num_versions != num_nodes and not is_config_svr:
msg = (("The number of binary versions specified: {} do not match the number of"\
" nodes in the replica set: {}.")).format(num_versions, num_nodes)
raise errors.ServerFailure(msg)
replset = _FIXTURES[self.REGISTERED_NAME](logger, job_num, fixturelib,
*args, **kwargs)
replset.set_fcv(fcv)
for i in range(replset.num_nodes):
node = self._new_mongod(replset, i, executables[i], classes[i])
replset.install_mongod(node)
if replset.start_initial_sync_node:
if not replset.initial_sync_node:
replset.initial_sync_node_idx = replset.num_nodes
# TODO: This adds the linear chain and steady state param now, is that ok?
replset.initial_sync_node = self._new_mongod(
replset, replset.initial_sync_node_idx, latest_mongod,
latest_class)
return replset