def _await_cmd_all_nodes(self, fn, msg, timeout_secs=30):
"""Run `fn` on all nodes until it returns a truthy value.
Return the node for which makes `fn` become truthy.
Two arguments are passed to fn: the client for a node and
the MongoDFixture corresponding to that node.
"""
start = time.time()
clients = {}
while True:
for node in self.nodes:
now = time.time()
if (now - start) >= timeout_secs:
msg = "Timed out while {} for replica set '{}'.".format(
msg, self.replset_name)
self.logger.error(msg)
raise errors.ServerFailure(msg)
try:
if node.port not in clients:
clients[node.port] = self.auth(node.mongo_client(),
self.auth_options)
if fn(clients[node.port], node):
return node
except pymongo.errors.AutoReconnect:
# AutoReconnect exceptions may occur if the primary stepped down since PyMongo
# last contacted it. We'll just try contacting the node again in the next round
# of isMaster requests.
continue
def _do_teardown(self, mode=None):
self.logger.info("Stopping all members of the replica set...")
running_at_start = self.is_running()
if not running_at_start:
self.logger.info(
"All members of the replica set were expected to be running, "
"but weren't.")
teardown_handler = interface.FixtureTeardownHandler(self.logger)
if self.initial_sync_node:
teardown_handler.teardown(self.initial_sync_node,
"initial sync node",
mode=mode)
# Terminate the secondaries first to reduce noise in the logs.
for node in reversed(self.nodes):
teardown_handler.teardown(node,
"replica set member on port %d" %
node.port,
mode=mode)
if teardown_handler.was_successful():
self.logger.info(
"Successfully stopped all members of the replica set.")
else:
self.logger.error("Stopping the replica set fixture failed.")
raise errors.ServerFailure(teardown_handler.get_error_message())
def __init__( # pylint: disable=too-many-arguments, too-many-locals
self,
logger,
job_num,
mongod_options=None,
dbpath_prefix=None,
preserve_dbpath=False,
num_nodes=2,
start_initial_sync_node=False,
write_concern_majority_journal_default=None,
auth_options=None,
replset_config_options=None,
voting_secondaries=True,
all_nodes_electable=False,
use_replica_set_connection_string=None,
linear_chain=False,
mixed_bin_versions=None,
default_read_concern=None,
default_write_concern=None):
"""Initialize ReplicaSetFixture."""
interface.ReplFixture.__init__(self,
logger,
job_num,
dbpath_prefix=dbpath_prefix)
self.mongod_options = utils.default_if_none(mongod_options, {})
self.preserve_dbpath = preserve_dbpath
self.start_initial_sync_node = start_initial_sync_node
self.write_concern_majority_journal_default = write_concern_majority_journal_default
self.auth_options = auth_options
self.replset_config_options = utils.default_if_none(
replset_config_options, {})
self.voting_secondaries = voting_secondaries
self.all_nodes_electable = all_nodes_electable
self.use_replica_set_connection_string = use_replica_set_connection_string
self.default_read_concern = default_read_concern
self.default_write_concern = default_write_concern
self.mixed_bin_versions = utils.default_if_none(
mixed_bin_versions, config.MIXED_BIN_VERSIONS)
# Use the values given from the command line if they exist for linear_chain and num_nodes.
linear_chain_option = utils.default_if_none(config.LINEAR_CHAIN,
linear_chain)
self.linear_chain = linear_chain_option if linear_chain_option else linear_chain
num_replset_nodes = config.NUM_REPLSET_NODES
self.num_nodes = num_replset_nodes if num_replset_nodes else num_nodes
if self.mixed_bin_versions is not None:
mongod_executable = utils.default_if_none(
config.MONGOD_EXECUTABLE, config.DEFAULT_MONGOD_EXECUTABLE)
latest_mongod = mongod_executable
last_stable_mongod = mongod_executable + "-" \
+ ReplicaSetFixture._LAST_STABLE_BIN_VERSION
is_config_svr = "configsvr" in self.replset_config_options and self.replset_config_options[
"configsvr"]
if not is_config_svr:
self.mixed_bin_versions = [
latest_mongod if (x == "new") else last_stable_mongod
for x in self.mixed_bin_versions
]
else:
# Our documented recommended path for upgrading shards lets us assume that config
# server nodes will always be fully upgraded before shard nodes.
self.mixed_bin_versions = [latest_mongod, latest_mongod]
num_versions = len(self.mixed_bin_versions)
if num_versions != self.num_nodes and not is_config_svr:
msg = (("The number of binary versions specified: {} do not match the number of"\
" nodes in the replica set: {}.")).format(num_versions, self.num_nodes)
raise errors.ServerFailure(msg)
# By default, we only use a replica set connection string if all nodes are capable of being
# elected primary.
if self.use_replica_set_connection_string is None:
self.use_replica_set_connection_string = self.all_nodes_electable
if self.default_write_concern is True:
self.default_write_concern = {
"w": "majority",
# Use a "signature" value that won't typically match a value assigned in normal use.
# This way the wtimeout set by this override is distinguishable in the server logs.
"wtimeout": 5 * 60 * 1000 + 321, # 300321ms
}
# Set the default oplogSize to 511MB.
self.mongod_options.setdefault("oplogSize", 511)
# The dbpath in mongod_options is used as the dbpath prefix for replica set members and
# takes precedence over other settings. The ShardedClusterFixture uses this parameter to
# create replica sets and assign their dbpath structure explicitly.
if "dbpath" in self.mongod_options:
self._dbpath_prefix = self.mongod_options.pop("dbpath")
else:
self._dbpath_prefix = os.path.join(self._dbpath_prefix,
config.FIXTURE_SUBDIR)
self.nodes = []
self.replset_name = None
self.initial_sync_node = None
self.initial_sync_node_idx = -1
def setup(self): # pylint: disable=too-many-branches,too-many-statements
"""Set up the replica set."""
self.replset_name = self.mongod_options.get("replSet", "rs")
if not self.nodes:
for i in range(self.num_nodes):
node = self._new_mongod(i, self.replset_name)
self.nodes.append(node)
for i in range(self.num_nodes):
if self.linear_chain and i > 0:
self.nodes[i].mongod_options["set_parameters"][
"failpoint.forceSyncSourceCandidate"] = {
"mode": "alwaysOn",
"data": {
"hostAndPort":
self.nodes[i - 1].get_internal_connection_string()
}
}
self.nodes[i].setup()
if self.start_initial_sync_node:
if not self.initial_sync_node:
self.initial_sync_node_idx = len(self.nodes)
self.initial_sync_node = self._new_mongod(
self.initial_sync_node_idx, self.replset_name)
self.initial_sync_node.setup()
self.initial_sync_node.await_ready()
# We need only to wait to connect to the first node of the replica set because we first
# initiate it as a single node replica set.
self.nodes[0].await_ready()
# Initiate the replica set.
members = []
for (i, node) in enumerate(self.nodes):
member_info = {
"_id": i,
"host": node.get_internal_connection_string()
}
if i > 0:
if not self.all_nodes_electable:
member_info["priority"] = 0
if i >= 7 or not self.voting_secondaries:
# Only 7 nodes in a replica set can vote, so the other members must still be
# non-voting when this fixture is configured to have voting secondaries.
member_info["votes"] = 0
members.append(member_info)
if self.initial_sync_node:
members.append({
"_id":
self.initial_sync_node_idx,
"host":
self.initial_sync_node.get_internal_connection_string(),
"priority":
0,
"hidden":
1,
"votes":
0
})
repl_config = {"_id": self.replset_name, "protocolVersion": 1}
client = self.nodes[0].mongo_client()
self.auth(client, self.auth_options)
if client.local.system.replset.count():
# Skip initializing the replset if there is an existing configuration.
return
if self.write_concern_majority_journal_default is not None:
repl_config[
"writeConcernMajorityJournalDefault"] = self.write_concern_majority_journal_default
else:
server_status = client.admin.command({"serverStatus": 1})
cmd_line_opts = client.admin.command({"getCmdLineOpts": 1})
if not (server_status["storageEngine"]["persistent"]
and cmd_line_opts["parsed"].get("storage", {}).get(
"journal", {}).get("enabled", True)):
repl_config["writeConcernMajorityJournalDefault"] = False
if self.replset_config_options.get("configsvr", False):
repl_config["configsvr"] = True
if self.replset_config_options.get("settings"):
replset_settings = self.replset_config_options["settings"]
repl_config["settings"] = replset_settings
# If not all nodes are electable and no election timeout was specified, then we increase
# the election timeout to 24 hours to prevent spurious elections.
if not self.all_nodes_electable:
repl_config.setdefault("settings", {})
if "electionTimeoutMillis" not in repl_config["settings"]:
repl_config["settings"][
"electionTimeoutMillis"] = 24 * 60 * 60 * 1000
# Start up a single node replica set then reconfigure to the correct size (if the config
# contains more than 1 node), so the primary is elected more quickly.
repl_config["members"] = [members[0]]
self.logger.info("Issuing replSetInitiate command: %s", repl_config)
self._configure_repl_set(client, {"replSetInitiate": repl_config})
self._await_primary()
if self.mixed_bin_versions is not None:
if self.mixed_bin_versions[0] == "new":
fcv_response = client.admin.command({
"getParameter":
1,
"featureCompatibilityVersion":
1
})
fcv = fcv_response["featureCompatibilityVersion"]["version"]
if fcv != ReplicaSetFixture._LATEST_FCV:
msg = (("Server returned FCV{} when we expected FCV{}."
).format(fcv, ReplicaSetFixture._LATEST_FCV))
raise errors.ServerFailure(msg)
# Initiating a replica set with a single node will use "latest" FCV. This will
# cause IncompatibleServerVersion errors if additional "last-stable" binary version
# nodes are subsequently added to the set, since such nodes cannot set their FCV to
# "latest". Therefore, we make sure the primary is "last-stable" FCV before adding in
# nodes of different binary versions to the replica set.
client.admin.command({
"setFeatureCompatibilityVersion":
ReplicaSetFixture._LAST_STABLE_FCV
})
if self.nodes[1:]:
# Wait to connect to each of the secondaries before running the replSetReconfig
# command.
for node in self.nodes[1:]:
node.await_ready()
repl_config["version"] = 2
repl_config["members"] = members
self.logger.info("Issuing replSetReconfig command: %s",
repl_config)
self._configure_repl_set(client, {"replSetReconfig": repl_config})
self._await_secondaries()
def __init__( # pylint: disable=too-many-arguments, too-many-locals
self,
logger,
job_num,
mongod_options=None,
dbpath_prefix=None,
preserve_dbpath=False,
num_nodes=2,
start_initial_sync_node=False,
write_concern_majority_journal_default=None,
auth_options=None,
replset_config_options=None,
voting_secondaries=None,
all_nodes_electable=False,
use_replica_set_connection_string=None,
linear_chain=False,
mixed_bin_versions=None):
"""Initialize ReplicaSetFixture."""
interface.ReplFixture.__init__(self,
logger,
job_num,
dbpath_prefix=dbpath_prefix)
self.mongod_options = utils.default_if_none(mongod_options, {})
self.preserve_dbpath = preserve_dbpath
self.num_nodes = num_nodes
self.start_initial_sync_node = start_initial_sync_node
self.write_concern_majority_journal_default = write_concern_majority_journal_default
self.auth_options = auth_options
self.replset_config_options = utils.default_if_none(
replset_config_options, {})
self.voting_secondaries = voting_secondaries
self.all_nodes_electable = all_nodes_electable
self.use_replica_set_connection_string = use_replica_set_connection_string
self.linear_chain = linear_chain
self.mixed_bin_versions = utils.default_if_none(
mixed_bin_versions, config.MIXED_BIN_VERSIONS)
if self.mixed_bin_versions is not None:
mongod_executable = utils.default_if_none(
config.MONGOD_EXECUTABLE, config.DEFAULT_MONGOD_EXECUTABLE)
latest_mongod = mongod_executable
last_stable_mongod = mongod_executable + "-" \
+ ReplicaSetFixture._LAST_STABLE_BIN_VERSION
is_config_svr = "configsvr" in self.replset_config_options and self.replset_config_options[
"configsvr"]
if not is_config_svr:
self.mixed_bin_versions = [
latest_mongod if (x == "new") else last_stable_mongod
for x in self.mixed_bin_versions
]
else:
# Our documented recommended path for upgrading shards lets us assume that config
# server nodes will always be fully upgraded before shard nodes.
self.mixed_bin_versions = [latest_mongod, latest_mongod]
num_versions = len(self.mixed_bin_versions)
if num_versions != num_nodes and not is_config_svr:
msg = (("The number of binary versions specified: {} do not match the number of"\
" nodes in the replica set: {}.")).format(num_versions, num_nodes)
raise errors.ServerFailure(msg)
# If voting_secondaries has not been set, set a default. By default, secondaries have zero
# votes unless they are also nodes capable of being elected primary.
if self.voting_secondaries is None:
self.voting_secondaries = self.all_nodes_electable
# By default, we only use a replica set connection string if all nodes are capable of being
# elected primary.
if self.use_replica_set_connection_string is None:
self.use_replica_set_connection_string = self.all_nodes_electable
# Set the default oplogSize to 511MB.
self.mongod_options.setdefault("oplogSize", 511)
# The dbpath in mongod_options is used as the dbpath prefix for replica set members and
# takes precedence over other settings. The ShardedClusterFixture uses this parameter to
# create replica sets and assign their dbpath structure explicitly.
if "dbpath" in self.mongod_options:
self._dbpath_prefix = self.mongod_options.pop("dbpath")
else:
self._dbpath_prefix = os.path.join(self._dbpath_prefix,
config.FIXTURE_SUBDIR)
self.nodes = []
self.replset_name = None
self.initial_sync_node = None
self.initial_sync_node_idx = -1
