def _await_cmd_all_nodes(self, fn, msg, timeout_secs=30): """Run `fn` on all nodes until it returns a truthy value. Return the node for which makes `fn` become truthy. Two arguments are passed to fn: the client for a node and the MongoDFixture corresponding to that node. """ start = time.time() clients = {} while True: for node in self.nodes: now = time.time() if (now - start) >= timeout_secs: msg = "Timed out while {} for replica set '{}'.".format( msg, self.replset_name) self.logger.error(msg) raise errors.ServerFailure(msg) try: if node.port not in clients: clients[node.port] = self.auth(node.mongo_client(), self.auth_options) if fn(clients[node.port], node): return node except pymongo.errors.AutoReconnect: # AutoReconnect exceptions may occur if the primary stepped down since PyMongo # last contacted it. We'll just try contacting the node again in the next round # of isMaster requests. continue
def _do_teardown(self, mode=None): self.logger.info("Stopping all members of the replica set...") running_at_start = self.is_running() if not running_at_start: self.logger.info( "All members of the replica set were expected to be running, " "but weren't.") teardown_handler = interface.FixtureTeardownHandler(self.logger) if self.initial_sync_node: teardown_handler.teardown(self.initial_sync_node, "initial sync node", mode=mode) # Terminate the secondaries first to reduce noise in the logs. for node in reversed(self.nodes): teardown_handler.teardown(node, "replica set member on port %d" % node.port, mode=mode) if teardown_handler.was_successful(): self.logger.info( "Successfully stopped all members of the replica set.") else: self.logger.error("Stopping the replica set fixture failed.") raise errors.ServerFailure(teardown_handler.get_error_message())
def __init__( # pylint: disable=too-many-arguments, too-many-locals self, logger, job_num, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False, num_nodes=2, start_initial_sync_node=False, write_concern_majority_journal_default=None, auth_options=None, replset_config_options=None, voting_secondaries=True, all_nodes_electable=False, use_replica_set_connection_string=None, linear_chain=False, mixed_bin_versions=None, default_read_concern=None, default_write_concern=None): """Initialize ReplicaSetFixture.""" interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) self.mongod_options = utils.default_if_none(mongod_options, {}) self.preserve_dbpath = preserve_dbpath self.start_initial_sync_node = start_initial_sync_node self.write_concern_majority_journal_default = write_concern_majority_journal_default self.auth_options = auth_options self.replset_config_options = utils.default_if_none( replset_config_options, {}) self.voting_secondaries = voting_secondaries self.all_nodes_electable = all_nodes_electable self.use_replica_set_connection_string = use_replica_set_connection_string self.default_read_concern = default_read_concern self.default_write_concern = default_write_concern self.mixed_bin_versions = utils.default_if_none( mixed_bin_versions, config.MIXED_BIN_VERSIONS) # Use the values given from the command line if they exist for linear_chain and num_nodes. linear_chain_option = utils.default_if_none(config.LINEAR_CHAIN, linear_chain) self.linear_chain = linear_chain_option if linear_chain_option else linear_chain num_replset_nodes = config.NUM_REPLSET_NODES self.num_nodes = num_replset_nodes if num_replset_nodes else num_nodes if self.mixed_bin_versions is not None: mongod_executable = utils.default_if_none( config.MONGOD_EXECUTABLE, config.DEFAULT_MONGOD_EXECUTABLE) latest_mongod = mongod_executable last_stable_mongod = mongod_executable + "-" \ + ReplicaSetFixture._LAST_STABLE_BIN_VERSION is_config_svr = "configsvr" in self.replset_config_options and self.replset_config_options[ "configsvr"] if not is_config_svr: self.mixed_bin_versions = [ latest_mongod if (x == "new") else last_stable_mongod for x in self.mixed_bin_versions ] else: # Our documented recommended path for upgrading shards lets us assume that config # server nodes will always be fully upgraded before shard nodes. self.mixed_bin_versions = [latest_mongod, latest_mongod] num_versions = len(self.mixed_bin_versions) if num_versions != self.num_nodes and not is_config_svr: msg = (("The number of binary versions specified: {} do not match the number of"\ " nodes in the replica set: {}.")).format(num_versions, self.num_nodes) raise errors.ServerFailure(msg) # By default, we only use a replica set connection string if all nodes are capable of being # elected primary. if self.use_replica_set_connection_string is None: self.use_replica_set_connection_string = self.all_nodes_electable if self.default_write_concern is True: self.default_write_concern = { "w": "majority", # Use a "signature" value that won't typically match a value assigned in normal use. # This way the wtimeout set by this override is distinguishable in the server logs. "wtimeout": 5 * 60 * 1000 + 321, # 300321ms } # Set the default oplogSize to 511MB. self.mongod_options.setdefault("oplogSize", 511) # The dbpath in mongod_options is used as the dbpath prefix for replica set members and # takes precedence over other settings. The ShardedClusterFixture uses this parameter to # create replica sets and assign their dbpath structure explicitly. if "dbpath" in self.mongod_options: self._dbpath_prefix = self.mongod_options.pop("dbpath") else: self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.nodes = [] self.replset_name = None self.initial_sync_node = None self.initial_sync_node_idx = -1
def setup(self): # pylint: disable=too-many-branches,too-many-statements """Set up the replica set.""" self.replset_name = self.mongod_options.get("replSet", "rs") if not self.nodes: for i in range(self.num_nodes): node = self._new_mongod(i, self.replset_name) self.nodes.append(node) for i in range(self.num_nodes): if self.linear_chain and i > 0: self.nodes[i].mongod_options["set_parameters"][ "failpoint.forceSyncSourceCandidate"] = { "mode": "alwaysOn", "data": { "hostAndPort": self.nodes[i - 1].get_internal_connection_string() } } self.nodes[i].setup() if self.start_initial_sync_node: if not self.initial_sync_node: self.initial_sync_node_idx = len(self.nodes) self.initial_sync_node = self._new_mongod( self.initial_sync_node_idx, self.replset_name) self.initial_sync_node.setup() self.initial_sync_node.await_ready() # We need only to wait to connect to the first node of the replica set because we first # initiate it as a single node replica set. self.nodes[0].await_ready() # Initiate the replica set. members = [] for (i, node) in enumerate(self.nodes): member_info = { "_id": i, "host": node.get_internal_connection_string() } if i > 0: if not self.all_nodes_electable: member_info["priority"] = 0 if i >= 7 or not self.voting_secondaries: # Only 7 nodes in a replica set can vote, so the other members must still be # non-voting when this fixture is configured to have voting secondaries. member_info["votes"] = 0 members.append(member_info) if self.initial_sync_node: members.append({ "_id": self.initial_sync_node_idx, "host": self.initial_sync_node.get_internal_connection_string(), "priority": 0, "hidden": 1, "votes": 0 }) repl_config = {"_id": self.replset_name, "protocolVersion": 1} client = self.nodes[0].mongo_client() self.auth(client, self.auth_options) if client.local.system.replset.count(): # Skip initializing the replset if there is an existing configuration. return if self.write_concern_majority_journal_default is not None: repl_config[ "writeConcernMajorityJournalDefault"] = self.write_concern_majority_journal_default else: server_status = client.admin.command({"serverStatus": 1}) cmd_line_opts = client.admin.command({"getCmdLineOpts": 1}) if not (server_status["storageEngine"]["persistent"] and cmd_line_opts["parsed"].get("storage", {}).get( "journal", {}).get("enabled", True)): repl_config["writeConcernMajorityJournalDefault"] = False if self.replset_config_options.get("configsvr", False): repl_config["configsvr"] = True if self.replset_config_options.get("settings"): replset_settings = self.replset_config_options["settings"] repl_config["settings"] = replset_settings # If not all nodes are electable and no election timeout was specified, then we increase # the election timeout to 24 hours to prevent spurious elections. if not self.all_nodes_electable: repl_config.setdefault("settings", {}) if "electionTimeoutMillis" not in repl_config["settings"]: repl_config["settings"][ "electionTimeoutMillis"] = 24 * 60 * 60 * 1000 # Start up a single node replica set then reconfigure to the correct size (if the config # contains more than 1 node), so the primary is elected more quickly. repl_config["members"] = [members[0]] self.logger.info("Issuing replSetInitiate command: %s", repl_config) self._configure_repl_set(client, {"replSetInitiate": repl_config}) self._await_primary() if self.mixed_bin_versions is not None: if self.mixed_bin_versions[0] == "new": fcv_response = client.admin.command({ "getParameter": 1, "featureCompatibilityVersion": 1 }) fcv = fcv_response["featureCompatibilityVersion"]["version"] if fcv != ReplicaSetFixture._LATEST_FCV: msg = (("Server returned FCV{} when we expected FCV{}." ).format(fcv, ReplicaSetFixture._LATEST_FCV)) raise errors.ServerFailure(msg) # Initiating a replica set with a single node will use "latest" FCV. This will # cause IncompatibleServerVersion errors if additional "last-stable" binary version # nodes are subsequently added to the set, since such nodes cannot set their FCV to # "latest". Therefore, we make sure the primary is "last-stable" FCV before adding in # nodes of different binary versions to the replica set. client.admin.command({ "setFeatureCompatibilityVersion": ReplicaSetFixture._LAST_STABLE_FCV }) if self.nodes[1:]: # Wait to connect to each of the secondaries before running the replSetReconfig # command. for node in self.nodes[1:]: node.await_ready() repl_config["version"] = 2 repl_config["members"] = members self.logger.info("Issuing replSetReconfig command: %s", repl_config) self._configure_repl_set(client, {"replSetReconfig": repl_config}) self._await_secondaries()
def __init__( # pylint: disable=too-many-arguments, too-many-locals self, logger, job_num, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False, num_nodes=2, start_initial_sync_node=False, write_concern_majority_journal_default=None, auth_options=None, replset_config_options=None, voting_secondaries=None, all_nodes_electable=False, use_replica_set_connection_string=None, linear_chain=False, mixed_bin_versions=None): """Initialize ReplicaSetFixture.""" interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) self.mongod_options = utils.default_if_none(mongod_options, {}) self.preserve_dbpath = preserve_dbpath self.num_nodes = num_nodes self.start_initial_sync_node = start_initial_sync_node self.write_concern_majority_journal_default = write_concern_majority_journal_default self.auth_options = auth_options self.replset_config_options = utils.default_if_none( replset_config_options, {}) self.voting_secondaries = voting_secondaries self.all_nodes_electable = all_nodes_electable self.use_replica_set_connection_string = use_replica_set_connection_string self.linear_chain = linear_chain self.mixed_bin_versions = utils.default_if_none( mixed_bin_versions, config.MIXED_BIN_VERSIONS) if self.mixed_bin_versions is not None: mongod_executable = utils.default_if_none( config.MONGOD_EXECUTABLE, config.DEFAULT_MONGOD_EXECUTABLE) latest_mongod = mongod_executable last_stable_mongod = mongod_executable + "-" \ + ReplicaSetFixture._LAST_STABLE_BIN_VERSION is_config_svr = "configsvr" in self.replset_config_options and self.replset_config_options[ "configsvr"] if not is_config_svr: self.mixed_bin_versions = [ latest_mongod if (x == "new") else last_stable_mongod for x in self.mixed_bin_versions ] else: # Our documented recommended path for upgrading shards lets us assume that config # server nodes will always be fully upgraded before shard nodes. self.mixed_bin_versions = [latest_mongod, latest_mongod] num_versions = len(self.mixed_bin_versions) if num_versions != num_nodes and not is_config_svr: msg = (("The number of binary versions specified: {} do not match the number of"\ " nodes in the replica set: {}.")).format(num_versions, num_nodes) raise errors.ServerFailure(msg) # If voting_secondaries has not been set, set a default. By default, secondaries have zero # votes unless they are also nodes capable of being elected primary. if self.voting_secondaries is None: self.voting_secondaries = self.all_nodes_electable # By default, we only use a replica set connection string if all nodes are capable of being # elected primary. if self.use_replica_set_connection_string is None: self.use_replica_set_connection_string = self.all_nodes_electable # Set the default oplogSize to 511MB. self.mongod_options.setdefault("oplogSize", 511) # The dbpath in mongod_options is used as the dbpath prefix for replica set members and # takes precedence over other settings. The ShardedClusterFixture uses this parameter to # create replica sets and assign their dbpath structure explicitly. if "dbpath" in self.mongod_options: self._dbpath_prefix = self.mongod_options.pop("dbpath") else: self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.nodes = [] self.replset_name = None self.initial_sync_node = None self.initial_sync_node_idx = -1