def _write_hosts_file(self, output): hosts = host_utils.extract_hosts("all_hosts", self.config) LOG.debug("Write /etc/hosts on all hosts.", hosts=hosts) run_threads([ partial(_write_hosts_file_thread, host_info, output) for host_info in hosts ])
def _run_host_command(host_list, command, config, prefix): """ For each host in the list, make a parallelized call to make_host_runner to make the appropriate host and run the set of commands. :param list host_list: List of ip addresses to connect to :param command: The command to execute. If str, run that command. If dict, type is one of upload_repo_files, upload_files, retrieve_files, exec, or exec_mongo_shell. :type command: str, dict :param ConfigDict config: The system configuration :param str prefix: The id for the test related to the current command. If there is not a specific test related to the current command, the value of prefix should reflect the hook that the command belongs to, such as between_tests, post_task, and so on. """ if not host_list: return LOG.debug("Calling run command for %s with command %s", str(host_list), str(command)) mongodb_auth_settings = mongodb_setup_helpers.mongodb_auth_settings(config) use_tls = mongodb_setup_helpers.mongodb_tls_settings( config["mongodb_setup"]["mongod_config_file"]) thread_commands = [] for host_info in host_list: thread_commands.append( partial(make_host_runner, host_info, command, prefix, mongodb_auth_settings, use_tls)) run_threads(thread_commands, daemon=True)
def destroy(self, max_time_ms): """Kills the remote replica members. For the max_time_ms parameter, see :method:`Host.exec_command` """ run_threads( [partial(node.destroy, max_time_ms) for node in self.nodes], daemon=True)
def destroy(self, max_time_ms): """Kills the remote cluster members. For the max_time_ms parameter, see :method:`Host.exec_command` """ run_threads( [partial(shard.destroy, max_time_ms) for shard in self.shards], daemon=True) self.config_svr.destroy(max_time_ms) run_threads( [partial(mongos.destroy, max_time_ms) for mongos in self.mongoses], daemon=True)
def _start_auth_explicit( self, is_restart=False, restart_clean_db_dir=None, restart_clean_logs=None, enable_auth=False, ): """ Complete the remaining start (either clean or restart) operations. Any Shutdown, destroy or downloading has been handled by the caller( See :method:`start` or See :method:`restart`). :param is_restart This is a restart of the cluster, not the first start. :param restart_clean_db_dir Should we clean db dir. If not specified, uses value from ConfigDict. :param restart_clean_logs Should we clean logs and diagnostic data. If not specified, uses value from ConfigDict. """ if not all( run_threads( [ partial( self.start_cluster, cluster=cluster, is_restart=is_restart, restart_clean_db_dir=restart_clean_db_dir, restart_clean_logs=restart_clean_logs, enable_auth=enable_auth, ) for cluster in self.clusters ], daemon=True, )): LOG.error("Could not start clusters in _start. Shutting down...") self.shutdown(self.shutdown_ms) return False return True
def run_validate(config, current_test_id=None, reports_dir="reports"): """ Validate the DB after a test :param dict(ConfigDict) config: The system configuration. :param string current_test_id: Indicates the id for the test related to the current set of commands. If there is not a specific test related to the current set of commands, the value of current_test_id will be None. :param string reports_dir: the report directory. """ LOG.debug("In run_validate before jstests_dir check") # If jstests_dir doesn't exist or is falsey, don't do anything. if "jstests_dir" not in config[ "test_control"] or not config["test_control"]["jstests_dir"]: LOG.info("No jstests_dir specified. Skipping validate.") return # If there is no validate entry in the mongodb_setup config, don't do anything. if "validate" not in config["mongodb_setup"]: LOG.warning("No validate entry in mongodb_setup. Skipping validate.") return # If jstests_dir doesn't actually exist, skip validate. # (v3.2 branch as well as official release archives.) # We check this on the workload_client, not locally. if not _remote_exists(config): LOG.warning("%s not found. Skipping validate.", config["test_control"]["jstests_dir"]) return LOG.info("In run_validate") # Run the checks if "standalone" in config["mongodb_setup"]["validate"]: run_threads([ partial(validate_one_host, config, primary, reports_dir, current_test_id, False) for primary in config["mongodb_setup"]["validate"]["standalone"] ]) if "primaries" in config["mongodb_setup"]["validate"]: run_threads([ partial(validate_one_host, config, primary, reports_dir, current_test_id, True) for primary in config["mongodb_setup"]["validate"]["primaries"] ])
def setup_host(self, restart_clean_db_dir=None, restart_clean_logs=None): return all( run_threads( [ partial( node.setup_host, restart_clean_db_dir=restart_clean_db_dir, restart_clean_logs=restart_clean_logs, ) for node in self.nodes ], daemon=True, ))
def shutdown(self, max_time_ms, auth_enabled=None, retries=20): """Shutdown gracefully For the max_time_ms parameter, see :method:`Host.exec_command` """ return all( run_threads( [ partial(node.shutdown, max_time_ms, auth_enabled) for node in self.nodes ], daemon=True, ))
def destroy(self, max_time_ms): """Kill all launched mongo programs For the max_time_ms parameter, see :method:`Host.exec_command` """ LOG.info("calling destroy") result = all( run_threads([ partial(cluster.destroy, max_time_ms) for cluster in self.clusters ], daemon=True)) if not result: LOG.warning("destroy: failed") return result
def shutdown(self, max_time_ms, auth_enabled=None, retries=20): """Shutdown the mongodb cluster gracefully. For the max_time_ms parameter, see :method:`Host.exec_command` """ commands = [] commands.extend( partial(shard.shutdown, max_time_ms, auth_enabled) for shard in self.shards) commands.append( partial(self.config_svr.shutdown, max_time_ms, auth_enabled)) commands.extend( partial(mongos.shutdown, max_time_ms, auth_enabled) for mongos in self.mongoses) return all(run_threads(commands, daemon=True))
def download_and_extract(self): """Download self.mongodb_binary_archive, extract it, and create some symlinks. :return: True if no mongodb_binary_archive was provided or all the commands completed successfully on all servers. """ if not self.mongodb_binary_archive: LOG.warning("DownloadMongodb: download_and_extract() was called, " "but mongodb_binary_archive isn't defined.") return True to_download = [] for host in self.hosts: commands = self._remote_commands(host) to_download.append(partial(host.run, commands)) return all(run_threads(to_download, daemon=True))
def shutdown(self, max_time_ms, auth_enabled=None): """Shutdown all launched mongo programs For the max_time_ms parameter, see :method:`Host.exec_command` """ LOG.info("Calling shutdown for %s clusters", len(self.clusters)) result = all( run_threads( [ partial(cluster.shutdown, max_time_ms, auth_enabled) for cluster in self.clusters ], daemon=True, )) LOG.warning("shutdown: %s", "succeeded" if result else "failed") return result
def setup_host(self, restart_clean_db_dir=None, restart_clean_logs=None): commands = [ partial( self.config_svr.setup_host, restart_clean_db_dir=restart_clean_db_dir, restart_clean_logs=restart_clean_logs, ) ] commands.extend( partial( shard.setup_host, restart_clean_db_dir=restart_clean_db_dir, restart_clean_logs=restart_clean_logs, ) for shard in self.shards) commands.extend( partial( mongos.setup_host, restart_clean_db_dir=restart_clean_db_dir, restart_clean_logs=restart_clean_logs, ) for mongos in self.mongoses) return all(run_threads(commands, daemon=True))
def launch(self, initialize=True, use_numactl=True, enable_auth=False): """Starts the replica set. :param boolean initialize: Initialize the replica set""" if not all( run_threads( [ partial(node.launch, initialize, use_numactl=use_numactl, enable_auth=enable_auth) for node in self.nodes ], daemon=True, )): return False self._set_explicit_priorities() if initialize: LOG.info("Configuring replica set: %s", self.topology_config.name) if not self.run_mongo_shell( self.topology_config.get_init_code(self.nodes)): return False # Wait for all nodes to be up return self.wait_until_up()
def launch(self, initialize=True, use_numactl=True, enable_auth=False): """Starts the sharded cluster. :param boolean initialize: Initialize the cluster """ LOG.info("Launching sharded cluster...") commands = [ partial( self.config_svr.launch, initialize=initialize, use_numactl=False, enable_auth=enable_auth, ) ] commands.extend( partial( shard.launch, initialize=initialize, use_numactl=use_numactl, enable_auth=enable_auth, ) for shard in self.shards) commands.extend( partial( mongos.launch, initialize=initialize, use_numactl=use_numactl, enable_auth=enable_auth, ) for mongos in self.mongoses) if not all(run_threads(commands, daemon=True)): return False if initialize: if not self._add_shards(): return False if self.sharded_config.disable_balancer and not self.run_mongo_shell( "sh.stopBalancer();"): return False return self.wait_until_up()
def establish_delays(self): """ Overrides grandparent method for establishing delays. """ all_nodes = self.shards + self.mongoses + [self.config_svr] run_threads([node.establish_delays for node in all_nodes], daemon=True)
def establish_delays(self): """ Overrides grandparent method for setting delays. """ run_threads([node.establish_delays for node in self.nodes], daemon=True)
def close(self): """Close connections to all hosts.""" run_threads([cluster.close for cluster in self.clusters], daemon=True)
def close(self): """Closes SSH connections to remote hosts.""" run_threads([node.close for node in self.nodes], daemon=True)
def close(self): """Closes SSH connections to remote hosts.""" run_threads([shard.close for shard in self.shards], daemon=True) self.config_svr.close() run_threads([mongos.close for mongos in self.mongoses], daemon=True)
def reset_delays(self): """ Overrides grandparent method for resetting delays. """ run_threads([node.reset_delays for node in self.nodes], daemon=True)