def copy_timeseries(config):
""" copy the files required for timeseries analysis from
their legacy mission-control locations to the new locations used by host.py.
:param dict(ConfigDict) config: The system configuration
"""
hosts = extract_hosts("all_servers", config)
for root, _, files in os.walk("./reports"):
for name in files:
# The following generator find the first host with an ip that matches
# the filename. The (.. for in if in ) generator will return 0 or more
# matches.
#
# In the non matching case, next would throw StopIteration . The
# None final param ensures that something 'Falsey' is returned instead.
host = next(
(host for host in hosts if name.endswith(host.public_ip)),
None)
if host:
source = os.path.join(root, name)
alias = "{category}.{offset}".format(category=host.category,
offset=host.offset)
destination = "{}-{}".format(
os.path.basename(source).split("--")[0],
os.path.basename(root))
destination = os.path.join("reports", alias, destination)
shutil.copyfile(source, destination)
def _write_hosts_file(self, output):
hosts = host_utils.extract_hosts("all_hosts", self.config)
LOG.debug("Write /etc/hosts on all hosts.", hosts=hosts)
run_threads([
partial(_write_hosts_file_thread, host_info, output)
for host_info in hosts
])
def _remote_exists(config):
"""
Check on remote workload_client whether jstests_dir exists.
"""
host_info = extract_hosts("workload_client", config)[0]
remote_host = make_host(host_info)
remote_command = [
"[ -e {} ]".format(config["test_control"]["jstests_dir"])
]
return remote_host.run(remote_command)
def make_workload_runner_host(config):
"""
Convenience function to make a host to connect to the workload runner node.
:param ConfigDict config: The system configuration
"""
host_info = host_utils.extract_hosts("workload_client", config)[0]
mongodb_auth_settings = mongodb_setup_helpers.mongodb_auth_settings(config)
use_tls = mongodb_setup_helpers.mongodb_tls_configured(
config["mongodb_setup"]["mongod_config_file"])
return host_factory.make_host(host_info, mongodb_auth_settings, use_tls)
def run_host_command(target, command, config, prefix):
"""
Sets up and runs a command for use on the appropriate hosts.
:param str target: The target to run the command on
:param dict command: The action to run
:param ConfigDict config: The system configuration
:param str prefix: The id for the test related to the current command. If there
is not a specific test related to the current command, the value of prefix should reflect the
hook that the command belongs to, such as between_tests, post_task, and so on.
"""
assert isinstance(command, MutableMapping), "command isn't a dict"
assert target.startswith("on_")
keys = list(command.keys())
target = target[3:]
hosts = host_utils.extract_hosts(target, config)
LOG.info("Running command(s) %s on %s", keys, target)
_run_host_command(hosts, command, config, prefix)
LOG.debug("Done running command(s) %s on %s", keys, target)
def test_extract_hosts(self):
""" Test extract hosts using config info """
default_host_info = host_utils.HostInfo(
public_ip=None,
# These are the user and key files used by this test.
ssh_user="******",
ssh_key_file=os.path.join(os.path.expanduser("~"), ".ssh", "linustorvalds.pem"),
category=None,
offset=-1,
)
def customize_host_info(new_ip, new_category, offset):
new_host_info = copy.copy(default_host_info)
new_host_info.public_ip = new_ip
new_host_info.category = new_category
new_host_info.offset = offset
return new_host_info
mongods = [customize_host_info("53.1.1.{}".format(i + 1), "mongod", i) for i in range(0, 9)]
configsvrs = [
customize_host_info("53.1.1.{}".format(i + 51), "configsvr", i) for i in range(0, 3)
]
mongos = [
customize_host_info("53.1.1.{}".format(i + 100), "mongos", i) for i in range(0, 3)
]
workload_clients = [customize_host_info("53.1.1.101", "workload_client", 0)]
localhost = [host_utils.HostInfo(public_ip="localhost", category="localhost", offset=0)]
self.assertEqual(host_utils.extract_hosts("localhost", self.config), localhost)
self.assertEqual(host_utils.extract_hosts("workload_client", self.config), workload_clients)
self.assertEqual(host_utils.extract_hosts("mongod", self.config), mongods)
self.assertEqual(host_utils.extract_hosts("mongos", self.config), mongos)
self.assertEqual(host_utils.extract_hosts("configsvr", self.config), configsvrs)
self.assertEqual(
host_utils.extract_hosts("all_servers", self.config), mongods + mongos + configsvrs
)
self.assertEqual(
host_utils.extract_hosts("all_hosts", self.config),
mongods + mongos + configsvrs + workload_clients,
)
def __init__(self, config):
self.config = config
self.mongodb_binary_archive = config["mongodb_setup"].get(
"mongodb_binary_archive", "")
# If mongodb_binary_archive is set in runtime, use it instead. Previously we only did that
# if the mongodb_binary_archive was "", but we have since added a useable default value.
# This second if statement is necessary for backwards compatibility after changes
# introducted in PERF-1044. It may be removed after all verions of system_perf.yml,
# longevity.yml specify the mongodb_binary_archive in bootstrap.yml instead of runtime.yml.
if "runtime" in list(config.keys()):
self.mongodb_binary_archive = config["runtime"].get(
"mongodb_binary_archive", self.mongodb_binary_archive)
LOG.info("Download url is %s", self.mongodb_binary_archive)
self.hosts = []
for host_info in host_utils.extract_hosts("all_hosts", self.config):
self.hosts.append(host_factory.make_host(host_info))
if self.mongodb_binary_archive:
LOG.debug("DownloadMongodb initialized with url: %s",
self.mongodb_binary_archive)