def setUp(self):
""" Init a ConfigDict object and load the configuration files from docs/config-specs/ """
self.config = ConfigDict("mongodb_setup", whereami.dsi_repo_path("docs", "config-specs"))
self.config.load()
self.parent_dir = os.path.join(os.path.expanduser("~"), "checkout_repos_test")
self._delete_fixtures()
def setUp(self):
""" Load self.config (ConfigDict) and set some other common values """
self.config = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
self.config.load()
cookiejar = requests.cookies.RequestsCookieJar()
request = requests.Request("GET", "http://ip.42.pl/raw")
request.prepare()
self.response_state = {
"cookies": cookiejar,
"_content": b"ip.42.hostname",
"encoding": "UTF-8",
"url": "http://ip.42.pl/raw",
"status_code": 200,
"request": request,
"elapsed": datetime.timedelta(0, 0, 615501),
"headers": {
"Content-Length": "14",
"X-Powered-By": "PHP/5.6.27",
"Keep-Alive": "timeout=5, max=100",
"Server":
"Apache/2.4.23 (FreeBSD) OpenSSL/1.0.1l-freebsd PHP/5.6.27",
"Connection": "Keep-Alive",
"Date": "Tue, 25 Jul 2017 14:20:06 GMT",
"Content-Type": "text/html; charset=UTF-8",
},
"reason": "OK",
"history": [],
}
def test_setup_overrides_no_file_config_vals(self):
"""
Testing setup_overrides where path = False and config vals given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
master_overrides = {}
master_overrides.update({
"infrastructure_provisioning": {
"tfvars": {
"ssh_key_file": "test_ssh_key_file.pem",
"ssh_key_name": "test_ssh_key_name",
"tags": {
"owner": "testuser",
"expire-on-delta": 24
},
}
}
})
real_configdict.raw["bootstrap"] = {}
real_configdict.raw["bootstrap"]["overrides"] = master_overrides
test_override_path = whereami.dsi_repo_path("dsi", "tests")
test_override_dict = {}
# Call to setup_overrides creates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
with open(os.path.join(test_override_path, "overrides.yml"),
"r") as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, master_overrides)
# Removing created file
os.remove(os.path.join(test_override_path, "overrides.yml"))
def test_setup_overrides_default_username(self):
"""
Testing setup_overrides fails if user doesn't change username
from bootstrap.example.yml default
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
real_configdict.raw["bootstrap"]["overrides"] = {
"infrastructure_provisioning": {
"tfvars": {
"tags": {
"owner": "your.username"
}
}
}
}
test_override_path = whereami.dsi_repo_path("dsi", "tests")
with self.assertRaises(AssertionError):
bootstrap.setup_overrides(real_configdict, test_override_path)
# Removing created file
try:
os.remove(os.path.join(test_override_path, "overrides.yml"))
except OSError:
pass
def test_build_hosts_file(self):
expected = [
"10.2.1.1\tmd md0 mongod0 mongod0.dsitest.dev",
"10.2.1.2\tmd1 mongod1 mongod1.dsitest.dev",
"10.2.1.3\tmd2 mongod2 mongod2.dsitest.dev",
"10.2.1.4\tmd3 mongod3 mongod3.dsitest.dev",
"10.2.1.5\tmd4 mongod4 mongod4.dsitest.dev",
"10.2.1.6\tmd5 mongod5 mongod5.dsitest.dev",
"10.2.1.7\tmd6 mongod6 mongod6.dsitest.dev",
"10.2.1.8\tmd7 mongod7 mongod7.dsitest.dev",
"10.2.1.9\tmd8 mongod8 mongod8.dsitest.dev",
"10.2.1.100\tms ms0 mongos0 mongos0.dsitest.dev",
"10.2.1.101\tms1 mongos1 mongos1.dsitest.dev",
"10.2.1.102\tms2 mongos2 mongos2.dsitest.dev",
"10.2.1.51\tcs cs0 configsvr0 configsvr0.dsitest.dev",
"10.2.1.52\tcs1 configsvr1 configsvr1.dsitest.dev",
"10.2.1.53\tcs2 configsvr2 configsvr2.dsitest.dev",
"10.2.1.10\twc wc0 workload_client0 workload_client0.dsitest.dev",
]
real_config_dict = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
real_config_dict.load()
real_config_dict.save = MagicMock(name="save")
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
hosts_contents = provisioner._build_hosts_file()
self.assertEqual(expected, hosts_contents)
def test_setup_overrides_file_exists_empty_config(self):
"""
Testing setup_overrides, path = True and config vals not given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
test_override_path = whereami.dsi_repo_path("dsi", "tests")
test_override_str = yaml.dump({}, default_flow_style=False)
# Creating 'overrides.yml' in current dir
with open(os.path.join(test_override_path, "overrides.yml"),
"w+") as test_override_file:
test_override_file.write(test_override_str)
# Call to setup_overrides updates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
test_override_dict = {}
with open(os.path.join(test_override_path, "overrides.yml"),
"r") as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, {})
# Removing created file
os.remove(os.path.join(test_override_path, "overrides.yml"))
def main(argv):
""" Main function. parse args and execute
:param argv list the command line arguments excluding the program name. Default is
sys.argv[1:]
"""
parser, args = parse_args(argv)
config = ConfigDict("infrastructure_provisioning").load()
if len(args.host) == 1:
host = alias.expand(args.host[0])
host = alias.unalias(host)
cmd = ";".join(args.command)
remote_cmd(host, cmd, config, args)
else:
if not args.command:
print("You must provide a command with more than one host\n\n")
parser.print_help()
sys.exit(1)
cmd = ";".join(args.command)
threads = []
for host in args.host:
host = alias.expand(host)
host = alias.unalias(host)
thread = threading.Thread(target=remote_cmd,
args=(host, cmd, config, args))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
def main():
""" Main function """
args = parse_command_line()
setup_logging(args.debug, args.log_file)
config = ConfigDict("infrastructure_provisioning")
config.load()
provisioner = Provisioner(config, verbose=args.debug)
provisioner.provision_resources()
def test_variable_reference_contains_invalid_id(self):
"""
Variable references cannot evaluate to blocks containing duplicate ids.
"""
with in_dir(FIXTURE_FILES.fixture_file_path("nested-invalid-ids")):
with self.assertRaises(config.InvalidConfigurationException):
conf = ConfigDict("mongodb_setup")
conf.load()
def test_find_nested_config_dicts(self):
"""
We check for duplicate ids in lists of lists correctly.
"""
with in_dir(FIXTURE_FILES.fixture_file_path("invalid-ids-in-lists")):
with self.assertRaises(config.InvalidConfigurationException):
conf = ConfigDict("mongodb_setup")
conf.load()
def test_setup_overrides_file_exists_config_vals(self):
"""
Testing setup_overrides where path = True and config vals given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
test_override_path = whereami.dsi_repo_path("dsi", "tests")
master_overrides = {}
master_overrides.update({
"infrastructure_provisioning": {
"tfvars": {
"ssh_key_file": "test_ssh_key_file1.pem",
"ssh_key_name": "test_ssh_key_name1",
"tags": {
"owner": "testuser1",
"expire-on-delta": 24
},
}
}
})
real_configdict.raw["bootstrap"]["overrides"] = master_overrides
test_override_str = yaml.dump(
{
"infrastructure_provisioning": {
"tfvars": {
"ssh_key_file": "test_ssh_key_file2.pem",
"ssh_key_name": "test_ssh_key_name2",
"tags": {
"owner": "testuser2",
"expire-on-delta": 48
},
}
}
},
default_flow_style=False,
)
# Creating 'overrides.yml' in current dir
with open(os.path.join(test_override_path, "overrides.yml"),
"w") as test_override_file:
test_override_file.write(test_override_str)
# Call to setup_overrides updates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
test_override_dict = {}
with open(os.path.join(test_override_path, "overrides.yml"),
"r") as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, master_overrides)
# Removing created file
os.remove(os.path.join(test_override_path, "overrides.yml"))
def setUp(self):
"""
Setup basic environment
"""
# Mocking `ConfigDict.assert_valid_ids` because it enforces structural constraints on yaml
# files that aren't necessary here.
with patch("dsi.common.config.ConfigDict.assert_valid_ids"
) as mock_assert_valid_ids:
self.config = ConfigDict(
"test_control",
FIXTURE_FILES.fixture_file_path("config_test_control"))
self.config.load()
mock_assert_valid_ids.assert_called_once()
def test_load_new(self):
"""Test loading ConfigDict with old naming convention .yml files"""
test_conf = ConfigDict(
"bootstrap",
whereami.dsi_repo_path("dsi", "tests", "test_config_files",
"new_format"))
test_conf.load()
self.assertFalse("cluster_type" in test_conf.raw["bootstrap"])
self.assertTrue(
"infrastructure_provisioning" in test_conf.raw["bootstrap"])
self.assertFalse("cluster_type" in test_conf.defaults["bootstrap"])
self.assertTrue(
"infrastructure_provisioning" in test_conf.defaults["bootstrap"])
def test_setup_overrides_no_file_empty_config(self):
"""
Testing setup_overrides, path = False and config vals not given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
test_override_path = whereami.dsi_repo_path("dsi", "tests")
# Call to setup_overrides creates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
self.assertFalse(
os.path.exists(os.path.join(test_override_path, "overrides.yml")))
def load_config_dict(module):
"""
Load ConfigDict for the given module with id checks mocked out.
:param str module: Name of module for ConfigDict.
"""
# pylint: disable=import-outside-toplevel
from dsi.common.config import ConfigDict
with patch("dsi.common.config.ConfigDict.assert_valid_ids") as mock_assert_valid_ids:
conf = ConfigDict(module)
conf.load()
mock_assert_valid_ids.assert_called_once()
return conf
def setUp(self):
""" Init a ConfigDict object and load the configuration files from docs/config-specs/ """
self.old_dir = os.getcwd() # Save the old path to restore
self.config = ConfigDict(
"mongodb_setup", whereami.dsi_repo_path("docs", "config-specs"))
self.config.load()
self.parent_dir = os.path.join(os.path.expanduser("~"),
"checkout_repos_test")
self._delete_fixtures()
self.reports_container = os.path.join(
FIXTURE_FILES.fixture_file_path(), "container")
self.reports_path = os.path.join(self.reports_container,
"reports_tests")
mkdir_p(self.reports_path)
def test_validate_delays(self):
blacklisted_configs = ["mongodb_setup.atlas.yml"] # Some configs don't have topologies.
directory = whereami.dsi_repo_path("configurations", "mongodb_setup")
errors = []
# There are a few files that aren't configuration files.
names = [name for name in os.listdir(directory) if name.startswith("mongodb_setup")]
# We need references to provisioning output
infrastructure_provisioning = whereami.dsi_repo_path(
"docs/config-specs/infrastructure_provisioning.out.yml"
)
with copied_file(infrastructure_provisioning, "infrastructure_provisioning.out.yml"):
for conf_name in names:
if conf_name in blacklisted_configs:
continue
with copied_file(os.path.join(directory, conf_name), "mongodb_setup.yml"):
config = ConfigDict("mongodb_setup")
config.load()
topologies = config["mongodb_setup"]["topology"]
network_delays = config["mongodb_setup"]["network_delays"]
delay_configs = network_delays["clusters"]
try:
# The DelayNodes throw exceptions when given bad delays, so we
# can validate the configuration by simply constructing a DelayGraph
# pylint: disable=unused-variable
DelayGraph.client_ip = config["infrastructure_provisioning"]["out"][
"workload_client"
][0]["private_ip"]
version_flag = str_to_version_flag(
network_delays.get("version_flag", "default")
)
DelayGraph.client_node = DelayNode(version_flag)
delays = DelayGraph.from_topologies(topologies, delay_configs, version_flag)
# pylint: disable=broad-except
except Exception as e:
errors.append(e)
# Reset the delay graph's client variables.
DelayGraph.client_node = DelayNode()
DelayGraph.client_ip = "workload_client"
self.assertEqual(errors, [])
def main(argv=sys.argv[1:]):
"""
Parse args and call workload_setup.yml operations
"""
parser = argparse.ArgumentParser(description="Workload Setup")
parser.add_argument("-d", "--debug", action="store_true", help="enable debug output")
parser.add_argument("--log-file", help="path to log file")
args = parser.parse_args(argv)
setup_logging(args.debug, args.log_file)
config = ConfigDict("workload_setup")
config.load()
setup = WorkloadSetupRunner(config)
setup.setup_workloads()
def main(argv):
""" Main function. parse args and execute
:param argv list the command line arguments excluding the program name. Default is
sys.argv[1:]
"""
args = parse_args(argv)
config = ConfigDict("infrastructure_provisioning").load()
host = args.host
expanded = expand(host)
unaliased = unalias(expanded)
ip_address = lookup_host(unaliased, config)
template = "{ip_address}"
if args.export:
template = "export {host}={ip_address}"
print(template.format(host=host, ip_address=ip_address))
def test_setup_hostnames(self, mock_exec_command, mock_create_file,
mock_ssh):
_ = mock_ssh
real_config_dict = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
real_config_dict.load()
real_config_dict.save = MagicMock(name="save")
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
provisioner.setup_hostnames()
out = provisioner.config["infrastructure_provisioning"]["out"]
self.assertEqual(out["mongod"][0]["private_hostname"],
"mongod0.dsitest.dev")
self.assertEqual(out["configsvr"][2]["private_hostname"],
"configsvr2.dsitest.dev")
self.assertEqual(mock_create_file.call_count, 16)
self.assertEqual(mock_exec_command.call_count, 16)
def main(argv):
""" Main function. Parse command line options, and run tests.
:returns: int the exit status to return to the caller (0 for OK)
"""
parser = argparse.ArgumentParser(description="DSI Test runner")
parser.add_argument("-d",
"--debug",
action="store_true",
help="enable debug output")
parser.add_argument("--log-file", help="path to log file")
args = parser.parse_args(argv)
log.setup_logging(args.debug, args.log_file)
config = ConfigDict("test_control")
config.load()
error = run_tests(config)
return 1 if error else 0
def test_setup_overrides_type_error(self):
"""
Testing setup_overrides doesn't throw TypeError.
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
real_configdict.raw["bootstrap"]["overrides"] = {
"infrastructure_provisioning": {
"tfvars": {
"tags": None
}
}
}
test_override_path = whereami.dsi_repo_path("dsi", "tests")
bootstrap.setup_overrides(real_configdict, test_override_path)
# Removing created file
try:
os.remove(os.path.join(test_override_path, "overrides.yml"))
except OSError:
pass
def main(argv):
""" Main function. Parse command line options, and run analysis.
Note that the return value here determines whether Evergreen considers the entire task passed
or failed. Non-zero return value means failure.
:returns: int the exit status to return to the caller (0 for OK)
"""
parser = argparse.ArgumentParser(description="Analyze DSI test results.")
parser.add_argument("-d",
"--debug",
action="store_true",
help="enable debug output")
parser.add_argument("--log-file", help="path to log file")
args = parser.parse_args(argv)
setup_logging(args.debug, args.log_file)
config = ConfigDict("analysis")
config.load()
analyzer = ResultsAnalyzer(config)
analyzer.analyze_all()
return 1 if analyzer.failures > 0 else 0
def main():
""" Handle the main functionality (parse args /setup logging ) and then start the mongodb
cluster."""
args = parse_command_line()
setup_logging(args.debug, args.log_file)
config = ConfigDict("mongodb_setup")
config.load()
# Start MongoDB cluster(s) using config given in mongodb_setup.topology (if any).
# Note: This also installs mongo client binary onto workload client.
mongo = MongodbSetup(config=config)
# Reset delays before starting so delays don't break setup.
mongo.client.reset_delays()
for cluster in mongo.clusters:
cluster.reset_delays()
start_cluster(mongo, config)
# Establish delays *after* the cluster is started so delays won't interfere with setup.
mongo.client.establish_delays()
for cluster in mongo.clusters:
cluster.establish_delays()
def setUp(self):
"""Setup so config dict works properly"""
self.config = ConfigDict(
"infrastructure_provisioning", whereami.dsi_repo_path("docs", "config-specs")
)
self.config.load()
def load_bootstrap(config, directory):
"""
Move specified bootstrap.yml file to correct location for read_runtime_values
"""
# Create directory if it doesn't exist
if not os.path.exists(directory):
os.makedirs(directory)
if "bootstrap_file" in config:
bootstrap_path = os.path.abspath(
os.path.expanduser(config["bootstrap_file"]))
if os.path.isfile(bootstrap_path):
if not bootstrap_path == os.path.abspath(
os.path.join(directory, "bootstrap.yml")):
if os.path.isfile(
os.path.abspath(
os.path.join(directory, "bootstrap.yml"))):
LOGGER.critical(
"Attempting to overwrite existing bootstrap.yml file. Aborting.",
directory=directory,
)
assert False
shutil.copyfile(bootstrap_path,
os.path.join(directory, "bootstrap.yml"))
else:
LOGGER.critical("Location specified for bootstrap.yml is invalid.")
assert False
else:
bootstrap_path = os.path.abspath(
os.path.expanduser(os.path.join(os.getcwd(), "bootstrap.yml")))
if os.path.isfile(bootstrap_path):
if not bootstrap_path == os.path.abspath(
os.path.join(directory, "bootstrap.yml")):
if os.path.isfile(
os.path.abspath(
os.path.join(directory, "bootstrap.yml"))):
LOGGER.critical(
"Attempting to overwrite existing bootstrap.yml file in %s. "
"Aborting.",
directory,
)
assert False
shutil.copyfile(bootstrap_path,
os.path.join(directory, "bootstrap.yml"))
expansions.write_if_necessary(directory)
current_path = os.getcwd()
os.chdir(directory)
config_dict = ConfigDict("bootstrap")
config_dict.load()
for key in config_dict["bootstrap"].keys():
config[key] = config_dict["bootstrap"][key]
# terraform required_version must be specified, we fail hard if user has tried to unset
config["terraform_version_check"] = config_dict[
"infrastructure_provisioning"]["terraform"]["required_version"]
config["terraform_linux_download"] = config_dict[
"infrastructure_provisioning"]["terraform"]["linux_download"]
config["terraform_mac_download"] = config_dict[
"infrastructure_provisioning"]["terraform"]["mac_download"]
os.chdir(current_path)
return config_dict
def setUp(self):
"""Init a ConfigDict object and load the configuration files from docs/config-specs/"""
self.conf = ConfigDict("mongodb_setup",
whereami.dsi_repo_path("docs", "config-specs"))
self.conf.load()
self.assertEqual(self.conf.module, "mongodb_setup")
def test_load_yaml_invalid_keys(self):
"""can't even get bad keys from yaml"""
with in_dir(FIXTURE_FILES.fixture_file_path("invalid-config")):
with self.assertRaises(config.InvalidConfigurationException):
ConfigDict("mongodb_setup").load()
