def test_destroy_resources_no_cluster_json(self, mock_find_terraform,
mock_os, mock_glob):
""" Test infrastructure_teardown.destroy_resources when there is no cluster.json file """
mock_os.path.dirname.return_value = "teardown/script/path"
mock_os.getcwd.return_value = "previous/directory"
mock_os.path.isfile.return_value = False
mock_glob.return_value = True
mock_find_terraform.return_value = "test/path/terraform"
with LogCapture(level=logging.CRITICAL) as critical:
with self.assertRaises(UserWarning):
infrastructure_teardown.destroy_resources()
critical.check((
"dsi.infrastructure_teardown",
"CRITICAL",
"In infrastructure_teardown.py and cluster.json does not exist. Giving up.",
))
mock_glob.assert_called_with(
os.path.join(whereami.dsi_repo_path("dsi"), "provisioned.*"))
chdir_calls = [
call(whereami.dsi_repo_path("dsi")),
call("previous/directory")
]
mock_os.chdir.assert_has_calls(chdir_calls)
mock_os.path.isfile.assert_called_with("cluster.json")
def test_load_bootstrap_different_filename(self):
"""
Testing that load_bootstrap works with alternate file names
"""
bootstrap_path = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"bootstrap2.yml")
directory = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"testdir/")
config = {"bootstrap_file": bootstrap_path, "production": False}
with open(bootstrap_path, "w") as bootstrap_file:
bootstrap_file.write("owner: test_owner")
bootstrap.load_bootstrap(config, directory)
self.assertEqual(config["owner"], "test_owner")
def test_load_bootstrap_local_file_makedir(self):
"""
Testing that load_bootstrap makes nonexistent directory and copies into it
"""
bootstrap_path = os.path.join(
os.path.dirname(whereami.dsi_repo_path("dsi", "tests")),
"bootstrap.yml")
directory = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"testdir/")
config = {"bootstrap_file": bootstrap_path, "production": False}
with open(bootstrap_path, "w", encoding="utf-8") as bootstrap_file:
bootstrap_file.write("owner: test_owner")
bootstrap.load_bootstrap(config, directory)
self.assertEqual(config["owner"], "test_owner")
def tearDown(self):
"""
Cleaning up directories and files
"""
paths = [
"tests/test_dsipath",
"tests/test_directory",
"tests/test_credentials",
"tests/testdir",
"tests/test_old_dir",
"tests/test_new_dir",
"tests/test_cred_path",
"tests/overrides.yml",
"bootstrap.yml",
"tests/bootstrap.yml",
"tests/bootstrap2.yml",
"testdir",
]
for path in paths:
try:
path = os.path.join(whereami.dsi_repo_path("dsi"), path)
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
except OSError:
pass
def test_setup_overrides_default_username(self):
"""
Testing setup_overrides fails if user doesn't change username
from bootstrap.example.yml default
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
real_configdict.raw["bootstrap"]["overrides"] = {
"infrastructure_provisioning": {
"tfvars": {
"tags": {
"owner": "your.username"
}
}
}
}
test_override_path = whereami.dsi_repo_path("dsi", "tests")
with self.assertRaises(AssertionError):
bootstrap.setup_overrides(real_configdict, test_override_path)
# Removing created file
try:
os.remove(os.path.join(test_override_path, "overrides.yml"))
except OSError:
pass
def test_setup_overrides_file_exists_empty_config(self):
"""
Testing setup_overrides, path = True and config vals not given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
test_override_path = whereami.dsi_repo_path("dsi", "tests")
test_override_str = yaml.dump({}, default_flow_style=False)
# Creating 'overrides.yml' in current dir
with open(os.path.join(test_override_path, "overrides.yml"),
"w+") as test_override_file:
test_override_file.write(test_override_str)
# Call to setup_overrides updates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
test_override_dict = {}
with open(os.path.join(test_override_path, "overrides.yml"),
"r") as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, {})
# Removing created file
os.remove(os.path.join(test_override_path, "overrides.yml"))
def test_setup_overrides_no_file_config_vals(self):
"""
Testing setup_overrides where path = False and config vals given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
master_overrides = {}
master_overrides.update({
"infrastructure_provisioning": {
"tfvars": {
"ssh_key_file": "test_ssh_key_file.pem",
"ssh_key_name": "test_ssh_key_name",
"tags": {
"owner": "testuser",
"expire-on-delta": 24
},
}
}
})
real_configdict.raw["bootstrap"] = {}
real_configdict.raw["bootstrap"]["overrides"] = master_overrides
test_override_path = whereami.dsi_repo_path("dsi", "tests")
test_override_dict = {}
# Call to setup_overrides creates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
with open(os.path.join(test_override_path, "overrides.yml"),
"r") as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, master_overrides)
# Removing created file
os.remove(os.path.join(test_override_path, "overrides.yml"))
def test_write_dsienv(self):
"""
Testing write_dsienv
"""
tests_path = whereami.dsi_repo_path("dsi", "tests")
directory = tests_path
terraform = "/Users/test_user/terraform"
master_dsienv = ("export PATH=" + whereami.dsi_repo_path("dsi") +
":$PATH\n"
"export TERRAFORM=/Users/test_user/terraform\n")
bootstrap.write_dsienv(directory, terraform)
with open(os.path.join(directory, "dsienv.sh")) as dsienv:
test_dsienv = dsienv.read()
self.assertEqual(test_dsienv[:len(master_dsienv)], master_dsienv)
os.remove(os.path.join(directory, "dsienv.sh"))
def test_upload_files_dir(self, mock_connected_ssh):
"""We can upload directories of files"""
ssh = mock.MagicMock(name="ssh")
ftp = mock.MagicMock(name="ftp")
channel = mock.MagicMock(name="channel")
ssh.exec_command.return_value = channel, channel, channel
mock_connected_ssh.return_value = (ssh, ftp)
remote = remote_ssh_host.RemoteSSHHost(hostname=None,
username=None,
pem_file=None)
remote._perform_exec = mock.MagicMock(name="_perform_exec")
remote._perform_exec.return_value = 0
local_path = whereami.dsi_repo_path("testscripts")
remote_path = "/foo/bar"
remote.upload_file(local_path, remote_path)
ssh.exec_command.assert_has_calls(
[
call("mkdir -p /foo/bar", get_pty=False),
call("tar xf /foo/bar.tar -C /foo/bar", get_pty=False),
call("rm /foo/bar.tar", get_pty=False),
],
any_order=False,
)
ftp.assert_has_calls(
[call.put(ANY, "/foo/bar.tar"),
call.chmod("/foo/bar.tar", ANY)],
any_order=False)
def setUp(self):
""" Load self.config (ConfigDict) and set some other common values """
self.config = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
self.config.load()
cookiejar = requests.cookies.RequestsCookieJar()
request = requests.Request("GET", "http://ip.42.pl/raw")
request.prepare()
self.response_state = {
"cookies": cookiejar,
"_content": b"ip.42.hostname",
"encoding": "UTF-8",
"url": "http://ip.42.pl/raw",
"status_code": 200,
"request": request,
"elapsed": datetime.timedelta(0, 0, 615501),
"headers": {
"Content-Length": "14",
"X-Powered-By": "PHP/5.6.27",
"Keep-Alive": "timeout=5, max=100",
"Server":
"Apache/2.4.23 (FreeBSD) OpenSSL/1.0.1l-freebsd PHP/5.6.27",
"Connection": "Keep-Alive",
"Date": "Tue, 25 Jul 2017 14:20:06 GMT",
"Content-Type": "text/html; charset=UTF-8",
},
"reason": "OK",
"history": [],
}
def test_set_some_values(self):
"""Set some values and write out file"""
self.conf["mongodb_setup"]["out"] = {"foo": "bar"}
# Read the value multiple times, because once upon a time that didn't work (believe it or not)
self.assert_equal_dicts(self.conf["mongodb_setup"]["out"],
{"foo": "bar"})
self.assert_equal_dicts(self.conf["mongodb_setup"]["out"],
{"foo": "bar"})
self.assert_equal_dicts(self.conf["mongodb_setup"]["out"],
{"foo": "bar"})
self.conf["mongodb_setup"]["out"]["zoo"] = "zar"
self.assert_equal_dicts(self.conf["mongodb_setup"]["out"], {
"foo": "bar",
"zoo": "zar"
})
with self.assertRaises(KeyError):
self.conf["foo"] = "bar"
# Write the out file only if it doesn't already exist, and delete it when done
file_name = os.path.join(
whereami.dsi_repo_path("docs", "config-specs"),
"mongodb_setup.out.yml")
if os.path.exists(file_name):
self.fail(
"Cannot test writing docs/config-specs/mongodb_setup.out.yml file, file already exists."
)
else:
self.conf.save()
file_handle = open(file_name)
saved_out_file = yaml.safe_load(file_handle)
file_handle.close()
self.assert_equal_dicts({"out": self.conf["mongodb_setup"]["out"]},
saved_out_file)
os.remove(file_name)
def test_repo_root_different_cwd(self):
existing = os.getcwd()
try:
os.chdir(os.path.dirname(ACTUAL_ROOT))
self.assertEqual(whereami.dsi_repo_path(), ACTUAL_ROOT)
finally:
os.chdir(existing)
def test_upload_repo_files(self):
""" Test run command map upload_repo_files """
root = whereami.dsi_repo_path() + os.sep
# test upload_repo_files
with patch("dsi.common.remote_host.RemoteHost") as mongod:
command = {
"upload_repo_files": [{
"target": "remote_path",
"source": "mongos.log"
}]
}
command_runner._run_host_command_map(mongod, command, "test_id")
mongod.upload_file.assert_called_once_with(root + "mongos.log",
"remote_path")
with patch("dsi.common.remote_host.RemoteHost") as mongod:
command = {
"upload_repo_files": [
{
"target": "remote_path",
"source": "mongos.log"
},
{
"target": "to",
"source": "from"
},
]
}
command_runner._run_host_command_map(mongod, command, "test_id")
calls = [
mock.call(root + "mongos.log", "remote_path"),
mock.call(root + "from", "to")
]
mongod.upload_file.assert_has_calls(calls, any_order=True)
def test_build_hosts_file(self):
expected = [
"10.2.1.1\tmd md0 mongod0 mongod0.dsitest.dev",
"10.2.1.2\tmd1 mongod1 mongod1.dsitest.dev",
"10.2.1.3\tmd2 mongod2 mongod2.dsitest.dev",
"10.2.1.4\tmd3 mongod3 mongod3.dsitest.dev",
"10.2.1.5\tmd4 mongod4 mongod4.dsitest.dev",
"10.2.1.6\tmd5 mongod5 mongod5.dsitest.dev",
"10.2.1.7\tmd6 mongod6 mongod6.dsitest.dev",
"10.2.1.8\tmd7 mongod7 mongod7.dsitest.dev",
"10.2.1.9\tmd8 mongod8 mongod8.dsitest.dev",
"10.2.1.100\tms ms0 mongos0 mongos0.dsitest.dev",
"10.2.1.101\tms1 mongos1 mongos1.dsitest.dev",
"10.2.1.102\tms2 mongos2 mongos2.dsitest.dev",
"10.2.1.51\tcs cs0 configsvr0 configsvr0.dsitest.dev",
"10.2.1.52\tcs1 configsvr1 configsvr1.dsitest.dev",
"10.2.1.53\tcs2 configsvr2 configsvr2.dsitest.dev",
"10.2.1.10\twc wc0 workload_client0 workload_client0.dsitest.dev",
]
real_config_dict = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
real_config_dict.load()
real_config_dict.save = MagicMock(name="save")
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
hosts_contents = provisioner._build_hosts_file()
self.assertEqual(expected, hosts_contents)
def setUp(self):
""" Init a ConfigDict object and load the configuration files from docs/config-specs/ """
self.config = ConfigDict("mongodb_setup", whereami.dsi_repo_path("docs", "config-specs"))
self.config.load()
self.parent_dir = os.path.join(os.path.expanduser("~"), "checkout_repos_test")
self._delete_fixtures()
def repo_root_file_path(self, *args):
"""
Return the absolute path of a file at `file_path` with respect to the repo root.
:param str file_path: The name of the file path.
:rtype: str.
"""
return whereami.dsi_repo_path(*args)
def test_validate_delays(self):
blacklisted_configs = ["mongodb_setup.atlas.yml"] # Some configs don't have topologies.
directory = whereami.dsi_repo_path("configurations", "mongodb_setup")
errors = []
# There are a few files that aren't configuration files.
names = [name for name in os.listdir(directory) if name.startswith("mongodb_setup")]
# We need references to provisioning output
infrastructure_provisioning = whereami.dsi_repo_path(
"docs/config-specs/infrastructure_provisioning.out.yml"
)
with copied_file(infrastructure_provisioning, "infrastructure_provisioning.out.yml"):
for conf_name in names:
if conf_name in blacklisted_configs:
continue
with copied_file(os.path.join(directory, conf_name), "mongodb_setup.yml"):
config = ConfigDict("mongodb_setup")
config.load()
topologies = config["mongodb_setup"]["topology"]
network_delays = config["mongodb_setup"]["network_delays"]
delay_configs = network_delays["clusters"]
try:
# The DelayNodes throw exceptions when given bad delays, so we
# can validate the configuration by simply constructing a DelayGraph
# pylint: disable=unused-variable
DelayGraph.client_ip = config["infrastructure_provisioning"]["out"][
"workload_client"
][0]["private_ip"]
version_flag = str_to_version_flag(
network_delays.get("version_flag", "default")
)
DelayGraph.client_node = DelayNode(version_flag)
delays = DelayGraph.from_topologies(topologies, delay_configs, version_flag)
# pylint: disable=broad-except
except Exception as e:
errors.append(e)
# Reset the delay graph's client variables.
DelayGraph.client_node = DelayNode()
DelayGraph.client_ip = "workload_client"
self.assertEqual(errors, [])
def setUp(self):
self.test_cred_path = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "creds")
with open(self.test_cred_path, "w") as test_cred_file:
test_cred_file.write(
"[default]\naws_access_key_id = "
"test_aws_access_key1\naws_secret_access_key = "
"test_aws_secret_key1")
def test_load_bootstrap_copy_file_to_local(self):
"""
Testing that load_bootstrap copies specified file in 'testdir' to local directory
"""
bootstrap_path = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"bootstrap.yml")
bootstrap_directory = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "testdir/")
os.mkdir(bootstrap_directory)
config = {"bootstrap_file": bootstrap_path, "production": False}
with open(bootstrap_path, "w") as bootstrap_file:
bootstrap_file.write("owner: test_owner")
bootstrap.load_bootstrap(config,
whereami.dsi_repo_path("dsi", "tests"))
# confirms that load_bootstrap copies file into working directory correctly
self.assertEqual(config["owner"], "test_owner")
def __init__(self,
config,
log_file=TF_LOG_PATH,
provisioning_file=PROVISION_LOG_PATH,
verbose=False):
self.config = config
ssh_key_file = config["infrastructure_provisioning"]["tfvars"][
"ssh_key_file"]
ssh_key_file = os.path.expanduser(ssh_key_file)
self.ssh_key_file = ssh_key_file
self.ssh_key_name = config["infrastructure_provisioning"]["tfvars"][
"ssh_key_name"]
self.aws_access_key, self.aws_secret_key = utils.read_aws_credentials(
config)
self.cluster = config["infrastructure_provisioning"]["tfvars"].get(
"cluster_name", "missing_cluster_name")
self.var_file = None
self.parallelism = "-parallelism=" + str(TERRAFORM_PARALLELISM)
self.terraform = utils.find_terraform()
LOG.info("Using terraform binary:", path=self.terraform)
self.tf_log_path = TF_LOG_PATH
self.hostnames_method = (config["infrastructure_provisioning"].get(
"hostnames", {}).get("method"))
os.environ["TF_LOG"] = "DEBUG"
os.environ["TF_LOG_PATH"] = TF_LOG_PATH
self.dsi_dir = whereami.dsi_repo_path()
self.bin_dir = whereami.dsi_repo_path("dsi")
self.log_file = log_file
self.verbose = verbose
self.provisioning_file = provisioning_file
# Counter-intuitively, _None_ has the following stream semantics.
# "With the default settings of None, no redirection will occur; the child's file handles
# will be inherited from the parent"
# @see subprocess.Popen
if self.verbose:
self.stdout = self.stderr = None
else:
LOG.info("Redirecting terraform output to file",
path=self.provisioning_file)
self.stdout = self.stderr = open(self.provisioning_file, "w")
def test_load_bootstrap_copy_file_default_to_local(self):
"""
Testing that load_bootstrap uses local file if --bootstrap-file flag not used
"""
bootstrap_path = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"bootstrap.yml")
bootstrap_directory = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "testdir/")
os.mkdir(bootstrap_directory)
config = {"production": False}
with open(bootstrap_path, "w") as bootstrap_file:
bootstrap_file.write("owner: test_owner")
current_path = os.getcwd()
os.chdir(os.path.join(bootstrap_directory, ".."))
bootstrap.load_bootstrap(config, bootstrap_directory)
os.chdir(current_path)
# confirms that load_bootstrap copies local file into working directory if not specified
self.assertEqual(config["owner"], "test_owner")
def test_setup_overrides_file_exists_config_vals(self):
"""
Testing setup_overrides where path = True and config vals given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
test_override_path = whereami.dsi_repo_path("dsi", "tests")
master_overrides = {}
master_overrides.update({
"infrastructure_provisioning": {
"tfvars": {
"ssh_key_file": "test_ssh_key_file1.pem",
"ssh_key_name": "test_ssh_key_name1",
"tags": {
"owner": "testuser1",
"expire-on-delta": 24
},
}
}
})
real_configdict.raw["bootstrap"]["overrides"] = master_overrides
test_override_str = yaml.dump(
{
"infrastructure_provisioning": {
"tfvars": {
"ssh_key_file": "test_ssh_key_file2.pem",
"ssh_key_name": "test_ssh_key_name2",
"tags": {
"owner": "testuser2",
"expire-on-delta": 48
},
}
}
},
default_flow_style=False,
)
# Creating 'overrides.yml' in current dir
with open(os.path.join(test_override_path, "overrides.yml"),
"w") as test_override_file:
test_override_file.write(test_override_str)
# Call to setup_overrides updates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
test_override_dict = {}
with open(os.path.join(test_override_path, "overrides.yml"),
"r") as test_override_file:
test_override_dict = yaml.load(test_override_file)
self.assertEqual(test_override_dict, master_overrides)
# Removing created file
os.remove(os.path.join(test_override_path, "overrides.yml"))
def test_load_bootstrap_copy_same_file(self):
"""
Testing that load_bootstrap copies specified file in 'testdir' to
local directory and fails on collision
"""
bootstrap_path = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"testdir/bootstrap.yml")
wrong_bootstrap_path = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "./bootstrap.yml")
bootstrap_directory = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "testdir/")
os.mkdir(bootstrap_directory)
config = {"bootstrap_file": bootstrap_path, "production": False}
with open(bootstrap_path, "w") as bootstrap_file:
bootstrap_file.write("owner: test_owner")
with open(wrong_bootstrap_path, "w") as wrong_bootstrap_file:
wrong_bootstrap_file.write("owner: test_owner")
with self.assertRaises(AssertionError):
bootstrap.load_bootstrap(config,
whereami.dsi_repo_path("dsi", "tests"))
def test_load_bootstrap_given_file_and_dir(self):
"""
Testing that load_bootstrap copies file from 'test_old_dir' into
'test_new_dir' without collisions
"""
bootstrap_path = os.path.join(whereami.dsi_repo_path("dsi", "tests"),
"test_old_dir/bootstrap.yml")
bootstrap_new_directory = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "test_new_dir/")
bootstrap_old_directory = os.path.join(
whereami.dsi_repo_path("dsi", "tests"), "test_old_dir/")
os.mkdir(bootstrap_new_directory)
os.mkdir(bootstrap_old_directory)
config = {"bootstrap_file": bootstrap_path, "production": False}
with open(bootstrap_path, "w") as bootstrap_file:
bootstrap_file.write("platform: test_platform")
bootstrap.load_bootstrap(config, bootstrap_new_directory)
# confirms that load_bootstrap copies file into working directory correctly
self.assertEqual(config["platform"], "test_platform")
def test_bootstrap_respects_existing_expansions_file(
self, mock_check_output):
"""
Test we don't create a new expansions yaml file if already there
"""
mock_check_output.return_value = "Terraform v0.12.16"
bootstrap_path = os.path.join(
os.path.dirname(whereami.dsi_repo_path("dsi", "tests")),
"bootstrap.yml")
directory = os.path.join(
os.path.dirname(whereami.dsi_repo_path("dsi", "tests")),
"testdir/")
expansions_file = os.path.join(directory, "expansions.yml")
os.mkdir(directory)
secret_overrides_path = os.path.join(directory, "overrides.yml")
with open(secret_overrides_path, "w") as config:
config.writelines([
"runtime_secret: { aws_access_key: dummy, aws_secret_key: dummy }"
])
with open(bootstrap_path, "w") as config:
config.writelines([
# this is the only entry that's actually needed
"infrastructure_provisioning: single"
])
with open(expansions_file, "w+") as expansions:
expansions.writelines(["from_expansions: true"])
self.assertTrue(os.path.exists(expansions_file))
bootstrap.run_bootstrap({
"directory": directory,
"bootstrap_file": bootstrap_path
})
with open(expansions_file, "r") as expansions:
self.assertEqual(expansions.read(), "from_expansions: true")
def symlink_bindir(directory):
"""
Create symlink to dsi_repo/dsi.
:param str directory: The work directory.
"""
src = whereami.dsi_repo_path("dsi")
dest = os.path.join(directory, ".bin")
if os.path.exists(dest):
LOGGER.warning("Removing old symlink to binaries.", dest=dest)
os.remove(dest)
LOGGER.info("Creating symlink to binaries.", src=src, dest=dest)
os.symlink(src, dest)
def test_load_new(self):
"""Test loading ConfigDict with old naming convention .yml files"""
test_conf = ConfigDict(
"bootstrap",
whereami.dsi_repo_path("dsi", "tests", "test_config_files",
"new_format"))
test_conf.load()
self.assertFalse("cluster_type" in test_conf.raw["bootstrap"])
self.assertTrue(
"infrastructure_provisioning" in test_conf.raw["bootstrap"])
self.assertFalse("cluster_type" in test_conf.defaults["bootstrap"])
self.assertTrue(
"infrastructure_provisioning" in test_conf.defaults["bootstrap"])
def test_setup_overrides_no_file_empty_config(self):
"""
Testing setup_overrides, path = False and config vals not given
"""
real_configdict = ConfigDict("bootstrap")
real_configdict.load()
test_override_path = whereami.dsi_repo_path("dsi", "tests")
# Call to setup_overrides creates 'overrides.yml' in current dir
bootstrap.setup_overrides(real_configdict, test_override_path)
self.assertFalse(
os.path.exists(os.path.join(test_override_path, "overrides.yml")))
def load(self):
"""
Populate with contents of module_name.yml, module_name.out.yml, overrides.yml.
Note: exceptions may be raised by the lower layer, see :method:
`ConfigDict.assert_valid_ids`, `_yaml_load`.
"""
loaded_files = []
# defaults.yml
file_name = whereami.dsi_repo_path("configurations", "defaults.yml")
with open(file_name) as file_handle:
self.defaults = _yaml_load(file_handle, file_name)
loaded_files.append(file_name)
# All module_name.yml and module_name.out.yml
for module_name in self.modules:
file_name = os.path.join(self.config_root, module_name + ".yml")
if os.path.isfile(file_name):
with open(file_name) as file_handle:
self.raw[module_name] = _yaml_load(file_handle, file_name)
loaded_files.append(file_name)
elif module_name != "_internal":
# Allow code to assume that first level of keys always exists
self.raw[module_name] = {}
file_name = os.path.join(self.config_root,
module_name + ".out.yml")
if os.path.isfile(file_name):
with open(file_name) as file_handle:
# Note: The .out.yml files will add a single key: ['module_name']['out']
out = _yaml_load(file_handle, file_name)
if isinstance(out, dict):
if module_name in self.raw:
self.raw[module_name].update(out)
else:
self.raw.update({module_name: out})
loaded_files.append(file_name)
# overrides.yml
file_name = os.path.join(self.config_root, "overrides.yml")
if os.path.isfile(file_name):
file_handle = open(file_name)
self.overrides = _yaml_load(file_handle, file_name)
file_handle.close()
loaded_files.append(file_name)
LOG.info("Loaded DSI config files: %s", loaded_files)
self.assert_valid_ids()
return self
def setUp(self):
self.os_environ_patcher = patch(
"dsi.infrastructure_provisioning.os.environ")
self.mock_environ = self.os_environ_patcher.start()
self.dsi_path = whereami.dsi_repo_path()
self.reset_mock_objects()
# pylint: disable=line-too-long
self.config = {
"bootstrap": {
"infrastructure_provisioning": "single"
},
"infrastructure_provisioning": {
"hostnames": {
"method": "/etc/hosts"
},
"terraform": {
"aws_required_version": "test_aws_version"
},
"tfvars": {
"cluster_name": "single",
"ssh_key_file": "aws_ssh_key.pem",
"ssh_key_name": "serverteam-perf-ssh-key",
},
"post_provisioning": [{
"on_all_hosts": {
"exec":
"""
# set ulimit nofile for user
echo "${infrastructure_provisioning.tfvars.ssh_user} soft nofile 65535" | sudo tee -a /etc/security/limits.conf
echo "${infrastructure_provisioning.tfvars.ssh_user} hard nofile 65535" | sudo tee -a /etc/security/limits.conf
echo "${infrastructure_provisioning.tfvars.ssh_user} soft core unlimited" | sudo tee -a /etc/security/limits.conf
echo "${infrastructure_provisioning.tfvars.ssh_user} hard core unlimited" | sudo tee -a /etc/security/limits.conf
"""
}
}],
},
"runtime_secret": {
"aws_access_key": "test_access_key",
"aws_secret_key": "test_secret_key",
},
}
# pylint: enable=line-too-long
# create a provision log path that can be safely deleted post test
self.provision_log_path = os.path.join(
FIXTURE_FILES.fixture_file_path(), ip.PROVISION_LOG_PATH)
# Setup logging so that structlog uses stdlib, and LogCapture works
structlog_for_test.setup_logging()
