def test_build_hosts_file(self):
expected = [
"10.2.1.1\tmd md0 mongod0 mongod0.dsitest.dev",
"10.2.1.2\tmd1 mongod1 mongod1.dsitest.dev",
"10.2.1.3\tmd2 mongod2 mongod2.dsitest.dev",
"10.2.1.4\tmd3 mongod3 mongod3.dsitest.dev",
"10.2.1.5\tmd4 mongod4 mongod4.dsitest.dev",
"10.2.1.6\tmd5 mongod5 mongod5.dsitest.dev",
"10.2.1.7\tmd6 mongod6 mongod6.dsitest.dev",
"10.2.1.8\tmd7 mongod7 mongod7.dsitest.dev",
"10.2.1.9\tmd8 mongod8 mongod8.dsitest.dev",
"10.2.1.100\tms ms0 mongos0 mongos0.dsitest.dev",
"10.2.1.101\tms1 mongos1 mongos1.dsitest.dev",
"10.2.1.102\tms2 mongos2 mongos2.dsitest.dev",
"10.2.1.51\tcs cs0 configsvr0 configsvr0.dsitest.dev",
"10.2.1.52\tcs1 configsvr1 configsvr1.dsitest.dev",
"10.2.1.53\tcs2 configsvr2 configsvr2.dsitest.dev",
"10.2.1.10\twc wc0 workload_client0 workload_client0.dsitest.dev",
]
real_config_dict = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
real_config_dict.load()
real_config_dict.save = MagicMock(name="save")
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
hosts_contents = provisioner._build_hosts_file()
self.assertEqual(expected, hosts_contents)
def test_setup_cluster_failure(
self,
mock_subprocess,
mock_terraform_configuration,
mock_terraform_output_parser,
mock_setup_terraform_tf,
):
"""
Test Provisioner.setup_cluster when an error happens. Ensure that the cluster is torn
down.
"""
# NOTE: This tests the majority of the functionality of the infrastructure_provisioning.py
mock_open_file = mock_open()
with patch("dsi.infrastructure_provisioning.open",
mock_open_file,
create=True):
with patch("dsi.infrastructure_provisioning.destroy_resources"
) as mock_destroy:
provisioner = ip.Provisioner(
self.config, provisioning_file=self.provision_log_path)
mock_subprocess.check_call.side_effect = [
1, CalledProcessError(1, ["cmd"]), 1
]
with self.assertRaises(CalledProcessError):
provisioner.setup_cluster()
mock_setup_terraform_tf.assert_called()
mock_destroy.assert_called()
self.assertFalse(mock_terraform_output_parser.return_value.
write_output_files.called)
mock_terraform_configuration.return_value.to_json.assert_called_with(
file_name="cluster.json")
self.reset_mock_objects()
def test_setup_security_tf(self):
"""
Testing setup_security_tf creates security.tf file
"""
key_name = self.config["infrastructure_provisioning"]["tfvars"][
"ssh_key_name"]
key_file = self.config["infrastructure_provisioning"]["tfvars"][
"ssh_key_file"]
master_tf_str = ('provider "aws" {{ '
'access_key = "test_aws_access_key" '
'secret_key = "test_aws_secret_key" '
"region = var.region"
'version = "test_aws_version" }}'
'variable "key_name" {{ '
'default = "{}"}}'
'variable "key_file" {{ '
'default = "{}"}}').format(key_name, key_file)
master_tf_str = master_tf_str.replace("\n", "").replace(" ", "")
provisioner = ip.Provisioner(self.config,
provisioning_file=self.provision_log_path)
provisioner.aws_access_key = "test_aws_access_key"
provisioner.aws_secret_key = "test_aws_secret_key"
# Creating 'security.tf' file in current dir to test, reading to string
provisioner.setup_security_tf()
test_tf_str = ""
with open("security.tf", "r") as test_tf_file:
test_tf_str = test_tf_file.read().replace("\n",
"").replace(" ", "")
self.assertEqual(test_tf_str, master_tf_str)
# Removing created file
os.remove("security.tf")
def test_provisioner_init(self):
"""
Test Provisioner.__init__
"""
# Check that the correct default provisioning log is created
mock_open_file = mock_open()
with patch("dsi.infrastructure_provisioning.open",
mock_open_file,
create=True):
ip.Provisioner(self.config)
mock_open_file.assert_called_with(ip.PROVISION_LOG_PATH, "w")
# Check when TERRAFORM is an environment variable
provisioner = ip.Provisioner(self.config,
provisioning_file=self.provision_log_path)
self.assertEqual(provisioner.cluster, "single")
self.assertEqual(provisioner.dsi_dir, self.dsi_path)
self.assertEqual(provisioner.parallelism, "-parallelism=20")
self.assertEqual(provisioner.terraform, "test/path/terraform")
# Check when TERRAFORM is not environment variable
os_environ_missing_terraform = self.os_environ.copy()
del os_environ_missing_terraform["TERRAFORM"]
os_environ_missing_terraform["PATH"] = "/foo:/bar"
self.mock_environ.__getitem__.side_effect = os_environ_missing_terraform.__getitem__
self.mock_environ.__contains__.side_effect = os_environ_missing_terraform.__contains__
with self.assertRaises(utils.TerraformNotFound):
provisioner_missing_terraform = ip.Provisioner(
self.config, provisioning_file=self.provision_log_path)
self.assertEqual(provisioner_missing_terraform.cluster, "single")
self.assertEqual(provisioner_missing_terraform.dsi_dir,
self.dsi_path)
self.assertEqual(provisioner_missing_terraform.parallelism,
"-parallelism=20")
self.assertEqual(provisioner_missing_terraform.terraform,
"./terraform")
self.reset_mock_objects()
def test_print_terraform_errors(self):
"""
Test infrastructure_provisioning.print_terraform_errors()
"""
provisioner = ip.Provisioner(self.config,
provisioning_file=self.provision_log_path)
provisioner.tf_log_path = FIXTURE_FILES.fixture_file_path(
"terraform.log.short")
logger = MagicMock(name="logger")
tailer = MagicMock(name="tailer")
provisioner.print_terraform_errors(logger, tailer)
logger_calls = [
call(
"2018-03-05T15:45:36.018+0200 [DEBUG] plugin.terraform-provider-aws_v1.6.0_x4: "
"<Response><Errors><Error><Code>InsufficientInstanceCapacity</Code><Message>"
"Insufficient capacity.</Message></Error></Errors>"
"<RequestID>bd5b4071-755d-440e-8381-aa09bad52d69</RequestID></Response>"
),
call(
"2018-03-05T15:45:36.914+0200 [DEBUG] plugin.terraform-provider-aws_v1.6.0_x4: "
"<Response><Errors><Error><Code>RequestLimitExceeded</Code><Message>"
"Request limit exceeded.</Message></Error></Errors>"
"<RequestID>6280e71d-9be4-442c-8ddf-00265efeafe6</RequestID></Response>"
),
call(
"2018-03-05T15:48:36.336+0200 [DEBUG] plugin.terraform-provider-aws_v1.6.0_x4: "
"<Response><Errors><Error><Code>InvalidRouteTableID.NotFound</Code><Message>"
"The routeTable ID 'rtb-509f1528' does not exist</Message></Error></Errors>"
"<RequestID>54256eb4-d706-4084-86dc-b7f581006f9f</RequestID></Response>"
),
call(
"2018-03-05T15:48:47.258+0200 [DEBUG] plugin.terraform-provider-aws_v1.6.0_x4: "
"<Response><Errors><Error><Code>DependencyViolation</Code><Message>"
"Network vpc-a9ed8bd0 has some mapped public address(es). Please unmap those public address(es) "
"before detaching the gateway.</Message></Error></Errors>"
"<RequestID>cd102bc6-d598-4bae-80f5-25e62103f9a4</RequestID></Response>"
),
call(
"2018-03-05T15:49:29.084+0200 [DEBUG] plugin.terraform-provider-aws_v1.6.0_x4: "
"<Response><Errors><Error><Code>InvalidPlacementGroup.Unknown</Code><Message>"
"The Placement Group 'shard-8665ea69-9e76-483a-937b-af68d41d54dd' is unknown."
"</Message></Error></Errors><RequestID>4264aef8-ae91-40f0-bc16-d914a8dc2cf8</RequestID></Response>"
),
call("For more info, see:",
path=FIXTURE_FILES.fixture_file_path("terraform.log.short")),
]
logger.error.assert_has_calls(logger_calls)
def test_setup_hostnames(self, mock_exec_command, mock_create_file,
mock_ssh):
_ = mock_ssh
real_config_dict = ConfigDict(
"infrastructure_provisioning",
whereami.dsi_repo_path("docs", "config-specs"))
real_config_dict.load()
real_config_dict.save = MagicMock(name="save")
provisioner = ip.Provisioner(real_config_dict,
provisioning_file=self.provision_log_path)
provisioner.setup_hostnames()
out = provisioner.config["infrastructure_provisioning"]["out"]
self.assertEqual(out["mongod"][0]["private_hostname"],
"mongod0.dsitest.dev")
self.assertEqual(out["configsvr"][2]["private_hostname"],
"configsvr2.dsitest.dev")
self.assertEqual(mock_create_file.call_count, 16)
self.assertEqual(mock_exec_command.call_count, 16)
def test_setup_terraform_tf(self):
"""
Test setup_terraform_tf creates the correct directories and files
"""
# Create temporary directory and get correct paths
directory = "temp_test"
if os.path.exists(directory):
shutil.rmtree(directory)
os.mkdir(directory)
cluster_path = os.path.join(self.dsi_path, "clusters", "default")
remote_scripts_path = os.path.join(self.dsi_path, "clusters",
"remote-scripts")
remote_scripts_target = os.path.join(directory, "remote-scripts")
modules_path = os.path.join(self.dsi_path, "clusters", "modules")
modules_target = os.path.join(directory, "modules")
# Check files copied correctly
with patch("dsi.infrastructure_provisioning.os.getcwd",
return_value="temp_test"):
provisioner = ip.Provisioner(
self.config, provisioning_file=self.provision_log_path)
provisioner.setup_terraform_tf()
for filename in glob.glob(os.path.join(cluster_path, "*")):
self.assertTrue(
os.path.exists(os.path.join(directory,
filename.split("/")[-1])))
for filename in glob.glob(os.path.join(remote_scripts_path, "*")):
self.assertTrue(
os.path.exists(
os.path.join(remote_scripts_target,
filename.split("/")[-1])))
for filename in glob.glob(os.path.join(modules_path, "*")):
self.assertTrue(
os.path.exists(
os.path.join(modules_target,
filename.split("/")[-1])))
# Remove temporary directory
shutil.rmtree(directory)
def test_setup_cluster(
self,
mock_subprocess,
mock_save_output,
mock_terraform_configuration,
mock_terraform_output_parser,
mock_setup_security_tf,
mock_setup_terraform_tf,
mock_pre_post_commands,
):
"""
Test Provisioner.setup_cluster
"""
# NOTE: This tests the majority of the functionality of the infrastructure_provisioning.py
# mock.mock_open is needed to effectively mock out the open() function in python
mock_open_file = mock_open()
with patch("dsi.infrastructure_provisioning.open",
mock_open_file,
create=True):
provisioner = ip.Provisioner(
self.config, provisioning_file=self.provision_log_path)
provisioner.cluster = "initialsync-logkeeper"
provisioner.hostnames_method = None
provisioner.setup_cluster()
mock_setup_security_tf.assert_called()
mock_setup_terraform_tf.assert_called()
mock_terraform_configuration.return_value.to_json.assert_called_with(
file_name="cluster.json")
# __enter__ and __exit__ are checked to see if the files were opened
# as context managers.
open_file_calls = [
call("infrastructure_provisioning.out.yml", "r"),
call().read()
]
mock_open_file.assert_has_calls(open_file_calls, any_order=True)
# If the cluster is initialsync-logkeeper, then terraform should be run twice
terraform = self.os_environ["TERRAFORM"]
check_call_calls = [
call(
[terraform, "init", "-upgrade"],
stdout=provisioner.stdout,
stderr=provisioner.stderr,
),
call(
[
terraform,
"apply",
"-var-file=cluster.json",
provisioner.parallelism,
"-auto-approve",
"-var=mongod_ebs_instance_count=0",
"-var=workload_instance_count=0",
],
stdout=provisioner.stdout,
stderr=provisioner.stderr,
),
call(
[
terraform,
"apply",
"-var-file=cluster.json",
provisioner.parallelism,
"-auto-approve",
],
stdout=provisioner.stdout,
stderr=provisioner.stderr,
),
call(
[terraform, "refresh", "-var-file=cluster.json"],
stdout=provisioner.stdout,
stderr=provisioner.stderr,
),
call(
[
terraform, "plan", "-detailed-exitcode",
"-var-file=cluster.json"
],
stdout=provisioner.stdout,
stderr=provisioner.stderr,
),
]
mock_subprocess.check_call.assert_has_calls(check_call_calls)
mock_save_output.assert_called_with([terraform, "output"])
self.assertTrue(mock_terraform_output_parser.return_value.
write_output_files.called)
mock_pre_post_commands.assert_called()
self.reset_mock_objects()