def test_get_magic_castle_configuration_with_dns_provider():
assert DnsManager("calculquebec.cloud").get_magic_castle_configuration() == {
"dns": {
"email": "*****@*****.**",
"source": f"git::https://github.com/ComputeCanada/magic_castle.git//dns/cloudflare?ref={MAGIC_CASTLE_VERSION_TAG}",
"name": "${module.openstack.cluster_name}",
"domain": "${module.openstack.domain}",
"public_ip": "${module.openstack.ip}",
"login_ids": "${module.openstack.login_ids}",
"rsa_public_key": "${module.openstack.rsa_public_key}",
"ssh_private_key": "${module.openstack.ssh_private_key}",
"sudoer_username": "******",
}
}
assert DnsManager("c3.ca").get_magic_castle_configuration() == {
"dns": {
"email": "*****@*****.**",
"project": "your-project-name",
"zone_name": "your-zone-name",
"source": f"git::https://github.com/ComputeCanada/magic_castle.git//dns/gcloud?ref={MAGIC_CASTLE_VERSION_TAG}",
"name": "${module.openstack.cluster_name}",
"domain": "${module.openstack.domain}",
"public_ip": "${module.openstack.ip}",
"login_ids": "${module.openstack.login_ids}",
"rsa_public_key": "${module.openstack.rsa_public_key}",
"ssh_private_key": "${module.openstack.ssh_private_key}",
"sudoer_username": "******",
}
}
def test_get_environment_variables_with_dns_provider():
assert DnsManager("calculquebec.cloud").get_environment_variables() == {
"CLOUDFLARE_API_TOKEN": "EXAMPLE_TOKEN",
"CLOUDFLARE_ZONE_API_TOKEN": "EXAMPLE_TOKEN",
"CLOUDFLARE_DNS_API_TOKEN": "EXAMPLE_TOKEN",
}
assert DnsManager("c3.ca").get_environment_variables() == {
"GOOGLE_CREDENTIALS": "/home/mcu/credentials/gcloud-service-account.json",
"GCE_SERVICE_ACCOUNT_FILE": "/home/mcu/credentials/gcloud-service-account.json",
}
def get_available_resources(self):
"""
Retrieves the available cloud resources including resources from OpenStack
and available domains.
"""
available_resources = self.__openstack_manager.get_available_resources(
)
available_resources["possible_resources"][
"domain"] = DnsManager.get_available_domains()
return available_resources
def update_main_tf_json_file(self):
"""
Formats the configuration and writes it to the cluster's main.tf.json file.
"""
main_tf_configuration = {
"terraform": {"required_version": TERRAFORM_REQUIRED_VERSION},
"module": {
"openstack": {
"source": f"{MAGIC_CASTLE_MODULE_SOURCE}//openstack?ref={MAGIC_CASTLE_VERSION_TAG}",
"generate_ssh_key": True,
"config_git_url": MAGIC_CASTLE_PUPPET_CONFIGURATION_URL,
"config_version": MAGIC_CASTLE_VERSION_TAG,
**deepcopy(self.__configuration),
}
},
}
main_tf_configuration["module"].update(
DnsManager(self.__configuration["domain"]).get_magic_castle_configuration()
)
# "node" is the only instance category that needs to be encapsulated in a list
main_tf_configuration["module"]["openstack"]["instances"]["node"] = [
main_tf_configuration["module"]["openstack"]["instances"]["node"]
]
# Magic Castle does not support an empty string in the hieradata field
if (
main_tf_configuration["module"]["openstack"].get("hieradata") is not None
and len(main_tf_configuration["module"]["openstack"]["hieradata"]) == 0
):
del main_tf_configuration["module"]["openstack"]["hieradata"]
with open(
get_cluster_path(self.get_hostname(), MAIN_TERRAFORM_FILENAME), "w"
) as main_terraform_file:
json.dump(main_tf_configuration, main_terraform_file)
def terraform_apply(destroy: bool):
try:
self.__rotate_terraform_logs(apply=True)
with open(
self.__get_cluster_path(TERRAFORM_APPLY_LOG_FILENAME),
"w") as output_file:
environment_variables = environ.copy()
dns_manager = DnsManager(self.get_domain())
environment_variables.update(
dns_manager.get_environment_variables())
environment_variables["OS_CLOUD"] = DEFAULT_CLOUD
if destroy:
environment_variables["TF_WARN_OUTPUT_ERRORS"] = "1"
run(
[
"terraform",
"apply",
"-input=false",
"-no-color",
"-auto-approve",
self.__get_cluster_path(
TERRAFORM_PLAN_BINARY_FILENAME),
],
cwd=self.__get_cluster_path(),
stdout=output_file,
stderr=output_file,
check=True,
env=environment_variables,
)
if destroy:
# Removes the content of the cluster's folder, even if not empty
rmtree(self.__get_cluster_path(), ignore_errors=True)
with DatabaseManager.connect() as database_connection:
database_connection.execute(
"DELETE FROM magic_castles WHERE hostname = ?",
(self.get_hostname(), ),
)
database_connection.commit()
else:
self.__update_status(
ClusterStatusCode.PROVISIONING_RUNNING)
if not destroy:
provisioning_manager = ProvisioningManager(
self.get_hostname())
# Avoid multiple threads polling the same cluster
if not provisioning_manager.is_busy():
try:
provisioning_manager.poll_until_success()
status_code = ClusterStatusCode.PROVISIONING_SUCCESS
except PuppetTimeoutException:
status_code = ClusterStatusCode.PROVISIONING_ERROR
self.__update_status(status_code)
except CalledProcessError:
logging.info(
"An error occurred while running terraform apply.")
self.__update_status(ClusterStatusCode.DESTROY_ERROR if destroy
else ClusterStatusCode.BUILD_ERROR)
finally:
self.__remove_existing_plan()
def __plan(self, *, destroy, existing_cluster):
plan_type = PlanType.DESTROY if destroy else PlanType.BUILD
if existing_cluster:
self.__remove_existing_plan()
previous_status = self.get_status()
else:
with DatabaseManager.connect() as database_connection:
database_connection.execute(
"INSERT INTO magic_castles (hostname, cluster_name, domain, status, plan_type, owner) VALUES (?, ?, ?, ?, ?, ?)",
(
self.get_hostname(),
self.get_cluster_name(),
self.get_domain(),
ClusterStatusCode.CREATED.value,
plan_type.value,
self.get_owner(),
),
)
database_connection.commit()
mkdir(self.__get_cluster_path())
previous_status = ClusterStatusCode.CREATED
self.__update_status(ClusterStatusCode.PLAN_RUNNING)
self.__update_plan_type(plan_type)
if not destroy:
self.__configuration.update_main_tf_json_file()
try:
run(
["terraform", "init", "-no-color", "-input=false"],
cwd=self.__get_cluster_path(),
capture_output=True,
check=True,
)
except CalledProcessError:
self.__update_status(previous_status)
raise PlanException(
"An error occurred while initializing Terraform.",
additional_details=f"hostname: {self.get_hostname()}",
)
self.__rotate_terraform_logs(apply=False)
with open(self.__get_cluster_path(TERRAFORM_PLAN_LOG_FILENAME),
"w") as output_file:
environment_variables = environ.copy()
dns_manager = DnsManager(self.get_domain())
environment_variables.update(
dns_manager.get_environment_variables())
environment_variables["OS_CLOUD"] = DEFAULT_CLOUD
try:
run(
[
"terraform",
"plan",
"-input=false",
"-no-color",
"-destroy=" + ("true" if destroy else "false"),
"-out=" + self.__get_cluster_path(
TERRAFORM_PLAN_BINARY_FILENAME),
],
cwd=self.__get_cluster_path(),
env=environment_variables,
stdout=output_file,
stderr=output_file,
check=True,
)
except CalledProcessError:
if destroy:
# Terraform returns an error if we try to destroy a cluster when the image
# it was created with does not exist anymore (e.g. CentOS-7-x64-2019-07). In these cases,
# not refreshing the terraform state (refresh=false) solves the issue.
try:
run(
[
"terraform",
"plan",
"-refresh=false",
"-input=false",
"-no-color",
"-destroy=" + ("true" if destroy else "false"),
"-out=" + self.__get_cluster_path(
TERRAFORM_PLAN_BINARY_FILENAME),
],
cwd=self.__get_cluster_path(),
env=environment_variables,
stdout=output_file,
stderr=output_file,
check=True,
)
except CalledProcessError:
# terraform plan fails even without refreshing the state
self.__update_status(previous_status)
raise PlanException(
"An error occurred while planning changes.",
additional_details=
f"hostname: {self.get_hostname()}",
)
else:
self.__update_status(previous_status)
raise PlanException(
"An error occurred while planning changes.",
additional_details=f"hostname: {self.get_hostname()}",
)
with open(self.__get_cluster_path(TERRAFORM_PLAN_JSON_FILENAME),
"w") as output_file:
try:
run(
[
"terraform",
"show",
"-no-color",
"-json",
TERRAFORM_PLAN_BINARY_FILENAME,
],
cwd=self.__get_cluster_path(),
stdout=output_file,
check=True,
)
except CalledProcessError:
self.__update_status(previous_status)
raise PlanException(
"An error occurred while exporting planned changes.",
additional_details=f"hostname: {self.get_hostname()}",
)
self.__update_status(previous_status)
def validate_domain(domain):
return domain in DnsManager.get_available_domains()
def test_initialize_disallowed_domain():
with pytest.raises(KeyError):
DnsManager("invalid.com")
def test_get_magic_castle_configuration_no_dns_provider():
assert DnsManager("sub.example.com").get_magic_castle_configuration() == {}
def test_get_environment_variables_no_dns_provider():
assert DnsManager("sub.example.com").get_environment_variables() == {}
def test_get_available_domains():
assert [
"calculquebec.cloud",
"c3.ca",
"sub.example.com",
] == DnsManager.get_available_domains()