def test_setup_bucket_with_bucket_configuration_failure(
mocker,
check_bucket_is_bootstrapped_error,
bucket_configure_error,
upload_bootstrapped_file_error,
cluster_action_error,
):
"""Verify that create_bucket_with_batch_resources behaves as expected in case of bucket configuration failure."""
mock_aws_api(mocker)
# mock bucket initialization
mock_bucket(mocker)
# mock bucket utils
mock_bucket_utils(mocker,
configure_bucket_side_effect=bucket_configure_error)
cluster = _mock_cluster(mocker, "slurm")
# mock bucket object utils
mock_bucket_object_utils(
mocker,
check_bucket_is_bootstrapped_side_effect=
check_bucket_is_bootstrapped_error,
upload_bootstrapped_file_side_effect=upload_bootstrapped_file_error,
)
with pytest.raises(ClusterActionError, match=cluster_action_error):
bucket_name = cluster.bucket.name
assert_that(bucket_name).is_equal_to(
"parallelcluster-a69601b5ee1fc2f2-v1-do-not-delete")
def test_setup_bucket_with_resources_success(
mocker,
scheduler,
cluster_name,
expected_config,
expected_template,
expected_dirs,
mock_generated_bucket_name,
expected_bucket_name,
provided_bucket_name,
):
"""Verify that create_bucket_with_batch_resources behaves as expected."""
# mock calls for bucket property in cluster object
artifact_dir = f"parallelcluster/clusters/{cluster_name}-abc123"
mock_aws_api(mocker)
# mock bucket initialization
mock_bucket(mocker)
# mock bucket object utils
mock_dict = mock_bucket_object_utils(mocker)
upload_config_mock = mock_dict.get("upload_config")
upload_template_mock = mock_dict.get("upload_cfn_template")
upload_custom_resources_mock = mock_dict.get("upload_resources")
# mock bucket utils
check_bucket_mock = mock_bucket_utils(
mocker,
root_service_dir=f"{cluster_name}-abc123")["check_bucket_exists"]
if provided_bucket_name:
cluster = _mock_cluster(mocker,
scheduler,
bucket_name=provided_bucket_name,
artifact_directory=artifact_dir)
cluster.config.custom_s3_bucket = provided_bucket_name
else:
cluster = _mock_cluster(mocker,
scheduler,
bucket_name=mock_generated_bucket_name,
artifact_directory=artifact_dir)
cluster.bucket.upload_config(expected_config, "fake_config_name")
cluster.bucket.upload_cfn_template(expected_template, "fake_template_name")
for dir in expected_dirs:
cluster.bucket.upload_resources(dir)
check_bucket_mock.assert_called_with()
# assert upload has been called
upload_config_mock.assert_called_with(expected_config, "fake_config_name")
upload_template_mock.assert_called_with(expected_template,
"fake_template_name")
upload_custom_resources_mock.assert_has_calls(
[mocker.call(dir) for dir in expected_dirs])
# assert bucket properties
assert_that(cluster.bucket.name).is_equal_to(expected_bucket_name)
assert_that(cluster.bucket.artifact_directory).is_equal_to(artifact_dir)
assert_that(cluster.bucket._root_directory).is_equal_to("parallelcluster")
def test_cw_dashboard_builder(mocker, test_datadir, config_file_name):
mock_aws_api(mocker)
mocker.patch(
"pcluster.config.cluster_config.HeadNodeNetworking.availability_zone",
new_callable=PropertyMock(return_value="us-east-1a"),
)
# mock bucket initialization parameters
mock_bucket(mocker)
input_yaml = load_yaml_dict(test_datadir / config_file_name)
cluster_config = ClusterSchema(cluster_name="clustername").load(input_yaml)
print(cluster_config)
generated_template = CDKTemplateBuilder().build_cluster_template(
cluster_config=cluster_config,
bucket=dummy_cluster_bucket(),
stack_name="clustername")
output_yaml = yaml.dump(generated_template, width=float("inf"))
print(output_yaml)
if cluster_config.is_cw_dashboard_enabled:
if cluster_config.shared_storage:
_verify_ec2_metrics_conditions(cluster_config, output_yaml)
if cluster_config.is_cw_logging_enabled:
_verify_head_node_logs_conditions(cluster_config, output_yaml)
else:
assert_that(output_yaml).does_not_contain("Head Node Logs")
def test_persist_stack_resources(self, cluster, mocker, template):
"""Verify that _persist_stack_resources behaves as expected."""
mocker.patch("pcluster.models.cluster.Cluster._get_artifact_dir")
mocker.patch("pcluster.models.cluster.Cluster._get_stack_template",
return_value=template)
update_stack_template_mock = mocker.patch(
"pcluster.models.cluster.Cluster._update_stack_template")
mock_aws_api(mocker)
mocker.patch("pcluster.aws.cfn.CfnClient.update_stack_from_url")
mock_bucket(mocker)
mock_bucket_utils(mocker)
mock_bucket_object_utils(mocker)
if "Resources" not in template:
expected_error_message = "Resources"
elif "key" not in template.get("Resources"):
expected_error_message = "key"
else:
expected_error_message = None
if expected_error_message:
with pytest.raises(KeyError, match=expected_error_message):
cluster._persist_stack_resources(["key"])
assert_that(update_stack_template_mock.called).is_false()
else:
cluster._persist_stack_resources(["key"])
assert_that(update_stack_template_mock.called).is_true()
assert_that(cluster._get_stack_template()["Resources"]["key"]
["DeletionPolicy"]).is_equal_to("Retain")
def test_persist_cloudwatch_log_groups(self, cluster, mocker, caplog, template, expected_retain, fail_on_persist):
"""Verify that _persist_cloudwatch_log_groups behaves as expected."""
mocker.patch("pcluster.models.cluster.Cluster._get_artifact_dir")
mocker.patch("pcluster.models.cluster.Cluster._get_stack_template", return_value=template)
client_error = AWSClientError("function", "Generic error.")
update_template_mock = mocker.patch.object(
cluster, "_update_stack_template", side_effect=client_error if fail_on_persist else None
)
mock_aws_api(mocker)
mocker.patch("pcluster.aws.cfn.CfnClient.update_stack_from_url")
mock_bucket(mocker)
mock_bucket_utils(mocker)
mock_bucket_object_utils(mocker)
if expected_retain:
keys = ["key"]
else:
keys = []
get_unretained_cw_log_group_resource_keys_mock = mocker.patch.object(
cluster, "_get_unretained_cw_log_group_resource_keys", return_value=keys
)
if fail_on_persist:
with pytest.raises(ClusterActionError) as e:
cluster._persist_cloudwatch_log_groups()
assert_that(str(e)).contains("Unable to persist logs")
else:
cluster._persist_cloudwatch_log_groups()
assert_that(get_unretained_cw_log_group_resource_keys_mock.call_count).is_equal_to(1)
assert_that(update_template_mock.call_count).is_equal_to(1 if expected_retain else 0)
def test_cluster_builder_from_configuration_file(mocker, config_file_name):
mock_aws_api(mocker)
# mock bucket initialization parameters
mock_bucket(mocker)
input_yaml, cluster = load_cluster_model_from_yaml(config_file_name)
generated_template = CDKTemplateBuilder().build_cluster_template(
cluster_config=cluster, bucket=dummy_cluster_bucket(), stack_name="clustername"
)
print(yaml.dump(generated_template))
def test_awsbatch_cluster_builder(mocker):
mock_aws_api(mocker)
# mock bucket initialization parameters
mock_bucket(mocker)
generated_template = CDKTemplateBuilder().build_cluster_template(
cluster_config=dummy_awsbatch_cluster_config(mocker),
bucket=dummy_cluster_bucket(),
stack_name="clustername")
print(yaml.dump(generated_template))
def test_setup_bucket_with_resources_upload_failure(mocker, cluster_name,
scheduler,
mock_generated_bucket_name,
expected_bucket_name,
provided_bucket_name):
"""Verify that create_bucket_with_batch_resources behaves as expected in case of upload failure."""
upload_config_cluster_action_error = "Unable to upload cluster config to the S3 bucket"
upload_resource_cluster_action_error = "Unable to upload cluster resources to the S3 bucket"
upload_awsclient_error = AWSClientError(
function_name="put_object",
message="Unable to put file to the S3 bucket")
upload_fileobj_awsclient_error = AWSClientError(
function_name="upload_fileobj",
message="Unable to upload file to the S3 bucket")
mock_aws_api(mocker)
# mock bucket initialization
mock_bucket(mocker)
# mock bucket utils
check_bucket_mock = mock_bucket_utils(
mocker,
bucket_name=provided_bucket_name,
root_service_dir=f"{cluster_name}-abc123",
)["check_bucket_exists"]
# mock bucket object utils
mock_bucket_object_utils(
mocker,
upload_config_side_effect=upload_awsclient_error,
upload_template_side_effect=upload_awsclient_error,
upload_resources_side_effect=upload_fileobj_awsclient_error,
)
if provided_bucket_name:
cluster = _mock_cluster(mocker, scheduler, provided_bucket_name)
cluster.config.cluster_s3_bucket = provided_bucket_name
else:
cluster = _mock_cluster(mocker, scheduler)
with pytest.raises(ClusterActionError,
match=upload_config_cluster_action_error):
cluster._upload_config()
with pytest.raises(ClusterActionError,
match=upload_resource_cluster_action_error):
cluster._upload_artifacts()
check_bucket_mock.assert_called_with()
def test_scheduler_plugin_substack(mocker, config_file_name, expected_scheduler_plugin_stack, test_datadir):
mock_aws_api(mocker)
# mock bucket initialization parameters
mock_bucket(mocker)
if config_file_name == "scheduler_plugin.full.yaml":
input_yaml, cluster = load_cluster_model_from_yaml(config_file_name)
else:
input_yaml, cluster = load_cluster_model_from_yaml(config_file_name, test_datadir)
generated_template = CDKTemplateBuilder().build_cluster_template(
cluster_config=cluster, bucket=dummy_cluster_bucket(), stack_name="clustername"
)
print(yaml.dump(generated_template))
assert_that(generated_template["Resources"].get("SchedulerPluginStack", {})).is_equal_to(
expected_scheduler_plugin_stack
)
def test_configure_s3_bucket(mocker, put_bucket_versioning_error,
put_bucket_encryption_error,
put_bucket_policy_error):
mock_aws_api(mocker)
mock_bucket(mocker)
bucket = dummy_cluster_bucket()
mocker.patch("pcluster.aws.s3.S3Client.put_bucket_versioning",
side_effect=put_bucket_versioning_error)
mocker.patch("pcluster.aws.s3.S3Client.put_bucket_encryption",
side_effect=put_bucket_encryption_error)
mocker.patch("pcluster.aws.s3.S3Client.put_bucket_policy",
side_effect=put_bucket_policy_error)
if put_bucket_versioning_error or put_bucket_encryption_error or put_bucket_policy_error:
with pytest.raises(AWSClientError, match="An error occurred"):
bucket.configure_s3_bucket()
def test_update_stack_template(self, cluster, mocker, error_message):
"""Verify that _update_stack_template behaves as expected."""
template_body = {"TemplateKey": "TemplateValue"}
template_url = "https://{bucket_name}.s3.{region}.amazonaws.com{partition_suffix}/{template_key}"
response = error_message or {"StackId": "stack ID"}
mock_aws_api(mocker)
mocker.patch("pcluster.aws.cfn.CfnClient.get_stack_template",
return_value=template_body)
mocker.patch(
"pcluster.aws.cfn.CfnClient.update_stack_from_url",
return_value=response,
expected_params={
"stack_name": FAKE_NAME,
"template_url": template_url,
},
side_effect=AWSClientError(function_name="update_stack_from_url",
message=error_message)
if error_message is not None else None,
)
# mock bucket initialize
mock_bucket(mocker)
# mock bucket utils
mock_bucket_utils(mocker)
# mock bucket object utils
mock_bucket_object_utils(mocker)
wait_for_update_mock = mocker.patch.object(cluster,
"_wait_for_stack_update")
if error_message is None or "no updates are to be performed" in error_message.lower(
):
cluster._update_stack_template(template_body)
if error_message is None or "no updates are to be performed" not in error_message.lower(
):
assert_that(wait_for_update_mock.called).is_true()
else:
assert_that(wait_for_update_mock.called).is_false()
else:
full_error_message = "Unable to update stack template for stack {stack_name}: {emsg}".format(
stack_name=FAKE_NAME, emsg=error_message)
with pytest.raises(AWSClientError,
match=full_error_message) as sysexit:
cluster._update_stack_template(template_url)
assert_that(sysexit.value.code).is_not_equal_to(0)
def test_create_s3_bucket(region, create_error, mocker):
bucket_name = "test"
expected_params = {"Bucket": bucket_name}
os.environ["AWS_DEFAULT_REGION"] = region
if region != "us-east-1":
# LocationConstraint specifies the region where the bucket will be created.
# When the region is us-east-1 we are not specifying this parameter because it's the default region.
expected_params["CreateBucketConfiguration"] = {
"LocationConstraint": region
}
mock_aws_api(mocker)
mocker.patch("pcluster.aws.s3.S3Client.create_bucket",
side_effect=create_error)
mock_bucket(mocker)
bucket = dummy_cluster_bucket(bucket_name=bucket_name)
if create_error:
with pytest.raises(AWSClientError, match="An error occurred"):
bucket.create_bucket()
def test_setup_bucket_with_resources_creation_failure(
mocker, provided_bucket_name, check_bucket_exists_error,
create_bucket_error, cluster_action_error):
"""Verify that create_bucket_with_batch_resources behaves as expected in case of bucket initialization failure."""
mock_aws_api(mocker)
# mock bucket initialization
mock_bucket(mocker)
if provided_bucket_name:
# mock bucket utils
mock_bucket_utils(
mocker, check_bucket_exists_side_effect=check_bucket_exists_error)
cluster = _mock_cluster(mocker,
"slurm",
bucket_name=provided_bucket_name)
cluster.config.custom_s3_bucket = provided_bucket_name
else:
# mock bucket utils
mock_bucket_utils(
mocker,
create_bucket_side_effect=create_bucket_error,
check_bucket_exists_side_effect=check_bucket_exists_error,
)
cluster = _mock_cluster(mocker, "slurm")
# mock bucket object utils
mocker.patch(
"pcluster.models.s3_bucket.S3Bucket.check_bucket_is_bootstrapped")
# assert failures
if provided_bucket_name:
with pytest.raises(ClusterActionError, match=cluster_action_error):
bucket_name = cluster.bucket.name
assert_that(bucket_name).is_equal_to(provided_bucket_name)
else:
with pytest.raises(ClusterActionError, match=cluster_action_error):
bucket_name = cluster.bucket.name
assert_that(bucket_name).is_equal_to(
"parallelcluster-a69601b5ee1fc2f2-v1-do-not-delete")