def test_setup_bucket_with_bucket_configuration_failure( mocker, check_bucket_is_bootstrapped_error, bucket_configure_error, upload_bootstrapped_file_error, cluster_action_error, ): """Verify that create_bucket_with_batch_resources behaves as expected in case of bucket configuration failure.""" mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) # mock bucket utils mock_bucket_utils(mocker, configure_bucket_side_effect=bucket_configure_error) cluster = _mock_cluster(mocker, "slurm") # mock bucket object utils mock_bucket_object_utils( mocker, check_bucket_is_bootstrapped_side_effect= check_bucket_is_bootstrapped_error, upload_bootstrapped_file_side_effect=upload_bootstrapped_file_error, ) with pytest.raises(ClusterActionError, match=cluster_action_error): bucket_name = cluster.bucket.name assert_that(bucket_name).is_equal_to( "parallelcluster-a69601b5ee1fc2f2-v1-do-not-delete")
def test_setup_bucket_with_resources_success( mocker, scheduler, cluster_name, expected_config, expected_template, expected_dirs, mock_generated_bucket_name, expected_bucket_name, provided_bucket_name, ): """Verify that create_bucket_with_batch_resources behaves as expected.""" # mock calls for bucket property in cluster object artifact_dir = f"parallelcluster/clusters/{cluster_name}-abc123" mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) # mock bucket object utils mock_dict = mock_bucket_object_utils(mocker) upload_config_mock = mock_dict.get("upload_config") upload_template_mock = mock_dict.get("upload_cfn_template") upload_custom_resources_mock = mock_dict.get("upload_resources") # mock bucket utils check_bucket_mock = mock_bucket_utils( mocker, root_service_dir=f"{cluster_name}-abc123")["check_bucket_exists"] if provided_bucket_name: cluster = _mock_cluster(mocker, scheduler, bucket_name=provided_bucket_name, artifact_directory=artifact_dir) cluster.config.custom_s3_bucket = provided_bucket_name else: cluster = _mock_cluster(mocker, scheduler, bucket_name=mock_generated_bucket_name, artifact_directory=artifact_dir) cluster.bucket.upload_config(expected_config, "fake_config_name") cluster.bucket.upload_cfn_template(expected_template, "fake_template_name") for dir in expected_dirs: cluster.bucket.upload_resources(dir) check_bucket_mock.assert_called_with() # assert upload has been called upload_config_mock.assert_called_with(expected_config, "fake_config_name") upload_template_mock.assert_called_with(expected_template, "fake_template_name") upload_custom_resources_mock.assert_has_calls( [mocker.call(dir) for dir in expected_dirs]) # assert bucket properties assert_that(cluster.bucket.name).is_equal_to(expected_bucket_name) assert_that(cluster.bucket.artifact_directory).is_equal_to(artifact_dir) assert_that(cluster.bucket._root_directory).is_equal_to("parallelcluster")
def test_cw_dashboard_builder(mocker, test_datadir, config_file_name): mock_aws_api(mocker) mocker.patch( "pcluster.config.cluster_config.HeadNodeNetworking.availability_zone", new_callable=PropertyMock(return_value="us-east-1a"), ) # mock bucket initialization parameters mock_bucket(mocker) input_yaml = load_yaml_dict(test_datadir / config_file_name) cluster_config = ClusterSchema(cluster_name="clustername").load(input_yaml) print(cluster_config) generated_template = CDKTemplateBuilder().build_cluster_template( cluster_config=cluster_config, bucket=dummy_cluster_bucket(), stack_name="clustername") output_yaml = yaml.dump(generated_template, width=float("inf")) print(output_yaml) if cluster_config.is_cw_dashboard_enabled: if cluster_config.shared_storage: _verify_ec2_metrics_conditions(cluster_config, output_yaml) if cluster_config.is_cw_logging_enabled: _verify_head_node_logs_conditions(cluster_config, output_yaml) else: assert_that(output_yaml).does_not_contain("Head Node Logs")
def test_persist_stack_resources(self, cluster, mocker, template): """Verify that _persist_stack_resources behaves as expected.""" mocker.patch("pcluster.models.cluster.Cluster._get_artifact_dir") mocker.patch("pcluster.models.cluster.Cluster._get_stack_template", return_value=template) update_stack_template_mock = mocker.patch( "pcluster.models.cluster.Cluster._update_stack_template") mock_aws_api(mocker) mocker.patch("pcluster.aws.cfn.CfnClient.update_stack_from_url") mock_bucket(mocker) mock_bucket_utils(mocker) mock_bucket_object_utils(mocker) if "Resources" not in template: expected_error_message = "Resources" elif "key" not in template.get("Resources"): expected_error_message = "key" else: expected_error_message = None if expected_error_message: with pytest.raises(KeyError, match=expected_error_message): cluster._persist_stack_resources(["key"]) assert_that(update_stack_template_mock.called).is_false() else: cluster._persist_stack_resources(["key"]) assert_that(update_stack_template_mock.called).is_true() assert_that(cluster._get_stack_template()["Resources"]["key"] ["DeletionPolicy"]).is_equal_to("Retain")
def test_persist_cloudwatch_log_groups(self, cluster, mocker, caplog, template, expected_retain, fail_on_persist): """Verify that _persist_cloudwatch_log_groups behaves as expected.""" mocker.patch("pcluster.models.cluster.Cluster._get_artifact_dir") mocker.patch("pcluster.models.cluster.Cluster._get_stack_template", return_value=template) client_error = AWSClientError("function", "Generic error.") update_template_mock = mocker.patch.object( cluster, "_update_stack_template", side_effect=client_error if fail_on_persist else None ) mock_aws_api(mocker) mocker.patch("pcluster.aws.cfn.CfnClient.update_stack_from_url") mock_bucket(mocker) mock_bucket_utils(mocker) mock_bucket_object_utils(mocker) if expected_retain: keys = ["key"] else: keys = [] get_unretained_cw_log_group_resource_keys_mock = mocker.patch.object( cluster, "_get_unretained_cw_log_group_resource_keys", return_value=keys ) if fail_on_persist: with pytest.raises(ClusterActionError) as e: cluster._persist_cloudwatch_log_groups() assert_that(str(e)).contains("Unable to persist logs") else: cluster._persist_cloudwatch_log_groups() assert_that(get_unretained_cw_log_group_resource_keys_mock.call_count).is_equal_to(1) assert_that(update_template_mock.call_count).is_equal_to(1 if expected_retain else 0)
def test_cluster_builder_from_configuration_file(mocker, config_file_name): mock_aws_api(mocker) # mock bucket initialization parameters mock_bucket(mocker) input_yaml, cluster = load_cluster_model_from_yaml(config_file_name) generated_template = CDKTemplateBuilder().build_cluster_template( cluster_config=cluster, bucket=dummy_cluster_bucket(), stack_name="clustername" ) print(yaml.dump(generated_template))
def test_awsbatch_cluster_builder(mocker): mock_aws_api(mocker) # mock bucket initialization parameters mock_bucket(mocker) generated_template = CDKTemplateBuilder().build_cluster_template( cluster_config=dummy_awsbatch_cluster_config(mocker), bucket=dummy_cluster_bucket(), stack_name="clustername") print(yaml.dump(generated_template))
def test_setup_bucket_with_resources_upload_failure(mocker, cluster_name, scheduler, mock_generated_bucket_name, expected_bucket_name, provided_bucket_name): """Verify that create_bucket_with_batch_resources behaves as expected in case of upload failure.""" upload_config_cluster_action_error = "Unable to upload cluster config to the S3 bucket" upload_resource_cluster_action_error = "Unable to upload cluster resources to the S3 bucket" upload_awsclient_error = AWSClientError( function_name="put_object", message="Unable to put file to the S3 bucket") upload_fileobj_awsclient_error = AWSClientError( function_name="upload_fileobj", message="Unable to upload file to the S3 bucket") mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) # mock bucket utils check_bucket_mock = mock_bucket_utils( mocker, bucket_name=provided_bucket_name, root_service_dir=f"{cluster_name}-abc123", )["check_bucket_exists"] # mock bucket object utils mock_bucket_object_utils( mocker, upload_config_side_effect=upload_awsclient_error, upload_template_side_effect=upload_awsclient_error, upload_resources_side_effect=upload_fileobj_awsclient_error, ) if provided_bucket_name: cluster = _mock_cluster(mocker, scheduler, provided_bucket_name) cluster.config.cluster_s3_bucket = provided_bucket_name else: cluster = _mock_cluster(mocker, scheduler) with pytest.raises(ClusterActionError, match=upload_config_cluster_action_error): cluster._upload_config() with pytest.raises(ClusterActionError, match=upload_resource_cluster_action_error): cluster._upload_artifacts() check_bucket_mock.assert_called_with()
def test_scheduler_plugin_substack(mocker, config_file_name, expected_scheduler_plugin_stack, test_datadir): mock_aws_api(mocker) # mock bucket initialization parameters mock_bucket(mocker) if config_file_name == "scheduler_plugin.full.yaml": input_yaml, cluster = load_cluster_model_from_yaml(config_file_name) else: input_yaml, cluster = load_cluster_model_from_yaml(config_file_name, test_datadir) generated_template = CDKTemplateBuilder().build_cluster_template( cluster_config=cluster, bucket=dummy_cluster_bucket(), stack_name="clustername" ) print(yaml.dump(generated_template)) assert_that(generated_template["Resources"].get("SchedulerPluginStack", {})).is_equal_to( expected_scheduler_plugin_stack )
def test_configure_s3_bucket(mocker, put_bucket_versioning_error, put_bucket_encryption_error, put_bucket_policy_error): mock_aws_api(mocker) mock_bucket(mocker) bucket = dummy_cluster_bucket() mocker.patch("pcluster.aws.s3.S3Client.put_bucket_versioning", side_effect=put_bucket_versioning_error) mocker.patch("pcluster.aws.s3.S3Client.put_bucket_encryption", side_effect=put_bucket_encryption_error) mocker.patch("pcluster.aws.s3.S3Client.put_bucket_policy", side_effect=put_bucket_policy_error) if put_bucket_versioning_error or put_bucket_encryption_error or put_bucket_policy_error: with pytest.raises(AWSClientError, match="An error occurred"): bucket.configure_s3_bucket()
def test_update_stack_template(self, cluster, mocker, error_message): """Verify that _update_stack_template behaves as expected.""" template_body = {"TemplateKey": "TemplateValue"} template_url = "https://{bucket_name}.s3.{region}.amazonaws.com{partition_suffix}/{template_key}" response = error_message or {"StackId": "stack ID"} mock_aws_api(mocker) mocker.patch("pcluster.aws.cfn.CfnClient.get_stack_template", return_value=template_body) mocker.patch( "pcluster.aws.cfn.CfnClient.update_stack_from_url", return_value=response, expected_params={ "stack_name": FAKE_NAME, "template_url": template_url, }, side_effect=AWSClientError(function_name="update_stack_from_url", message=error_message) if error_message is not None else None, ) # mock bucket initialize mock_bucket(mocker) # mock bucket utils mock_bucket_utils(mocker) # mock bucket object utils mock_bucket_object_utils(mocker) wait_for_update_mock = mocker.patch.object(cluster, "_wait_for_stack_update") if error_message is None or "no updates are to be performed" in error_message.lower( ): cluster._update_stack_template(template_body) if error_message is None or "no updates are to be performed" not in error_message.lower( ): assert_that(wait_for_update_mock.called).is_true() else: assert_that(wait_for_update_mock.called).is_false() else: full_error_message = "Unable to update stack template for stack {stack_name}: {emsg}".format( stack_name=FAKE_NAME, emsg=error_message) with pytest.raises(AWSClientError, match=full_error_message) as sysexit: cluster._update_stack_template(template_url) assert_that(sysexit.value.code).is_not_equal_to(0)
def test_create_s3_bucket(region, create_error, mocker): bucket_name = "test" expected_params = {"Bucket": bucket_name} os.environ["AWS_DEFAULT_REGION"] = region if region != "us-east-1": # LocationConstraint specifies the region where the bucket will be created. # When the region is us-east-1 we are not specifying this parameter because it's the default region. expected_params["CreateBucketConfiguration"] = { "LocationConstraint": region } mock_aws_api(mocker) mocker.patch("pcluster.aws.s3.S3Client.create_bucket", side_effect=create_error) mock_bucket(mocker) bucket = dummy_cluster_bucket(bucket_name=bucket_name) if create_error: with pytest.raises(AWSClientError, match="An error occurred"): bucket.create_bucket()
def test_setup_bucket_with_resources_creation_failure( mocker, provided_bucket_name, check_bucket_exists_error, create_bucket_error, cluster_action_error): """Verify that create_bucket_with_batch_resources behaves as expected in case of bucket initialization failure.""" mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) if provided_bucket_name: # mock bucket utils mock_bucket_utils( mocker, check_bucket_exists_side_effect=check_bucket_exists_error) cluster = _mock_cluster(mocker, "slurm", bucket_name=provided_bucket_name) cluster.config.custom_s3_bucket = provided_bucket_name else: # mock bucket utils mock_bucket_utils( mocker, create_bucket_side_effect=create_bucket_error, check_bucket_exists_side_effect=check_bucket_exists_error, ) cluster = _mock_cluster(mocker, "slurm") # mock bucket object utils mocker.patch( "pcluster.models.s3_bucket.S3Bucket.check_bucket_is_bootstrapped") # assert failures if provided_bucket_name: with pytest.raises(ClusterActionError, match=cluster_action_error): bucket_name = cluster.bucket.name assert_that(bucket_name).is_equal_to(provided_bucket_name) else: with pytest.raises(ClusterActionError, match=cluster_action_error): bucket_name = cluster.bucket.name assert_that(bucket_name).is_equal_to( "parallelcluster-a69601b5ee1fc2f2-v1-do-not-delete")