def test_setup_bucket_with_bucket_configuration_failure( mocker, check_bucket_is_bootstrapped_error, bucket_configure_error, upload_bootstrapped_file_error, cluster_action_error, ): """Verify that create_bucket_with_batch_resources behaves as expected in case of bucket configuration failure.""" mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) # mock bucket utils mock_bucket_utils(mocker, configure_bucket_side_effect=bucket_configure_error) cluster = _mock_cluster(mocker, "slurm") # mock bucket object utils mock_bucket_object_utils( mocker, check_bucket_is_bootstrapped_side_effect= check_bucket_is_bootstrapped_error, upload_bootstrapped_file_side_effect=upload_bootstrapped_file_error, ) with pytest.raises(ClusterActionError, match=cluster_action_error): bucket_name = cluster.bucket.name assert_that(bucket_name).is_equal_to( "parallelcluster-a69601b5ee1fc2f2-v1-do-not-delete")
def test_persist_stack_resources(self, cluster, mocker, template): """Verify that _persist_stack_resources behaves as expected.""" mocker.patch("pcluster.models.cluster.Cluster._get_artifact_dir") mocker.patch("pcluster.models.cluster.Cluster._get_stack_template", return_value=template) update_stack_template_mock = mocker.patch( "pcluster.models.cluster.Cluster._update_stack_template") mock_aws_api(mocker) mocker.patch("pcluster.aws.cfn.CfnClient.update_stack_from_url") mock_bucket(mocker) mock_bucket_utils(mocker) mock_bucket_object_utils(mocker) if "Resources" not in template: expected_error_message = "Resources" elif "key" not in template.get("Resources"): expected_error_message = "key" else: expected_error_message = None if expected_error_message: with pytest.raises(KeyError, match=expected_error_message): cluster._persist_stack_resources(["key"]) assert_that(update_stack_template_mock.called).is_false() else: cluster._persist_stack_resources(["key"]) assert_that(update_stack_template_mock.called).is_true() assert_that(cluster._get_stack_template()["Resources"]["key"] ["DeletionPolicy"]).is_equal_to("Retain")
def test_persist_cloudwatch_log_groups(self, cluster, mocker, caplog, template, expected_retain, fail_on_persist): """Verify that _persist_cloudwatch_log_groups behaves as expected.""" mocker.patch("pcluster.models.cluster.Cluster._get_artifact_dir") mocker.patch("pcluster.models.cluster.Cluster._get_stack_template", return_value=template) client_error = AWSClientError("function", "Generic error.") update_template_mock = mocker.patch.object( cluster, "_update_stack_template", side_effect=client_error if fail_on_persist else None ) mock_aws_api(mocker) mocker.patch("pcluster.aws.cfn.CfnClient.update_stack_from_url") mock_bucket(mocker) mock_bucket_utils(mocker) mock_bucket_object_utils(mocker) if expected_retain: keys = ["key"] else: keys = [] get_unretained_cw_log_group_resource_keys_mock = mocker.patch.object( cluster, "_get_unretained_cw_log_group_resource_keys", return_value=keys ) if fail_on_persist: with pytest.raises(ClusterActionError) as e: cluster._persist_cloudwatch_log_groups() assert_that(str(e)).contains("Unable to persist logs") else: cluster._persist_cloudwatch_log_groups() assert_that(get_unretained_cw_log_group_resource_keys_mock.call_count).is_equal_to(1) assert_that(update_template_mock.call_count).is_equal_to(1 if expected_retain else 0)
def test_setup_bucket_with_resources_success( mocker, scheduler, cluster_name, expected_config, expected_template, expected_dirs, mock_generated_bucket_name, expected_bucket_name, provided_bucket_name, ): """Verify that create_bucket_with_batch_resources behaves as expected.""" # mock calls for bucket property in cluster object artifact_dir = f"parallelcluster/clusters/{cluster_name}-abc123" mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) # mock bucket object utils mock_dict = mock_bucket_object_utils(mocker) upload_config_mock = mock_dict.get("upload_config") upload_template_mock = mock_dict.get("upload_cfn_template") upload_custom_resources_mock = mock_dict.get("upload_resources") # mock bucket utils check_bucket_mock = mock_bucket_utils( mocker, root_service_dir=f"{cluster_name}-abc123")["check_bucket_exists"] if provided_bucket_name: cluster = _mock_cluster(mocker, scheduler, bucket_name=provided_bucket_name, artifact_directory=artifact_dir) cluster.config.custom_s3_bucket = provided_bucket_name else: cluster = _mock_cluster(mocker, scheduler, bucket_name=mock_generated_bucket_name, artifact_directory=artifact_dir) cluster.bucket.upload_config(expected_config, "fake_config_name") cluster.bucket.upload_cfn_template(expected_template, "fake_template_name") for dir in expected_dirs: cluster.bucket.upload_resources(dir) check_bucket_mock.assert_called_with() # assert upload has been called upload_config_mock.assert_called_with(expected_config, "fake_config_name") upload_template_mock.assert_called_with(expected_template, "fake_template_name") upload_custom_resources_mock.assert_has_calls( [mocker.call(dir) for dir in expected_dirs]) # assert bucket properties assert_that(cluster.bucket.name).is_equal_to(expected_bucket_name) assert_that(cluster.bucket.artifact_directory).is_equal_to(artifact_dir) assert_that(cluster.bucket._root_directory).is_equal_to("parallelcluster")
def test_update_stack_template(self, cluster, mocker, error_message): """Verify that _update_stack_template behaves as expected.""" template_body = {"TemplateKey": "TemplateValue"} template_url = "https://{bucket_name}.s3.{region}.amazonaws.com{partition_suffix}/{template_key}" response = error_message or {"StackId": "stack ID"} mock_aws_api(mocker) mocker.patch("pcluster.aws.cfn.CfnClient.get_stack_template", return_value=template_body) mocker.patch( "pcluster.aws.cfn.CfnClient.update_stack_from_url", return_value=response, expected_params={ "stack_name": FAKE_NAME, "template_url": template_url, }, side_effect=AWSClientError(function_name="update_stack_from_url", message=error_message) if error_message is not None else None, ) # mock bucket initialize mock_bucket(mocker) # mock bucket utils mock_bucket_utils(mocker) # mock bucket object utils mock_bucket_object_utils(mocker) wait_for_update_mock = mocker.patch.object(cluster, "_wait_for_stack_update") if error_message is None or "no updates are to be performed" in error_message.lower( ): cluster._update_stack_template(template_body) if error_message is None or "no updates are to be performed" not in error_message.lower( ): assert_that(wait_for_update_mock.called).is_true() else: assert_that(wait_for_update_mock.called).is_false() else: full_error_message = "Unable to update stack template for stack {stack_name}: {emsg}".format( stack_name=FAKE_NAME, emsg=error_message) with pytest.raises(AWSClientError, match=full_error_message) as sysexit: cluster._update_stack_template(template_url) assert_that(sysexit.value.code).is_not_equal_to(0)
def test_setup_bucket_with_resources_creation_failure( mocker, provided_bucket_name, check_bucket_exists_error, create_bucket_error, cluster_action_error): """Verify that create_bucket_with_batch_resources behaves as expected in case of bucket initialization failure.""" mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) if provided_bucket_name: # mock bucket utils mock_bucket_utils( mocker, check_bucket_exists_side_effect=check_bucket_exists_error) cluster = _mock_cluster(mocker, "slurm", bucket_name=provided_bucket_name) cluster.config.custom_s3_bucket = provided_bucket_name else: # mock bucket utils mock_bucket_utils( mocker, create_bucket_side_effect=create_bucket_error, check_bucket_exists_side_effect=check_bucket_exists_error, ) cluster = _mock_cluster(mocker, "slurm") # mock bucket object utils mocker.patch( "pcluster.models.s3_bucket.S3Bucket.check_bucket_is_bootstrapped") # assert failures if provided_bucket_name: with pytest.raises(ClusterActionError, match=cluster_action_error): bucket_name = cluster.bucket.name assert_that(bucket_name).is_equal_to(provided_bucket_name) else: with pytest.raises(ClusterActionError, match=cluster_action_error): bucket_name = cluster.bucket.name assert_that(bucket_name).is_equal_to( "parallelcluster-a69601b5ee1fc2f2-v1-do-not-delete")
def test_setup_bucket_with_resources_upload_failure(mocker, cluster_name, scheduler, mock_generated_bucket_name, expected_bucket_name, provided_bucket_name): """Verify that create_bucket_with_batch_resources behaves as expected in case of upload failure.""" upload_config_cluster_action_error = "Unable to upload cluster config to the S3 bucket" upload_resource_cluster_action_error = "Unable to upload cluster resources to the S3 bucket" upload_awsclient_error = AWSClientError( function_name="put_object", message="Unable to put file to the S3 bucket") upload_fileobj_awsclient_error = AWSClientError( function_name="upload_fileobj", message="Unable to upload file to the S3 bucket") mock_aws_api(mocker) # mock bucket initialization mock_bucket(mocker) # mock bucket utils check_bucket_mock = mock_bucket_utils( mocker, bucket_name=provided_bucket_name, root_service_dir=f"{cluster_name}-abc123", )["check_bucket_exists"] # mock bucket object utils mock_bucket_object_utils( mocker, upload_config_side_effect=upload_awsclient_error, upload_template_side_effect=upload_awsclient_error, upload_resources_side_effect=upload_fileobj_awsclient_error, ) if provided_bucket_name: cluster = _mock_cluster(mocker, scheduler, provided_bucket_name) cluster.config.cluster_s3_bucket = provided_bucket_name else: cluster = _mock_cluster(mocker, scheduler) with pytest.raises(ClusterActionError, match=upload_config_cluster_action_error): cluster._upload_config() with pytest.raises(ClusterActionError, match=upload_resource_cluster_action_error): cluster._upload_artifacts() check_bucket_mock.assert_called_with()