def dummy_awsbatch_cluster_config(mocker): """Generate dummy cluster.""" image = Image(os="alinux2") head_node = dummy_head_node(mocker) compute_resources = [ AwsBatchComputeResource(name="dummy_compute_resource1", instance_types=["dummyc5.xlarge", "optimal"]) ] queue_networking = AwsBatchQueueNetworking(subnet_ids=["dummy-subnet-1"], security_groups=["sg-1", "sg-2"]) queues = [AwsBatchQueue(name="queue1", networking=queue_networking, compute_resources=compute_resources)] scheduling = AwsBatchScheduling(queues=queues) # shared storage shared_storage: List[Resource] = [] shared_storage.append(dummy_fsx()) shared_storage.append(dummy_ebs("/ebs1")) shared_storage.append(dummy_ebs("/ebs2", volume_id="vol-abc")) shared_storage.append(dummy_ebs("/ebs3", raid=Raid(raid_type=1, number_of_volumes=5))) shared_storage.append(dummy_efs("/efs1", file_system_id="fs-efs-1")) shared_storage.append(dummy_raid("/raid1")) cluster = _DummyAwsBatchClusterConfig( image=image, head_node=head_node, scheduling=scheduling, shared_storage=shared_storage ) cluster.custom_s3_bucket = "s3://dummy-s3-bucket" cluster.additional_resources = "https://additional.template.url" cluster.config_version = "1.0" cluster.iam = ClusterIam() cluster.tags = [Tag(key="test", value="testvalue")] return cluster
def test_tags(self, cluster, mocker, existing_tags): """Verify that the function to get the tags list behaves as expected.""" mock_aws_api(mocker) cluster.config = dummy_slurm_cluster_config(mocker) # Populate config with list of existing tags existing_tags_list = [ Tag(key=tag_name, value=tag_value) for tag_name, tag_value in existing_tags.items() ] cluster.config.tags = existing_tags_list # Expected tags: installed_version = "FakeInstalledVersion" tags = existing_tags tags["parallelcluster:version"] = installed_version expected_tags_list = self._sort_tags([ Tag(key=tag_name, value=tag_value) for tag_name, tag_value in tags.items() ]) # Test method to add version tag get_version_patch = mocker.patch( "pcluster.models.cluster.get_installed_version", return_value=installed_version) cluster._add_version_tag() assert_that(get_version_patch.call_count).is_equal_to(1) assert_that(len(cluster.config.tags)).is_equal_to( len(expected_tags_list)) assert_that( all([ source.value == target.value for source, target in zip( self._sort_tags(cluster.config.tags), expected_tags_list) ])).is_true() # Test method to retrieve CFN tags expected_cfn_tags = self._sort_cfn_tags([{ "Key": tag_name, "Value": tag_value } for tag_name, tag_value in tags.items()]) cfn_tags = self._sort_cfn_tags(cluster._get_cfn_tags()) assert_that(len(cfn_tags)).is_equal_to(len(expected_cfn_tags)) assert_that( all([ source["Value"] == target["Value"] for source, target in zip(cfn_tags, expected_cfn_tags) ])).is_true()
def test_generate_tag_specifications(input_tags): """Verify function to generate tag specifications for dry runs of RunInstances works as expected.""" input_tags = [Tag(tag.get("key"), tag.get("value")) for tag in input_tags] if input_tags: expected_output_tags = [ {"ResourceType": "instance", "Tags": [{"Key": tag.key, "Value": tag.value} for tag in input_tags]} ] else: expected_output_tags = [] assert_that(_LaunchTemplateValidator._generate_tag_specifications(input_tags)).is_equal_to(expected_output_tags)
def _add_version_tag(self): """Add version tag to the stack.""" if self.config.tags is None: self.config.tags = [] # Remove PCLUSTER_VERSION_TAG if already exists self.config.tags = [ tag for tag in self.config.tags if tag.key != PCLUSTER_VERSION_TAG ] # Add PCLUSTER_VERSION_TAG self.config.tags.append( Tag(key=PCLUSTER_VERSION_TAG, value=get_installed_version()))
def dummy_slurm_cluster_config(mocker): """Generate dummy cluster.""" image = Image(os="alinux2") head_node = dummy_head_node(mocker) queue_iam = Iam(s3_access=[ S3Access("dummy-readonly-bucket", enable_write_access=True), S3Access("dummy-readwrite-bucket"), ]) compute_resources = [ SlurmComputeResource(name="dummy_compute_resource1", instance_type="dummyc5.xlarge") ] queue_networking1 = QueueNetworking(subnet_ids=["dummy-subnet-1"], security_groups=["sg-1", "sg-2"]) queue_networking2 = QueueNetworking(subnet_ids=["dummy-subnet-1"], security_groups=["sg-1", "sg-3"]) queue_networking3 = QueueNetworking(subnet_ids=["dummy-subnet-1"], security_groups=None) queues = [ SlurmQueue(name="queue1", networking=queue_networking1, compute_resources=compute_resources, iam=queue_iam), SlurmQueue(name="queue2", networking=queue_networking2, compute_resources=compute_resources), SlurmQueue(name="queue3", networking=queue_networking3, compute_resources=compute_resources), ] scheduling = SlurmScheduling(queues=queues) # shared storage shared_storage: List[Resource] = [] shared_storage.append(dummy_fsx()) shared_storage.append(dummy_ebs("/ebs1")) shared_storage.append(dummy_ebs("/ebs2", volume_id="vol-abc")) shared_storage.append( dummy_ebs("/ebs3", raid=Raid(raid_type=1, number_of_volumes=5))) shared_storage.append(dummy_efs("/efs1", file_system_id="fs-efs-1")) shared_storage.append(dummy_raid("/raid1")) cluster = _DummySlurmClusterConfig(image=image, head_node=head_node, scheduling=scheduling, shared_storage=shared_storage) cluster.custom_s3_bucket = "s3://dummy-s3-bucket" cluster.additional_resources = "https://additional.template.url" cluster.config_version = "1.0" cluster.iam = ClusterIam() cluster.tags = [Tag(key="test", value="testvalue")] return cluster
class TestBaseClusterConfig: @pytest.fixture() def base_cluster_config(self): return BaseClusterConfig( cluster_name="clustername", image=Image("alinux2"), head_node=HeadNode("c5.xlarge", HeadNodeNetworking("subnet")), ) @pytest.fixture() def base_slurm_cluster_config(self): return SlurmClusterConfig( cluster_name="clustername", image=Image("alinux2"), head_node=HeadNode("c5.xlarge", HeadNodeNetworking("subnet")), scheduling=SlurmScheduling([ SlurmQueue( name="queue0", networking=SlurmQueueNetworking(subnet_ids=["subnet"]), compute_resources=[ SlurmComputeResource(name="compute_resource_1", instance_type="c5.xlarge") ], ) ]), ) @pytest.mark.parametrize( "global_custom_ami, head_node_custom_ami, ami_filters", [ (None, None, None), ("ami-custom", None, AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])), (None, "ami-custom", AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])), ("ami-custom-1", "ami-custom-2", AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])), ], ) def test_head_node_ami_id(self, base_cluster_config, aws_api_mock, global_custom_ami, head_node_custom_ami, ami_filters): if global_custom_ami: base_cluster_config.image.custom_ami = global_custom_ami if head_node_custom_ami: base_cluster_config.head_node.image = HeadNodeImage( custom_ami=head_node_custom_ami) if ami_filters: base_cluster_config.dev_settings = ClusterDevSettings( ami_search_filters=ami_filters) expected_ami = head_node_custom_ami or global_custom_ami or "official-ami-id" aws_api_mock.ec2.get_official_image_id.return_value = "official-ami-id" ami_id = base_cluster_config.head_node_ami assert_that(ami_id).is_equal_to(expected_ami) if not (global_custom_ami or head_node_custom_ami): aws_api_mock.ec2.get_official_image_id.assert_called_with( "alinux2", "x86_64", ami_filters) else: aws_api_mock.ec2.get_official_image_id.assert_not_called() @pytest.mark.parametrize( "global_custom_ami, compute_custom_ami, ami_filters", [ (None, None, None), ("ami-custom", None, AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])), (None, "ami-custom", AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])), ("ami-custom-1", "ami-custom-2", AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])), ], ) def test_compute_node_ami_id(self, base_slurm_cluster_config, aws_api_mock, global_custom_ami, compute_custom_ami, ami_filters): if global_custom_ami: base_slurm_cluster_config.image.custom_ami = global_custom_ami if compute_custom_ami: queues = base_slurm_cluster_config.scheduling.queues for queue in queues: queue.image = QueueImage(custom_ami=compute_custom_ami) if ami_filters: base_slurm_cluster_config.dev_settings = ClusterDevSettings( ami_search_filters=ami_filters) expected_ami = compute_custom_ami or global_custom_ami or "official-ami-id" aws_api_mock.ec2.get_official_image_id.return_value = "official-ami-id" image_dict = base_slurm_cluster_config.image_dict for queue in base_slurm_cluster_config.scheduling.queues: assert_that(image_dict[queue.name]).is_equal_to(expected_ami) if not (global_custom_ami or compute_custom_ami): aws_api_mock.ec2.get_official_image_id.assert_called_with( "alinux2", "x86_64", ami_filters) else: aws_api_mock.ec2.get_official_image_id.assert_not_called()
"x86_64", AmiSearchFilters(owner="self"), { "Images": [{ "ImageId": "ami-00e87074e52e6", "CreationDate": "2018-11-09T01:21:00.000Z" }] }, None, ), ( "alinux2", "x86_64", AmiSearchFilters( owner="self", tags=[Tag("key1", "value1"), Tag("key2", "value2")]), { "Images": [{ "ImageId": "ami-00e87074e52e6", "CreationDate": "2018-11-09T01:21:00.000Z" }] }, None, ), ("alinux2", "arm64", None, Exception("error message"), "error message"), ("alinux2", "arm64", None, { "Images": [] }, "Cannot find official ParallelCluster AMI"), (