def dummy_awsbatch_cluster_config(mocker):
"""Generate dummy cluster."""
image = Image(os="alinux2")
head_node = dummy_head_node(mocker)
compute_resources = [
AwsBatchComputeResource(name="dummy_compute_resource1", instance_types=["dummyc5.xlarge", "optimal"])
]
queue_networking = AwsBatchQueueNetworking(subnet_ids=["dummy-subnet-1"], security_groups=["sg-1", "sg-2"])
queues = [AwsBatchQueue(name="queue1", networking=queue_networking, compute_resources=compute_resources)]
scheduling = AwsBatchScheduling(queues=queues)
# shared storage
shared_storage: List[Resource] = []
shared_storage.append(dummy_fsx())
shared_storage.append(dummy_ebs("/ebs1"))
shared_storage.append(dummy_ebs("/ebs2", volume_id="vol-abc"))
shared_storage.append(dummy_ebs("/ebs3", raid=Raid(raid_type=1, number_of_volumes=5)))
shared_storage.append(dummy_efs("/efs1", file_system_id="fs-efs-1"))
shared_storage.append(dummy_raid("/raid1"))
cluster = _DummyAwsBatchClusterConfig(
image=image, head_node=head_node, scheduling=scheduling, shared_storage=shared_storage
)
cluster.custom_s3_bucket = "s3://dummy-s3-bucket"
cluster.additional_resources = "https://additional.template.url"
cluster.config_version = "1.0"
cluster.iam = ClusterIam()
cluster.tags = [Tag(key="test", value="testvalue")]
return cluster
def test_tags(self, cluster, mocker, existing_tags):
"""Verify that the function to get the tags list behaves as expected."""
mock_aws_api(mocker)
cluster.config = dummy_slurm_cluster_config(mocker)
# Populate config with list of existing tags
existing_tags_list = [
Tag(key=tag_name, value=tag_value)
for tag_name, tag_value in existing_tags.items()
]
cluster.config.tags = existing_tags_list
# Expected tags:
installed_version = "FakeInstalledVersion"
tags = existing_tags
tags["parallelcluster:version"] = installed_version
expected_tags_list = self._sort_tags([
Tag(key=tag_name, value=tag_value)
for tag_name, tag_value in tags.items()
])
# Test method to add version tag
get_version_patch = mocker.patch(
"pcluster.models.cluster.get_installed_version",
return_value=installed_version)
cluster._add_version_tag()
assert_that(get_version_patch.call_count).is_equal_to(1)
assert_that(len(cluster.config.tags)).is_equal_to(
len(expected_tags_list))
assert_that(
all([
source.value == target.value for source, target in zip(
self._sort_tags(cluster.config.tags), expected_tags_list)
])).is_true()
# Test method to retrieve CFN tags
expected_cfn_tags = self._sort_cfn_tags([{
"Key": tag_name,
"Value": tag_value
} for tag_name, tag_value in tags.items()])
cfn_tags = self._sort_cfn_tags(cluster._get_cfn_tags())
assert_that(len(cfn_tags)).is_equal_to(len(expected_cfn_tags))
assert_that(
all([
source["Value"] == target["Value"]
for source, target in zip(cfn_tags, expected_cfn_tags)
])).is_true()
def test_generate_tag_specifications(input_tags):
"""Verify function to generate tag specifications for dry runs of RunInstances works as expected."""
input_tags = [Tag(tag.get("key"), tag.get("value")) for tag in input_tags]
if input_tags:
expected_output_tags = [
{"ResourceType": "instance", "Tags": [{"Key": tag.key, "Value": tag.value} for tag in input_tags]}
]
else:
expected_output_tags = []
assert_that(_LaunchTemplateValidator._generate_tag_specifications(input_tags)).is_equal_to(expected_output_tags)
def _add_version_tag(self):
"""Add version tag to the stack."""
if self.config.tags is None:
self.config.tags = []
# Remove PCLUSTER_VERSION_TAG if already exists
self.config.tags = [
tag for tag in self.config.tags if tag.key != PCLUSTER_VERSION_TAG
]
# Add PCLUSTER_VERSION_TAG
self.config.tags.append(
Tag(key=PCLUSTER_VERSION_TAG, value=get_installed_version()))
def dummy_slurm_cluster_config(mocker):
"""Generate dummy cluster."""
image = Image(os="alinux2")
head_node = dummy_head_node(mocker)
queue_iam = Iam(s3_access=[
S3Access("dummy-readonly-bucket", enable_write_access=True),
S3Access("dummy-readwrite-bucket"),
])
compute_resources = [
SlurmComputeResource(name="dummy_compute_resource1",
instance_type="dummyc5.xlarge")
]
queue_networking1 = QueueNetworking(subnet_ids=["dummy-subnet-1"],
security_groups=["sg-1", "sg-2"])
queue_networking2 = QueueNetworking(subnet_ids=["dummy-subnet-1"],
security_groups=["sg-1", "sg-3"])
queue_networking3 = QueueNetworking(subnet_ids=["dummy-subnet-1"],
security_groups=None)
queues = [
SlurmQueue(name="queue1",
networking=queue_networking1,
compute_resources=compute_resources,
iam=queue_iam),
SlurmQueue(name="queue2",
networking=queue_networking2,
compute_resources=compute_resources),
SlurmQueue(name="queue3",
networking=queue_networking3,
compute_resources=compute_resources),
]
scheduling = SlurmScheduling(queues=queues)
# shared storage
shared_storage: List[Resource] = []
shared_storage.append(dummy_fsx())
shared_storage.append(dummy_ebs("/ebs1"))
shared_storage.append(dummy_ebs("/ebs2", volume_id="vol-abc"))
shared_storage.append(
dummy_ebs("/ebs3", raid=Raid(raid_type=1, number_of_volumes=5)))
shared_storage.append(dummy_efs("/efs1", file_system_id="fs-efs-1"))
shared_storage.append(dummy_raid("/raid1"))
cluster = _DummySlurmClusterConfig(image=image,
head_node=head_node,
scheduling=scheduling,
shared_storage=shared_storage)
cluster.custom_s3_bucket = "s3://dummy-s3-bucket"
cluster.additional_resources = "https://additional.template.url"
cluster.config_version = "1.0"
cluster.iam = ClusterIam()
cluster.tags = [Tag(key="test", value="testvalue")]
return cluster
class TestBaseClusterConfig:
@pytest.fixture()
def base_cluster_config(self):
return BaseClusterConfig(
cluster_name="clustername",
image=Image("alinux2"),
head_node=HeadNode("c5.xlarge", HeadNodeNetworking("subnet")),
)
@pytest.fixture()
def base_slurm_cluster_config(self):
return SlurmClusterConfig(
cluster_name="clustername",
image=Image("alinux2"),
head_node=HeadNode("c5.xlarge", HeadNodeNetworking("subnet")),
scheduling=SlurmScheduling([
SlurmQueue(
name="queue0",
networking=SlurmQueueNetworking(subnet_ids=["subnet"]),
compute_resources=[
SlurmComputeResource(name="compute_resource_1",
instance_type="c5.xlarge")
],
)
]),
)
@pytest.mark.parametrize(
"global_custom_ami, head_node_custom_ami, ami_filters",
[
(None, None, None),
("ami-custom", None,
AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])),
(None, "ami-custom",
AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])),
("ami-custom-1", "ami-custom-2",
AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])),
],
)
def test_head_node_ami_id(self, base_cluster_config, aws_api_mock,
global_custom_ami, head_node_custom_ami,
ami_filters):
if global_custom_ami:
base_cluster_config.image.custom_ami = global_custom_ami
if head_node_custom_ami:
base_cluster_config.head_node.image = HeadNodeImage(
custom_ami=head_node_custom_ami)
if ami_filters:
base_cluster_config.dev_settings = ClusterDevSettings(
ami_search_filters=ami_filters)
expected_ami = head_node_custom_ami or global_custom_ami or "official-ami-id"
aws_api_mock.ec2.get_official_image_id.return_value = "official-ami-id"
ami_id = base_cluster_config.head_node_ami
assert_that(ami_id).is_equal_to(expected_ami)
if not (global_custom_ami or head_node_custom_ami):
aws_api_mock.ec2.get_official_image_id.assert_called_with(
"alinux2", "x86_64", ami_filters)
else:
aws_api_mock.ec2.get_official_image_id.assert_not_called()
@pytest.mark.parametrize(
"global_custom_ami, compute_custom_ami, ami_filters",
[
(None, None, None),
("ami-custom", None,
AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])),
(None, "ami-custom",
AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])),
("ami-custom-1", "ami-custom-2",
AmiSearchFilters(owner="owner", tags=[Tag("key1", "value1")])),
],
)
def test_compute_node_ami_id(self, base_slurm_cluster_config, aws_api_mock,
global_custom_ami, compute_custom_ami,
ami_filters):
if global_custom_ami:
base_slurm_cluster_config.image.custom_ami = global_custom_ami
if compute_custom_ami:
queues = base_slurm_cluster_config.scheduling.queues
for queue in queues:
queue.image = QueueImage(custom_ami=compute_custom_ami)
if ami_filters:
base_slurm_cluster_config.dev_settings = ClusterDevSettings(
ami_search_filters=ami_filters)
expected_ami = compute_custom_ami or global_custom_ami or "official-ami-id"
aws_api_mock.ec2.get_official_image_id.return_value = "official-ami-id"
image_dict = base_slurm_cluster_config.image_dict
for queue in base_slurm_cluster_config.scheduling.queues:
assert_that(image_dict[queue.name]).is_equal_to(expected_ami)
if not (global_custom_ami or compute_custom_ami):
aws_api_mock.ec2.get_official_image_id.assert_called_with(
"alinux2", "x86_64", ami_filters)
else:
aws_api_mock.ec2.get_official_image_id.assert_not_called()
"x86_64",
AmiSearchFilters(owner="self"),
{
"Images": [{
"ImageId": "ami-00e87074e52e6",
"CreationDate": "2018-11-09T01:21:00.000Z"
}]
},
None,
),
(
"alinux2",
"x86_64",
AmiSearchFilters(
owner="self",
tags=[Tag("key1", "value1"),
Tag("key2", "value2")]),
{
"Images": [{
"ImageId": "ami-00e87074e52e6",
"CreationDate": "2018-11-09T01:21:00.000Z"
}]
},
None,
),
("alinux2", "arm64", None, Exception("error message"),
"error message"),
("alinux2", "arm64", None, {
"Images": []
}, "Cannot find official ParallelCluster AMI"),
(