def _gpu_test_scaleup(remote_command_executor, region, asg_name, stack_name,
scaledown_idletime, num_gpus):
"""Test cluster is scaling up correctly and GPU jobs are not aborted on slurmctld restart."""
logging.info("Testing cluster scales correctly with GPU jobs")
slurm_commands = SlurmCommands(remote_command_executor)
# Assert initial conditions
_assert_asg_has_no_node(region, asg_name)
_assert_no_nodes_in_scheduler(slurm_commands)
# g3.8xlarge has 32 vcpus and 2 GPUs, hardcoding tests for g3.8xlarge
job_ids = []
# sbatch --wrap 'sleep 10' -G 3
result = slurm_commands.submit_command(command="sleep 10",
nodes=-1,
other_options="-G 3")
job_ids.append(slurm_commands.assert_job_submitted(result.stdout))
# Nodes/resources available after this job:
# [{cpu:31, gpu:0}, {cpu:31, gpu:0}]
# sbatch --wrap 'sleep 10' --cpus-per-gpu=10 --gpus-per-task=1
result = slurm_commands.submit_command(
command="sleep 10",
nodes=-1,
other_options="--cpus-per-gpu=10 --gpus-per-task=1")
job_ids.append(slurm_commands.assert_job_submitted(result.stdout))
# Nodes/resources available after this job:
# [{cpu:31, gpu:0}, {cpu:31, gpu:0}, {cpu:22, gpu:1}]
# sbatch --wrap 'sleep 10' -N 1 --gpus-per-node=1 -c 22 -n 1
result = slurm_commands.submit_command(
command="sleep 10",
nodes=1,
slots=1,
other_options="--gpus-per-node=1 -c 23")
job_ids.append(slurm_commands.assert_job_submitted(result.stdout))
# Nodes/resources available after this job:
# [{cpu:31, gpu:0}, {cpu:31, gpu:0}, {cpu:22, gpu:1}, {cpu:19, gpu:1}]
# sbatch --wrap 'sleep 10' -c 31 -n 1
result = slurm_commands.submit_command(command="sleep 10",
nodes=-1,
slots=1,
other_options="-c 31")
job_ids.append(slurm_commands.assert_job_submitted(result.stdout))
# Nodes/resources available after this job:
# [{cpu:0, gpu:0}, {cpu:31, gpu:0}, {cpu:22, gpu:1}, {cpu:19, gpu:1}]
# Assert scaling worked as expected
assert_scaling_worked(slurm_commands,
region,
stack_name,
scaledown_idletime,
expected_max=4,
expected_final=0)
# Assert jobs were completed
for job_id in job_ids:
slurm_commands.assert_job_succeeded(job_id)
def _test_job_dependencies(remote_command_executor, region, stack_name,
scaledown_idletime, max_queue_size):
logging.info(
"Testing cluster doesn't scale when job dependencies are not satisfied"
)
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 60", nodes=1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
result = remote_command_executor.run_remote_command(
"sbatch -N 1 --wrap='sleep 1' -d afterok:{0}".format(job_id))
dependent_job_id = slurm_commands.assert_job_submitted(result.stdout)
# Wait for reason to be computed
time.sleep(3)
assert_that(slurm_commands.get_job_info(job_id)).contains(
"JobState=PENDING Reason=Nodes_required_for_job_are_DOWN,_DRAINED"
"_or_reserved_for_jobs_in_higher_priority_partitions")
assert_that(slurm_commands.get_job_info(dependent_job_id)).contains(
"JobState=PENDING Reason=Dependency")
assert_scaling_worked(slurm_commands,
region,
stack_name,
scaledown_idletime,
expected_max=1,
expected_final=0)
# Assert scheduler configuration is correct
_assert_dummy_nodes(remote_command_executor, max_queue_size)
assert_that(
_retrieve_slurm_compute_nodes_from_config(
remote_command_executor)).is_empty()
# Assert jobs were completed
_assert_job_completed(remote_command_executor, job_id)
_assert_job_completed(remote_command_executor, dependent_job_id)
def _test_job_dependencies(remote_command_executor, region, stack_name, scaledown_idletime):
logging.info("Testing cluster doesn't scale when job dependencies are not satisfied")
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 60", nodes=1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
result = remote_command_executor.run_remote_command("sbatch -N 1 --wrap='sleep 1' -d afterok:{0}".format(job_id))
dependent_job_id = slurm_commands.assert_job_submitted(result.stdout)
# Wait for reason to be computed
time.sleep(3)
assert_that(_get_job_info(remote_command_executor, job_id)).contains(
"JobState=PENDING Reason=Nodes_required_for_job_are_DOWN,_DRAINED"
"_or_reserved_for_jobs_in_higher_priority_partitions"
)
assert_that(_get_job_info(remote_command_executor, dependent_job_id)).contains("JobState=PENDING Reason=Dependency")
jobs_execution_time = 1
estimated_scaleup_time = 5
estimated_scaledown_time = 20
asg_capacity_time_series, compute_nodes_time_series, timestamps = get_compute_nodes_allocation(
scheduler_commands=slurm_commands,
region=region,
stack_name=stack_name,
max_monitoring_time=minutes(jobs_execution_time)
+ minutes(scaledown_idletime)
+ minutes(estimated_scaleup_time)
+ minutes(estimated_scaledown_time),
)
assert_that(max(asg_capacity_time_series)).is_equal_to(1)
assert_that(max(compute_nodes_time_series)).is_equal_to(1)
assert_that(asg_capacity_time_series[-1]).is_equal_to(0)
assert_that(compute_nodes_time_series[-1]).is_equal_to(0)
def _test_dynamic_dummy_nodes(remote_command_executor, max_queue_size):
logging.info("Testing dummy nodes are automatically reconfigured based on actual compute nodes")
_assert_dummy_nodes(remote_command_executor, max_queue_size)
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 1", nodes=1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
slurm_commands.wait_job_completed(job_id)
_assert_dummy_nodes(remote_command_executor, max_queue_size - 1)
def _test_dynamic_dummy_nodes(remote_command_executor, max_queue_size):
logging.info("Testing dummy nodes are automatically reconfigured based on actual compute nodes")
_assert_dummy_nodes(remote_command_executor, max_queue_size)
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 1", nodes=1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
slurm_commands.wait_job_completed(job_id)
_assert_dummy_nodes(remote_command_executor, max_queue_size - 1)
def _test_cluster_limits(remote_command_executor, max_queue_size, region, asg_name):
logging.info("Testing cluster doesn't scale when job requires a capacity that is higher than the max available")
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 1", nodes=max_queue_size + 1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
# Wait for reason to be computed
time.sleep(3)
assert_that(_get_job_info(remote_command_executor, job_id)).contains("JobState=PENDING Reason=PartitionNodeLimit")
# Check we are not scaling
time.sleep(60)
asg_client = boto3.client("autoscaling", region_name=region)
asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]).get("AutoScalingGroups")[0]
assert_that(asg.get("DesiredCapacity")).is_equal_to(0)
def test_replace_compute_on_failure(region, pcluster_config_reader, clusters_factory, s3_bucket_factory, test_datadir):
"""
Test that compute nodes get replaced on userdata failures and logs get saved in shared directory.
The failure is caused by a post_install script that exits with errors on compute nodes.
"""
bucket_name = s3_bucket_factory()
bucket = boto3.resource("s3", region_name=region).Bucket(bucket_name)
bucket.upload_file(str(test_datadir / "post_install.sh"), "post_install.sh")
cluster_config = pcluster_config_reader(bucket_name=bucket_name)
cluster = clusters_factory(cluster_config)
remote_command_executor = RemoteCommandExecutor(cluster)
# submit a job to spin up a compute node that will fail due to post_install script
sge_commands = SlurmCommands(remote_command_executor)
sge_commands.submit_command("sleep 1")
instance_id = wait_compute_log(remote_command_executor)
# extract logs and check one of them
_assert_compute_logs(remote_command_executor, instance_id)
# check that instance got already replaced or is marked as Unhealthy
assert_instance_replaced_or_terminating(instance_id, region)
def _gpu_test_cluster_limits(remote_command_executor, max_queue_size,
num_gpus):
"""Test edge cases regarding the number of GPUs."""
logging.info(
"Testing scheduler does not accept jobs when requesting for more GPUs than available"
)
slurm_commands = SlurmCommands(remote_command_executor)
# Expect commands below to fail with exit 1
_submit_and_assert_job_rejected_node_config(
remote_command_executor,
"sbatch -N 1 --wrap='sleep 1' --gpus-per-task {0}".format(num_gpus +
1))
_submit_and_assert_job_rejected_node_config(
remote_command_executor,
"sbatch -N 1 --wrap='sleep 1' --gres=gpu:{0}".format(num_gpus + 1))
_submit_and_assert_job_rejected_node_config(
remote_command_executor,
"sbatch -G {0} --wrap='sleep 1'".format(num_gpus * max_queue_size + 1))
# Commands below should be correctly submitted
result = slurm_commands.submit_command(
"sleep 1",
nodes=1,
slots=num_gpus,
other_options="-G {0} --gpus-per-task=1".format(num_gpus))
slurm_commands.assert_job_submitted(result.stdout)
result = slurm_commands.submit_command(
"sleep 1", nodes=1, other_options="--gres=gpu:{0}".format(num_gpus))
slurm_commands.assert_job_submitted(result.stdout)
# Submit job without '-N' option(nodes=-1)
result = slurm_commands.submit_command(
"sleep 1",
nodes=-1,
other_options="-G {0} --gpus-per-node={1}".format(
num_gpus * max_queue_size, num_gpus))
slurm_commands.assert_job_submitted(result.stdout)
def _test_cluster_limits(remote_command_executor, max_queue_size, region, asg_name):
logging.info("Testing cluster doesn't scale when job requires a capacity that is higher than the max available")
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 1000", nodes=max_queue_size + 1)
max_nodes_job_id = slurm_commands.assert_job_submitted(result.stdout)
result = remote_command_executor.run_remote_command("sbatch -N 1 --wrap='sleep 1' --cpus-per-task 5")
max_cpu_job_id = slurm_commands.assert_job_submitted(result.stdout)
# Check we are not scaling
time.sleep(60)
assert_asg_desired_capacity(region, asg_name, expected=0)
assert_that(_get_job_info(remote_command_executor, max_nodes_job_id)).contains(
"JobState=PENDING Reason=PartitionNodeLimit"
)
assert_that(_get_job_info(remote_command_executor, max_cpu_job_id)).contains(
"JobState=PENDING Reason=Nodes_required_for_job_are_DOWN,_DRAINED"
"_or_reserved_for_jobs_in_higher_priority_partitions"
)
def _test_dynamic_dummy_nodes(remote_command_executor,
region,
asg_name,
max_queue_size,
slots=4,
gpus=0):
logging.info(
"Testing dummy nodes are automatically reconfigured based on actual compute nodes"
)
slurm_commands = SlurmCommands(remote_command_executor)
# Assert initial conditions
_assert_asg_has_no_node(region, asg_name)
_assert_no_nodes_in_scheduler(slurm_commands)
_assert_dummy_nodes(remote_command_executor, max_queue_size, slots, gpus)
result = slurm_commands.submit_command("sleep 1", nodes=1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
slurm_commands.wait_job_completed(job_id)
_assert_dummy_nodes(remote_command_executor, max_queue_size - 1, slots,
gpus)
def _test_cluster_limits(remote_command_executor, max_queue_size, region,
asg_name):
logging.info(
"Testing cluster doesn't scale when job requires a capacity that is higher than the max available"
)
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 1", nodes=max_queue_size + 1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
# Wait for reason to be computed
time.sleep(3)
assert_that(_get_job_info(
remote_command_executor,
job_id)).contains("JobState=PENDING Reason=PartitionNodeLimit")
# Check we are not scaling
time.sleep(60)
asg_client = boto3.client("autoscaling", region_name=region)
asg = asg_client.describe_auto_scaling_groups(
AutoScalingGroupNames=[asg_name]).get("AutoScalingGroups")[0]
assert_that(asg.get("DesiredCapacity")).is_equal_to(0)
def _test_job_dependencies(remote_command_executor, region, stack_name,
scaledown_idletime):
logging.info(
"Testing cluster doesn't scale when job dependencies are not satisfied"
)
slurm_commands = SlurmCommands(remote_command_executor)
result = slurm_commands.submit_command("sleep 60", nodes=1)
job_id = slurm_commands.assert_job_submitted(result.stdout)
result = remote_command_executor.run_remote_command(
"sbatch -N 1 --wrap='sleep 1' -d afterok:{0}".format(job_id))
dependent_job_id = slurm_commands.assert_job_submitted(result.stdout)
# Wait for reason to be computed
time.sleep(3)
assert_that(_get_job_info(remote_command_executor, job_id)).contains(
"JobState=PENDING Reason=Nodes_required_for_job_are_DOWN,_DRAINED"
"_or_reserved_for_jobs_in_higher_priority_partitions")
assert_that(_get_job_info(
remote_command_executor,
dependent_job_id)).contains("JobState=PENDING Reason=Dependency")
jobs_execution_time = 1
estimated_scaleup_time = 5
estimated_scaledown_time = 20
asg_capacity_time_series, compute_nodes_time_series, timestamps = get_compute_nodes_allocation(
scheduler_commands=slurm_commands,
region=region,
stack_name=stack_name,
max_monitoring_time=minutes(jobs_execution_time) +
minutes(scaledown_idletime) + minutes(estimated_scaleup_time) +
minutes(estimated_scaledown_time),
)
assert_that(max(asg_capacity_time_series)).is_equal_to(1)
assert_that(max(compute_nodes_time_series)).is_equal_to(1)
assert_that(asg_capacity_time_series[-1]).is_equal_to(0)
assert_that(compute_nodes_time_series[-1]).is_equal_to(0)
def test_update_hit(region, scheduler, pcluster_config_reader,
clusters_factory, test_datadir, s3_bucket_factory):
# Create S3 bucket for pre/post install scripts
bucket_name = s3_bucket_factory()
bucket = boto3.resource("s3", region_name=region).Bucket(bucket_name)
bucket.upload_file(str(test_datadir / "preinstall.sh"),
"scripts/preinstall.sh")
bucket.upload_file(str(test_datadir / "postinstall.sh"),
"scripts/postinstall.sh")
# Create cluster with initial configuration
init_config_file = pcluster_config_reader(resource_bucket=bucket_name)
cluster = clusters_factory(init_config_file)
# Update cluster with the same configuration, command should not result any error even if not using force update
cluster.config_file = str(init_config_file)
cluster.update(force=True)
# Command executors
command_executor = RemoteCommandExecutor(cluster)
slurm_commands = SlurmCommands(command_executor)
# Create shared dir for script results
command_executor.run_remote_command("mkdir -p /shared/script_results")
initial_queues_config = {
"queue1": {
"compute_resources": {
"queue1_i1": {
"instance_type": "c5.xlarge",
"expected_running_instances": 1,
"expected_power_saved_instances": 1,
"enable_efa": False,
"disable_hyperthreading": False,
},
"queue1_i2": {
"instance_type": "t2.micro",
"expected_running_instances": 1,
"expected_power_saved_instances": 9,
"enable_efa": False,
"disable_hyperthreading": False,
},
},
"compute_type": "ondemand",
},
"queue2": {
"compute_resources": {
"queue2_i1": {
"instance_type": "c5n.18xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"enable_efa": False,
"disable_hyperthreading": False,
},
},
"compute_type": "ondemand",
},
}
_assert_scheduler_nodes(queues_config=initial_queues_config,
slurm_commands=slurm_commands)
_assert_launch_templates_config(queues_config=initial_queues_config,
cluster_name=cluster.name,
region=region)
# Submit a job in order to verify that jobs are not affected by an update of the queue size
result = slurm_commands.submit_command("sleep infinity",
constraint="static")
job_id = slurm_commands.assert_job_submitted(result.stdout)
# Update cluster with new configuration
updated_config_file = pcluster_config_reader(
config_file="pcluster.config.update.ini",
bucket=bucket_name,
resource_bucket=bucket_name)
cluster.config_file = str(updated_config_file)
cluster.update()
# Here is the expected list of nodes. Note that queue1-dy-t2micro-1 comes from the initial_count set when creating
# the cluster:
# queue1-dy-t2micro-1
# queue1-st-c5xlarge-1
# queue1-st-c5xlarge-2
assert_initial_conditions(slurm_commands, 2, 1, partition="queue1")
updated_queues_config = {
"queue1": {
"compute_resources": {
"queue1_i1": {
"instance_type": "c5.xlarge",
"expected_running_instances": 2,
"expected_power_saved_instances": 2,
"disable_hyperthreading": False,
"enable_efa": False,
},
"queue1_i2": {
"instance_type": "c5.2xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": False,
"enable_efa": False,
},
"queue1_i3": {
"instance_type": "t2.micro",
"expected_running_instances":
1, # This comes from initial_count before update
"expected_power_saved_instances": 9,
"disable_hyperthreading": False,
"enable_efa": False,
},
},
"compute_type": "spot",
},
"queue2": {
"compute_resources": {
"queue2_i1": {
"instance_type": "c5n.18xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 1,
"enable_efa": True,
"disable_hyperthreading": True,
},
},
"compute_type": "ondemand",
},
"queue3": {
"compute_resources": {
"queue3_i1": {
"instance_type": "c5n.18xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": True,
"enable_efa": True,
},
"queue3_i2": {
"instance_type": "t2.xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": False,
"enable_efa": False,
},
},
"compute_type": "ondemand",
},
}
_assert_scheduler_nodes(queues_config=updated_queues_config,
slurm_commands=slurm_commands)
_assert_launch_templates_config(queues_config=updated_queues_config,
cluster_name=cluster.name,
region=region)
# Read updated configuration
updated_config = configparser.ConfigParser()
updated_config.read(updated_config_file)
# Check new S3 resources
check_s3_read_resource(
region, cluster,
updated_config.get("cluster default", "s3_read_resource"))
check_s3_read_write_resource(
region, cluster,
updated_config.get("cluster default", "s3_read_write_resource"))
# Check new Additional IAM policies
_check_role_attached_policy(
region, cluster,
updated_config.get("cluster default", "additional_iam_policies"))
# Assert that the job submitted before the update is still running
assert_that(
slurm_commands.get_job_info(job_id)).contains("JobState=RUNNING")
def test_update_slurm(region, pcluster_config_reader, s3_bucket_factory,
clusters_factory, test_datadir):
# Create S3 bucket for pre/post install scripts
bucket_name = s3_bucket_factory()
bucket = boto3.resource("s3", region_name=region).Bucket(bucket_name)
for script in [
"preinstall.sh", "postinstall.sh", "updated_preinstall.sh",
"updated_postinstall.sh"
]:
bucket.upload_file(str(test_datadir / script), f"scripts/{script}")
# Create cluster with initial configuration
init_config_file = pcluster_config_reader(resource_bucket=bucket_name,
bucket=bucket_name)
cluster = clusters_factory(init_config_file)
# Update cluster with the same configuration, command should not result any error even if not using force update
cluster.update(str(init_config_file), force_update="true")
# Command executors
command_executor = RemoteCommandExecutor(cluster)
slurm_commands = SlurmCommands(command_executor)
# Create shared dir for script results
command_executor.run_remote_command("mkdir -p /shared/script_results")
initial_queues_config = {
"queue1": {
"compute_resources": {
"queue1-i1": {
"instance_type": "c5.xlarge",
"expected_running_instances": 1,
"expected_power_saved_instances": 1,
"enable_efa": False,
"disable_hyperthreading": False,
},
"queue1-i2": {
"instance_type": "t2.micro",
"expected_running_instances": 1,
"expected_power_saved_instances": 9,
"enable_efa": False,
"disable_hyperthreading": False,
},
},
"compute_type": "ondemand",
},
"queue2": {
"compute_resources": {
"queue2-i1": {
"instance_type": "c5n.18xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"enable_efa": False,
"disable_hyperthreading": False,
}
},
"compute_type": "ondemand",
},
}
_assert_scheduler_nodes(queues_config=initial_queues_config,
slurm_commands=slurm_commands)
_assert_launch_templates_config(queues_config=initial_queues_config,
cluster_name=cluster.name,
region=region)
# submit job in queue1 to verify original pre/post-install script execution
initial_compute_nodes = slurm_commands.get_compute_nodes(
filter_by_partition="queue1")
_check_script(command_executor, slurm_commands, initial_compute_nodes[0],
"preinstall", "QWE")
_check_script(command_executor, slurm_commands, initial_compute_nodes[0],
"postinstall", "RTY")
# Submit a job in order to verify that jobs are not affected by an update of the queue size
result = slurm_commands.submit_command("sleep infinity",
constraint="static&c5.xlarge")
job_id = slurm_commands.assert_job_submitted(result.stdout)
# Update cluster with new configuration
additional_policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonAppStreamServiceAccess"
updated_config_file = pcluster_config_reader(
config_file="pcluster.config.update.yaml",
bucket=bucket_name,
resource_bucket=bucket_name,
additional_policy_arn=additional_policy_arn,
)
cluster.update(str(updated_config_file), force_update="true")
# Here is the expected list of nodes.
# the cluster:
# queue1-st-c5xlarge-1
# queue1-st-c5xlarge-2
assert_initial_conditions(slurm_commands, 2, 0, partition="queue1")
updated_queues_config = {
"queue1": {
"compute_resources": {
"queue1-i1": {
"instance_type": "c5.xlarge",
"expected_running_instances": 2,
"expected_power_saved_instances": 2,
"disable_hyperthreading": False,
"enable_efa": False,
},
"queue1-i2": {
"instance_type": "c5.2xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": False,
"enable_efa": False,
},
"queue1-i3": {
"instance_type": "t2.micro",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": False,
"enable_efa": False,
},
},
"compute_type": "spot",
},
"queue2": {
"compute_resources": {
"queue2-i1": {
"instance_type": "c5n.18xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 1,
"enable_efa": True,
"disable_hyperthreading": True,
}
},
"compute_type": "ondemand",
"networking": {
"placement_group": {
"enabled": False
}
},
},
"queue3": {
"compute_resources": {
"queue3-i1": {
"instance_type": "c5n.18xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": True,
"enable_efa": True,
},
"queue3-i2": {
"instance_type": "t2.xlarge",
"expected_running_instances": 0,
"expected_power_saved_instances": 10,
"disable_hyperthreading": False,
"enable_efa": False,
},
},
"compute_type": "ondemand",
"networking": {
"placement_group": {
"enabled": False
}
},
},
}
_assert_scheduler_nodes(queues_config=updated_queues_config,
slurm_commands=slurm_commands)
_assert_launch_templates_config(queues_config=updated_queues_config,
cluster_name=cluster.name,
region=region)
# Read updated configuration
with open(updated_config_file, encoding="utf-8") as conf_file:
updated_config = yaml.safe_load(conf_file)
# Check new S3 resources
check_s3_read_resource(
region, cluster,
get_policy_resources(updated_config, enable_write_access=False))
check_s3_read_write_resource(
region, cluster,
get_policy_resources(updated_config, enable_write_access=True))
# Check new Additional IAM policies
_check_role_attached_policy(region, cluster, additional_policy_arn)
# Assert that the job submitted before the update is still running
assert_that(
slurm_commands.get_job_info(job_id)).contains("JobState=RUNNING")
_check_volume(cluster, updated_config, region)
# Launch a new instance for queue1 and test updated pre/post install script execution and extra json update
# Add a new dynamic node t2.micro to queue1-i3
new_compute_node = _add_compute_nodes(slurm_commands, "queue1", "t2.micro")
assert_that(len(new_compute_node)).is_equal_to(1)
_check_script(command_executor, slurm_commands, new_compute_node[0],
"updated_preinstall", "ABC")
_check_script(command_executor, slurm_commands, new_compute_node[0],
"updated_postinstall", "DEF")
# check new extra json
_check_extra_json(command_executor, slurm_commands, new_compute_node[0],
"test_value")