def test_cluster_in_private_subnet(region, pcluster_config_reader, clusters_factory):
# This test just creates a cluster in the private subnet and just checks that no failures occur
cluster_config = pcluster_config_reader()
cluster = clusters_factory(cluster_config)
assert_that(cluster).is_not_none()
assert_that(len(get_compute_nodes_instance_ids(cluster.cfn_name, region))).is_equal_to(1)
def _test_reset_terminated_nodes(scheduler_commands, cluster_name, region,
partition, num_static_nodes,
num_dynamic_nodes, dynamic_instance_type):
"""
Test that slurm nodes are reset if instances are terminated manually.
Static capacity should be replaced and dynamic capacity power saved.
"""
logging.info(
"Testing that nodes are reset when instances are terminated manually")
init_job_id = submit_initial_job(
scheduler_commands,
"sleep 300",
partition,
dynamic_instance_type,
num_dynamic_nodes,
other_options="--no-requeue",
)
static_nodes, dynamic_nodes = assert_initial_conditions(
scheduler_commands,
num_static_nodes,
num_dynamic_nodes,
partition,
cancel_job_id=init_job_id)
instance_ids = get_compute_nodes_instance_ids(cluster_name, region)
# terminate all instances manually
_terminate_nodes_manually(instance_ids, region)
# Assert that cluster replaced static node and reset dynamic nodes
_wait_for_node_reset(scheduler_commands, static_nodes, dynamic_nodes)
assert_num_instances_in_cluster(cluster_name, region, len(static_nodes))
def test_efa(region, scheduler, instance, os, pcluster_config_reader,
clusters_factory, test_datadir):
"""
Test all EFA Features.
Grouped all tests in a single function so that cluster can be reused for all of them.
"""
max_queue_size = 2
slots_per_instance = fetch_instance_slots(region, instance)
cluster_config = pcluster_config_reader(max_queue_size=max_queue_size)
cluster = clusters_factory(cluster_config)
remote_command_executor = RemoteCommandExecutor(cluster)
scheduler_commands = get_scheduler_commands(scheduler,
remote_command_executor)
_test_efa_installed(scheduler_commands, remote_command_executor)
_test_mpi(remote_command_executor, slots_per_instance, scheduler, os)
logging.info("Running on Instances: {0}".format(
get_compute_nodes_instance_ids(cluster.cfn_name, region)))
_test_osu_benchmarks("openmpi", remote_command_executor,
scheduler_commands, test_datadir, slots_per_instance)
_test_osu_benchmarks("intelmpi", remote_command_executor,
scheduler_commands, test_datadir, slots_per_instance)
_test_shm_transfer_is_enabled(scheduler_commands, remote_command_executor)
assert_no_errors_in_logs(remote_command_executor,
["/var/log/sqswatcher", "/var/log/jobwatcher"])
def _test_compute_node_nics(cluster, region, remote_command_executor,
scheduler_commands):
compute_instance_id = get_compute_nodes_instance_ids(
cluster.cfn_name, region)[0]
# Get compute node's IP addresses
compute_ip_addresses = _get_private_ip_addresses(compute_instance_id,
region,
remote_command_executor)
for ip_address in compute_ip_addresses:
_test_compute_node_nic(ip_address, remote_command_executor,
scheduler_commands)
def test_nodewatcher_terminates_failing_node(scheduler, region,
pcluster_config_reader,
clusters_factory, test_datadir):
# slurm test use more nodes because of internal request to test in multi-node settings
initial_queue_size = 1
maintain_initial_size = "true"
environ["AWS_DEFAULT_REGION"] = region
cluster_config = pcluster_config_reader(
initial_queue_size=initial_queue_size,
maintain_initial_size=maintain_initial_size)
cluster = clusters_factory(cluster_config)
remote_command_executor = RemoteCommandExecutor(cluster)
scheduler_commands = get_scheduler_commands(scheduler,
remote_command_executor)
compute_nodes = scheduler_commands.get_compute_nodes()
instance_ids = get_compute_nodes_instance_ids(cluster.cfn_name, region)
hostname_to_instance_id = get_instance_ids_compute_hostnames_conversion_dict(
instance_ids, id_to_hostname=False)
logging.info(
"Testing that nodewatcher will terminate a node in failing state")
# submit a job to run on all nodes
scheduler_commands.submit_command("sleep infinity",
nodes=initial_queue_size)
expected_num_nodes_killed = 1
# simulate unexpected hardware failure by killing first x nodes
nodes_to_remove = compute_nodes[:expected_num_nodes_killed]
for node in nodes_to_remove:
remote_command_executor.run_remote_script(str(
test_datadir / "{0}_kill_scheduler_job.sh".format(scheduler)),
args=[node])
# assert failing nodes are terminated according to ASG
_assert_failing_nodes_terminated(nodes_to_remove, hostname_to_instance_id,
region)
nodes_to_retain = [
compute for compute in compute_nodes if compute not in nodes_to_remove
]
# verify that desired capacity is still the initial_queue_size
assert_that(get_desired_asg_capacity(
region, cluster.cfn_name)).is_equal_to(initial_queue_size)
# assert failing nodes are removed from scheduler config
_assert_nodes_removed_and_replaced_in_scheduler(
scheduler_commands,
nodes_to_remove,
nodes_to_retain,
desired_capacity=initial_queue_size)
assert_no_errors_in_logs(remote_command_executor, scheduler)
test_maintain_initial_size(cluster.cfn_name, region, maintain_initial_size,
initial_queue_size)
def test_cluster_in_private_subnet(region, os, scheduler,
pcluster_config_reader, clusters_factory,
bastion_instance):
# This test just creates a cluster in the private subnet and just checks that no failures occur
fsx_mount_dir = "/fsx_mount"
cluster_config = pcluster_config_reader(fsx_mount_dir=fsx_mount_dir)
cluster = clusters_factory(cluster_config)
assert_that(cluster).is_not_none()
assert_that(len(get_compute_nodes_instance_ids(cluster.cfn_name,
region))).is_equal_to(1)
_test_fsx_in_private_subnet(cluster, os, region, scheduler, fsx_mount_dir,
bastion_instance)
def test_retain_on_deletion(pcluster_config_reader, clusters_factory, region,
os):
cluster_config = pcluster_config_reader()
cluster = clusters_factory(cluster_config)
stack_arn = cluster.cfn_stack_arn
retained_volume = cluster.cfn_resources["EBS0"]
head_node_instance_id = get_head_node_instance_id(cluster)
head_node_root_volume = get_root_volume_id(head_node_instance_id, region,
os)
compute_node_instance_ids = get_compute_nodes_instance_ids(
cluster.name, region)
logging.info("Checking at least one compute node is running")
assert_that(len(compute_node_instance_ids)).is_greater_than_or_equal_to(1)
compute_root_volumes = []
for compute_node in compute_node_instance_ids:
compute_root_volumes.append(
get_root_volume_id(compute_node, region, os))
logging.info("Compute root volume %s", compute_root_volumes)
ec2_client = boto3.client("ec2")
logging.info(
"Checking no snapshot with the tag of stack id is created before stack deletion"
)
snapshots = _get_snapshots(ec2_client, stack_arn)
assert_that(snapshots).is_length(0)
cluster.delete()
logging.info(
"Checking a snapshot with the tag of stack id is created after stack deletion"
)
snapshots = _get_snapshots(ec2_client, stack_arn)
assert_that(snapshots).is_length(1)
logging.info("Checking retained volume after stack deletion")
_check_volume(ec2_client, retained_volume)
logging.info(
"Checking retained head node root volume after stack deletion")
_check_volume(ec2_client, head_node_root_volume)
logging.info(
"Checking compute node root volumes are deleted after stack deletion")
with pytest.raises(ClientError, match="InvalidVolume.NotFound"):
ec2_client.describe_volumes(VolumeIds=compute_root_volumes)["Volumes"]
def test_sit_efa(
region,
scheduler,
instance,
pcluster_config_reader,
clusters_factory,
test_datadir,
architecture,
network_interfaces_count,
):
"""
Test all EFA Features.
Grouped all tests in a single function so that cluster can be reused for all of them.
"""
max_queue_size = 2
slots_per_instance = fetch_instance_slots(region, instance)
cluster_config = pcluster_config_reader(max_queue_size=max_queue_size)
cluster = clusters_factory(cluster_config)
remote_command_executor = RemoteCommandExecutor(cluster)
scheduler_commands = get_scheduler_commands(scheduler,
remote_command_executor)
_test_efa_installation(scheduler_commands,
remote_command_executor,
efa_installed=True)
_test_mpi(remote_command_executor, slots_per_instance, scheduler)
logging.info("Running on Instances: {0}".format(
get_compute_nodes_instance_ids(cluster.cfn_name, region)))
_test_osu_benchmarks_latency("openmpi", remote_command_executor,
scheduler_commands, test_datadir,
slots_per_instance)
if architecture == "x86_64":
_test_osu_benchmarks_latency("intelmpi", remote_command_executor,
scheduler_commands, test_datadir,
slots_per_instance)
_test_shm_transfer_is_enabled(scheduler_commands, remote_command_executor)
if network_interfaces_count > 1:
_test_osu_benchmarks_multiple_bandwidth(remote_command_executor,
scheduler_commands,
test_datadir,
slots_per_instance)
assert_no_errors_in_logs(remote_command_executor, scheduler)
def _run_mpi_jobs(mpi_variants, remote_command_executor, test_datadir,
slurm_commands, cluster, region):
for mpi_variant in mpi_variants:
logging.info(
f"Running OSU benchmark {OSU_BENCHMARK_VERSION} for {mpi_variant}")
compile_osu(mpi_variant, remote_command_executor)
submission_script = render_jinja_template(
template_file_path=test_datadir /
f"osu_pt2pt_submit_{mpi_variant}.sh",
osu_benchmark_version=OSU_BENCHMARK_VERSION,
)
result = slurm_commands.submit_script(str(submission_script))
job_id = slurm_commands.assert_job_submitted(result.stdout)
slurm_commands.wait_job_completed(job_id)
slurm_commands.assert_job_succeeded(job_id)
logging.info("Checking cluster has two nodes after running MPI jobs"
) # 1 static node + 1 dynamic node
assert_that(len(get_compute_nodes_instance_ids(cluster.cfn_name,
region))).is_equal_to(2)
def test_cluster_in_no_internet_subnet(
region,
scheduler,
pcluster_config_reader,
vpc_stack,
s3_bucket_factory,
clusters_factory,
test_datadir,
architecture,
os,
mpi_variants,
bastion_instance,
):
"""This test creates a cluster in a subnet with no internet, run osu latency and checks that no failures occur."""
bucket_name = s3_bucket_factory()
_upload_pre_install_script(bucket_name, test_datadir)
vpc_default_security_group_id = get_default_vpc_security_group(
vpc_stack.cfn_outputs["VpcId"], region)
cluster_config = pcluster_config_reader(
vpc_default_security_group_id=vpc_default_security_group_id,
bucket_name=bucket_name,
architecture=architecture)
cluster = clusters_factory(cluster_config)
logging.info("Checking cluster has one static node")
assert_that(len(get_compute_nodes_instance_ids(cluster.cfn_name,
region))).is_equal_to(1)
remote_command_executor = RemoteCommandExecutor(cluster,
bastion=bastion_instance)
slurm_commands = SlurmCommands(remote_command_executor)
_check_no_internet_access(remote_command_executor)
_check_hostname(remote_command_executor)
_run_mpi_jobs(mpi_variants, remote_command_executor, test_datadir,
slurm_commands, cluster, region)
utils.check_pcluster_list_cluster_log_streams(cluster, os)
assert_no_errors_in_logs(remote_command_executor, scheduler)
logging.info(
"Checking compute node is scaled down after scaledown idle time")
wait_for_num_instances_in_cluster(cluster.cfn_name, region, 1)
def test_efa(
os,
region,
scheduler,
instance,
pcluster_config_reader,
clusters_factory,
test_datadir,
architecture,
network_interfaces_count,
):
"""
Test all EFA Features.
Grouped all tests in a single function so that cluster can be reused for all of them.
"""
# We collected OSU benchmarks results for c5n.18xlarge only.
osu_benchmarks_instances = ["c5n.18xlarge"]
# 4 instances are required to see performance differences in collective OSU benchmarks.
# 2 instances are enough for other EFA tests.
max_queue_size = 4 if instance in osu_benchmarks_instances else 2
slots_per_instance = fetch_instance_slots(region, instance)
head_node_instance = "c5n.18xlarge" if architecture == "x86_64" else "c6gn.16xlarge"
cluster_config = pcluster_config_reader(max_queue_size=max_queue_size, head_node_instance=head_node_instance)
cluster = clusters_factory(cluster_config)
remote_command_executor = RemoteCommandExecutor(cluster)
scheduler_commands = get_scheduler_commands(scheduler, remote_command_executor)
_test_efa_installation(scheduler_commands, remote_command_executor, efa_installed=True, partition="efa-enabled")
_test_mpi(remote_command_executor, slots_per_instance, scheduler, partition="efa-enabled")
logging.info("Running on Instances: {0}".format(get_compute_nodes_instance_ids(cluster.cfn_name, region)))
if instance in osu_benchmarks_instances:
benchmark_failures = []
mpi_versions = ["openmpi"]
if architecture == "x86_64":
mpi_versions.append("intelmpi")
# Run OSU benchmarks in efa-enabled queue.
for mpi_version in mpi_versions:
benchmark_failures.extend(
_test_osu_benchmarks_pt2pt(
mpi_version,
remote_command_executor,
scheduler_commands,
test_datadir,
instance,
slots_per_instance,
partition="efa-enabled",
)
)
benchmark_failures.extend(
_test_osu_benchmarks_collective(
mpi_version,
remote_command_executor,
scheduler_commands,
test_datadir,
instance,
num_of_instances=max_queue_size,
slots_per_instance=slots_per_instance,
partition="efa-enabled",
)
)
assert_that(benchmark_failures, description="Some OSU benchmarks are failing").is_empty()
if network_interfaces_count > 1:
_test_osu_benchmarks_multiple_bandwidth(
remote_command_executor, scheduler_commands, test_datadir, slots_per_instance, partition="efa-enabled"
)
_test_shm_transfer_is_enabled(scheduler_commands, remote_command_executor, partition="efa-enabled")
if instance == "p4d.24xlarge" and "centos" not in os:
_test_nccl_benchmarks(remote_command_executor, test_datadir, "openmpi", scheduler_commands)
assert_no_errors_in_logs(remote_command_executor, scheduler)
def assert_num_instances_in_cluster(cluster_name, region, desired):
assert_that(len(get_compute_nodes_instance_ids(
cluster_name, region))).is_equal_to(desired)
