def test_generate_slurm_conf(self):
writer = cStringIO.StringIO()
partitions = OrderedDict()
partitions["hpc"] = Partition("custom_partition_name",
"hpc",
"Standard_D2_v2",
is_default=True,
is_hpc=True,
max_scaleset_size=3,
vcpu_count=2,
memory=4,
max_vm_count=8)
partitions["htc"] = Partition("htc",
"htc",
"Standard_D2_v3",
is_default=False,
max_scaleset_size=100,
is_hpc=False,
vcpu_count=2,
memory=3.5,
max_vm_count=8)
partitions["hpc"].node_list = "hpc-10[1-8]"
partitions["htc"].node_list = "htc-[1-8]"
with MockSubprocessModule() as mock_subprocess:
mock_subprocess.expect([
'scontrol', 'show', 'hostnames', 'hpc-10[1-8]'
], "hpc-101 hpc-102 hpc-103 hpc-104 hpc-105 hpc-106 hpc-107 hpc-108"
)
mock_subprocess.expect(
['scontrol', 'show', 'hostlist', "hpc-101,hpc-102,hpc-103"],
'hpc-10[1-3]')
mock_subprocess.expect(
['scontrol', 'show', 'hostlist', "hpc-104,hpc-105,hpc-106"],
'hpc-10[4-6]')
mock_subprocess.expect(
['scontrol', 'show', 'hostlist', "hpc-107,hpc-108"],
'hpc-10[7-8]')
mock_subprocess.expect(
['scontrol', 'show', 'hostnames', 'htc-[1-8]'],
"htc-1 htc-2 htc-3 htc-4 htc-5 htc-6 htc-7 htc-8")
mock_subprocess.expect([
'scontrol', 'show', 'hostlist',
"htc-1,htc-2,htc-3,htc-4,htc-5,htc-6,htc-7,htc-8"
], 'htc-[1-8]')
cyclecloud_slurm._generate_slurm_conf(partitions, writer,
mock_subprocess)
result = writer.getvalue().strip()
expected = '''
PartitionName=custom_partition_name Nodes=hpc-10[1-8] Default=YES MaxTime=INFINITE State=UP
Nodename=hpc-10[1-3] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=4096
Nodename=hpc-10[4-6] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=4096
Nodename=hpc-10[7-8] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=4096
PartitionName=htc Nodes=htc-[1-8] Default=NO MaxTime=INFINITE State=UP
Nodename=htc-[1-8] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=3584'''.strip()
for e, a in zip(result.splitlines(), expected.splitlines()):
assert e == a, "\n%s\n%s" % (e, a)
def test_generate_slurm_conf(self):
writer = cStringIO.StringIO()
partitions = OrderedDict()
# use dampen_memory = .02 to test overriding this
partitions["hpc"] = Partition("custom_partition_name",
"hpc",
"Standard_D2_v2",
is_default=True,
is_hpc=True,
max_scaleset_size=3,
vcpu_count=2,
memory=128,
max_vm_count=8,
dampen_memory=.02)
partitions["htc"] = Partition("htc",
"htc",
"Standard_D2_v3",
is_default=False,
max_scaleset_size=100,
is_hpc=False,
vcpu_count=2,
memory=3.5,
max_vm_count=8)
partitions["hpc"].node_list = "hpc-10[1-8]"
partitions["htc"].node_list = "htc-[1-8]"
with MockSubprocessModule() as mock_subprocess:
mock_subprocess.expect([
'scontrol', 'show', 'hostnames', 'hpc-10[1-8]'
], "hpc-101 hpc-102 hpc-103 hpc-104 hpc-105 hpc-106 hpc-107 hpc-108"
)
mock_subprocess.expect(
['scontrol', 'show', 'hostlist', "hpc-101,hpc-102,hpc-103"],
'hpc-10[1-3]')
mock_subprocess.expect(
['scontrol', 'show', 'hostlist', "hpc-104,hpc-105,hpc-106"],
'hpc-10[4-6]')
mock_subprocess.expect(
['scontrol', 'show', 'hostlist', "hpc-107,hpc-108"],
'hpc-10[7-8]')
mock_subprocess.expect(
['scontrol', 'show', 'hostnames', 'htc-[1-8]'],
"htc-1 htc-2 htc-3 htc-4 htc-5 htc-6 htc-7 htc-8")
mock_subprocess.expect([
'scontrol', 'show', 'hostlist',
"htc-1,htc-2,htc-3,htc-4,htc-5,htc-6,htc-7,htc-8"
], 'htc-[1-8]')
cyclecloud_slurm._generate_slurm_conf(partitions, writer,
mock_subprocess)
result = writer.getvalue().strip()
expected = '''
# Note: CycleCloud reported a RealMemory of 131072 but we reduced it by 2621 (i.e. max(1gb, 2%)) to account for OS/VM overhead which
# would result in the nodes being rejected by Slurm if they report a number less than defined here.
# To pick a different percentage to dampen, set slurm.dampen_memory=X in the nodearray's Configuration where X is percentage (5 = 5%).
PartitionName=custom_partition_name Nodes=hpc-10[1-8] Default=YES DefMemPerCPU=64225 MaxTime=INFINITE State=UP
Nodename=hpc-10[1-3] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=128450
Nodename=hpc-10[4-6] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=128450
Nodename=hpc-10[7-8] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=128450
# Note: CycleCloud reported a RealMemory of 3584 but we reduced it by 1024 (i.e. max(1gb, 5%)) to account for OS/VM overhead which
# would result in the nodes being rejected by Slurm if they report a number less than defined here.
# To pick a different percentage to dampen, set slurm.dampen_memory=X in the nodearray's Configuration where X is percentage (5 = 5%).
PartitionName=htc Nodes=htc-[1-8] Default=NO DefMemPerCPU=1280 MaxTime=INFINITE State=UP
Nodename=htc-[1-8] Feature=cloud STATE=CLOUD CPUs=2 RealMemory=2560'''.strip()
for e, a in zip(result.splitlines(), expected.splitlines()):
assert e == a, "\n%s\n%s" % (e, a)
def test_create_nodes(self):
partitions = {}
partitions["hpc"] = Partition("hpc",
"hpc",
"Standard_D2_v2",
is_default=True,
is_hpc=True,
max_scaleset_size=3,
vcpu_count=2,
memory=4,
max_vm_count=8)
partitions["htc"] = Partition("htc",
"htc",
"Standard_D2_v2",
is_default=False,
is_hpc=False,
max_scaleset_size=100,
vcpu_count=2,
memory=4,
max_vm_count=8)
mock_cluster = MockClusterModule()
cluster_wrapper = clusterwrapper.ClusterWrapper(
mock_cluster.name, DoNotUse(), DoNotUse(), mock_cluster)
subprocess_module = MockSubprocessModule()
# create 3 placement groups for hpc with sizes 3,3,2 (3 + 3 + 2 = 8, which is the max_vm_count_
# al
expected_request = {
'sets': [{
'count': 3,
'nameFormat': 'hpc-pg0-%d',
'nameOffset': 1,
'definition': {
'machineType': 'Standard_D2_v2'
},
'nodeAttributes': {
'StartAutomatically': False,
'Fixed': True
},
'nodearray': 'hpc',
'placementGroupId': 'hpc-Standard_D2_v2-pg0'
}, {
'count': 3,
'nameFormat': 'hpc-pg1-%d',
'nameOffset': 1,
'definition': {
'machineType': 'Standard_D2_v2'
},
'nodeAttributes': {
'StartAutomatically': False,
'Fixed': True
},
'nodearray': 'hpc',
'placementGroupId': 'hpc-Standard_D2_v2-pg1'
}, {
'count': 2,
'nameFormat': 'hpc-pg2-%d',
'nameOffset': 1,
'definition': {
'machineType': 'Standard_D2_v2'
},
'nodeAttributes': {
'StartAutomatically': False,
'Fixed': True
},
'nodearray': 'hpc',
'placementGroupId': 'hpc-Standard_D2_v2-pg2'
}, {
'count': 8,
'nameFormat': 'htc-%d',
'nameOffset': 1,
'definition': {
'machineType': 'Standard_D2_v2'
},
'nodeAttributes': {
'StartAutomatically': False,
'Fixed': True
},
'nodearray': 'htc'
}]
}
mock_cluster.expected_create_nodes_request = expected_request
cyclecloud_slurm._create_nodes(partitions, cluster_wrapper,
subprocess_module)
# ok now bump max vm count 1 and try recreating.
partitions["hpc"].max_vm_count += 1
partitions[
"hpc"].node_list = "hpc-pg0-[1-3],hpc-pg1-[1-3],hpc-pg2-[1-2]"
for _ in range(3):
subprocess_module.expect([
'scontrol', 'show', 'hostnames',
'hpc-pg0-[1-3],hpc-pg1-[1-3],hpc-pg2-[1-2]'
], " ".join([
"hpc-pg0-1", "hpc-pg0-2", "hpc-pg0-3", "hpc-pg1-1",
"hpc-pg1-2", "hpc-pg1-3", "hpc-pg2-1", "hpc-pg2-2"
]))
# fails because existing node policy is default, error
self.assertRaises(
CyclecloudSlurmError, lambda: cyclecloud_slurm._create_nodes(
partitions,
cluster_wrapper,
subprocess_module,
existing_policy=ExistingNodePolicy.Error))
# succeeds because existing node policy is AllowExisting, so we fill in / add nodes
expected_request["sets"] = [{
'count':
1,
'nameFormat':
'hpc-pg2-%d',
'nameOffset':
3,
'definition': {
'machineType': 'Standard_D2_v2'
},
'nodeAttributes': {
'StartAutomatically': False,
'Fixed': True
},
'nodearray':
'hpc',
'placementGroupId':
'hpc-Standard_D2_v2-pg2'
}, {
'count': 8,
'nameFormat': 'htc-%d',
'nameOffset': 1,
'definition': {
'machineType': 'Standard_D2_v2'
},
'nodeAttributes': {
'StartAutomatically': False,
'Fixed': True
},
'nodearray': 'htc'
}]
cyclecloud_slurm._create_nodes(
partitions,
cluster_wrapper,
subprocess_module,
existing_policy=ExistingNodePolicy.AllowExisting)