def test_partition_delete():
"""Partition: Test partition().delete()."""
rc = pyslurm.partition().delete("part_test")
assert rc == 0
partition_ids = pyslurm.partition().ids()
assert "part_test" not in partition_ids
def test_partition_scontrol():
"""Partition: Compare scontrol values to PySlurm values."""
all_partition_ids = pyslurm.partition().ids()
test_partition = all_partition_ids[0]
test_partition_info = pyslurm.partition().find_id(test_partition)
assert test_partition == test_partition_info["name"]
sctl_dict = scontrol_show("partition", str(test_partition))
assert test_partition_info["allow_alloc_nodes"] == sctl_dict["AllocNodes"]
assert test_partition_info["allow_accounts"] == sctl_dict["AllowAccounts"]
assert test_partition_info["allow_groups"] == sctl_dict["AllowGroups"]
assert test_partition_info["allow_qos"] == sctl_dict["AllowQos"]
# assert test_partition_info["def_mem_per_node"] == sctl_dict["DefMemPerNode"]
assert test_partition_info["default_time_str"] == sctl_dict["DefaultTime"]
assert test_partition_info["grace_time"] == int(sctl_dict["GraceTime"])
assert test_partition_info["max_cpus_per_node"] == sctl_dict[
"MaxCPUsPerNode"]
assert test_partition_info["max_mem_per_node"] == sctl_dict[
"MaxMemPerNode"]
assert test_partition_info["max_nodes"] == int(sctl_dict["MaxNodes"])
assert test_partition_info["max_time_str"] == sctl_dict["MaxTime"]
assert test_partition_info["min_nodes"] == int(sctl_dict["MinNodes"])
assert test_partition_info["nodes"] == sctl_dict["Nodes"]
assert test_partition_info["name"] == sctl_dict["PartitionName"]
# assert test_partition_info["preempt_mode"] == sctl_dict["PreemptMode"]
assert test_partition_info["state"] == sctl_dict["State"]
assert test_partition_info["total_cpus"] == int(sctl_dict["TotalCPUs"])
assert test_partition_info["total_nodes"] == int(sctl_dict["TotalNodes"])
def test_partition_delete():
"""Partition: Test partition().delete()."""
rc = pyslurm.partition().delete("part_test")
assert_equals(rc, 0)
partition_ids = pyslurm.partition().ids()
assert_true("part_test" not in partition_ids)
def test_partition_scontrol():
"""Partition: Compare scontrol values to PySlurm values."""
all_partition_ids = pyslurm.partition().ids()
test_partition = all_partition_ids[0]
test_partition_info = pyslurm.partition().find_id(test_partition)
assert_equals(test_partition, test_partition_info["name"])
sctl = subprocess.Popen(["scontrol", "-d", "show", "partition", str(test_partition)],
stdout=subprocess.PIPE).communicate()
sctl_stdout = sctl[0].strip().decode("UTF-8").split()
sctl_dict = dict((value.split("=")[0], value.split("=")[1])
for value in sctl_stdout)
assert_equals(test_partition_info["allow_alloc_nodes"], sctl_dict["AllocNodes"])
assert_equals(test_partition_info["allow_accounts"], sctl_dict["AllowAccounts"])
assert_equals(test_partition_info["allow_groups"], sctl_dict["AllowGroups"])
assert_equals(test_partition_info["allow_qos"], sctl_dict["AllowQos"])
assert_equals(test_partition_info["def_mem_per_cpu"], int(sctl_dict["DefMemPerCPU"]))
assert_equals(test_partition_info["default_time_str"], sctl_dict["DefaultTime"])
assert_equals(test_partition_info["grace_time"], int(sctl_dict["GraceTime"]))
assert_equals(test_partition_info["max_cpus_per_node"], sctl_dict["MaxCPUsPerNode"])
assert_equals(test_partition_info["max_mem_per_node"], sctl_dict["MaxMemPerNode"])
assert_equals(test_partition_info["max_nodes"], int(sctl_dict["MaxNodes"]))
assert_equals(test_partition_info["max_time_str"], sctl_dict["MaxTime"])
assert_equals(test_partition_info["min_nodes"], int(sctl_dict["MinNodes"]))
assert_equals(test_partition_info["nodes"], sctl_dict["Nodes"])
assert_equals(test_partition_info["name"], sctl_dict["PartitionName"])
assert_equals(test_partition_info["preempt_mode"], sctl_dict["PreemptMode"])
assert_equals(test_partition_info["state"], sctl_dict["State"])
assert_equals(test_partition_info["total_cpus"], int(sctl_dict["TotalCPUs"]))
assert_equals(test_partition_info["total_nodes"], int(sctl_dict["TotalNodes"]))
def test_partition_delete():
"""Partition: Test partition().delete()."""
rc = pyslurm.partition().delete("part_test")
assert_equals(rc, 0)
partition_ids = pyslurm.partition().ids()
assert_true("part_test" not in partition_ids)
def test_partition_create():
"""Partition: Test partition().create()."""
part_test = {"Name": "part_test"}
rc = pyslurm.partition().create(part_test)
assert rc == 0
partition_ids = pyslurm.partition().ids()
assert "part_test" in partition_ids
def test_partition_create():
"""Partition: Test partition().create()."""
part_test = {"Name": "part_test"}
rc = pyslurm.partition().create(part_test)
assert_equals(rc, 0)
partition_ids = pyslurm.partition().ids()
assert_true("part_test" in partition_ids)
def test_partition_create():
"""Partition: Test partition().create()."""
part_test = {"Name": "part_test"}
rc = pyslurm.partition().create(part_test)
assert_equals(rc, 0)
partition_ids = pyslurm.partition().ids()
assert_true("part_test" in partition_ids)
def test_partition_update():
"""Partition: Test partition().update()."""
part_test_before = pyslurm.partition().find_id("part_test")
assert_equals(part_test_before["state"], "UP")
part_test_update = {"Name": "part_test", "State": "DOWN"}
rc = pyslurm.partition().update(part_test_update)
assert_equals(rc, 0)
part_test_after = pyslurm.partition().find_id("part_test")
assert_equals(part_test_after["state"], "DOWN")
def test_partition_update():
"""Partition: Test partition().update()."""
part_test_before = pyslurm.partition().find_id("part_test")
assert part_test_before["state"] == "UP"
part_test_update = {"Name": "part_test", "State": "DOWN"}
rc = pyslurm.partition().update(part_test_update)
assert rc == 0
part_test_after = pyslurm.partition().find_id("part_test")
assert part_test_after["state"] == "DOWN"
def test_partition_update():
"""Partition: Test partition().update()."""
part_test_before = pyslurm.partition().find_id("part_test")
assert_equals(part_test_before["state"], "UP")
part_test_update = {"Name": "part_test", "State": "DOWN"}
rc = pyslurm.partition().update(part_test_update)
assert_equals(rc, 0)
part_test_after = pyslurm.partition().find_id("part_test")
assert_equals(part_test_after["state"], "DOWN")
def test_partition_scontrol():
all_partition_ids = pyslurm.partition().ids()
test_partition = all_partition_ids[0]
#assert type(test_partition) is IntType
test_partition_info = pyslurm.partition().find_id(test_partition)
assert test_partition == test_partition_info["name"]
scontrol = subprocess.Popen(["scontrol", "-d", "show", "partition",
str(test_partition)],
stdout=subprocess.PIPE).communicate()
scontrol_stdout = scontrol[0].strip().split()
scontrol_dict = {value.split("=")[0]: value.split("=")[1]
for value in scontrol_stdout}
assert test_partition_info["allow_alloc_nodes"] == scontrol_dict["AllocNodes"]
assert test_partition_info["allow_accounts"] == scontrol_dict["AllowAccounts"]
assert test_partition_info["allow_groups"] == scontrol_dict["AllowGroups"]
assert test_partition_info["allow_qos"] == scontrol_dict["AllowQos"]
# 'DefMemPerCpu': 'UNLIMITED',
assert test_partition_info["def_mem_per_node"] == scontrol_dict["DefMemPerNode"]
# 'Default': 'YES',
assert test_partition_info["default_time_str"] == scontrol_dict["DefaultTime"]
# 'DisableRootJobs': 'NO',
# 'ExclusiveUser': '******',
assert test_partition_info["grace_time"] == int(scontrol_dict["GraceTime"])
# 'Hidden': 'NO',
# 'LLN': 'NO',
assert test_partition_info["max_cpus_per_node"] == scontrol_dict["MaxCPUsPerNode"]
# 'MaxMemPerCpu': 'UNLIMITED',
assert test_partition_info["max_mem_per_node"] == scontrol_dict["MaxMemPerNode"]
assert test_partition_info["max_nodes"] == int(scontrol_dict["MaxNodes"])
assert test_partition_info["max_time_str"] == scontrol_dict["MaxTime"]
assert test_partition_info["min_nodes"] == int(scontrol_dict["MinNodes"])
assert test_partition_info["nodes"] == scontrol_dict["Nodes"]
assert test_partition_info["name"] == scontrol_dict["PartitionName"]
assert test_partition_info["preempt_mode"] == scontrol_dict["PreemptMode"]
assert test_partition_info["priority"] == int(scontrol_dict["Priority"])
# 'QoS': 'N/A',
# 'ReqResv': 'NO',
# 'RootOnly': 'NO',
# 'SelectTypeParameters': 'N/A',
assert test_partition_info["flags"]["Shared"] == scontrol_dict["Shared"]
assert test_partition_info["state"] == scontrol_dict["State"]
assert test_partition_info["total_cpus"] == int(scontrol_dict["TotalCPUs"])
assert test_partition_info["total_nodes"] == int(scontrol_dict["TotalNodes"])
def test_partition_scontrol():
"""Partition: Compare scontrol values to PySlurm values."""
all_partition_ids = pyslurm.partition().ids()
test_partition = all_partition_ids[0]
test_partition_info = pyslurm.partition().find_id(test_partition)
assert_equals(test_partition, test_partition_info["name"])
sctl = subprocess.Popen(
["scontrol", "-d", "show", "partition",
str(test_partition)],
stdout=subprocess.PIPE).communicate()
sctl_stdout = sctl[0].strip().decode("UTF-8").split()
sctl_dict = dict(
(value.split("=")[0], value.split("=")[1]) for value in sctl_stdout)
assert_equals(test_partition_info["allow_alloc_nodes"],
sctl_dict["AllocNodes"])
assert_equals(test_partition_info["allow_accounts"],
sctl_dict["AllowAccounts"])
assert_equals(test_partition_info["allow_groups"],
sctl_dict["AllowGroups"])
assert_equals(test_partition_info["allow_qos"], sctl_dict["AllowQos"])
assert_equals(test_partition_info["def_mem_per_cpu"],
int(sctl_dict["DefMemPerCPU"]))
assert_equals(test_partition_info["default_time_str"],
sctl_dict["DefaultTime"])
assert_equals(test_partition_info["grace_time"],
int(sctl_dict["GraceTime"]))
assert_equals(test_partition_info["max_cpus_per_node"],
sctl_dict["MaxCPUsPerNode"])
assert_equals(test_partition_info["max_mem_per_node"],
sctl_dict["MaxMemPerNode"])
assert_equals(test_partition_info["max_nodes"], int(sctl_dict["MaxNodes"]))
assert_equals(test_partition_info["max_time_str"], sctl_dict["MaxTime"])
assert_equals(test_partition_info["min_nodes"], int(sctl_dict["MinNodes"]))
assert_equals(test_partition_info["nodes"], sctl_dict["Nodes"])
assert_equals(test_partition_info["name"], sctl_dict["PartitionName"])
assert_equals(test_partition_info["preempt_mode"],
sctl_dict["PreemptMode"])
assert_equals(test_partition_info["state"], sctl_dict["State"])
assert_equals(test_partition_info["total_cpus"],
int(sctl_dict["TotalCPUs"]))
assert_equals(test_partition_info["total_nodes"],
int(sctl_dict["TotalNodes"]))
def sinfo():
# Partition and node lists are required
# to compute sinfo informations
partitions = get_from_cache(pyslurm.partition().get, 'get_partitions')
nodes = get_from_cache(pyslurm.node().get, 'get_nodes')
# Retreiving the state of each nodes
nodes_state = dict(
(node.lower(), attributes['state'].lower())
for node, attributes in nodes.iteritems()
)
# For all partitions, retrieving the states of each nodes
sinfo_data = {}
for name, attr in partitions.iteritems():
for node in list(NodeSet(attr['nodes'])):
key = (name, nodes_state[node])
if key not in sinfo_data.keys():
sinfo_data[key] = []
sinfo_data[key].append(node)
# Preparing the response
resp = []
for k, nodes in sinfo_data.iteritems():
name, state = k
partition = partitions[name]
avail = partition['state'].lower()
min_nodes = partition['min_nodes']
max_nodes = partition['max_nodes']
total_nodes = partition['total_nodes']
job_size = "{0}-{1}".format(min_nodes, max_nodes)
job_size = job_size.replace('UNLIMITED', 'infinite')
time_limit = partition['max_time_str'].replace('UNLIMITED', 'infinite')
# Creating the nodeset
nodeset = NodeSet()
map(nodeset.update, nodes)
resp.append({
'name': name,
'avail': avail,
'job_size': job_size,
'time_limit': time_limit,
'nodes': total_nodes,
'state': state,
'nodelist': str(nodeset),
})
# Jsonify can not works on list, thus using json.dumps
# And making sure headers are properly set
return make_response(json.dumps(resp), mimetype='application/json')
def sinfo():
# Partition and node lists are required
# to compute sinfo informations
partitions = pyslurm.partition().get()
nodes = pyslurm.node().get()
# Retreiving the state of each nodes
nodes_state = dict(
(node.lower(), attributes['state'].lower())
for node, attributes in nodes.iteritems()
)
# For all partitions, retrieving the states of each nodes
sinfo_data = {}
for name, attr in partitions.iteritems():
for node in list(NodeSet(attr['nodes'])):
key = (name, nodes_state[node])
if key not in sinfo_data.keys():
sinfo_data[key] = []
sinfo_data[key].append(node)
# Preparing the response
resp = []
for k, nodes in sinfo_data.iteritems():
name, state = k
partition = partitions[name]
avail = partition['state'].lower()
min_nodes = partition['min_nodes']
max_nodes = partition['max_nodes']
total_nodes = partition['total_nodes']
job_size = "{0}-{1}".format(min_nodes, max_nodes)
job_size = job_size.replace('UNLIMITED', 'infinite')
time_limit = partition['max_time_str'].replace('UNLIMITED', 'infinite')
# Creating the nodeset
nodeset = NodeSet()
map(nodeset.update, nodes)
resp.append({
'name': name,
'avail': avail,
'job_size': job_size,
'time_limit': time_limit,
'nodes': total_nodes,
'state': state,
'nodelist': str(nodeset),
})
# Jsonify can not works on list, thus using json.dumps
# And making sure headers are properly set
return make_response(json.dumps(resp), mimetype='application/json')
def search(limit=100):
partitions = []
handler = pyslurm.partition()
try:
all_partitions = handler.get()
except Exception as e:
resp = {"code": 500, "message": str(e)}
return resp, 500
if len(all_partitions.values()) < 1:
return NoContent, 204
for this_part in all_partitions.values():
partitions.append(this_part)
return partitions[0:limit], 200
def get(id, limit=100):
handler = pyslurm.partition()
try:
all_partitions = handler.get()
except Exception as e:
resp = {"code": 500, "message": str(e)}
return resp, 500
all_partitions = dict(all_partitions)
if len(all_partitions.values()) < 1:
return NoContent, 204
for this_part in all_partitions.values():
if re.match(str(id), this_part['name']):
return dict(this_part), 200
return NoContent, 204
def collect(self):
# Metric declarations
PART_NODES = GaugeMetricFamily('slurm_partitions_total_nodes', 'Total numbers of nodes per partition grouped by {}'.format(', '.join(self.labels)), labels=self.labels)
PART_CPUS = GaugeMetricFamily('slurm_partitions_total_cpus', 'Total numbers of CPUs per partition grouped by {}'.format(', '.join(self.labels)), labels=self.labels)
PART_STATE = GaugeMetricFamily('slurm_partitions_state', 'Partition states grouped by {}'.format(', '.join(self.labels)), labels=self.labels)
# Load part info from Slurm
cluster = pyslurm.config().get()['cluster_name']
partitions = pyslurm.partition().get()
# Update the metrics
for partition in partitions.keys():
PART_NODES.add_metric([cluster, partition], partitions[partition]['total_nodes'])
PART_CPUS.add_metric( [cluster, partition], partitions[partition]['total_cpus'])
PART_STATE.add_metric([cluster, partition], int(partitions[partition]['state'] == 'UP'))
yield PART_NODES
yield PART_CPUS
yield PART_STATE
def run(self):
#pdb.set_trace()
logger.info("Start running PyslurmReader ...")
while True:
# pyslurm query
ts = int(datetime.now().timestamp())
job_dict = pyslurm.job().get()
node_dict= pyslurm.node().get()
part_dict= pyslurm.partition().get()
qos_dict = pyslurm.qos().get()
#res_dict = pyslurm.reservation().get()
res_dict = {} #TODO: pyslurm reservation coredump ERROR
#js_dict = pyslurm.jobstep().get()
#convert to points
points = []
for jid,job in job_dict.items():
self.slurmJob2point(ts, job, points)
finishJob = [jid for jid in self.sav_job_dict.keys() if jid not in job_dict.keys()]
#logger.debug ("PyslurmReader::run: Finish jobs {}".format(finishJob))
for jid in finishJob:
del self.sav_job_dict[jid]
for node in node_dict.values():
self.slurmNode2point(ts, node, points)
if json.dumps(part_dict) != json.dumps(self.sav_part_dict):
for pname, part in part_dict.items():
self.slurmPartition2point(ts, pname, part, points)
self.sav_part_dict = part_dict
if json.dumps(qos_dict) != json.dumps(self.sav_qos_dict):
for qname, qos in qos_dict.items():
self.slurmQOS2point(ts, qname, qos, points)
self.sav_qos_dict = qos_dict
if res_dict and (json.dumps(res_dict) != json.dumps(self.sav_res_dict)):
for rname, res in res_dict.items():
self.slurmReservation2point(ts, rname, res, points)
self.sav_res_dict = res_dict
with self.lock:
self.points.extend(points)
time.sleep (PyslurmReader.INTERVAL)
def get_partitions():
partitions = pyslurm.partition().get()
return partitions
def get_partitions():
partitions = pyslurm.partition().get()
return jsonify(partitions)
def get(self):
p = pyslurm.partition()
data = p.get()
return data
def get_partitions():
partitions = get_from_cache(pyslurm.partition().get, 'get_partitions')
return partitions
def get_partitions():
partitions = pyslurm.partition().get()
return partitions
def test_partition_count():
"""Partition: Test partition count."""
all_partitions = pyslurm.partition().get()
all_partition_ids = pyslurm.partition().ids()
assert_equals(len(all_partitions), len(all_partition_ids))
def test_partition_get():
all_partitions = pyslurm.partition().get()
assert type(all_partitions) is DictType
def test_partition_ids():
"""Partition: Test partition().ids() return type."""
all_partition_ids = pyslurm.partition().ids()
assert_true(isinstance(all_partition_ids, list))
def test_partition_count():
"""Partition: Test partition count."""
all_partitions = pyslurm.partition().get()
all_partition_ids = pyslurm.partition().ids()
assert_equals(len(all_partitions), len(all_partition_ids))
def test_partition_ids():
"""Partition: Test partition().ids() return type."""
all_partition_ids = pyslurm.partition().ids()
assert_true(isinstance(all_partition_ids, list))
def partitions():
return pyslurm.partition().get()
try:
pyslurmnode = pyslurm.node()
except:
sys.stderr.write('Failed to get Slurm data\n')
sys.exit(3)
if config['user_lookup']:
try:
ldap_c = ldap.initialize('ldaps://%s:636' % config['ldap_hostname'])
ldap_c.simple_bind_s(config['ldap_username'], config['ldap_password'])
except:
sys.stderr.write('Failed to bind to LDAP\n')
sys.exit(4)
groups = config['groups']
partitions = pyslurm.partition().get()
node_partitions = {}
metrics = {}
metrics['partition'] = {}
metrics['partition']['cpu_total'] = {}
metrics['partition']['cpu_usage'] = {}
metrics['partition']['cpu_usage_pc'] = {}
metrics['partition']['gpu_total'] = {}
metrics['partition']['gpu_usage'] = {}
metrics['partition']['gpu_usage_pc'] = {}
metrics['partition']['mem_total'] = {}
metrics['partition']['mem_usage'] = {}
metrics['partition']['mem_usage_pc'] = {}
metrics['partition']['jobs_running'] = {}
def test_partition_count():
all_partitions = pyslurm.partition().get()
all_partition_ids = pyslurm.partition().ids()
assert len(all_partitions) == len(all_partition_ids)
elif part_key in [ 'max_nodes', 'max_time', 'max_cpus_per_node']:
if value[part_key] == "UNLIMITED":
valStr = "Unlimited"
print("\t{0:<20} : {1}".format(part_key, valStr))
print('{0:*^80}'.format(''))
if __name__ == "__main__":
import pyslurm
import time
try:
a = pyslurm.partition()
part_dict = a.get()
except ValueError as e:
print("Partition error - {0}".format(e.args[0]))
else:
if len(part_dict) > 0:
display(part_dict)
print()
print("Partition IDs - {0}".format(a.ids()))
print()
else:
print("No partitions found !")
def test_partition_get():
"""Partition: Test partition().get() return type."""
all_partitions = pyslurm.partition().get()
assert_true(isinstance(all_partitions, dict))
def test_partition_ids():
all_partition_ids = pyslurm.partition().ids()
assert type(all_partition_ids) is ListType
def test_partition_get():
"""Partition: Test partition().get() return type."""
all_partitions = pyslurm.partition().get()
assert_true(isinstance(all_partitions, dict))
def _partitions(self):
"""Fetch fresh data."""
return pyslurm.partition().get()
def get_partitions():
if mocking:
return mock('partitions.json')
partitions = pyslurm.partition().get()
return partitions
xml_file.write(f"\t\t\t<{part_key[0]}>{part_key[1]}</{part_key[0]}>\n")
if primary and key in primary:
xml_file.write("\t\t\t<controller>Primary</controller>\n")
elif backup and key in backup:
xml_file.write("\t\t\t<controller>backup</controller>\n")
else:
xml_file.write("\t\t\t<controller></controller>\n")
xml_file.write("\t\t</node>\n")
xml_file.write("\t</nodes>\n")
###########################
# XML output of Partttions
###########################
a = pyslurm.partition()
part_dict = a.get()
xml_file.write("\t<partitions>\n")
for key, value in part_dict.items():
xml_file.write("\t\t<partition>\n")
for part_key, part_value in value.items():
xml_file.write(f"\t\t\t<{part_key}>{part_value}</{part_key}>\n")
xml_file.write("\t\t</partition>\n")
xml_file.write("\t</partitions>\n")
xml_file.write("</slurm>\n")
xml_file.flush()
xml_file.close()
def get_partitions():
if mocking:
return mock('partitions.json')
partitions = pyslurm.partition().get()
return partitions
