def _get_pbsnodes_values(orig_file, out_file, write_method):
check_empty_file(orig_file)
raw_blocks = _read_all_blocks(orig_file)
all_pbs_values = []
anonymize = anonymize_func()
for block in raw_blocks:
pbs_values = dict()
pbs_values['domainname'] = block['domainname'] if not options.ANONYMIZE else anonymize(block['domainname'], 'wns')
nextchar = block['state'][0]
state = (nextchar == 'f') and "-" or nextchar
pbs_values['state'] = state
try:
pbs_values['np'] = block['np']
except KeyError:
pbs_values['np'] = block['pcpus'] # handle torque cases # todo : to check
if block.get('gpus') > 0: # this should be rare.
pbs_values['gpus'] = block['gpus']
try: # this should turn up more often, hence the try/except.
_ = block['jobs']
except KeyError:
pass
else:
pbs_values['core_job_map'] = []
jobs = block['jobs'].split(',')
for job, core in _get_jobs_cores(jobs):
_d = dict()
_d['job'] = job
_d['core'] = core
pbs_values['core_job_map'].append(_d)
finally:
all_pbs_values.append(pbs_values)
return all_pbs_values
def _get_pbsnodes_values(orig_file, out_file, write_method):
check_empty_file(orig_file)
raw_blocks = _read_all_blocks(orig_file)
all_pbs_values = []
anonymize = anonymize_func()
for block in raw_blocks:
pbs_values = dict()
pbs_values['domainname'] = block[
'domainname'] if not options.ANONYMIZE else anonymize(
block['domainname'], 'wns')
nextchar = block['state'][0]
state = (nextchar == 'f') and "-" or nextchar
pbs_values['state'] = state
try:
pbs_values['np'] = block['np']
except KeyError:
pbs_values['np'] = block[
'pcpus'] # handle torque cases # todo : to check
if block.get('gpus') > 0: # this should be rare.
pbs_values['gpus'] = block['gpus']
try: # this should turn up more often, hence the try/except.
_ = block['jobs']
except KeyError:
pass
else:
pbs_values['core_job_map'] = []
jobs = block['jobs'].split(',')
for job, core in _get_jobs_cores(jobs):
_d = dict()
_d['job'] = job
_d['core'] = core
pbs_values['core_job_map'].append(_d)
finally:
all_pbs_values.append(pbs_values)
return all_pbs_values
def _get_statq_from_xml(fn, write_method=options.write_method):
logging.debug("Parsing tree of %s" % fn)
check_empty_file(fn)
anonymize = anonymize_func()
with open(fn, mode='rb') as fin:
try:
tree = etree.parse(fin)
except etree.ParseError:
logging.critical("Something happened during the parsing of the XML file. Exiting...")
except:
logging.debug("XML file state %s" % fin)
logging.debug("thinking...")
import sys
sys.exit(1)
root = tree.getroot()
qstatq_list = []
# for queue_elem in root.iter('Queue-List'): # python 2.7-only
for queue_elem in root.findall('queue_info/Queue-List'):
# queue_name = queue_elem.find('./resource[@name="qname"]').text # python 2.7-only
queue_names = queue_elem.findall('resource')
for _queue_name in queue_names:
if _queue_name.attrib.get('name') == 'qname':
queue_name = _queue_name.text if not options.ANONYMIZE else anonymize(_queue_name.text, 'qs')
break
else:
raise ValueError("No such resource")
FOUND = False
for exist_d in qstatq_list:
if queue_name == exist_d['queue_name']:
# exist_d['run'] += len(queue_elem.findall('./job_list[@state="running"]')) # python 2.7 only
jobs = queue_elem.findall('job_list')
run_count = 0
for _run in jobs:
if _run.attrib.get('state') == 'running':
run_count += 1
exist_d['run'] += run_count
FOUND = True
break
if FOUND:
continue
d = dict()
d['queue_name'] = queue_name
try:
d['state'] = queue_elem.find('./state').text
except AttributeError:
d['state'] = '?'
except:
raise
# d['run'] = len(queue_elem.findall('./job_list[@state="running"]')) # python 2.7 only
job_lists = queue_elem.findall('job_list')
run_count = 0
for _run in job_lists:
if _run.attrib.get('state') == 'running':
run_count += 1
d['run'] = run_count
d['lm'] = 0
d['queued'] = 0
qstatq_list.append(d)
total_running_jobs = str(sum([d['run'] for d in qstatq_list]))
logging.info('Total running jobs found: %s' % total_running_jobs)
for d in qstatq_list:
d['run'] = str(d['run'])
d['queued'] = str(d['queued'])
# total_queued_jobs = str(len(root.findall('.//job_list[@state="pending"]'))) # python 2.7 only
total_queued_jobs_elems = root.findall('job_info/job_list')
pending_count = 0
for job in total_queued_jobs_elems:
if job.attrib.get('state') == 'pending':
pending_count += 1
total_queued_jobs = str(pending_count)
logging.info('Total queued jobs found: %s' % total_queued_jobs)
qstatq_list.append({'run': '0', 'queued': total_queued_jobs, 'queue_name': 'Pending', 'state': 'Q', 'lm': '0'})
logging.debug('qstatq_list contains %s elements' % len(qstatq_list))
# TODO: check validity. 'state' shouldnt just be 'Q'!
logging.debug("Closing %s" % fn)
return total_running_jobs, total_queued_jobs, qstatq_list
def _calc_everything(fn, write_method):
logging.debug('Parsing tree of %s' % fn)
anonymize = anonymize_func()
with open(fn, 'rb') as fin:
tree = etree.parse(fin)
root = tree.getroot()
worker_nodes = list()
existing_node_names = set()
# for queue_elem in root.iter('Queue-List'): # 2.7-only
for queue_elem in root.findall('queue_info/Queue-List'):
worker_node = dict()
# worker_node['domainname'] = queue_elem.find('./resource[@name="hostname"]').text.split('.', 1)[0] # 2.7 only
resources = queue_elem.findall('resource')
# TODO: find a way to loop ONCE for both hostname and qname!!
try:
slots_used = int(queue_elem.find('./slots_used').text)
except AttributeError:
slots_used = 0
count = 0
# worker_node.setdefault('qname', [])
for resource in resources:
if resource.attrib.get('name') == 'hostname':
worker_node['domainname'] = resource.text if not options.ANONYMIZE else anonymize(resource.text, 'wns')
count += 1
if count == 2: break
elif resource.attrib.get('name') == 'qname':
if not slots_used:
worker_node['qname'] = set()
else:
worker_node['qname'] = set(resource.text[0]) \
if not options.ANONYMIZE else set(anonymize(resource.text[0], 'qs'))
count += 1
if count == 2: break
else:
raise ValueError("No such resource")
# worker_node['np'] = queue_elem.find('./resource[@name="num_proc"]').text # python 2.7 only
resources = queue_elem.findall('resource')
for resource in resources:
if resource.attrib.get('name') == 'num_proc':
worker_node['np'] = resource.text
break
else:
# TODO: check this for bugs, maybe raise an exception in the future?
worker_node['np'] = 0
try:
state = queue_elem.find('state').text
except AttributeError:
worker_node['state'] = '-'
else:
worker_node['state'] = state
if worker_node['domainname'] not in existing_node_names:
job_ids, usernames, job_states = extract_job_info(queue_elem, 'job_list')
worker_node['core_job_map'] = [{'core': idx, 'job': job_id} for idx, job_id in enumerate(job_ids)]
worker_node['existing_busy_cores'] = len(worker_node['core_job_map'])
existing_node_names.update([worker_node['domainname']])
worker_nodes.append(worker_node)
else:
for existing_wn in worker_nodes:
if worker_node['domainname'] != existing_wn['domainname']:
continue
job_ids, usernames, job_states = extract_job_info(queue_elem, 'job_list')
core_jobs = [{'core': idx, 'job': job_id}
for idx, job_id in enumerate(job_ids, existing_wn['existing_busy_cores'])]
existing_wn['core_job_map'].extend(core_jobs)
# don't change the node state to free.
# Just keep the state reported in the last queue mentioning the node.
existing_wn['state'] = (worker_node['state'] == '-') and existing_wn['state'] or worker_node['state']
existing_wn['qname'].update(worker_node['qname'])
break
logging.debug('Closing %s' % fn)
logging.info('worker_nodes contains %s entries' % len(worker_nodes))
return worker_nodes
