def _get_xml_tree(self, xml_file):
with open(xml_file, mode='rb') as fin:
try:
tree = etree.parse(fin)
except etree.ParseError:
logging.critical("Something happened during the parsing of the XML file. Exiting...")
except:
logging.debug("XML file state %s" % fin)
logging.debug("thinking...")
sys.exit(1)
return tree
def convert_qstat_to_yaml(self, orig_file, out_file, write_method):
out_file = out_file.rsplit('/', 1)[1]
try:
tree = etree.parse(orig_file)
except etree.ParseError:
logging.critical("This is an XML parse error (??)")
raise
except IOError:
raise
except:
print "File %(filename)s does not appear to contain a proper XML structure. Exiting.." % {
"filename": orig_file
}
raise
else:
root = tree.getroot()
# for queue_elem in root.iter('Queue-List'): # 2.7 only
for queue_elem in root.findall('queue_info/Queue-List'):
# queue_name = queue_elem.find('./resource[@name="qname"]').text # 2.7 only
queue_name_elems = queue_elem.findall('resource')
for queue_name_elem in queue_name_elems:
if queue_name_elem.attrib.get('name') == 'qname':
queue_name = queue_name_elem.text if not options.ANONYMIZE else self.anonymize(
queue_name_elem.text, 'qs')
# import wdb; wdb.set_trace()
break
else:
raise ValueError("No such queue name")
self._extract_job_info(
queue_elem, 'job_list',
queue_name=queue_name) # puts it into self.l
job_info_elem = root.find('./job_info')
if job_info_elem is None:
logging.debug('No pending jobs found!')
else:
self._extract_job_info(job_info_elem,
'job_list',
queue_name='Pending') # puts it into self.l
prefix, suffix = out_file.split('.')
prefix += '_'
suffix = '.' + suffix
SGEStatMaker.fd, SGEStatMaker.temp_filepath = get_new_temp_file(
prefix=prefix, suffix=suffix, config=self.config)
self.dump_all(SGEStatMaker.fd, self.stat_mapping[write_method])
def extract_qstat(self, orig_file):
all_values = list()
try:
tree = etree.parse(orig_file)
except etree.ParseError:
logging.critical("This is an XML parse error (??)")
raise
except IOError:
raise
except:
print "File %(filename)s does not appear to contain a proper XML structure. Exiting.." % {"filename": orig_file}
raise
else:
root = tree.getroot()
for queue_elem in root.findall('queue_info/Queue-List'):
queue_name_elems = queue_elem.findall('resource')
for queue_name_elem in queue_name_elems:
if queue_name_elem.attrib.get('name') == 'qname':
queue_name_elem.text = queue_name_elem.text if not options.ANONYMIZE else self.anonymize(queue_name_elem.text, 'qs')
break
else:
raise ValueError("No such queue name")
try:
all_values = self._extract_job_info(all_values, queue_elem, 'job_list', queue_name=queue_name_elem.text)
except ValueError:
logging.info('No jobs found in XML file!')
job_info_elem = root.find('./job_info')
if job_info_elem is None:
logging.debug('No pending jobs found!')
else:
try:
all_values = self._extract_job_info(all_values, job_info_elem, 'job_list', queue_name='Pending')
except ValueError:
logging.info('No jobs found in XML file!')
if options.SAMPLE >= 1:
tree.write(orig_file) # TODO anonymize rest of the sensitive information within xml file
return all_values
def convert_qstat_to_yaml(self, orig_file, out_file, write_method):
out_file = out_file.rsplit('/', 1)[1]
try:
tree = etree.parse(orig_file)
except etree.ParseError:
logging.critical("This is an XML parse error (??)")
raise
except IOError:
raise
except:
print("File %(filename)s does not appear to contain a proper XML structure. Exiting.." % {"filename": orig_file})
raise
else:
root = tree.getroot()
# for queue_elem in root.iter('Queue-List'): # 2.7 only
for queue_elem in root.findall('queue_info/Queue-List'):
# queue_name = queue_elem.find('./resource[@name="qname"]').text # 2.7 only
queue_name_elems = queue_elem.findall('resource')
for queue_name_elem in queue_name_elems:
if queue_name_elem.attrib.get('name') == 'qname':
queue_name = queue_name_elem.text if not options.ANONYMIZE else self.anonymize(queue_name_elem.text, 'qs')
# import wdb; wdb.set_trace()
break
else:
raise ValueError("No such queue name")
self._extract_job_info(queue_elem, 'job_list', queue_name=queue_name) # puts it into self.l
job_info_elem = root.find('./job_info')
if job_info_elem is None:
logging.debug('No pending jobs found!')
else:
self._extract_job_info(job_info_elem, 'job_list', queue_name='Pending') # puts it into self.l
prefix, suffix = out_file.split('.')
prefix += '_'
suffix = '.' + suffix
SGEStatMaker.fd, SGEStatMaker.temp_filepath = get_new_temp_file(prefix=prefix, suffix=suffix, config=self.config)
self.dump_all(SGEStatMaker.fd, self.stat_mapping[write_method])
def _get_statq_from_xml(fn, write_method=options.write_method):
logging.debug("Parsing tree of %s" % fn)
check_empty_file(fn)
anonymize = anonymize_func()
with open(fn, mode='rb') as fin:
try:
tree = etree.parse(fin)
except etree.ParseError:
logging.critical("Something happened during the parsing of the XML file. Exiting...")
except:
logging.debug("XML file state %s" % fin)
logging.debug("thinking...")
import sys
sys.exit(1)
root = tree.getroot()
qstatq_list = []
# for queue_elem in root.iter('Queue-List'): # python 2.7-only
for queue_elem in root.findall('queue_info/Queue-List'):
# queue_name = queue_elem.find('./resource[@name="qname"]').text # python 2.7-only
queue_names = queue_elem.findall('resource')
for _queue_name in queue_names:
if _queue_name.attrib.get('name') == 'qname':
queue_name = _queue_name.text if not options.ANONYMIZE else anonymize(_queue_name.text, 'qs')
break
else:
raise ValueError("No such resource")
FOUND = False
for exist_d in qstatq_list:
if queue_name == exist_d['queue_name']:
# exist_d['run'] += len(queue_elem.findall('./job_list[@state="running"]')) # python 2.7 only
jobs = queue_elem.findall('job_list')
run_count = 0
for _run in jobs:
if _run.attrib.get('state') == 'running':
run_count += 1
exist_d['run'] += run_count
FOUND = True
break
if FOUND:
continue
d = dict()
d['queue_name'] = queue_name
try:
d['state'] = queue_elem.find('./state').text
except AttributeError:
d['state'] = '?'
except:
raise
# d['run'] = len(queue_elem.findall('./job_list[@state="running"]')) # python 2.7 only
job_lists = queue_elem.findall('job_list')
run_count = 0
for _run in job_lists:
if _run.attrib.get('state') == 'running':
run_count += 1
d['run'] = run_count
d['lm'] = 0
d['queued'] = 0
qstatq_list.append(d)
total_running_jobs = str(sum([d['run'] for d in qstatq_list]))
logging.info('Total running jobs found: %s' % total_running_jobs)
for d in qstatq_list:
d['run'] = str(d['run'])
d['queued'] = str(d['queued'])
# total_queued_jobs = str(len(root.findall('.//job_list[@state="pending"]'))) # python 2.7 only
total_queued_jobs_elems = root.findall('job_info/job_list')
pending_count = 0
for job in total_queued_jobs_elems:
if job.attrib.get('state') == 'pending':
pending_count += 1
total_queued_jobs = str(pending_count)
logging.info('Total queued jobs found: %s' % total_queued_jobs)
qstatq_list.append({'run': '0', 'queued': total_queued_jobs, 'queue_name': 'Pending', 'state': 'Q', 'lm': '0'})
logging.debug('qstatq_list contains %s elements' % len(qstatq_list))
# TODO: check validity. 'state' shouldnt just be 'Q'!
logging.debug("Closing %s" % fn)
return total_running_jobs, total_queued_jobs, qstatq_list
