def _get_xml_tree(self, xml_file): with open(xml_file, mode='rb') as fin: try: tree = etree.parse(fin) except etree.ParseError: logging.critical("Something happened during the parsing of the XML file. Exiting...") except: logging.debug("XML file state %s" % fin) logging.debug("thinking...") sys.exit(1) return tree
def convert_qstat_to_yaml(self, orig_file, out_file, write_method): out_file = out_file.rsplit('/', 1)[1] try: tree = etree.parse(orig_file) except etree.ParseError: logging.critical("This is an XML parse error (??)") raise except IOError: raise except: print "File %(filename)s does not appear to contain a proper XML structure. Exiting.." % { "filename": orig_file } raise else: root = tree.getroot() # for queue_elem in root.iter('Queue-List'): # 2.7 only for queue_elem in root.findall('queue_info/Queue-List'): # queue_name = queue_elem.find('./resource[@name="qname"]').text # 2.7 only queue_name_elems = queue_elem.findall('resource') for queue_name_elem in queue_name_elems: if queue_name_elem.attrib.get('name') == 'qname': queue_name = queue_name_elem.text if not options.ANONYMIZE else self.anonymize( queue_name_elem.text, 'qs') # import wdb; wdb.set_trace() break else: raise ValueError("No such queue name") self._extract_job_info( queue_elem, 'job_list', queue_name=queue_name) # puts it into self.l job_info_elem = root.find('./job_info') if job_info_elem is None: logging.debug('No pending jobs found!') else: self._extract_job_info(job_info_elem, 'job_list', queue_name='Pending') # puts it into self.l prefix, suffix = out_file.split('.') prefix += '_' suffix = '.' + suffix SGEStatMaker.fd, SGEStatMaker.temp_filepath = get_new_temp_file( prefix=prefix, suffix=suffix, config=self.config) self.dump_all(SGEStatMaker.fd, self.stat_mapping[write_method])
def extract_qstat(self, orig_file): all_values = list() try: tree = etree.parse(orig_file) except etree.ParseError: logging.critical("This is an XML parse error (??)") raise except IOError: raise except: print "File %(filename)s does not appear to contain a proper XML structure. Exiting.." % {"filename": orig_file} raise else: root = tree.getroot() for queue_elem in root.findall('queue_info/Queue-List'): queue_name_elems = queue_elem.findall('resource') for queue_name_elem in queue_name_elems: if queue_name_elem.attrib.get('name') == 'qname': queue_name_elem.text = queue_name_elem.text if not options.ANONYMIZE else self.anonymize(queue_name_elem.text, 'qs') break else: raise ValueError("No such queue name") try: all_values = self._extract_job_info(all_values, queue_elem, 'job_list', queue_name=queue_name_elem.text) except ValueError: logging.info('No jobs found in XML file!') job_info_elem = root.find('./job_info') if job_info_elem is None: logging.debug('No pending jobs found!') else: try: all_values = self._extract_job_info(all_values, job_info_elem, 'job_list', queue_name='Pending') except ValueError: logging.info('No jobs found in XML file!') if options.SAMPLE >= 1: tree.write(orig_file) # TODO anonymize rest of the sensitive information within xml file return all_values
def convert_qstat_to_yaml(self, orig_file, out_file, write_method): out_file = out_file.rsplit('/', 1)[1] try: tree = etree.parse(orig_file) except etree.ParseError: logging.critical("This is an XML parse error (??)") raise except IOError: raise except: print("File %(filename)s does not appear to contain a proper XML structure. Exiting.." % {"filename": orig_file}) raise else: root = tree.getroot() # for queue_elem in root.iter('Queue-List'): # 2.7 only for queue_elem in root.findall('queue_info/Queue-List'): # queue_name = queue_elem.find('./resource[@name="qname"]').text # 2.7 only queue_name_elems = queue_elem.findall('resource') for queue_name_elem in queue_name_elems: if queue_name_elem.attrib.get('name') == 'qname': queue_name = queue_name_elem.text if not options.ANONYMIZE else self.anonymize(queue_name_elem.text, 'qs') # import wdb; wdb.set_trace() break else: raise ValueError("No such queue name") self._extract_job_info(queue_elem, 'job_list', queue_name=queue_name) # puts it into self.l job_info_elem = root.find('./job_info') if job_info_elem is None: logging.debug('No pending jobs found!') else: self._extract_job_info(job_info_elem, 'job_list', queue_name='Pending') # puts it into self.l prefix, suffix = out_file.split('.') prefix += '_' suffix = '.' + suffix SGEStatMaker.fd, SGEStatMaker.temp_filepath = get_new_temp_file(prefix=prefix, suffix=suffix, config=self.config) self.dump_all(SGEStatMaker.fd, self.stat_mapping[write_method])
def _get_statq_from_xml(fn, write_method=options.write_method): logging.debug("Parsing tree of %s" % fn) check_empty_file(fn) anonymize = anonymize_func() with open(fn, mode='rb') as fin: try: tree = etree.parse(fin) except etree.ParseError: logging.critical("Something happened during the parsing of the XML file. Exiting...") except: logging.debug("XML file state %s" % fin) logging.debug("thinking...") import sys sys.exit(1) root = tree.getroot() qstatq_list = [] # for queue_elem in root.iter('Queue-List'): # python 2.7-only for queue_elem in root.findall('queue_info/Queue-List'): # queue_name = queue_elem.find('./resource[@name="qname"]').text # python 2.7-only queue_names = queue_elem.findall('resource') for _queue_name in queue_names: if _queue_name.attrib.get('name') == 'qname': queue_name = _queue_name.text if not options.ANONYMIZE else anonymize(_queue_name.text, 'qs') break else: raise ValueError("No such resource") FOUND = False for exist_d in qstatq_list: if queue_name == exist_d['queue_name']: # exist_d['run'] += len(queue_elem.findall('./job_list[@state="running"]')) # python 2.7 only jobs = queue_elem.findall('job_list') run_count = 0 for _run in jobs: if _run.attrib.get('state') == 'running': run_count += 1 exist_d['run'] += run_count FOUND = True break if FOUND: continue d = dict() d['queue_name'] = queue_name try: d['state'] = queue_elem.find('./state').text except AttributeError: d['state'] = '?' except: raise # d['run'] = len(queue_elem.findall('./job_list[@state="running"]')) # python 2.7 only job_lists = queue_elem.findall('job_list') run_count = 0 for _run in job_lists: if _run.attrib.get('state') == 'running': run_count += 1 d['run'] = run_count d['lm'] = 0 d['queued'] = 0 qstatq_list.append(d) total_running_jobs = str(sum([d['run'] for d in qstatq_list])) logging.info('Total running jobs found: %s' % total_running_jobs) for d in qstatq_list: d['run'] = str(d['run']) d['queued'] = str(d['queued']) # total_queued_jobs = str(len(root.findall('.//job_list[@state="pending"]'))) # python 2.7 only total_queued_jobs_elems = root.findall('job_info/job_list') pending_count = 0 for job in total_queued_jobs_elems: if job.attrib.get('state') == 'pending': pending_count += 1 total_queued_jobs = str(pending_count) logging.info('Total queued jobs found: %s' % total_queued_jobs) qstatq_list.append({'run': '0', 'queued': total_queued_jobs, 'queue_name': 'Pending', 'state': 'Q', 'lm': '0'}) logging.debug('qstatq_list contains %s elements' % len(qstatq_list)) # TODO: check validity. 'state' shouldnt just be 'Q'! logging.debug("Closing %s" % fn) return total_running_jobs, total_queued_jobs, qstatq_list