Ejemplo n.º 1
0
def _get_pbsnodes_values(orig_file, out_file, write_method):
    check_empty_file(orig_file)
    raw_blocks = _read_all_blocks(orig_file)
    all_pbs_values = []
    anonymize = anonymize_func()
    for block in raw_blocks:
        pbs_values = dict()
        pbs_values['domainname'] = block['domainname'] if not options.ANONYMIZE else anonymize(block['domainname'], 'wns')

        nextchar = block['state'][0]
        state = (nextchar == 'f') and "-" or nextchar

        pbs_values['state'] = state
        try:
            pbs_values['np'] = block['np']
        except KeyError:
            pbs_values['np'] = block['pcpus']  # handle torque cases  # todo : to check

        if block.get('gpus') > 0:  # this should be rare.
            pbs_values['gpus'] = block['gpus']
        try:  # this should turn up more often, hence the try/except.
            _ = block['jobs']
        except KeyError:
            pass
        else:
            pbs_values['core_job_map'] = []
            jobs = block['jobs'].split(',')
            for job, core in _get_jobs_cores(jobs):
                _d = dict()
                _d['job'] = job
                _d['core'] = core
                pbs_values['core_job_map'].append(_d)
        finally:
            all_pbs_values.append(pbs_values)
    return all_pbs_values
Ejemplo n.º 2
0
def _get_pbsnodes_values(orig_file, out_file, write_method):
    check_empty_file(orig_file)
    raw_blocks = _read_all_blocks(orig_file)
    all_pbs_values = []
    anonymize = anonymize_func()
    for block in raw_blocks:
        pbs_values = dict()
        pbs_values['domainname'] = block[
            'domainname'] if not options.ANONYMIZE else anonymize(
                block['domainname'], 'wns')

        nextchar = block['state'][0]
        state = (nextchar == 'f') and "-" or nextchar

        pbs_values['state'] = state
        try:
            pbs_values['np'] = block['np']
        except KeyError:
            pbs_values['np'] = block[
                'pcpus']  # handle torque cases  # todo : to check

        if block.get('gpus') > 0:  # this should be rare.
            pbs_values['gpus'] = block['gpus']
        try:  # this should turn up more often, hence the try/except.
            _ = block['jobs']
        except KeyError:
            pass
        else:
            pbs_values['core_job_map'] = []
            jobs = block['jobs'].split(',')
            for job, core in _get_jobs_cores(jobs):
                _d = dict()
                _d['job'] = job
                _d['core'] = core
                pbs_values['core_job_map'].append(_d)
        finally:
            all_pbs_values.append(pbs_values)
    return all_pbs_values
Ejemplo n.º 3
0
def _get_statq_from_xml(fn, write_method=options.write_method):
    logging.debug("Parsing tree of %s" % fn)
    check_empty_file(fn)
    anonymize = anonymize_func()
    with open(fn, mode='rb') as fin:
        try:
            tree = etree.parse(fin)
        except etree.ParseError:
            logging.critical("Something happened during the parsing of the XML file. Exiting...")
        except:
            logging.debug("XML file state %s" % fin)
            logging.debug("thinking...")
            import sys
            sys.exit(1)

        root = tree.getroot()
        qstatq_list = []
        # for queue_elem in root.iter('Queue-List'):  # python 2.7-only
        for queue_elem in root.findall('queue_info/Queue-List'):
            # queue_name = queue_elem.find('./resource[@name="qname"]').text  # python 2.7-only
            queue_names = queue_elem.findall('resource')
            for _queue_name in queue_names:
                if _queue_name.attrib.get('name') == 'qname':
                    queue_name = _queue_name.text if not options.ANONYMIZE else anonymize(_queue_name.text, 'qs')
                    break
            else:
                raise ValueError("No such resource")
            FOUND = False
            for exist_d in qstatq_list:
                if queue_name == exist_d['queue_name']:
                    # exist_d['run'] += len(queue_elem.findall('./job_list[@state="running"]'))  # python 2.7 only
                    jobs = queue_elem.findall('job_list')
                    run_count = 0
                    for _run in jobs:
                        if _run.attrib.get('state') == 'running':
                            run_count += 1
                    exist_d['run'] += run_count
                    FOUND = True
                    break
            if FOUND:
                continue

            d = dict()
            d['queue_name'] = queue_name
            try:
                d['state'] = queue_elem.find('./state').text
            except AttributeError:
                d['state'] = '?'
            except:
                raise
            # d['run'] = len(queue_elem.findall('./job_list[@state="running"]'))  # python 2.7 only
            job_lists = queue_elem.findall('job_list')
            run_count = 0
            for _run in job_lists:
                if _run.attrib.get('state') == 'running':
                    run_count += 1
            d['run'] = run_count
            d['lm'] = 0
            d['queued'] = 0
            qstatq_list.append(d)

        total_running_jobs = str(sum([d['run'] for d in qstatq_list]))
        logging.info('Total running jobs found: %s' % total_running_jobs)
        for d in qstatq_list:
            d['run'] = str(d['run'])
            d['queued'] = str(d['queued'])
        # total_queued_jobs = str(len(root.findall('.//job_list[@state="pending"]')))  # python 2.7 only
        total_queued_jobs_elems = root.findall('job_info/job_list')
        pending_count = 0
        for job in total_queued_jobs_elems:
            if job.attrib.get('state') == 'pending':
                pending_count += 1
        total_queued_jobs = str(pending_count)
        logging.info('Total queued jobs found: %s' % total_queued_jobs)
        qstatq_list.append({'run': '0', 'queued': total_queued_jobs, 'queue_name': 'Pending', 'state': 'Q', 'lm': '0'})
        logging.debug('qstatq_list contains %s elements' % len(qstatq_list))
        # TODO: check validity. 'state' shouldnt just be 'Q'!
    logging.debug("Closing %s" % fn)
    return total_running_jobs, total_queued_jobs, qstatq_list
Ejemplo n.º 4
0
def _calc_everything(fn, write_method):
    logging.debug('Parsing tree of %s' % fn)
    anonymize = anonymize_func()
    with open(fn, 'rb') as fin:
        tree = etree.parse(fin)
        root = tree.getroot()
        worker_nodes = list()
        existing_node_names = set()
        # for queue_elem in root.iter('Queue-List'):  # 2.7-only
        for queue_elem in root.findall('queue_info/Queue-List'):
            worker_node = dict()
            # worker_node['domainname'] = queue_elem.find('./resource[@name="hostname"]').text.split('.', 1)[0]  # 2.7 only
            resources = queue_elem.findall('resource')
            # TODO: find a way to loop ONCE for both hostname and qname!!
            try:
                slots_used = int(queue_elem.find('./slots_used').text)
            except AttributeError:
                slots_used = 0
            count = 0
            # worker_node.setdefault('qname', [])
            for resource in resources:
                if resource.attrib.get('name') == 'hostname':
                    worker_node['domainname'] = resource.text if not options.ANONYMIZE else anonymize(resource.text, 'wns')
                    count += 1
                    if count == 2: break
                elif resource.attrib.get('name') == 'qname':
                    if not slots_used:
                        worker_node['qname'] = set()
                    else:
                        worker_node['qname'] = set(resource.text[0]) \
                            if not options.ANONYMIZE else set(anonymize(resource.text[0], 'qs'))
                    count += 1
                    if count == 2: break
            else:
                raise ValueError("No such resource")

            # worker_node['np'] = queue_elem.find('./resource[@name="num_proc"]').text  # python 2.7 only
            resources = queue_elem.findall('resource')
            for resource in resources:
                if resource.attrib.get('name') == 'num_proc':
                    worker_node['np'] = resource.text
                    break
            else:
                # TODO: check this for bugs, maybe raise an exception in the future?
                worker_node['np'] = 0

            try:
                state = queue_elem.find('state').text
            except AttributeError:
                worker_node['state'] = '-'
            else:
                worker_node['state'] = state

            if worker_node['domainname'] not in existing_node_names:
                job_ids, usernames, job_states = extract_job_info(queue_elem, 'job_list')
                worker_node['core_job_map'] = [{'core': idx, 'job': job_id} for idx, job_id in enumerate(job_ids)]
                worker_node['existing_busy_cores'] = len(worker_node['core_job_map'])
                existing_node_names.update([worker_node['domainname']])
                worker_nodes.append(worker_node)
            else:
                for existing_wn in worker_nodes:
                    if worker_node['domainname'] != existing_wn['domainname']:
                        continue
                    job_ids, usernames, job_states = extract_job_info(queue_elem, 'job_list')
                    core_jobs = [{'core': idx, 'job': job_id}
                                 for idx, job_id in enumerate(job_ids, existing_wn['existing_busy_cores'])]
                    existing_wn['core_job_map'].extend(core_jobs)
                    # don't change the node state to free.
                    # Just keep the state reported in the last queue mentioning the node.
                    existing_wn['state'] = (worker_node['state'] == '-') and existing_wn['state'] or worker_node['state']
                    existing_wn['qname'].update(worker_node['qname'])
                    break
    logging.debug('Closing %s' % fn)
    logging.info('worker_nodes contains %s entries' % len(worker_nodes))
    return worker_nodes