def acquire(self): """ Acquire jobs from the HTCondor Schedd :rtype: :obj:`~pd.DataFrame` """ dataframe = pandas.DataFrame() (collector_host, secondary_collectors) = htcondor_query.split_collector_host( self.collector_host) for schedd in self.schedds: try: condor_q = htcondor_query.CondorQ( schedd_name=schedd, pool_name=self.collector_host) condor_q.load(constraint=self.constraint, format_list=self.classad_attrs, condor_config=self.condor_config) df = pandas.DataFrame(condor_q.stored_data) if not df.empty: # Add schedd name and collector host to job records df['ScheddName'] = pandas.Series([schedd] * len(condor_q.stored_data)) df['CollectorHost'] = pandas.Series( [collector_host] * len(condor_q.stored_data)) dataframe = dataframe.append(df, ignore_index=True) except htcondor_query.QueryError: self.logger.warning( 'Query error fetching job classads from schedd "%s" in collector host(s) "%s".' % (schedd, collector_host)) except Exception: msg = 'Unexpected error fetching job classads from schedd "{}" in collector host(s) "{}".' self.logger.warning(msg.format(schedd, collector_host)) self.logger.error( msg.format(schedd, collector_host) + " Traceback: {}".format(traceback.format_exc())) return {'job_manifests': dataframe}
def test_condorq(self): condor_q = htcondor_query.CondorQ( schedd_name=config_cq.get("schedd_name"), pool_name=config_cq.get("pool_name")) with mock.patch.object(htcondor_query.CondorQ, "fetch") as f: f.return_value = utils.input_from_file(FIXTURE_FILE) condor_q.load() assert f.return_value == condor_q.stored_data
def test_condorq_queryerror(self): condor_q = htcondor_query.CondorQ( schedd_name=config_cq_bad.get("schedd_name"), pool_name=config_cq_bad.get("pool_name")) try: condor_q.load(constraint="procid < 2") assert False except htcondor_query.QueryError: assert True
def test_condorq(self): condor_q = htcondor_query.CondorQ( schedd_name=config_cq.get('schedd_name'), pool_name=config_cq.get('pool_name')) with mock.patch.object(htcondor_query.CondorQ, 'fetch') as f: f.return_value = utils.input_from_file('cq.fixture') condor_q.load() pprint.pprint('Fetched %s job classads' % len(condor_q.stored_data))
def test_condorq_queryerror(self): if not os.environ.get('CONDOR_CONFIG'): os.environ['CONDOR_CONFIG'] = config_cq_bad.get('condor_config') condor_q = htcondor_query.CondorQ( schedd_name=config_cq_bad.get('schedd_name'), pool_name=config_cq_bad.get('pool_name')) try: condor_q.load(constraint='procid < 2') assert False except htcondor_query.QueryError: assert True
def test_condorq_live(self): if not os.environ.get('CONDOR_CONFIG'): os.environ['CONDOR_CONFIG'] = config_cq.get('condor_config') condor_q = htcondor_query.CondorQ( schedd_name=config_cq.get('schedd_name'), pool_name=config_cq.get('pool_name')) condor_q.load(constraint='procid < 2') #pprint.pprint(condor_q.stored_data) pprint.pprint( 'Fetched %s job classads from schedd %s in collector %s' % (len(condor_q.stored_data), config_cq.get('schedd_name'), config_cq.get('pool_name')))
def acquire(self): """ Acquire jobs from the HTCondor Schedd :rtype: :obj:`~pd.DataFrame` """ logging.getLogger().debug("we are in source job_q acquire()") dataframe = pandas.DataFrame() (collector_host, secondary_collectors) = htcondor_query.split_collector_host( self.collector_host) for schedd in self.schedds: try: condor_q = htcondor_query.CondorQ( schedd_name=schedd, pool_name=self.collector_host) condor_q.load(constraint=self.constraint, format_list=self.classad_attrs, condor_config=self.condor_config) for eachDict in condor_q.stored_data: for key, value in self.correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value df = pandas.DataFrame(condor_q.stored_data) if not df.empty: # Add schedd name and collector host to job records df['ScheddName'] = pandas.Series([schedd] * len(condor_q.stored_data)) df['CollectorHost'] = pandas.Series( [collector_host] * len(condor_q.stored_data)) dataframe = dataframe.append(df, ignore_index=True) except htcondor_query.QueryError: self.logger.exception( f"Unexpected error fetching job classads from schedd {schedd} in collector host(s) {collector_host}." ) except Exception: self.logger.exception( f"Unexpected error fetching job classads from schedd {schedd} in collector host(s) {collector_host}." ) return {'job_manifests': dataframe}
def acquire(self): """ Acquire jobs from the HTCondor Schedd :rtype: :obj:`~pd.DataFrame` """ self.logger.debug("in JobQ acquire") dataframe = pandas.DataFrame() (collector_host, secondary_collectors) = htcondor_query.split_collector_host( self.collector_host) for schedd in self.schedds: try: condor_q = htcondor_query.CondorQ( schedd_name=schedd, pool_name=self.collector_host) condor_q.load(constraint=self.constraint, format_list=self.classad_attrs, condor_config=self.condor_config) for eachDict in condor_q.stored_data: for key, value in self.correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value df = pandas.DataFrame(condor_q.stored_data) if not df.empty: # Add schedd name and collector host to job records df["ScheddName"] = pandas.Series([schedd] * len(condor_q.stored_data)) df["CollectorHost"] = pandas.Series( [collector_host] * len(condor_q.stored_data)) dataframe = dataframe.append(df, ignore_index=True) except htcondor_query.QueryError: self.logger.warning( f'Query error fetching job classads from schedd "{schedd}" in collector host(s) "{collector_host}".' ) except Exception: msg = r'Unexpected error fetching job classads from schedd "{}" in collector host(s) "{}".' self.logger.warning(msg.format(schedd, collector_host)) self.logger.error( msg.format(schedd, collector_host) + f" Traceback: {traceback.format_exc()}") return {"job_manifests": dataframe}