def acquire(self):
"""
Acquire jobs from the HTCondor Schedd
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = pandas.DataFrame()
(collector_host,
secondary_collectors) = htcondor_query.split_collector_host(
self.collector_host)
for schedd in self.schedds:
try:
condor_q = htcondor_query.CondorQ(
schedd_name=schedd, pool_name=self.collector_host)
condor_q.load(constraint=self.constraint,
format_list=self.classad_attrs,
condor_config=self.condor_config)
df = pandas.DataFrame(condor_q.stored_data)
if not df.empty:
# Add schedd name and collector host to job records
df['ScheddName'] = pandas.Series([schedd] *
len(condor_q.stored_data))
df['CollectorHost'] = pandas.Series(
[collector_host] * len(condor_q.stored_data))
dataframe = dataframe.append(df, ignore_index=True)
except htcondor_query.QueryError:
self.logger.warning(
'Query error fetching job classads from schedd "%s" in collector host(s) "%s".'
% (schedd, collector_host))
except Exception:
msg = 'Unexpected error fetching job classads from schedd "{}" in collector host(s) "{}".'
self.logger.warning(msg.format(schedd, collector_host))
self.logger.error(
msg.format(schedd, collector_host) +
" Traceback: {}".format(traceback.format_exc()))
return {'job_manifests': dataframe}
def test_condorq(self):
condor_q = htcondor_query.CondorQ(
schedd_name=config_cq.get("schedd_name"),
pool_name=config_cq.get("pool_name"))
with mock.patch.object(htcondor_query.CondorQ, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
condor_q.load()
assert f.return_value == condor_q.stored_data
def test_condorq_queryerror(self):
condor_q = htcondor_query.CondorQ(
schedd_name=config_cq_bad.get("schedd_name"),
pool_name=config_cq_bad.get("pool_name"))
try:
condor_q.load(constraint="procid < 2")
assert False
except htcondor_query.QueryError:
assert True
def test_condorq(self):
condor_q = htcondor_query.CondorQ(
schedd_name=config_cq.get('schedd_name'),
pool_name=config_cq.get('pool_name'))
with mock.patch.object(htcondor_query.CondorQ, 'fetch') as f:
f.return_value = utils.input_from_file('cq.fixture')
condor_q.load()
pprint.pprint('Fetched %s job classads' %
len(condor_q.stored_data))
def test_condorq_queryerror(self):
if not os.environ.get('CONDOR_CONFIG'):
os.environ['CONDOR_CONFIG'] = config_cq_bad.get('condor_config')
condor_q = htcondor_query.CondorQ(
schedd_name=config_cq_bad.get('schedd_name'),
pool_name=config_cq_bad.get('pool_name'))
try:
condor_q.load(constraint='procid < 2')
assert False
except htcondor_query.QueryError:
assert True
def test_condorq_live(self):
if not os.environ.get('CONDOR_CONFIG'):
os.environ['CONDOR_CONFIG'] = config_cq.get('condor_config')
condor_q = htcondor_query.CondorQ(
schedd_name=config_cq.get('schedd_name'),
pool_name=config_cq.get('pool_name'))
condor_q.load(constraint='procid < 2')
#pprint.pprint(condor_q.stored_data)
pprint.pprint(
'Fetched %s job classads from schedd %s in collector %s' %
(len(condor_q.stored_data), config_cq.get('schedd_name'),
config_cq.get('pool_name')))
def acquire(self):
"""
Acquire jobs from the HTCondor Schedd
:rtype: :obj:`~pd.DataFrame`
"""
logging.getLogger().debug("we are in source job_q acquire()")
dataframe = pandas.DataFrame()
(collector_host,
secondary_collectors) = htcondor_query.split_collector_host(
self.collector_host)
for schedd in self.schedds:
try:
condor_q = htcondor_query.CondorQ(
schedd_name=schedd, pool_name=self.collector_host)
condor_q.load(constraint=self.constraint,
format_list=self.classad_attrs,
condor_config=self.condor_config)
for eachDict in condor_q.stored_data:
for key, value in self.correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
df = pandas.DataFrame(condor_q.stored_data)
if not df.empty:
# Add schedd name and collector host to job records
df['ScheddName'] = pandas.Series([schedd] *
len(condor_q.stored_data))
df['CollectorHost'] = pandas.Series(
[collector_host] * len(condor_q.stored_data))
dataframe = dataframe.append(df, ignore_index=True)
except htcondor_query.QueryError:
self.logger.exception(
f"Unexpected error fetching job classads from schedd {schedd} in collector host(s) {collector_host}."
)
except Exception:
self.logger.exception(
f"Unexpected error fetching job classads from schedd {schedd} in collector host(s) {collector_host}."
)
return {'job_manifests': dataframe}
def acquire(self):
"""
Acquire jobs from the HTCondor Schedd
:rtype: :obj:`~pd.DataFrame`
"""
self.logger.debug("in JobQ acquire")
dataframe = pandas.DataFrame()
(collector_host,
secondary_collectors) = htcondor_query.split_collector_host(
self.collector_host)
for schedd in self.schedds:
try:
condor_q = htcondor_query.CondorQ(
schedd_name=schedd, pool_name=self.collector_host)
condor_q.load(constraint=self.constraint,
format_list=self.classad_attrs,
condor_config=self.condor_config)
for eachDict in condor_q.stored_data:
for key, value in self.correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
df = pandas.DataFrame(condor_q.stored_data)
if not df.empty:
# Add schedd name and collector host to job records
df["ScheddName"] = pandas.Series([schedd] *
len(condor_q.stored_data))
df["CollectorHost"] = pandas.Series(
[collector_host] * len(condor_q.stored_data))
dataframe = dataframe.append(df, ignore_index=True)
except htcondor_query.QueryError:
self.logger.warning(
f'Query error fetching job classads from schedd "{schedd}" in collector host(s) "{collector_host}".'
)
except Exception:
msg = r'Unexpected error fetching job classads from schedd "{}" in collector host(s) "{}".'
self.logger.warning(msg.format(schedd, collector_host))
self.logger.error(
msg.format(schedd, collector_host) +
f" Traceback: {traceback.format_exc()}")
return {"job_manifests": dataframe}
