def test_condorstatus_queryerror():
condor_status = htcondor_query.CondorStatus(pool_name=config_cs_bad.get("pool_name"))
try:
condor_status.load()
assert False
except htcondor_query.QueryError:
assert True
def test_condorstatus(self):
condor_status = htcondor_query.CondorStatus(
subsystem_name="any", pool_name=config_cs.get("pool_name"))
with mock.patch.object(htcondor_query.CondorStatus, "fetch") as f:
f.return_value = utils.input_from_file(FIXTURE_FILE)
condor_status.load()
assert f.return_value == condor_status.stored_data
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
self.logger.debug("in FactoryGlobalManifests acquire")
dataframe = None
for factory in self.factories:
collector_host = factory.get("collector_host")
constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactoryglobal\")"
classad_attrs = []
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=["Name"],
logger=self.logger,
)
retry_wrapper(
partial(condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
max_retries=self.max_retries,
retry_interval=self.retry_interval,
logger=self.logger,
)
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df["CollectorHost"] = [col_host] * len(df)
if sec_cols != "":
df["CollectorHosts"] = [f"{col_host},{sec_cols}"
] * len(df)
else:
df["CollectorHosts"] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError:
self.logger.exception(
f'Failed to get glidefactoryglobal classads from collector host(s) "{collector_host}"'
)
except Exception:
self.logger.exception("Unexpected error fetching "
"glidefactoryglobal classads from "
"collector host(s) "
f"{collector_host}")
return {"factoryglobal_manifests": dataframe}
def test_condorstatus(self):
condor_status = htcondor_query.CondorStatus(
subsystem_name='any', pool_name=config_cs.get('pool_name'))
with mock.patch.object(htcondor_query.CondorStatus, 'fetch') as f:
f.return_value = utils.input_from_file('cs.fixture')
condor_status.load()
#pprint.pprint(condor_status.stored_data)
pprint.pprint('Fetched %s resource classads' %
len(condor_status.stored_data))
def test_condorstatus_live(self):
if not os.environ.get('CONDOR_CONFIG'):
os.environ['CONDOR_CONFIG'] = config_cs.get('condor_config')
condor_status = htcondor_query.CondorStatus(
pool_name=config_cs.get('pool_name'))
condor_status.load()
pprint.pprint(
'Fetched %s resource classads from collector %s' %
(len(condor_status.stored_data), config_cs.get('pool_name')))
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = None
for factory in self.factories:
collector_host = factory.get('collector_host')
constraint = '(%s)&&(glideinmytype=="glidefactoryglobal")' % \
factory.get('constraint', True)
classad_attrs = []
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=['Name'])
retry_wrapper(partial(
condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
nretries=self.nretries,
retry_interval=self.retry_interval)
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df['CollectorHost'] = [col_host] * len(df)
if sec_cols != '':
df['CollectorHosts'] = [
'%s,%s' % (col_host, sec_cols)
] * len(df)
else:
df['CollectorHosts'] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError as e:
self.logger.error('Failed to get glidefactoryglobal classads '
'from collector host(s) "{}": {}'.format(
collector_host, e))
except Exception:
self.logger.exception('Unexpected error fetching '
'glidefactoryglobal classads from '
'collector host(s) '
"{}"
''.format(collector_host))
return {'factoryglobal_manifests': dataframe}
def test_condorstatus_queryerror(self):
if not os.environ.get('CONDOR_CONFIG'):
os.environ['CONDOR_CONFIG'] = config_cs_bad.get('condor_config')
condor_status = htcondor_query.CondorStatus(
pool_name=config_cs_bad.get('pool_name'))
try:
condor_status.load()
assert False
except htcondor_query.QueryError:
assert True
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = None
for factory in self.factories:
collector_host = factory.get('collector_host')
constraint = '(%s)&&(glideinmytype=="glidefactoryglobal")' % factory.get(
'constraint', True)
classad_attrs = []
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=['Name'])
condor_status.load(constraint, classad_attrs,
self.condor_config)
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df['CollectorHost'] = [col_host] * len(df)
if sec_cols != '':
df['CollectorHosts'] = [
'%s,%s' % (col_host, sec_cols)
] * len(df)
else:
df['CollectorHosts'] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True)
except htcondor_query.QueryError:
self.logger.warning(
'Failed to get glidefactoryglobal classads from collector host(s) "%s"'
% collector_host)
self.logger.error(
'Failed to get glidefactoryglobal classads from collector host(s) "%s". Traceback: %s'
% (collector_host, traceback.format_exc()))
except Exception:
self.logger.warning(
'Unexpected error fetching glidefactoryglobal classads from collector host(s) "%s"'
% collector_host)
self.logger.error(
'Unexpected error fetching glidefactoryglobal classads from collector host(s) "%s". Traceback: %s'
% (collector_host, traceback.format_exc()))
return {PRODUCES[0]: dataframe}
def load(self):
"""
Acquire resource classads from the HTCondor Collector
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = pandas.DataFrame()
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=self.collector_host,
group_attr=self.group_attr)
condor_status.load(self.constraint, self.classad_attrs,
self.condor_config)
for eachDict in condor_status.stored_data:
for key, value in self.correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
dataframe = pandas.DataFrame(condor_status.stored_data)
if not dataframe.empty:
(collector_host,
secondary_collectors) = htcondor_query.split_collector_host(
self.collector_host)
dataframe['CollectorHost'] = [collector_host] * len(dataframe)
if secondary_collectors != '':
dataframe['CollectorHosts'] = [
'%s,%s' % (collector_host, secondary_collectors)
] * len(dataframe)
else:
dataframe['CollectorHosts'] = [collector_host
] * len(dataframe)
except htcondor_query.QueryError:
self.logger.warning(
'Query error fetching classads from collector host(s) "%s"' %
self.collector_host)
self.logger.error(
'Query error fetching classads from collector host(s) "%s". Traceback: %s'
% (self.collector_host, traceback.format_exc()))
except Exception:
self.logger.warning(
'Unexpected error fetching classads from collector host(s) "%s"'
% self.collector_host)
self.logger.error(
'Unexpected error fetching classads from collector host(s) "%s". Traceback: %s'
% (self.collector_host, traceback.format_exc()))
return dataframe
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = pandas.DataFrame()
for factory in self.factories:
collector_host = factory.get('collector_host')
constraint = '(%s)&&(glideinmytype=="glidefactory")' % factory.get('constraint', True)
classad_attrs = factory.get('classad_attrs')
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=['GLIDEIN_GridType'])
condor_status.load(constraint, classad_attrs, self.condor_config)
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols) = htcondor_query.split_collector_host(collector_host)
df['CollectorHost'] = [col_host] * len(df)
if sec_cols != '':
df['CollectorHosts'] = ['%s,%s' % (col_host, sec_cols)] * len(df)
else:
df['CollectorHosts'] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df], ignore_index=True)
except htcondor_query.QueryError:
self.logger.warning('Query error fetching glidefactory classads from collector host(s) "%s"' % collector_host)
self.logger.error('Query error fetching glidefactory classads from collector host(s) "%s". Traceback: %s' % (collector_host, traceback.format_exc()))
except Exception:
self.logger.warning('Unexpected error fetching glidefactory classads from collector host(s) "%s"' % collector_host)
self.logger.error('Unexpected error fetching glidefactory classads from collector host(s) "%s". Traceback: %s' % (collector_host, traceback.format_exc()))
results = {}
if not dataframe.empty:
for key, value in self._entry_gridtype_map.items():
results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin(list(value)))]
else:
# There were no entry classads in the factory collector or
# quering the collector failed
for entry_type in self.produces():
results[entry_type] = pandas.DataFrame()
return results
def load(self):
"""
Acquire resource classads from the HTCondor Collector
:rtype: :obj:`~pd.DataFrame`
"""
self.logger.debug("in ResourceManifests load")
dataframe = pandas.DataFrame()
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=self.collector_host,
group_attr=self.group_attr,
logger=self.logger,
)
condor_status.load(self.constraint, self.classad_attrs, self.condor_config)
for eachDict in condor_status.stored_data:
for key, value in self.correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
dataframe = pandas.DataFrame(condor_status.stored_data)
if not dataframe.empty:
(collector_host, secondary_collectors) = htcondor_query.split_collector_host(self.collector_host)
dataframe["CollectorHost"] = [collector_host] * len(dataframe)
if secondary_collectors != "":
dataframe["CollectorHosts"] = [f"{collector_host},{secondary_collectors}"] * len(dataframe)
else:
dataframe["CollectorHosts"] = [collector_host] * len(dataframe)
except htcondor_query.QueryError:
self.logger.warning(f'Query error fetching classads from collector host(s) "{self.collector_host}"')
self.logger.error(
f'Query error fetching classads from collector host(s) "{self.collector_host}". Traceback: {traceback.format_exc()}'
)
except Exception:
self.logger.warning(f'Unexpected error fetching classads from collector host(s) "{self.collector_host}"')
self.logger.error(
f'Unexpected error fetching classads from collector host(s) "{self.collector_host,}". Traceback: {traceback.format_exc()}'
)
return dataframe
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = pandas.DataFrame()
for factory in self.factories:
collector_host = factory.get('collector_host')
constraint = '(%s)&&(glideinmytype=="glidefactory")' % \
factory.get('constraint', True)
classad_attrs = factory.get('classad_attrs')
correction_map = factory.get('correction_map')
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=['GLIDEIN_GridType'])
retry_wrapper(partial(
condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
nretries=self.nretries,
retry_interval=self.retry_interval)
if correction_map is not None:
for eachDict in condor_status.stored_data:
for key, value in correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df['CollectorHost'] = [col_host] * len(df)
if sec_cols != '':
df['CollectorHosts'] = [
'%s,%s' % (col_host, sec_cols)
] * len(df)
else:
df['CollectorHosts'] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError as e:
self.logger.error('Failed to fetch glidefactory classads '
'from collector host(s) "{}": {}'.format(
collector_host, e))
except Exception:
self.logger.exception('Unexpected error fetching glidefactory '
'classads from collector host(s) '
'"{}"'.format(collector_host))
if dataframe.empty:
# There were no entry classads in the factory collector or
# quering the collector failed
return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame())
results = {}
for key, value in self._entry_gridtype_map.items():
results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin(
list(value)))]
return results
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
self.logger.debug("in FactoryEntries acquire")
dataframe = pandas.DataFrame()
for factory in self.factories:
collector_host = factory.get("collector_host")
constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactory\")"
classad_attrs = factory.get("classad_attrs")
correction_map = factory.get("correction_map")
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=["GLIDEIN_GridType"],
logger=self.logger,
)
retry_wrapper(
partial(condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
max_retries=self.max_retries,
retry_interval=self.retry_interval,
logger=self.logger,
)
if correction_map is not None:
for eachDict in condor_status.stored_data:
for key, value in correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df["CollectorHost"] = [col_host] * len(df)
if sec_cols != "":
df["CollectorHosts"] = [f"{col_host},{sec_cols}"
] * len(df)
else:
df["CollectorHosts"] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError:
self.logger.exception(
f"Failed to fetch glidefactory classads from collector host(s) {collector_host}"
)
except Exception:
self.logger.exception(
f"Unexpected error fetching glidefactory classads from collector host(s) {collector_host}"
)
if dataframe.empty:
# There were no entry classads in the factory collector or
# quering the collector failed
return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame())
results = {}
for key, value in self._entry_gridtype_map.items():
results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin(
list(value)))]
return results
