def send_dict(self,
namespace,
data,
debug_print=True,
send_data=True,
max_retries=2,
retry_interval=60):
"""send data contained in dictionary as {k: v} to graphite dataset
$namespace.k with current timestamp"""
if data is None:
if self.logger is not None:
self.logger.warning("Warning: send_dict called with no data")
return
now = int(time.time())
post_data = []
# turning data dict into [('$path.$key',($timestamp,$value)),...]]
for k, v in data.items():
t = (namespace + "." + k, (now, v))
post_data.append(t)
if debug_print and self.logger is not None:
self.logger.debug(f"{t}")
# pickle data
payload = pickle.dumps(post_data, protocol=2)
header = struct.pack("!L", len(payload))
message = header + payload
if not send_data:
return
# throw data at graphite
retry_wrapper(partial(self._send_to_graphite, message),
max_retries,
retry_interval,
logger=self.logger)
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
self.logger.debug("in FactoryGlobalManifests acquire")
dataframe = None
for factory in self.factories:
collector_host = factory.get("collector_host")
constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactoryglobal\")"
classad_attrs = []
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=["Name"],
logger=self.logger,
)
retry_wrapper(
partial(condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
max_retries=self.max_retries,
retry_interval=self.retry_interval,
logger=self.logger,
)
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df["CollectorHost"] = [col_host] * len(df)
if sec_cols != "":
df["CollectorHosts"] = [f"{col_host},{sec_cols}"
] * len(df)
else:
df["CollectorHosts"] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError:
self.logger.exception(
f'Failed to get glidefactoryglobal classads from collector host(s) "{collector_host}"'
)
except Exception:
self.logger.exception("Unexpected error fetching "
"glidefactoryglobal classads from "
"collector host(s) "
f"{collector_host}")
return {"factoryglobal_manifests": dataframe}
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = None
for factory in self.factories:
collector_host = factory.get('collector_host')
constraint = '(%s)&&(glideinmytype=="glidefactoryglobal")' % \
factory.get('constraint', True)
classad_attrs = []
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=['Name'])
retry_wrapper(partial(
condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
nretries=self.nretries,
retry_interval=self.retry_interval)
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df['CollectorHost'] = [col_host] * len(df)
if sec_cols != '':
df['CollectorHosts'] = [
'%s,%s' % (col_host, sec_cols)
] * len(df)
else:
df['CollectorHosts'] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError as e:
self.logger.error('Failed to get glidefactoryglobal classads '
'from collector host(s) "{}": {}'.format(
collector_host, e))
except Exception:
self.logger.exception('Unexpected error fetching '
'glidefactoryglobal classads from '
'collector host(s) '
"{}"
''.format(collector_host))
return {'factoryglobal_manifests': dataframe}
def acquire(self):
self.logger.debug("in GCEOccupancy acquire")
return retry_wrapper(self._acquire,
self.max_retries,
self.retry_timeout,
backoff=False,
logger=self.logger)
def condor_advertise(self,
classads,
collector_host=None,
update_ad_command=DEFAULT_UPDATE_AD_COMMAND):
return retry_wrapper(
partial(
self._condor_advertise, classads, **{
"collector_host": collector_host,
"update_ad_command": update_ad_command
}), self.nretries, self.retry_interval)
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
dataframe = pandas.DataFrame()
for factory in self.factories:
collector_host = factory.get('collector_host')
constraint = '(%s)&&(glideinmytype=="glidefactory")' % \
factory.get('constraint', True)
classad_attrs = factory.get('classad_attrs')
correction_map = factory.get('correction_map')
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=['GLIDEIN_GridType'])
retry_wrapper(partial(
condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
nretries=self.nretries,
retry_interval=self.retry_interval)
if correction_map is not None:
for eachDict in condor_status.stored_data:
for key, value in correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df['CollectorHost'] = [col_host] * len(df)
if sec_cols != '':
df['CollectorHosts'] = [
'%s,%s' % (col_host, sec_cols)
] * len(df)
else:
df['CollectorHosts'] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError as e:
self.logger.error('Failed to fetch glidefactory classads '
'from collector host(s) "{}": {}'.format(
collector_host, e))
except Exception:
self.logger.exception('Unexpected error fetching glidefactory '
'classads from collector host(s) '
'"{}"'.format(collector_host))
if dataframe.empty:
# There were no entry classads in the factory collector or
# quering the collector failed
return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame())
results = {}
for key, value in self._entry_gridtype_map.items():
results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin(
list(value)))]
return results
def acquire(self):
return retry_wrapper(self._acquire,
self.max_retries,
self.retry_timeout,
backoff=False)
def func_failure(self, foo, input2=3):
return retry_function.retry_wrapper(
partial(self._func_failure, *(foo, ), **{"input2": input2}),
self.max_retries, self.retry_interval)
def func_success(self, foo, input2=3):
return retry_function.retry_wrapper(
partial(self._func_success, *(foo, ), **{"input2": input2}),
self.nretries, self.retry_interval)
def acquire(self):
"""
Acquire factory entries from the factory collector
and return as pandas frame
:rtype: :obj:`~pd.DataFrame`
"""
self.logger.debug("in FactoryEntries acquire")
dataframe = pandas.DataFrame()
for factory in self.factories:
collector_host = factory.get("collector_host")
constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactory\")"
classad_attrs = factory.get("classad_attrs")
correction_map = factory.get("correction_map")
try:
condor_status = htcondor_query.CondorStatus(
subsystem_name=self.subsystem_name,
pool_name=collector_host,
group_attr=["GLIDEIN_GridType"],
logger=self.logger,
)
retry_wrapper(
partial(condor_status.load,
*(constraint, classad_attrs, self.condor_config)),
max_retries=self.max_retries,
retry_interval=self.retry_interval,
logger=self.logger,
)
if correction_map is not None:
for eachDict in condor_status.stored_data:
for key, value in correction_map.items():
if eachDict.get(key) is None:
eachDict[key] = value
df = pandas.DataFrame(condor_status.stored_data)
if not df.empty:
(col_host, sec_cols
) = htcondor_query.split_collector_host(collector_host)
df["CollectorHost"] = [col_host] * len(df)
if sec_cols != "":
df["CollectorHosts"] = [f"{col_host},{sec_cols}"
] * len(df)
else:
df["CollectorHosts"] = [col_host] * len(df)
dataframe = pandas.concat([dataframe, df],
ignore_index=True,
sort=True)
except htcondor_query.QueryError:
self.logger.exception(
f"Failed to fetch glidefactory classads from collector host(s) {collector_host}"
)
except Exception:
self.logger.exception(
f"Unexpected error fetching glidefactory classads from collector host(s) {collector_host}"
)
if dataframe.empty:
# There were no entry classads in the factory collector or
# quering the collector failed
return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame())
results = {}
for key, value in self._entry_gridtype_map.items():
results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin(
list(value)))]
return results