def send_dict(self, namespace, data, debug_print=True, send_data=True, max_retries=2, retry_interval=60): """send data contained in dictionary as {k: v} to graphite dataset $namespace.k with current timestamp""" if data is None: if self.logger is not None: self.logger.warning("Warning: send_dict called with no data") return now = int(time.time()) post_data = [] # turning data dict into [('$path.$key',($timestamp,$value)),...]] for k, v in data.items(): t = (namespace + "." + k, (now, v)) post_data.append(t) if debug_print and self.logger is not None: self.logger.debug(f"{t}") # pickle data payload = pickle.dumps(post_data, protocol=2) header = struct.pack("!L", len(payload)) message = header + payload if not send_data: return # throw data at graphite retry_wrapper(partial(self._send_to_graphite, message), max_retries, retry_interval, logger=self.logger)
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ self.logger.debug("in FactoryGlobalManifests acquire") dataframe = None for factory in self.factories: collector_host = factory.get("collector_host") constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactoryglobal\")" classad_attrs = [] try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=["Name"], logger=self.logger, ) retry_wrapper( partial(condor_status.load, *(constraint, classad_attrs, self.condor_config)), max_retries=self.max_retries, retry_interval=self.retry_interval, logger=self.logger, ) df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df["CollectorHost"] = [col_host] * len(df) if sec_cols != "": df["CollectorHosts"] = [f"{col_host},{sec_cols}" ] * len(df) else: df["CollectorHosts"] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError: self.logger.exception( f'Failed to get glidefactoryglobal classads from collector host(s) "{collector_host}"' ) except Exception: self.logger.exception("Unexpected error fetching " "glidefactoryglobal classads from " "collector host(s) " f"{collector_host}") return {"factoryglobal_manifests": dataframe}
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ dataframe = None for factory in self.factories: collector_host = factory.get('collector_host') constraint = '(%s)&&(glideinmytype=="glidefactoryglobal")' % \ factory.get('constraint', True) classad_attrs = [] try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=['Name']) retry_wrapper(partial( condor_status.load, *(constraint, classad_attrs, self.condor_config)), nretries=self.nretries, retry_interval=self.retry_interval) df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df['CollectorHost'] = [col_host] * len(df) if sec_cols != '': df['CollectorHosts'] = [ '%s,%s' % (col_host, sec_cols) ] * len(df) else: df['CollectorHosts'] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError as e: self.logger.error('Failed to get glidefactoryglobal classads ' 'from collector host(s) "{}": {}'.format( collector_host, e)) except Exception: self.logger.exception('Unexpected error fetching ' 'glidefactoryglobal classads from ' 'collector host(s) ' "{}" ''.format(collector_host)) return {'factoryglobal_manifests': dataframe}
def acquire(self): self.logger.debug("in GCEOccupancy acquire") return retry_wrapper(self._acquire, self.max_retries, self.retry_timeout, backoff=False, logger=self.logger)
def condor_advertise(self, classads, collector_host=None, update_ad_command=DEFAULT_UPDATE_AD_COMMAND): return retry_wrapper( partial( self._condor_advertise, classads, **{ "collector_host": collector_host, "update_ad_command": update_ad_command }), self.nretries, self.retry_interval)
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ dataframe = pandas.DataFrame() for factory in self.factories: collector_host = factory.get('collector_host') constraint = '(%s)&&(glideinmytype=="glidefactory")' % \ factory.get('constraint', True) classad_attrs = factory.get('classad_attrs') correction_map = factory.get('correction_map') try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=['GLIDEIN_GridType']) retry_wrapper(partial( condor_status.load, *(constraint, classad_attrs, self.condor_config)), nretries=self.nretries, retry_interval=self.retry_interval) if correction_map is not None: for eachDict in condor_status.stored_data: for key, value in correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df['CollectorHost'] = [col_host] * len(df) if sec_cols != '': df['CollectorHosts'] = [ '%s,%s' % (col_host, sec_cols) ] * len(df) else: df['CollectorHosts'] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError as e: self.logger.error('Failed to fetch glidefactory classads ' 'from collector host(s) "{}": {}'.format( collector_host, e)) except Exception: self.logger.exception('Unexpected error fetching glidefactory ' 'classads from collector host(s) ' '"{}"'.format(collector_host)) if dataframe.empty: # There were no entry classads in the factory collector or # quering the collector failed return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame()) results = {} for key, value in self._entry_gridtype_map.items(): results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin( list(value)))] return results
def acquire(self): return retry_wrapper(self._acquire, self.max_retries, self.retry_timeout, backoff=False)
def func_failure(self, foo, input2=3): return retry_function.retry_wrapper( partial(self._func_failure, *(foo, ), **{"input2": input2}), self.max_retries, self.retry_interval)
def func_success(self, foo, input2=3): return retry_function.retry_wrapper( partial(self._func_success, *(foo, ), **{"input2": input2}), self.nretries, self.retry_interval)
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ self.logger.debug("in FactoryEntries acquire") dataframe = pandas.DataFrame() for factory in self.factories: collector_host = factory.get("collector_host") constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactory\")" classad_attrs = factory.get("classad_attrs") correction_map = factory.get("correction_map") try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=["GLIDEIN_GridType"], logger=self.logger, ) retry_wrapper( partial(condor_status.load, *(constraint, classad_attrs, self.condor_config)), max_retries=self.max_retries, retry_interval=self.retry_interval, logger=self.logger, ) if correction_map is not None: for eachDict in condor_status.stored_data: for key, value in correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df["CollectorHost"] = [col_host] * len(df) if sec_cols != "": df["CollectorHosts"] = [f"{col_host},{sec_cols}" ] * len(df) else: df["CollectorHosts"] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError: self.logger.exception( f"Failed to fetch glidefactory classads from collector host(s) {collector_host}" ) except Exception: self.logger.exception( f"Unexpected error fetching glidefactory classads from collector host(s) {collector_host}" ) if dataframe.empty: # There were no entry classads in the factory collector or # quering the collector failed return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame()) results = {} for key, value in self._entry_gridtype_map.items(): results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin( list(value)))] return results