def test_condorstatus_queryerror(): condor_status = htcondor_query.CondorStatus(pool_name=config_cs_bad.get("pool_name")) try: condor_status.load() assert False except htcondor_query.QueryError: assert True
def test_condorstatus(self): condor_status = htcondor_query.CondorStatus( subsystem_name="any", pool_name=config_cs.get("pool_name")) with mock.patch.object(htcondor_query.CondorStatus, "fetch") as f: f.return_value = utils.input_from_file(FIXTURE_FILE) condor_status.load() assert f.return_value == condor_status.stored_data
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ self.logger.debug("in FactoryGlobalManifests acquire") dataframe = None for factory in self.factories: collector_host = factory.get("collector_host") constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactoryglobal\")" classad_attrs = [] try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=["Name"], logger=self.logger, ) retry_wrapper( partial(condor_status.load, *(constraint, classad_attrs, self.condor_config)), max_retries=self.max_retries, retry_interval=self.retry_interval, logger=self.logger, ) df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df["CollectorHost"] = [col_host] * len(df) if sec_cols != "": df["CollectorHosts"] = [f"{col_host},{sec_cols}" ] * len(df) else: df["CollectorHosts"] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError: self.logger.exception( f'Failed to get glidefactoryglobal classads from collector host(s) "{collector_host}"' ) except Exception: self.logger.exception("Unexpected error fetching " "glidefactoryglobal classads from " "collector host(s) " f"{collector_host}") return {"factoryglobal_manifests": dataframe}
def test_condorstatus(self): condor_status = htcondor_query.CondorStatus( subsystem_name='any', pool_name=config_cs.get('pool_name')) with mock.patch.object(htcondor_query.CondorStatus, 'fetch') as f: f.return_value = utils.input_from_file('cs.fixture') condor_status.load() #pprint.pprint(condor_status.stored_data) pprint.pprint('Fetched %s resource classads' % len(condor_status.stored_data))
def test_condorstatus_live(self): if not os.environ.get('CONDOR_CONFIG'): os.environ['CONDOR_CONFIG'] = config_cs.get('condor_config') condor_status = htcondor_query.CondorStatus( pool_name=config_cs.get('pool_name')) condor_status.load() pprint.pprint( 'Fetched %s resource classads from collector %s' % (len(condor_status.stored_data), config_cs.get('pool_name')))
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ dataframe = None for factory in self.factories: collector_host = factory.get('collector_host') constraint = '(%s)&&(glideinmytype=="glidefactoryglobal")' % \ factory.get('constraint', True) classad_attrs = [] try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=['Name']) retry_wrapper(partial( condor_status.load, *(constraint, classad_attrs, self.condor_config)), nretries=self.nretries, retry_interval=self.retry_interval) df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df['CollectorHost'] = [col_host] * len(df) if sec_cols != '': df['CollectorHosts'] = [ '%s,%s' % (col_host, sec_cols) ] * len(df) else: df['CollectorHosts'] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError as e: self.logger.error('Failed to get glidefactoryglobal classads ' 'from collector host(s) "{}": {}'.format( collector_host, e)) except Exception: self.logger.exception('Unexpected error fetching ' 'glidefactoryglobal classads from ' 'collector host(s) ' "{}" ''.format(collector_host)) return {'factoryglobal_manifests': dataframe}
def test_condorstatus_queryerror(self): if not os.environ.get('CONDOR_CONFIG'): os.environ['CONDOR_CONFIG'] = config_cs_bad.get('condor_config') condor_status = htcondor_query.CondorStatus( pool_name=config_cs_bad.get('pool_name')) try: condor_status.load() assert False except htcondor_query.QueryError: assert True
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ dataframe = None for factory in self.factories: collector_host = factory.get('collector_host') constraint = '(%s)&&(glideinmytype=="glidefactoryglobal")' % factory.get( 'constraint', True) classad_attrs = [] try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=['Name']) condor_status.load(constraint, classad_attrs, self.condor_config) df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df['CollectorHost'] = [col_host] * len(df) if sec_cols != '': df['CollectorHosts'] = [ '%s,%s' % (col_host, sec_cols) ] * len(df) else: df['CollectorHosts'] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True) except htcondor_query.QueryError: self.logger.warning( 'Failed to get glidefactoryglobal classads from collector host(s) "%s"' % collector_host) self.logger.error( 'Failed to get glidefactoryglobal classads from collector host(s) "%s". Traceback: %s' % (collector_host, traceback.format_exc())) except Exception: self.logger.warning( 'Unexpected error fetching glidefactoryglobal classads from collector host(s) "%s"' % collector_host) self.logger.error( 'Unexpected error fetching glidefactoryglobal classads from collector host(s) "%s". Traceback: %s' % (collector_host, traceback.format_exc())) return {PRODUCES[0]: dataframe}
def load(self): """ Acquire resource classads from the HTCondor Collector :rtype: :obj:`~pd.DataFrame` """ dataframe = pandas.DataFrame() try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=self.collector_host, group_attr=self.group_attr) condor_status.load(self.constraint, self.classad_attrs, self.condor_config) for eachDict in condor_status.stored_data: for key, value in self.correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value dataframe = pandas.DataFrame(condor_status.stored_data) if not dataframe.empty: (collector_host, secondary_collectors) = htcondor_query.split_collector_host( self.collector_host) dataframe['CollectorHost'] = [collector_host] * len(dataframe) if secondary_collectors != '': dataframe['CollectorHosts'] = [ '%s,%s' % (collector_host, secondary_collectors) ] * len(dataframe) else: dataframe['CollectorHosts'] = [collector_host ] * len(dataframe) except htcondor_query.QueryError: self.logger.warning( 'Query error fetching classads from collector host(s) "%s"' % self.collector_host) self.logger.error( 'Query error fetching classads from collector host(s) "%s". Traceback: %s' % (self.collector_host, traceback.format_exc())) except Exception: self.logger.warning( 'Unexpected error fetching classads from collector host(s) "%s"' % self.collector_host) self.logger.error( 'Unexpected error fetching classads from collector host(s) "%s". Traceback: %s' % (self.collector_host, traceback.format_exc())) return dataframe
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ dataframe = pandas.DataFrame() for factory in self.factories: collector_host = factory.get('collector_host') constraint = '(%s)&&(glideinmytype=="glidefactory")' % factory.get('constraint', True) classad_attrs = factory.get('classad_attrs') try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=['GLIDEIN_GridType']) condor_status.load(constraint, classad_attrs, self.condor_config) df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols) = htcondor_query.split_collector_host(collector_host) df['CollectorHost'] = [col_host] * len(df) if sec_cols != '': df['CollectorHosts'] = ['%s,%s' % (col_host, sec_cols)] * len(df) else: df['CollectorHosts'] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True) except htcondor_query.QueryError: self.logger.warning('Query error fetching glidefactory classads from collector host(s) "%s"' % collector_host) self.logger.error('Query error fetching glidefactory classads from collector host(s) "%s". Traceback: %s' % (collector_host, traceback.format_exc())) except Exception: self.logger.warning('Unexpected error fetching glidefactory classads from collector host(s) "%s"' % collector_host) self.logger.error('Unexpected error fetching glidefactory classads from collector host(s) "%s". Traceback: %s' % (collector_host, traceback.format_exc())) results = {} if not dataframe.empty: for key, value in self._entry_gridtype_map.items(): results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin(list(value)))] else: # There were no entry classads in the factory collector or # quering the collector failed for entry_type in self.produces(): results[entry_type] = pandas.DataFrame() return results
def load(self): """ Acquire resource classads from the HTCondor Collector :rtype: :obj:`~pd.DataFrame` """ self.logger.debug("in ResourceManifests load") dataframe = pandas.DataFrame() try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=self.collector_host, group_attr=self.group_attr, logger=self.logger, ) condor_status.load(self.constraint, self.classad_attrs, self.condor_config) for eachDict in condor_status.stored_data: for key, value in self.correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value dataframe = pandas.DataFrame(condor_status.stored_data) if not dataframe.empty: (collector_host, secondary_collectors) = htcondor_query.split_collector_host(self.collector_host) dataframe["CollectorHost"] = [collector_host] * len(dataframe) if secondary_collectors != "": dataframe["CollectorHosts"] = [f"{collector_host},{secondary_collectors}"] * len(dataframe) else: dataframe["CollectorHosts"] = [collector_host] * len(dataframe) except htcondor_query.QueryError: self.logger.warning(f'Query error fetching classads from collector host(s) "{self.collector_host}"') self.logger.error( f'Query error fetching classads from collector host(s) "{self.collector_host}". Traceback: {traceback.format_exc()}' ) except Exception: self.logger.warning(f'Unexpected error fetching classads from collector host(s) "{self.collector_host}"') self.logger.error( f'Unexpected error fetching classads from collector host(s) "{self.collector_host,}". Traceback: {traceback.format_exc()}' ) return dataframe
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ dataframe = pandas.DataFrame() for factory in self.factories: collector_host = factory.get('collector_host') constraint = '(%s)&&(glideinmytype=="glidefactory")' % \ factory.get('constraint', True) classad_attrs = factory.get('classad_attrs') correction_map = factory.get('correction_map') try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=['GLIDEIN_GridType']) retry_wrapper(partial( condor_status.load, *(constraint, classad_attrs, self.condor_config)), nretries=self.nretries, retry_interval=self.retry_interval) if correction_map is not None: for eachDict in condor_status.stored_data: for key, value in correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df['CollectorHost'] = [col_host] * len(df) if sec_cols != '': df['CollectorHosts'] = [ '%s,%s' % (col_host, sec_cols) ] * len(df) else: df['CollectorHosts'] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError as e: self.logger.error('Failed to fetch glidefactory classads ' 'from collector host(s) "{}": {}'.format( collector_host, e)) except Exception: self.logger.exception('Unexpected error fetching glidefactory ' 'classads from collector host(s) ' '"{}"'.format(collector_host)) if dataframe.empty: # There were no entry classads in the factory collector or # quering the collector failed return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame()) results = {} for key, value in self._entry_gridtype_map.items(): results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin( list(value)))] return results
def acquire(self): """ Acquire factory entries from the factory collector and return as pandas frame :rtype: :obj:`~pd.DataFrame` """ self.logger.debug("in FactoryEntries acquire") dataframe = pandas.DataFrame() for factory in self.factories: collector_host = factory.get("collector_host") constraint = f"({factory.get('constraint', True)})&&(glideinmytype==\"glidefactory\")" classad_attrs = factory.get("classad_attrs") correction_map = factory.get("correction_map") try: condor_status = htcondor_query.CondorStatus( subsystem_name=self.subsystem_name, pool_name=collector_host, group_attr=["GLIDEIN_GridType"], logger=self.logger, ) retry_wrapper( partial(condor_status.load, *(constraint, classad_attrs, self.condor_config)), max_retries=self.max_retries, retry_interval=self.retry_interval, logger=self.logger, ) if correction_map is not None: for eachDict in condor_status.stored_data: for key, value in correction_map.items(): if eachDict.get(key) is None: eachDict[key] = value df = pandas.DataFrame(condor_status.stored_data) if not df.empty: (col_host, sec_cols ) = htcondor_query.split_collector_host(collector_host) df["CollectorHost"] = [col_host] * len(df) if sec_cols != "": df["CollectorHosts"] = [f"{col_host},{sec_cols}" ] * len(df) else: df["CollectorHosts"] = [col_host] * len(df) dataframe = pandas.concat([dataframe, df], ignore_index=True, sort=True) except htcondor_query.QueryError: self.logger.exception( f"Failed to fetch glidefactory classads from collector host(s) {collector_host}" ) except Exception: self.logger.exception( f"Unexpected error fetching glidefactory classads from collector host(s) {collector_host}" ) if dataframe.empty: # There were no entry classads in the factory collector or # quering the collector failed return dict.fromkeys(self._entry_gridtype_map, pandas.DataFrame()) results = {} for key, value in self._entry_gridtype_map.items(): results[key] = dataframe.loc[(dataframe.GLIDEIN_GridType.isin( list(value)))] return results