def __init__(self, lastAcc=False): self.lastAccOpt = lastAcc popsettings = confSettings() self.interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE") + 'Interface' try: module = __import__('Apps.victorinterface.utils.serviceInterface', globals(), locals(), [self.interface], -1) self.replicaInterface = getattr(module, self.interface) except Exception as err: msg = 'unable to import replica interface: %s ' % self.interface logger.error(msg) raise Exception(msg) self.replicaHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST")
def __init__(self, lastAcc=False): self.lastAccOpt = lastAcc popsettings = confSettings() self.interface = popsettings.getSetting( "victorinterface", "DATASERVICE_INTERFACE") + 'Interface' try: module = __import__('Apps.victorinterface.utils.serviceInterface', globals(), locals(), [self.interface], -1) self.replicaInterface = getattr(module, self.interface) except Exception as err: msg = 'unable to import replica interface: %s ' % self.interface logger.error(msg) raise Exception(msg) self.replicaHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST")
def replicaPopularity(opts): popsettings = confSettings() dasHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST") popHost = popsettings.getSetting("victorinterface", "POPULARITY_HOST") #dasHost = 'https://cmsweb.cern.ch' #popHost = 'http://cms-popularity-prod.cern.ch' debug = 0 sitename = opts.SiteName incomplete = 0 #opts.incomplete timestart = opts.TStart timestop = opts.TStop if not sitename: raise Exception('You must provide input sitename') combiner = ReplicaPopularity(debug=debug,dasHost=dasHost,popHost=popHost) outdict = combiner.combine(sitename, timestart, timestop, incomplete, debug) logger.debug(outdict) return outdict
def replicaPopularity(opts): popsettings = confSettings() dasHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST") popHost = popsettings.getSetting("victorinterface", "POPULARITY_HOST") #dasHost = 'https://cmsweb.cern.ch' #popHost = 'http://cms-popularity-prod.cern.ch' debug = 0 sitename = opts.SiteName incomplete = 0 #opts.incomplete timestart = opts.TStart timestop = opts.TStop if not sitename: raise Exception('You must provide input sitename') combiner = ReplicaPopularity(debug=debug, dasHost=dasHost, popHost=popHost) outdict = combiner.combine(sitename, timestart, timestop, incomplete, debug) logger.debug(outdict) return outdict
def combine(self, sitename, timestart, timestop, incomplete=0, debug=0): self.validateSiteName(sitename) try: popsettings = confSettings() interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE") logger.info("using interface: %s - host: %s" % (interface, self._dasHost)) #self.get_das_blockreplica_data(sitename) #COMMENT ABOVE AND UNCOMMENT BELOW TO SWITCH TO PHEDEX if (interface == 'phedex'): try: self.get_phedex_blockreplica_data(sitename) except Exception, err: logger.error("Unable to fetch replica information from PhEDEx") logger.error(err) raise err elif (interface == 'das'): try: self.get_das_blockreplica_data(sitename) except Exception, err: logger.error("Unable to fetch replica information from DAS") logger.error(err) raise err
from __future__ import absolute_import from Apps.popCommon.utils.confSettings import confSettings import os popsettings = confSettings() # Django settings for popDBDjango project. os.environ['PYTHON_EGG_CACHE'] = popsettings.getSetting( "popularity_project", "base_dir") + '/cache' DEBUG = False if popsettings.getSetting("popularity_project", "debug_mode") == "True": DEBUG = True TEMPLATE_DEBUG = DEBUG ADMINS = ( # ('Your Name', '*****@*****.**'), ) MANAGERS = ADMINS DATABASES = popsettings.getNestedConfigSectionMap('db_sections') # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # although not all choices may be available on all operating systems. # On Unix systems, a value of None will cause Django to use the same # timezone as the operating system. # If running in a Windows environment this must be set to the same as your # system time zone.
from django.db import connection from Apps.popCommon.utils import utility from Apps.popCommon.PopularityException import PopularityDBException from Apps.popCommon.utils.confSettings import confSettings import logging logger = logging.getLogger(__name__) popsettings = confSettings() DBUSER = popsettings.getSetting("popCommon", "DBUSER") #DBUSER = '******' def DSStatInTimeWindow(params, MView): #cursor = connection.cursor() if MView == 'DataTier': table = "%s.%s" % (DBUSER, "MV_DS_STAT0_AGGR2") elif MView == 'DS': table = "%s.%s" % (DBUSER, "MV_DS_STAT0_AGGR1") elif MView == 'DSName': table = "%s.%s" % (DBUSER, "MV_DS_STAT0_AGGR4") vars = '''collName , sum(numAccesses) as nAcc, round(sum(totCPU)/3600,0) as totCPU, sum(numUsers) as nUsers''' whereCondition = '''TDay >= to_date('%s','YYYY-MM-DD') and TDay <= to_date('%s','YYYY-MM-DD') ''' % (params.TStart, params.TStop) groupBy = "collName" if params.orderVar == 'naccess' : orderBy = "nAcc desc"
def combine(self, sitename, timestart, timestop, incomplete=0, debug=0): self.validateSiteName(sitename) try: popsettings = confSettings() interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE") logger.info("using interface: %s - host: %s" % (interface, self._dasHost)) #self.get_das_blockreplica_data(sitename) #COMMENT ABOVE AND UNCOMMENT BELOW TO SWITCH TO PHEDEX if (interface == 'phedex'): try: self.get_phedex_blockreplica_data(sitename) except Exception as err: logger.error("Unable to fetch replica information from PhEDEx") logger.error(err) raise err elif (interface == 'das'): try: self.get_das_blockreplica_data(sitename) except Exception as err: logger.error("Unable to fetch replica information from DAS") logger.error(err) raise err else: logger.error("Unable to fetch replica information - Unknow module specified in config file: %s" % interface) raise Exception("Unable to fetch replica information") except PopularityConfigException as err: logger.error(err) raise err try: self.get_pop_data(sitename,timestart,timestop) except Exception as err: logger.error("Unable to fetch popularity information from PopDB") logger.error(err) raise err try: data = self.das_data['data'] except KeyError as err: logger.warning("No data in DAS query result") raise err if not data: logger.warning('WARNING: no replicas found at %s' % sitename) msg='WARNING: no replicas found at %s' % sitename raise ValueError(msg) popdict=self.pop_data outcoll={} for row in data: if not incomplete and row['block']['replica']['complete'] == 'n': continue outrep={} for val in ('group','custodial','creation_time','nfiles','size'): outrep[val]=row['block']['replica'][val] try: outrep['popularitynacc']=popdict[sitename][row['block']['name']]['NACC'] outrep['popularitycpu']=popdict[sitename][row['block']['name']]['TOTCPU'] logger.debug(row['block']['name']+" POPULAR") del popdict[sitename][row['block']['name']] except KeyError: outrep['popularitynacc']=0 outrep['popularitycpu']=0 logger.debug(row['block']['name']+" UNPOPULAR") outcoll[row['block']['name']]=outrep if len(popdict) != 0: for row in popdict[sitename]: logger.debug(row+" NOTINPHEDEX") outdict={sitename:outcoll,'popularitytstart':timestart,'popularitytstop':timestop} return outdict
def combine(self, sitename, timestart, timestop, incomplete=0, debug=0): self.validateSiteName(sitename) try: popsettings = confSettings() interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE") logger.info("using interface: %s - host: %s" % (interface, self._dasHost)) #self.get_das_blockreplica_data(sitename) #COMMENT ABOVE AND UNCOMMENT BELOW TO SWITCH TO PHEDEX if (interface == 'phedex'): try: self.get_phedex_blockreplica_data(sitename) except Exception as err: logger.error( "Unable to fetch replica information from PhEDEx") logger.error(err) raise err elif (interface == 'das'): try: self.get_das_blockreplica_data(sitename) except Exception as err: logger.error( "Unable to fetch replica information from DAS") logger.error(err) raise err else: logger.error( "Unable to fetch replica information - Unknow module specified in config file: %s" % interface) raise Exception("Unable to fetch replica information") except PopularityConfigException as err: logger.error(err) raise err try: self.get_pop_data(sitename, timestart, timestop) except Exception as err: logger.error("Unable to fetch popularity information from PopDB") logger.error(err) raise err try: data = self.das_data['data'] except KeyError as err: logger.warning("No data in DAS query result") raise err if not data: logger.warning('WARNING: no replicas found at %s' % sitename) msg = 'WARNING: no replicas found at %s' % sitename raise ValueError(msg) popdict = self.pop_data outcoll = {} for row in data: if not incomplete and row['block']['replica']['complete'] == 'n': continue outrep = {} for val in ('group', 'custodial', 'creation_time', 'nfiles', 'size'): outrep[val] = row['block']['replica'][val] try: outrep['popularitynacc'] = popdict[sitename][row['block'] ['name']]['NACC'] outrep['popularitycpu'] = popdict[sitename][row['block'] ['name']]['TOTCPU'] logger.debug(row['block']['name'] + " POPULAR") del popdict[sitename][row['block']['name']] except KeyError: outrep['popularitynacc'] = 0 outrep['popularitycpu'] = 0 logger.debug(row['block']['name'] + " UNPOPULAR") outcoll[row['block']['name']] = outrep if len(popdict) != 0: for row in popdict[sitename]: logger.debug(row + " NOTINPHEDEX") outdict = { sitename: outcoll, 'popularitytstart': timestart, 'popularitytstop': timestop } return outdict