def __init__(self, lastAcc=False):
self.lastAccOpt = lastAcc
popsettings = confSettings()
self.interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE") + 'Interface'
try:
module = __import__('Apps.victorinterface.utils.serviceInterface', globals(), locals(), [self.interface], -1)
self.replicaInterface = getattr(module, self.interface)
except Exception as err:
msg = 'unable to import replica interface: %s ' % self.interface
logger.error(msg)
raise Exception(msg)
self.replicaHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST")
def __init__(self, lastAcc=False):
self.lastAccOpt = lastAcc
popsettings = confSettings()
self.interface = popsettings.getSetting(
"victorinterface", "DATASERVICE_INTERFACE") + 'Interface'
try:
module = __import__('Apps.victorinterface.utils.serviceInterface',
globals(), locals(), [self.interface], -1)
self.replicaInterface = getattr(module, self.interface)
except Exception as err:
msg = 'unable to import replica interface: %s ' % self.interface
logger.error(msg)
raise Exception(msg)
self.replicaHost = popsettings.getSetting("victorinterface",
"DATASERVICE_HOST")
def replicaPopularity(opts):
popsettings = confSettings()
dasHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST")
popHost = popsettings.getSetting("victorinterface", "POPULARITY_HOST")
#dasHost = 'https://cmsweb.cern.ch'
#popHost = 'http://cms-popularity-prod.cern.ch'
debug = 0
sitename = opts.SiteName
incomplete = 0 #opts.incomplete
timestart = opts.TStart
timestop = opts.TStop
if not sitename:
raise Exception('You must provide input sitename')
combiner = ReplicaPopularity(debug=debug,dasHost=dasHost,popHost=popHost)
outdict = combiner.combine(sitename, timestart, timestop, incomplete, debug)
logger.debug(outdict)
return outdict
def replicaPopularity(opts):
popsettings = confSettings()
dasHost = popsettings.getSetting("victorinterface", "DATASERVICE_HOST")
popHost = popsettings.getSetting("victorinterface", "POPULARITY_HOST")
#dasHost = 'https://cmsweb.cern.ch'
#popHost = 'http://cms-popularity-prod.cern.ch'
debug = 0
sitename = opts.SiteName
incomplete = 0 #opts.incomplete
timestart = opts.TStart
timestop = opts.TStop
if not sitename:
raise Exception('You must provide input sitename')
combiner = ReplicaPopularity(debug=debug, dasHost=dasHost, popHost=popHost)
outdict = combiner.combine(sitename, timestart, timestop, incomplete,
debug)
logger.debug(outdict)
return outdict
def combine(self, sitename, timestart, timestop, incomplete=0, debug=0):
self.validateSiteName(sitename)
try:
popsettings = confSettings()
interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE")
logger.info("using interface: %s - host: %s" % (interface, self._dasHost))
#self.get_das_blockreplica_data(sitename)
#COMMENT ABOVE AND UNCOMMENT BELOW TO SWITCH TO PHEDEX
if (interface == 'phedex'):
try:
self.get_phedex_blockreplica_data(sitename)
except Exception, err:
logger.error("Unable to fetch replica information from PhEDEx")
logger.error(err)
raise err
elif (interface == 'das'):
try:
self.get_das_blockreplica_data(sitename)
except Exception, err:
logger.error("Unable to fetch replica information from DAS")
logger.error(err)
raise err
from __future__ import absolute_import
from Apps.popCommon.utils.confSettings import confSettings
import os
popsettings = confSettings()
# Django settings for popDBDjango project.
os.environ['PYTHON_EGG_CACHE'] = popsettings.getSetting(
"popularity_project", "base_dir") + '/cache'
DEBUG = False
if popsettings.getSetting("popularity_project", "debug_mode") == "True":
DEBUG = True
TEMPLATE_DEBUG = DEBUG
ADMINS = (
# ('Your Name', '*****@*****.**'),
)
MANAGERS = ADMINS
DATABASES = popsettings.getNestedConfigSectionMap('db_sections')
# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# On Unix systems, a value of None will cause Django to use the same
# timezone as the operating system.
# If running in a Windows environment this must be set to the same as your
# system time zone.
from django.db import connection
from Apps.popCommon.utils import utility
from Apps.popCommon.PopularityException import PopularityDBException
from Apps.popCommon.utils.confSettings import confSettings
import logging
logger = logging.getLogger(__name__)
popsettings = confSettings()
DBUSER = popsettings.getSetting("popCommon", "DBUSER")
#DBUSER = '******'
def DSStatInTimeWindow(params, MView):
#cursor = connection.cursor()
if MView == 'DataTier':
table = "%s.%s" % (DBUSER, "MV_DS_STAT0_AGGR2")
elif MView == 'DS':
table = "%s.%s" % (DBUSER, "MV_DS_STAT0_AGGR1")
elif MView == 'DSName':
table = "%s.%s" % (DBUSER, "MV_DS_STAT0_AGGR4")
vars = '''collName , sum(numAccesses) as nAcc, round(sum(totCPU)/3600,0)
as totCPU, sum(numUsers) as nUsers'''
whereCondition = '''TDay >= to_date('%s','YYYY-MM-DD')
and TDay <= to_date('%s','YYYY-MM-DD')
''' % (params.TStart, params.TStop)
groupBy = "collName"
if params.orderVar == 'naccess' :
orderBy = "nAcc desc"
def combine(self, sitename, timestart, timestop, incomplete=0, debug=0):
self.validateSiteName(sitename)
try:
popsettings = confSettings()
interface = popsettings.getSetting("victorinterface", "DATASERVICE_INTERFACE")
logger.info("using interface: %s - host: %s" % (interface, self._dasHost))
#self.get_das_blockreplica_data(sitename)
#COMMENT ABOVE AND UNCOMMENT BELOW TO SWITCH TO PHEDEX
if (interface == 'phedex'):
try:
self.get_phedex_blockreplica_data(sitename)
except Exception as err:
logger.error("Unable to fetch replica information from PhEDEx")
logger.error(err)
raise err
elif (interface == 'das'):
try:
self.get_das_blockreplica_data(sitename)
except Exception as err:
logger.error("Unable to fetch replica information from DAS")
logger.error(err)
raise err
else:
logger.error("Unable to fetch replica information - Unknow module specified in config file: %s" % interface)
raise Exception("Unable to fetch replica information")
except PopularityConfigException as err:
logger.error(err)
raise err
try:
self.get_pop_data(sitename,timestart,timestop)
except Exception as err:
logger.error("Unable to fetch popularity information from PopDB")
logger.error(err)
raise err
try:
data = self.das_data['data']
except KeyError as err:
logger.warning("No data in DAS query result")
raise err
if not data:
logger.warning('WARNING: no replicas found at %s' % sitename)
msg='WARNING: no replicas found at %s' % sitename
raise ValueError(msg)
popdict=self.pop_data
outcoll={}
for row in data:
if not incomplete and row['block']['replica']['complete'] == 'n':
continue
outrep={}
for val in ('group','custodial','creation_time','nfiles','size'):
outrep[val]=row['block']['replica'][val]
try:
outrep['popularitynacc']=popdict[sitename][row['block']['name']]['NACC']
outrep['popularitycpu']=popdict[sitename][row['block']['name']]['TOTCPU']
logger.debug(row['block']['name']+" POPULAR")
del popdict[sitename][row['block']['name']]
except KeyError:
outrep['popularitynacc']=0
outrep['popularitycpu']=0
logger.debug(row['block']['name']+" UNPOPULAR")
outcoll[row['block']['name']]=outrep
if len(popdict) != 0:
for row in popdict[sitename]:
logger.debug(row+" NOTINPHEDEX")
outdict={sitename:outcoll,'popularitytstart':timestart,'popularitytstop':timestop}
return outdict
def combine(self, sitename, timestart, timestop, incomplete=0, debug=0):
self.validateSiteName(sitename)
try:
popsettings = confSettings()
interface = popsettings.getSetting("victorinterface",
"DATASERVICE_INTERFACE")
logger.info("using interface: %s - host: %s" %
(interface, self._dasHost))
#self.get_das_blockreplica_data(sitename)
#COMMENT ABOVE AND UNCOMMENT BELOW TO SWITCH TO PHEDEX
if (interface == 'phedex'):
try:
self.get_phedex_blockreplica_data(sitename)
except Exception as err:
logger.error(
"Unable to fetch replica information from PhEDEx")
logger.error(err)
raise err
elif (interface == 'das'):
try:
self.get_das_blockreplica_data(sitename)
except Exception as err:
logger.error(
"Unable to fetch replica information from DAS")
logger.error(err)
raise err
else:
logger.error(
"Unable to fetch replica information - Unknow module specified in config file: %s"
% interface)
raise Exception("Unable to fetch replica information")
except PopularityConfigException as err:
logger.error(err)
raise err
try:
self.get_pop_data(sitename, timestart, timestop)
except Exception as err:
logger.error("Unable to fetch popularity information from PopDB")
logger.error(err)
raise err
try:
data = self.das_data['data']
except KeyError as err:
logger.warning("No data in DAS query result")
raise err
if not data:
logger.warning('WARNING: no replicas found at %s' % sitename)
msg = 'WARNING: no replicas found at %s' % sitename
raise ValueError(msg)
popdict = self.pop_data
outcoll = {}
for row in data:
if not incomplete and row['block']['replica']['complete'] == 'n':
continue
outrep = {}
for val in ('group', 'custodial', 'creation_time', 'nfiles',
'size'):
outrep[val] = row['block']['replica'][val]
try:
outrep['popularitynacc'] = popdict[sitename][row['block']
['name']]['NACC']
outrep['popularitycpu'] = popdict[sitename][row['block']
['name']]['TOTCPU']
logger.debug(row['block']['name'] + " POPULAR")
del popdict[sitename][row['block']['name']]
except KeyError:
outrep['popularitynacc'] = 0
outrep['popularitycpu'] = 0
logger.debug(row['block']['name'] + " UNPOPULAR")
outcoll[row['block']['name']] = outrep
if len(popdict) != 0:
for row in popdict[sitename]:
logger.debug(row + " NOTINPHEDEX")
outdict = {
sitename: outcoll,
'popularitytstart': timestart,
'popularitytstop': timestop
}
return outdict
