def execute(self, *args, **kwargs):
totalevents = kwargs['task']['tm_totalunits']
firstEvent = 1
lastEvent = totalevents
firstLumi = 1
lastLumi = 10
# Set a default of 100 events per lumi. This is set as a task
# property, as the splitting considers it independently of the file
# information provided by the fake dataset.
if not kwargs['task']['tm_events_per_lumi']:
kwargs['task']['tm_events_per_lumi'] = 100
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCFakeFileSet")
newFile = File("MCFakeFile", size = 1000, events = totalevents)
if hasattr(self.config.Sites, 'available'):
newFile.setLocation(self.config.Sites.available)
else:
sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
"cert":self.config.TaskWorker.cmscert})
newFile.setLocation(sbj.getAllCMSNames())
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["block"] = 'MCFackBlock'
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
singleMCFileset.addFile(newFile)
return Result(task=kwargs['task'], result=singleMCFileset)
def execute(self, *args, **kwargs):
totalevents = kwargs['task']['tm_totalunits']
firstEvent = 1
lastEvent = totalevents
firstLumi = 1
lastLumi = 10
# Set a default of 100 events per lumi. This is set as a task
# property, as the splitting considers it independently of the file
# information provided by the fake dataset.
if not kwargs['task']['tm_events_per_lumi']:
kwargs['task']['tm_events_per_lumi'] = 100
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCFakeFileSet")
newFile = File("MCFakeFile", size = 1000, events = totalevents)
sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
"cert":self.config.TaskWorker.cmscert})
newFile.setLocation(sbj.getAllCMSNames())
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["block"] = 'MCFakeBlock'
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
singleMCFileset.addFile(newFile)
return Result(task=kwargs['task'], result=singleMCFileset)
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
except Exception, ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'),
config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
except Exception, ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
def sites():
"Return known CMS site list from SiteDB"
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
sites = sorted(sitedb.getAllCMSNames())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc)
raise Exception(msg)
return sites
def sites():
"Return known CMS site list from SiteDB"
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
sites = sorted(sitedb.getAllCMSNames())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(
exc)
raise Exception(msg)
return sites
def execute(self, *args, **kwargs):
self.logger.info(
"Data discovery and splitting for %s using user-provided files" %
kwargs['task']['tm_taskname'])
userfiles = kwargs['task']['tm_arguments'].get('userfiles')
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs[
'task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
self.logger.error("Setting %s as failed: %s" %
(kwargs['task']['tm_taskname'], msg))
configreq = {
'workflow': kwargs['task']['tm_taskname'],
'status': "FAILED",
'subresource': 'failure',
'failure': b64encode(msg)
}
self.server.post(self.resturi, data=urllib.urlencode(configreq))
raise StopHandler(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
sbj = SiteDBJSON({
"key": self.config.TaskWorker.cmskey,
"cert": self.config.TaskWorker.cmscert
})
locations = sbj.getAllCMSNames()
userFileset = Fileset(name=kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." %
len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size=1000, events=1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task=kwargs['task'], result=userFileset)
class MakeFakeFileSet(TaskAction):
"""This is needed to make WMCore.JobSplitting lib working...
do not like very much. Given that all is fake here I am
quite sure we only need total number of events said that I
set all the other parmas to dummy values. We may want to set
them in the future"""
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
self.sbj = SiteDBJSON({
"key": self.config.TaskWorker.cmskey,
"cert": self.config.TaskWorker.cmscert
})
def getListOfSites(self):
""" Get the list of sites to use for PrivateMC workflows.
For the moment we are filtering out T1_ since they are precious resources
and don't want to overtake production (WMAgent) jobs there. In the
future we would like to take this list from the SSB.
"""
sites = self.sbj.getAllCMSNames()
filteredSites = [site for site in sites if not site.startswith("T1_")]
return filteredSites
#even though args is not used we call "handler.actionWork(args, kwargs)" in Handler
def execute(self, *args, **kwargs): #pylint: disable=unused-argument
# since https://github.com/dmwm/CRABServer/issues/5633 totalunits can be a float
# but that would confuse WMCore, therefore cast to int
totalevents = int(kwargs['task']['tm_totalunits'])
firstEvent = 1
lastEvent = totalevents
firstLumi = 1
lastLumi = 10
# Set a default of 100 events per lumi. This is set as a task
# property, as the splitting considers it independently of the file
# information provided by the fake dataset.
if not kwargs['task']['tm_events_per_lumi']:
kwargs['task']['tm_events_per_lumi'] = 100
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name="MCFakeFileSet")
newFile = File("MCFakeFile", size=1000, events=totalevents)
newFile.setLocation(self.getListOfSites())
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["block"] = 'MCFakeBlock'
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
singleMCFileset.addFile(newFile)
return Result(task=kwargs['task'], result=singleMCFileset)
def insertAllSEs(self,
siteName,
pendingSlots=0,
runningSlots=0,
ceName=None,
plugin=None,
taskList=[]):
"""
_insertAllSEs_
Insert all SEs into WMBS ResourceControl
This uses the Services.SiteDB to insert all SEs under a common
CE. It is meant to be used with WMS submission.
Sites will be named siteName_SEName
It expects a taskList of the following form:
[{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots, 'pendingSlots' : pendingSlots}]
for each entry in the taskList, a threshold is inserted into the database
for EVERY SE
"""
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
siteDB = SiteDBJSON()
cmsNames = siteDB.getAllCMSNames()
for cmsName in cmsNames:
seNames = siteDB.cmsNametoSE(cmsName)
for SE in seNames:
sName = '%s_%s' % (siteName, SE)
self.insertSite(siteName=sName,
pendingSlots=pendingSlots,
seName=SE,
runningSlots=runningSlots,
ceName=ceName,
cmsName=cmsName,
plugin=plugin)
for task in taskList:
if not task.has_key('maxSlots') or not task.has_key('taskType') \
or not task.has_key('priority'):
msg = "Incomplete task in taskList for ResourceControl.insertAllSEs\n"
msg += task
raise ResourceControlException(msg)
self.insertThreshold(siteName=sName,
taskType=task['taskType'],
maxSlots=task['maxSlots'],
pendingSlots=task['pendingSlots'],
priority=task['priority'])
return
class MakeFakeFileSet(TaskAction):
"""This is needed to make WMCore.JobSplitting lib working...
do not like very much. Given that all is fake here I am
quite sure we only need total number of events said that I
set all the other parmas to dummy values. We may want to set
them in the future"""
def __init__(self, *args, **kwargs):
TaskAction.__init__(self, *args, **kwargs)
self.sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
"cert":self.config.TaskWorker.cmscert})
def getListOfSites(self):
""" Get the list of sites to use for PrivateMC workflows.
For the moment we are filtering out T1_ since they are precious resources
and don't want to overtake production (WMAgent) jobs there. In the
future we would like to take this list from the SSB.
"""
sites = self.sbj.getAllCMSNames()
filteredSites = [site for site in sites if not site.startswith("T1_")]
return filteredSites
#even though args is not used we call "handler.actionWork(args, kwargs)" in Handler
def execute(self, *args, **kwargs): #pylint: disable=unused-argument
totalevents = kwargs['task']['tm_totalunits']
firstEvent = 1
lastEvent = totalevents
firstLumi = 1
lastLumi = 10
# Set a default of 100 events per lumi. This is set as a task
# property, as the splitting considers it independently of the file
# information provided by the fake dataset.
if not kwargs['task']['tm_events_per_lumi']:
kwargs['task']['tm_events_per_lumi'] = 100
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCFakeFileSet")
newFile = File("MCFakeFile", size = 1000, events = totalevents)
newFile.setLocation(self.getListOfSites())
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["block"] = 'MCFakeBlock'
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
singleMCFileset.addFile(newFile)
return Result(task=kwargs['task'], result=singleMCFileset)
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
global __sitedb
if not __sitedb:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
try:
__cmsSiteNames = __sitedb.getAllCMSNames()
except:
pass
return __cmsSiteNames
def sites():
"Return known CMS site list from SiteDB"
try:
# Download a list of all the sites from SiteDB, uses v2 API.
if os.getenv("WMAGENT_USE_CRIC", False) or os.getenv("WMCORE_USE_CRIC", False):
cric = CRIC()
site_list = sorted(cric.getAllPSNs())
else:
sitedb = SiteDBJSON()
site_list = sorted(sitedb.getAllCMSNames())
except Exception as exc:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % str(exc)
raise Exception(msg)
return site_list
def cmsSiteNames():
"""Get all cms sites"""
global __cmsSiteNames
if __cmsSiteNames:
return __cmsSiteNames
global __sitedb
if not __sitedb:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
try:
__cmsSiteNames = __sitedb.getAllCMSNames()
except:
pass
return __cmsSiteNames
def execute(self, *args, **kwargs):
self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname'])
if 'tm_user_files' in kwargs['task'] and kwargs['task']['tm_user_files']:
userfiles = kwargs['task']['tm_user_files']
else: ## For backward compatibility only.
userfiles = kwargs['task']['tm_arguments'].get('userfiles')
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
self.logger.error("Setting %s as failed: %s" % (kwargs['task']['tm_taskname'], msg))
configreq = {'workflow': kwargs['task']['tm_taskname'],
'status': "FAILED",
'subresource': 'failure',
'failure': b64encode(msg)}
self.server.post(self.resturi, data = urllib.urlencode(configreq))
raise StopHandler(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
"cert":self.config.TaskWorker.cmscert})
locations = sbj.getAllCMSNames()
userFileset = Fileset(name = kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." % len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size = 1000, events = 1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task = kwargs['task'], result = userFileset)
def execute(self, *args, **kwargs):
self.logger.info(
"Data discovery and splitting for %s using user-provided files" %
kwargs['task']['tm_taskname'])
userfiles = kwargs['task']['tm_user_files']
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs[
'task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
raise TaskWorkerException(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
sbj = SiteDBJSON({
"key": self.config.TaskWorker.cmskey,
"cert": self.config.TaskWorker.cmscert
})
locations = sbj.getAllCMSNames()
userFileset = Fileset(name=kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." %
len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size=1000, events=1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task=kwargs['task'], result=userFileset)
def insertAllSEs(self, siteName, pendingSlots = 0, runningSlots = 0,
ceName = None, plugin = None,
taskList = []):
"""
_insertAllSEs_
Insert all SEs into WMBS ResourceControl
This uses the Services.SiteDB to insert all SEs under a common
CE. It is meant to be used with WMS submission.
Sites will be named siteName_SEName
It expects a taskList of the following form:
[{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots}]
for each entry in the taskList, a threshold is inserted into the database
for EVERY SE
"""
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
siteDB = SiteDBJSON()
cmsNames = siteDB.getAllCMSNames()
for cmsName in cmsNames:
seNames = siteDB.cmsNametoSE(cmsName)
for SE in seNames:
sName = '%s_%s' % (siteName, SE)
self.insertSite(siteName = sName, pendingSlots = pendingSlots,
seName = SE, runningSlots = runningSlots,
ceName = ceName, cmsName = cmsName, plugin = plugin)
for task in taskList:
if not task.has_key('maxSlots') or not task.has_key('taskType') \
or not task.has_key('priority'):
msg = "Incomplete task in taskList for ResourceControl.insertAllSEs\n"
msg += task
raise ResourceControlException(msg)
self.insertThreshold(siteName = sName, taskType = task['taskType'],
maxSlots = task['maxSlots'], priority = task['priority'])
return
def getSiteInfo(config):
sitedb = SiteDBJSON()
sites = sitedb.getAllCMSNames()
sites.sort()
wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'})
wildcardSites = {}
for k in wildcardKeys.keys():
reValue = wildcardKeys.get(k)
found = False
for s in sites:
if re.search(reValue, s):
found = True
if not k in wildcardSites.keys():
wildcardSites[k] = []
wildcardSites[k].append(s)
if found:
sites.append(k)
return sites
def execute(self, *args, **kwargs):
self.logger.info("Data discovery and splitting for %s using user-provided files" % kwargs['task']['tm_taskname'])
userfiles = kwargs['task']['tm_user_files']
splitting = kwargs['task']['tm_split_algo']
total_units = kwargs['task']['tm_totalunits']
if not userfiles or splitting != 'FileBased':
if not userfiles:
msg = "No files specified to process for task %s." % kwargs['task']['tm_taskname']
if splitting != 'FileBased':
msg = "Data.splitting must be set to 'FileBased' when using a custom set of files."
raise TaskWorkerException(msg)
if hasattr(self.config.Sites, 'available'):
locations = self.config.Sites.available
else:
sbj = SiteDBJSON({"key":self.config.TaskWorker.cmskey,
"cert":self.config.TaskWorker.cmscert})
locations = sbj.getAllCMSNames()
userFileset = Fileset(name = kwargs['task']['tm_taskname'])
self.logger.info("There are %d files specified by the user." % len(userfiles))
if total_units > 0:
self.logger.info("Will run over the first %d files." % total_units)
file_counter = 0
for userfile, idx in zip(userfiles, range(len(userfiles))):
newFile = File(userfile, size = 1000, events = 1)
newFile.setLocation(locations)
newFile.addRun(Run(1, idx))
newFile["block"] = 'UserFilesFakeBlock'
newFile["first_event"] = 1
newFile["last_event"] = 2
userFileset.addFile(newFile)
file_counter += 1
if total_units > 0 and file_counter >= total_units:
break
return Result(task = kwargs['task'], result = userFileset)
def getSiteInfo(config):
sitedb = SiteDBJSON()
sites = sitedb.getAllCMSNames()
sites.sort()
wildcardKeys = getattr(config, 'wildcardKeys', {
'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'
})
wildcardSites = {}
for k in wildcardKeys.keys():
reValue = wildcardKeys.get(k)
found = False
for s in sites:
if re.search(reValue, s):
found = True
if not k in wildcardSites.keys():
wildcardSites[k] = []
wildcardSites[k].append(s)
if found:
sites.append(k)
return sites
#! /bin/env python
import ConfigParser
import sys
ConfigFile = '/etc/ciconnect/resources.ini'
# Get CMS Sites from SiteDB
# This requires the user to setup CMSSW and CRAB3 or WMCore
# before being able to use the script
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
db = SiteDBJSON({'cacheduration': 24})
cmsnames = db.getAllCMSNames()
if len(cmsnames) > 10:
sitelist = ",".join(cmsnames)
else:
sys.exit("[Error]: Couldn't get a reasonable list of sites")
# print sitelist
cfg = ConfigParser.RawConfigParser()
cfg.read(ConfigFile)
if cfg.has_section('resources'):
cfg.set('resources', 'ListOfSites', sitelist)
resfile = open(ConfigFile,'w')
cfg.write(resfile)
print "All Done."
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True)
self.phedexNodes.sort()
except Exception as ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
else:
self.sites = []
# store result lfn base with all Physics group
storeResultLFNBase = [
"/store/results/analysisops",
"/store/results/b_physics",
"/store/results/b_tagging",
"/store/results/b2g",
"/store/results/e_gamma_ecal",
"/store/results/ewk",
"/store/results/exotica",
"/store/results/forward",
"/store/results/heavy_ions",
"/store/results/higgs",
"/store/results/jets_met_hcal",
"/store/results/muon",
"/store/results/qcd",
"/store/results/susy",
"/store/results/tau_pflow",
"/store/results/top",
"/store/results/tracker_dpg",
"/store/results/tracker_pog",
"/store/results/trigger",
]
# yet 0.9.40 had also another self.mergedLFNBases which was differentiating
# list of mergedLFNBases based on type of request, removed and all bases
# will be displayed regardless of the request type (discussion with Edgar)
self.allMergedLFNBases = [
"/store/backfill/1",
"/store/backfill/2",
"/store/data",
"/store/mc",
"/store/generator",
"/store/relval",
"/store/hidata",
"/store/himc",
]
self.allMergedLFNBases.extend(storeResultLFNBase)
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe("start_thread", self.initThread)
self.wildcardKeys = getattr(config, "wildcardKeys", {"T1*": "T1_*", "T2*": "T2_*", "T3*": "T3_*"})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True)
self.phedexNodes.sort()
except Exception as ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
else:
self.sites = []
# store result lfn base with all Physics group
storeResultLFNBase = ["/store/results/analysisops",
"/store/results/b_physics",
"/store/results/b_tagging",
"/store/results/b2g",
"/store/results/e_gamma_ecal",
"/store/results/ewk",
"/store/results/exotica",
"/store/results/forward",
"/store/results/heavy_ions",
"/store/results/higgs",
"/store/results/jets_met_hcal",
"/store/results/muon",
"/store/results/qcd",
"/store/results/susy",
"/store/results/tau_pflow",
"/store/results/top",
"/store/results/tracker_dpg",
"/store/results/tracker_pog",
"/store/results/trigger"]
# yet 0.9.40 had also another self.mergedLFNBases which was differentiating
# list of mergedLFNBases based on type of request, removed and all bases
# will be displayed regardless of the request type (discussion with Edgar)
self.allMergedLFNBases = [
"/store/backfill/1",
"/store/backfill/2",
"/store/data",
"/store/mc",
"/store/generator",
"/store/relval",
"/store/hidata",
"/store/himc"]
self.allMergedLFNBases.extend(storeResultLFNBase)
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)