def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True)
siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {
'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'
}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config=self.config.UnitTests.views.active.rest,
noSiteDB=True)
siteList = [
'T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL',
'T2_US_CIT'
]
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys, assign.sites,
assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'],
['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'],
['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(
assign.wildcardSites['US*'],
['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
def __init__(self, config, noSiteDB = False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.clipboardDB = config.clipboardDB
cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl)
self.clipboardUrl = "%s/%s/_design/OpsClipboard/index.html" % (cleanUrl, self.clipboardDB)
self.opshold = config.opshold
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
if not noSiteDB:
self.sites = Utilities.sites(config.sitedb)
else:
self.sites = []
self.allMergedLFNBases = [
"/store/backfill/1", "/store/backfill/2",
"/store/data", "/store/mc", "/store/generator", "/store/relval"]
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.mergedLFNBases = {
"ReReco" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
"DataProcessing" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
"ReDigi" : ["/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc"],
"MonteCarlo" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"RelValMC" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"Resubmission" : ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data"],
"MonteCarloFromGEN" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"TaskChain": ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data", "/store/relval"],
"LHEStepZero": ["/store/backfill/1", "/store/backfill/2", "/store/generator"]}
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True)
self.phedexNodes.sort()
except Exception as ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
else:
self.sites = []
# store result lfn base with all Physics group
storeResultLFNBase = [
"/store/results/analysisops",
"/store/results/b_physics",
"/store/results/b_tagging",
"/store/results/b2g",
"/store/results/e_gamma_ecal",
"/store/results/ewk",
"/store/results/exotica",
"/store/results/forward",
"/store/results/heavy_ions",
"/store/results/higgs",
"/store/results/jets_met_hcal",
"/store/results/muon",
"/store/results/qcd",
"/store/results/susy",
"/store/results/tau_pflow",
"/store/results/top",
"/store/results/tracker_dpg",
"/store/results/tracker_pog",
"/store/results/trigger",
]
# yet 0.9.40 had also another self.mergedLFNBases which was differentiating
# list of mergedLFNBases based on type of request, removed and all bases
# will be displayed regardless of the request type (discussion with Edgar)
self.allMergedLFNBases = [
"/store/backfill/1",
"/store/backfill/2",
"/store/data",
"/store/mc",
"/store/generator",
"/store/relval",
"/store/hidata",
"/store/himc",
]
self.allMergedLFNBases.extend(storeResultLFNBase)
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe("start_thread", self.initThread)
self.wildcardKeys = getattr(config, "wildcardKeys", {"T1*": "T1_*", "T2*": "T2_*", "T3*": "T3_*"})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
]
self.allMergedLFNBases.extend(storeResultLFNBase)
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {
'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'
})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites,
self.wildcardSites)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def validate(self, v, name=""):
"""
_validate_
Checks different fields with different Lexicon methods,
if not the field name is not known then apply the identifier check
"""
"/store/backfill/2",
"/store/data",
"/store/mc",
"/store/generator",
"/store/relval",
"/store/hidata",
"/store/himc",
]
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe("start_thread", self.initThread)
self.wildcardKeys = getattr(config, "wildcardKeys", {"T1*": "T1_*", "T2*": "T2_*", "T3*": "T3_*"})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
# myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def validate(self, v, name=""):
"""
_validate_
Checks different fields with different Lexicon methods,
if not the field name is not known then apply the identifier check
"""
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True)
self.phedexNodes.sort()
except Exception as ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
else:
self.sites = []
# store result lfn base with all Physics group
storeResultLFNBase = ["/store/results/analysisops",
"/store/results/b_physics",
"/store/results/b_tagging",
"/store/results/b2g",
"/store/results/e_gamma_ecal",
"/store/results/ewk",
"/store/results/exotica",
"/store/results/forward",
"/store/results/heavy_ions",
"/store/results/higgs",
"/store/results/jets_met_hcal",
"/store/results/muon",
"/store/results/qcd",
"/store/results/susy",
"/store/results/tau_pflow",
"/store/results/top",
"/store/results/tracker_dpg",
"/store/results/tracker_pog",
"/store/results/trigger"]
# yet 0.9.40 had also another self.mergedLFNBases which was differentiating
# list of mergedLFNBases based on type of request, removed and all bases
# will be displayed regardless of the request type (discussion with Edgar)
self.allMergedLFNBases = [
"/store/backfill/1",
"/store/backfill/2",
"/store/data",
"/store/mc",
"/store/generator",
"/store/relval",
"/store/hidata",
"/store/himc"]
self.allMergedLFNBases.extend(storeResultLFNBase)
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl)
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'),
config.wmstatDBName)
if not noSiteDB:
self.sites = Utilities.sites(config.sitedb)
else:
self.sites = []
self.allMergedLFNBases = [
"/store/backfill/1", "/store/backfill/2", "/store/data",
"/store/mc", "/store/generator", "/store/relval", "/store/hidata",
"/store/himc"
]
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.mergedLFNBases = {
"ReReco": [
"/store/backfill/1", "/store/backfill/2", "/store/data",
"/store/hidata"
],
"DataProcessing": [
"/store/backfill/1", "/store/backfill/2", "/store/data",
"/store/hidata"
],
"ReDigi": [
"/store/backfill/1", "/store/backfill/2", "/store/data",
"/store/mc", "/store/himc"
],
"MonteCarlo": [
"/store/backfill/1", "/store/backfill/2", "/store/mc",
"/store/himc"
],
"RelValMC": [
"/store/backfill/1", "/store/backfill/2", "/store/mc",
"/store/himc"
],
"Resubmission": [
"/store/backfill/1", "/store/backfill/2", "/store/mc",
"/store/data", "/store/hidata"
],
"MonteCarloFromGEN": [
"/store/backfill/1", "/store/backfill/2", "/store/mc",
"/store/himc"
],
"TaskChain": [
"/store/backfill/1", "/store/backfill/2", "/store/mc",
"/store/data", "/store/relval"
],
"LHEStepZero":
["/store/backfill/1", "/store/backfill/2", "/store/generator"]
}
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {
'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'
})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites,
self.wildcardSites)