def testA_SiteWhitelist(self): """ _SiteWhitelist_ Test to see if we can get the siteWhitelist to work properly. """ secconfig = getattr(self.config, "SecurityModule") cherrypy.server.environment = 'test' cherrypy.tools.secmodv2 = NullAuth(secconfig) self.config.UnitTests.views.active.rest.templates = 'templateDir' self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot' self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*', 'US*': '_US_'} from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True) siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'] self.assertEqual(assign.sites, []) assign.sites.extend(siteList) Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites) for s in siteList: self.assertTrue(s in assign.sites) self.assertTrue('T1*' in assign.sites) self.assertTrue('T2*' in assign.sites) self.assertFalse('T3*' in assign.sites) self.assertTrue('US*' in assign.sites) self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL']) self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']) self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
def testA_SiteWhitelist(self): """ _SiteWhitelist_ Test to see if we can get the siteWhitelist to work properly. """ secconfig = getattr(self.config, "SecurityModule") cherrypy.server.environment = 'test' cherrypy.tools.secmodv2 = NullAuth(secconfig) self.config.UnitTests.views.active.rest.templates = 'templateDir' self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot' self.config.UnitTests.views.active.rest.wildcardKeys = { 'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*', 'US*': '_US_' } from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign assign = Assign(config=self.config.UnitTests.views.active.rest, noSiteDB=True) siteList = [ 'T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT' ] self.assertEqual(assign.sites, []) assign.sites.extend(siteList) Utilities.addSiteWildcards(assign.wildcardKeys, assign.sites, assign.wildcardSites) for s in siteList: self.assertTrue(s in assign.sites) self.assertTrue('T1*' in assign.sites) self.assertTrue('T2*' in assign.sites) self.assertFalse('T3*' in assign.sites) self.assertTrue('US*' in assign.sites) self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL']) self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']) self.assertEqual( assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
def __init__(self, config, noSiteDB = False): """ _init_ Note, noSiteDB added for TESTING PURPOSED ONLY! """ WebAPI.__init__(self, config) ReqMgrAuth.assign_roles = config.security_roles # Take a guess self.templatedir = config.templates self.couchUrl = config.couchUrl self.clipboardDB = config.clipboardDB cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl) self.clipboardUrl = "%s/%s/_design/OpsClipboard/index.html" % (cleanUrl, self.clipboardDB) self.opshold = config.opshold self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) if not noSiteDB: self.sites = Utilities.sites(config.sitedb) else: self.sites = [] self.allMergedLFNBases = [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc", "/store/generator", "/store/relval"] self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.mergedLFNBases = { "ReReco" : ["/store/backfill/1", "/store/backfill/2", "/store/data"], "DataProcessing" : ["/store/backfill/1", "/store/backfill/2", "/store/data"], "ReDigi" : ["/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc"], "MonteCarlo" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"], "RelValMC" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"], "Resubmission" : ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data"], "MonteCarloFromGEN" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"], "TaskChain": ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data", "/store/relval"], "LHEStepZero": ["/store/backfill/1", "/store/backfill/2", "/store/generator"]} self.yuiroot = config.yuiroot cherrypy.engine.subscribe('start_thread', self.initThread) self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*'}) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def __init__(self, config, noSiteDB=False): """ _init_ Note, noSiteDB added for TESTING PURPOSED ONLY! """ WebAPI.__init__(self, config) ReqMgrAuth.assign_roles = config.security_roles # Take a guess self.templatedir = config.templates self.couchUrl = config.couchUrl self.configDBName = config.configDBName self.workloadDBName = config.workloadDBName self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip("/"), config.wmstatDBName) if not noSiteDB: try: # Download a list of all the sites from SiteDB, uses v2 API. sitedb = SiteDBJSON() self.sites = sitedb.getAllCMSNames() self.sites.sort() self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True) self.phedexNodes.sort() except Exception as ex: msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex cherrypy.log(msg) raise else: self.sites = [] # store result lfn base with all Physics group storeResultLFNBase = [ "/store/results/analysisops", "/store/results/b_physics", "/store/results/b_tagging", "/store/results/b2g", "/store/results/e_gamma_ecal", "/store/results/ewk", "/store/results/exotica", "/store/results/forward", "/store/results/heavy_ions", "/store/results/higgs", "/store/results/jets_met_hcal", "/store/results/muon", "/store/results/qcd", "/store/results/susy", "/store/results/tau_pflow", "/store/results/top", "/store/results/tracker_dpg", "/store/results/tracker_pog", "/store/results/trigger", ] # yet 0.9.40 had also another self.mergedLFNBases which was differentiating # list of mergedLFNBases based on type of request, removed and all bases # will be displayed regardless of the request type (discussion with Edgar) self.allMergedLFNBases = [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc", "/store/generator", "/store/relval", "/store/hidata", "/store/himc", ] self.allMergedLFNBases.extend(storeResultLFNBase) self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.yuiroot = config.yuiroot cherrypy.engine.subscribe("start_thread", self.initThread) self.wildcardKeys = getattr(config, "wildcardKeys", {"T1*": "T1_*", "T2*": "T2_*", "T3*": "T3_*"}) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
] self.allMergedLFNBases.extend(storeResultLFNBase) self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.yuiroot = config.yuiroot cherrypy.engine.subscribe('start_thread', self.initThread) self.wildcardKeys = getattr(config, 'wildcardKeys', { 'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*' }) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() #myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def validate(self, v, name=""): """ _validate_ Checks different fields with different Lexicon methods, if not the field name is not known then apply the identifier check """
"/store/backfill/2", "/store/data", "/store/mc", "/store/generator", "/store/relval", "/store/hidata", "/store/himc", ] self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.yuiroot = config.yuiroot cherrypy.engine.subscribe("start_thread", self.initThread) self.wildcardKeys = getattr(config, "wildcardKeys", {"T1*": "T1_*", "T2*": "T2_*", "T3*": "T3_*"}) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() # myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def validate(self, v, name=""): """ _validate_ Checks different fields with different Lexicon methods, if not the field name is not known then apply the identifier check """
def __init__(self, config, noSiteDB=False): """ _init_ Note, noSiteDB added for TESTING PURPOSED ONLY! """ WebAPI.__init__(self, config) ReqMgrAuth.assign_roles = config.security_roles # Take a guess self.templatedir = config.templates self.couchUrl = config.couchUrl self.configDBName = config.configDBName self.workloadDBName = config.workloadDBName self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) if not noSiteDB: try: # Download a list of all the sites from SiteDB, uses v2 API. sitedb = SiteDBJSON() self.sites = sitedb.getAllCMSNames() self.sites.sort() self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True) self.phedexNodes.sort() except Exception as ex: msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex cherrypy.log(msg) raise else: self.sites = [] # store result lfn base with all Physics group storeResultLFNBase = ["/store/results/analysisops", "/store/results/b_physics", "/store/results/b_tagging", "/store/results/b2g", "/store/results/e_gamma_ecal", "/store/results/ewk", "/store/results/exotica", "/store/results/forward", "/store/results/heavy_ions", "/store/results/higgs", "/store/results/jets_met_hcal", "/store/results/muon", "/store/results/qcd", "/store/results/susy", "/store/results/tau_pflow", "/store/results/top", "/store/results/tracker_dpg", "/store/results/tracker_pog", "/store/results/trigger"] # yet 0.9.40 had also another self.mergedLFNBases which was differentiating # list of mergedLFNBases based on type of request, removed and all bases # will be displayed regardless of the request type (discussion with Edgar) self.allMergedLFNBases = [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc", "/store/generator", "/store/relval", "/store/hidata", "/store/himc"] self.allMergedLFNBases.extend(storeResultLFNBase) self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.yuiroot = config.yuiroot cherrypy.engine.subscribe('start_thread', self.initThread) self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*'}) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def __init__(self, config, noSiteDB=False): """ _init_ Note, noSiteDB added for TESTING PURPOSED ONLY! """ WebAPI.__init__(self, config) ReqMgrAuth.assign_roles = config.security_roles # Take a guess self.templatedir = config.templates self.couchUrl = config.couchUrl cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl) self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) if not noSiteDB: self.sites = Utilities.sites(config.sitedb) else: self.sites = [] self.allMergedLFNBases = [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc", "/store/generator", "/store/relval", "/store/hidata", "/store/himc" ] self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.mergedLFNBases = { "ReReco": [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/hidata" ], "DataProcessing": [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/hidata" ], "ReDigi": [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc", "/store/himc" ], "MonteCarlo": [ "/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/himc" ], "RelValMC": [ "/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/himc" ], "Resubmission": [ "/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data", "/store/hidata" ], "MonteCarloFromGEN": [ "/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/himc" ], "TaskChain": [ "/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data", "/store/relval" ], "LHEStepZero": ["/store/backfill/1", "/store/backfill/2", "/store/generator"] } self.yuiroot = config.yuiroot cherrypy.engine.subscribe('start_thread', self.initThread) self.wildcardKeys = getattr(config, 'wildcardKeys', { 'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*' }) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)