def handleAnnounce(self, **kwargs):
""" Handler for announcing requests """
requests = self.requestNamesFromCheckboxes(kwargs)
# the distinction good/bad datasets was based upon contacting
# wrong DBS url so it was always giving rubbish,
# perhaps the entire page is just not used at all
datasets = []
goodDatasets = []
badDatasets = []
for requestName in requests:
WMCore.Lexicon.identifier(requestName)
Utilities.changeStatus(requestName,
'announced',
wmstatUrl=self.wmstatWriteURL,
acdcUrl=self.acdcURL)
datasets.extend(Utilities.getOutputForRequest(requestName))
for dataset in datasets:
# was needed for the DBS call to wrong DBS URL
# check code before 2013-04-04
#toks = dataset.split('/')
#data = {'primary_ds_name': toks[0], 'processed_ds_name': toks[1],
# 'data_tier_name': toks[2], 'is_dataset_valid': 1}
goodDatasets.append(dataset)
return self.templatepage("Announce",
requests=requests,
goodDatasets=goodDatasets,
badDatasets=badDatasets)
def testA_SoftwareManagement(self):
"""
_SoftwareManagement_
Test the SoftwareManagement code
"""
self.assertEqual(SoftwareManagement.listSoftware(), {})
softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
result = SoftwareManagement.listSoftware()
for scramArch in result.keys():
self.assertEqual(set(result[scramArch]), set(softwareVersions[scramArch]))
# Now for each scramArch insert a blank set
# Because of the way that updateSoftware works, this interprets a blank list
# as telling it that no softwareVersions are available.
# It deletes every software version it is not handed, so it should give nothing out.
for scramArch in result.keys():
SoftwareManagement.updateSoftware(softwareNames = [], scramArch = scramArch)
self.assertEqual(SoftwareManagement.listSoftware(), {})
from WMCore.HTTPFrontEnd.RequestManager import Admin
setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
self.config.section_('templates')
self.config.section_('html')
admin = Admin.Admin(self.config)
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
return
def __init__(self, config):
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.clipboardDB = config.clipboardDB
cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl)
self.clipboardUrl = "%s/%s/_design/OpsClipboard/index.html" % (cleanUrl, self.clipboardDB)
self.hold = config.hold
self.configDBName = config.configDBName
self.sites = Utilities.sites(config.sitedb)
self.allMergedLFNBases = [
"/store/backfill/1", "/store/backfill/2",
"/store/data", "/store/mc"]
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.mergedLFNBases = {
"ReReco" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
"DataProcessing" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
"ReDigi" : ["/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc"],
"MonteCarlo" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"RelValMC" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"Resubmission" : ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data"],
"MonteCarloFromGEN" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"]}
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config = self.config.UnitTests.views.active.rest, noSiteDB = True)
siteList = ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT']
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys,assign.sites,assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'], ['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'], ['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(assign.wildcardSites['US*'], ['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
requestType = request["RequestType"]
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
dashboardActivity = helper.getDashboardActivity()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage("Assign", requests=[request], teams=teams,
assignments=assignments, sites=self.sites,
mergedLFNBases=self.mergedLFNBases[requestType],
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra = acqEra, procVer = procVer,
dashboardActivity=dashboardActivity,
badRequests=[])
def makeSchema(self, **schema):
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
try:
request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName)
except RuntimeError, e:
raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
def javascript(self, *args):
if args[0] == "external":
return Utilities.serveFile('application/javascript',
os.path.join(self.config.javascript), *args)
return Utilities.serveFile('application/javascript',
os.path.join(self.config.javascript,
'WMCore', 'WebTools'), *args)
def makeSchema(self, **schema):
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." % (decodedSchema, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL)
# catching here KeyError is just terrible
except (RuntimeError, KeyError, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
baseURL = cherrypy.request.base
raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
def testA_SoftwareManagement(self):
"""
_SoftwareManagement_
Test the SoftwareManagement code
"""
self.assertEqual(SoftwareManagement.listSoftware(), {})
softwareVersions = ReqMgrWebTools.allScramArchsAndVersions()
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
result = SoftwareManagement.listSoftware()
for scramArch in result.keys():
self.assertEqual(set(result[scramArch]),
set(softwareVersions[scramArch]))
# Now for each scramArch insert a blank set
# Because of the way that updateSoftware works, this interprets a blank list
# as telling it that no softwareVersions are available.
# It deletes every software version it is not handed, so it should give nothing out.
for scramArch in result.keys():
SoftwareManagement.updateSoftware(softwareNames=[],
scramArch=scramArch)
self.assertEqual(SoftwareManagement.listSoftware(), {})
# import has to be here, otherwise getting:
# AttributeError: 'Toolbox' object has no attribute 'secmodv2' from the Admin module
from WMCore.HTTPFrontEnd.RequestManager import Admin
setattr(self.config, 'database', self.testInit.coreConfig.CoreDatabase)
self.config.section_('templates')
self.config.section_('html')
admin = Admin.Admin(self.config)
ReqMgrWebTools.updateScramArchsAndCMSSWVersions()
self.assertTrue('slc5_amd64_gcc434' in admin.scramArchs())
def javascript(self, *args):
if args[0] == "external":
return Utilities.serveFile('application/javascript',
os.path.join(self.config.javascript),
*args)
return Utilities.serveFile(
'application/javascript',
os.path.join(self.config.javascript, 'WMCore', 'WebTools'), *args)
def cloneRequest(self, requestName):
"""
Input assumes an existing request, checks that.
The original existing request is not touched.
A new request is generated.
The cloned request has a newly generated RequestName, new timestamp,
RequestDate, however -everything- else is copied from the original request.
Addition: since Edgar changed his mind, he no longer
wants this cloned request be in the 'new' state but put
straight into 'assignment-approved' state.
"""
request = None
if requestName:
self.info("Cloning request: request name: '%s'" % requestName)
requestOrigDict = self.getRequest(requestName)
# add OriginalRequestName to the new request schema
requestOrigDict['OriginalRequestName'] = requestName
if requestOrigDict:
self.info("Request found, cloning ...")
newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
# since we cloned the request, all the attributes
# manipulation in Utilities.makeRequest() should not be necessary
# the cloned request shall be identical but following arguments
toRemove = ["RequestName", "RequestDate", "timeStamp"]
toRemove.extend(deprecatedRequestArgs)
for remove in toRemove:
try:
del requestOrigDict[remove]
except KeyError:
pass
newReqSchema.update(requestOrigDict) # clone
# problem that priority wasn't preserved went down from here:
# via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
request = Utilities.buildWorkloadAndCheckIn(
self,
newReqSchema,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL,
clone=True)
# change the clone request state as desired
Utilities.changeStatus(newReqSchema["RequestName"],
"assignment-approved",
self.wmstatWriteURL, self.acdcURL)
return request
else:
msg = "Request '%s' not found." % requestName
self.warning(msg)
raise cherrypy.HTTPError(404, msg)
else:
msg = "Received empty request name: '%s', exit." % requestName
self.warn(msg)
raise cherrypy.HTTPError(400, msg)
def cloneRequest(self, requestName):
"""
Input assumes an existing request, checks that.
The original existing request is not touched.
A new request is generated.
The cloned request has a newly generated RequestName, new timestamp,
RequestDate, however -everything- else is copied from the original request.
Addition: since Edgar changed his mind, he no longer
wants this cloned request be in the 'new' state but put
straight into 'assignment-approved' state.
"""
request = None
if requestName:
self.info("Cloning request: request name: '%s'" % requestName)
requestOrigDict = self.getRequest(requestName)
# add OriginalRequestName to the new request schema
requestOrigDict['OriginalRequestName'] = requestName
if requestOrigDict:
self.info("Request found, cloning ...")
newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
# since we cloned the request, all the attributes
# manipulation in Utilities.makeRequest() should not be necessary
# the cloned request shall be identical but following arguments
toRemove = ["RequestName",
"RequestDate",
"timeStamp"]
toRemove.extend(deprecatedRequestArgs)
for remove in toRemove:
try:
del requestOrigDict[remove]
except KeyError:
pass
newReqSchema.update(requestOrigDict) # clone
# problem that priority wasn't preserved went down from here:
# via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
request = Utilities.buildWorkloadAndCheckIn(self, newReqSchema,
self.couchUrl, self.workloadDBName,
self.wmstatWriteURL, clone=True)
# change the clone request state as desired
Utilities.changeStatus(newReqSchema["RequestName"],
"assignment-approved",
self.wmstatWriteURL,
self.acdcURL)
return request
else:
msg = "Request '%s' not found." % requestName
self.warning(msg)
raise cherrypy.HTTPError(404, msg)
else:
msg = "Received empty request name: '%s', exit." % requestName
self.warn(msg)
raise cherrypy.HTTPError(400, msg)
def handleSplittingPage(self, requestName, splittingTask, splittingAlgo,
**submittedParams):
"""
_handleSplittingPage_
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
splitParams = {}
if splittingAlgo == "FileBased":
splitParams["files_per_job"] = int(submittedParams["files_per_job"])
elif splittingAlgo == "TwoFileBased":
splitParams["files_per_job"] = int(submittedParams["two_files_per_job"])
elif splittingAlgo == "LumiBased":
splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"])
if str(submittedParams["halt_job_on_file_boundaries"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventAwareLumiBased":
splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"])
splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"])
if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventBased":
splitParams["events_per_job"] = int(submittedParams["events_per_job"])
if "events_per_lumi" in submittedParams:
splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"])
if "lheInputFiles" in submittedParams:
if str(submittedParams["lheInputFiles"]) == "True":
splitParams["lheInputFiles"] = True
else:
splitParams["lheInputFiles"] = False
elif splittingAlgo == "Harvest":
splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"])
elif 'Merg' in splittingTask:
for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']:
splitParams[field] = int(submittedParams[field])
if "include_parents" in submittedParams.keys():
if str(submittedParams["include_parents"]) == "True":
splitParams["include_parents"] = True
else:
splitParams["include_parents"] = False
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams))
helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams)
Utilities.saveWorkload(helper, request['RequestWorkflow'])
return "Successfully updated splitting parameters for " + splittingTask \
+ " " + detailsBackLink(requestName)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist=whiteList,
siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(
int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 167000)),
int(kwargs.get("GracePeriod", 300)))
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
if "AutoApprove" in kwargs:
autoApproveList = nonCustodialList
else:
autoApproveList = []
priority = kwargs.get("Priority", "Low")
if priority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
helper.setSubscriptionInformationWildCards(
wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
priority=priority)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'],
self.wmstatWriteURL)
def testA_ParseRunList(self):
l0 = ''
l1 = ' [1, 2, 3 ] '
l2 = '1, 2, 3 '
l3 = u'1, 2, 3 '
l4 = [1,2,3]
l5 = {1:2, 3:4}
self.assertEqual(ReqMgrWebTools.parseRunList(l0), [])
self.assertEqual(ReqMgrWebTools.parseRunList(l1), [1,2,3])
self.assertEqual(ReqMgrWebTools.parseRunList(l2), [1,2,3])
self.assertEqual(ReqMgrWebTools.parseRunList(l3), [1,2,3])
self.assertEqual(ReqMgrWebTools.parseRunList(l4), [1,2,3])
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
#Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except:
#Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, types.StringTypes):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl = self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(400, "Must assign to one or more teams")
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(requestName, team, wmstatUrl = self.wmstatWriteURL)
priority = kwargs.get(requestName+':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
participle=kwargs['action']+'ed'
if self.opshold and kwargs['action'] == 'Assign':
participle='put into "ops-hold" state (see <a href="%s">OpsClipboard</a>)' % self.clipboardUrl
# this, previously used, call made all requests injected into OpsClipboard to
# have campaign_id null since the call doesn't propagate request's
# CampaignName at all, AcquisitionEra remains default null and probably
# a bunch of other things is wrong too
#requests = [GetRequest.getRequestByName(requestName) for requestName in requestNames]
requests = [Utilities.requestDetails(requestName) for requestName in requestNames]
OpsClipboard.inject(self.couchUrl, self.clipboardDB, *requests)
for request in requestNames:
ChangeState.changeRequestStatus(requestName, 'ops-hold', wmstatUrl = self.wmstatWriteURL)
return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def showWorkload(self, requestName):
""" Displays the workload """
self.validate(requestName)
try:
request = GetRequest.getRequestByName(requestName)
except (Exception, RuntimeError) as ex:
raise cherrypy.HTTPError(400, "Invalid request. %s" % str(ex))
request = Utilities.prepareForTable(request)
helper = Utilities.loadWorkload(request)
workloadText = str(helper.data)
return cgi.escape(workloadText).replace("\n", "<br/>\n")
def showWorkload(self, requestName):
""" Displays the workload """
self.validate(requestName)
try:
request = GetRequest.getRequestByName(requestName)
except (Exception, RuntimeError) as ex:
raise cherrypy.HTTPError(400, "Invalid request.")
request = Utilities.prepareForTable(request)
helper = Utilities.loadWorkload(request)
workloadText = str(helper.data)
return cgi.escape(workloadText).replace("\n", "<br/>\n")
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
procString = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString():
procString = helper.getProcessingString()
dashboardActivity = helper.getDashboardActivity()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage(
"Assign",
requests=[request],
teams=teams,
assignments=assignments,
sites=self.sites,
phedexNodes=self.phedexNodes,
mergedLFNBases=self.allMergedLFNBases,
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra,
procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=[],
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents,
)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
for field in ["AcquisitionEra", "ProcessingVersion"]:
if type(kwargs[field]) == dict:
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs[field], field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
helper.setSiteWildcardsLists(siteWhitelist = whiteList, siteBlacklist = blackList,
wildcardDict = self.wildcardSites)
# Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts
helper.setProcessingVersion(kwargs["ProcessingVersion"])
helper.setAcquisitionEra(kwargs["AcquisitionEra"])
#FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(int(kwargs.get("maxRSS", 2411724)),
int(kwargs.get("maxVSize", 2411724)),
int(kwargs.get("SoftTimeout", 171600)),
int(kwargs.get("GracePeriod", 300)))
# Check whether we should check location for the data
if "useSiteListAsLocation" in kwargs:
helper.setLocationDataSourceFlag()
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
if "AutoApprove" in kwargs:
autoApproveList = nonCustodialList
else:
autoApproveList = []
priority = kwargs.get("Priority", "Low")
if priority not in ["Low", "Normal", "High"]:
raise cherrypy.HTTPError(400, "Invalid subscription priority")
helper.setSubscriptionInformationWildCards(wildcardDict = self.wildcardSites,
custodialSites = custodialList,
nonCustodialSites = nonCustodialList,
autoApproveSites = autoApproveList,
priority = priority)
helper.setDashboardActivity(kwargs.get("dashboard", ""))
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
class Approve(BulkOperations):
""" Page for Physics group leaders to approve requests """
def __init__(self, config):
BulkOperations.__init__(self, config)
self.wmstatWriteURL = "%s/%s" % (config.couchUrl.rstrip('/'),
config.wmstatDBName)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def index(self):
""" Page for approving requests """
return self.draw(self.requests())
def requests(self):
""" Base list of the requests """
return Utilities.requestsWhichCouldLeadTo('assignment-approved')
def draw(self, requests):
return self.templatepage("BulkOperations",
operation="Approve",
actions=["Approve", "Reject"],
searchFields=["RequestName", "RequestType"],
requests=requests)
@cherrypy.expose
#@cherrypy.tools.secmodv2() security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleApprove(self, **kwargs):
""" Handler for approving requests """
requests = self.requestNamesFromCheckboxes(kwargs)
particple = ''
for requestName in requests:
if kwargs['action'] == 'Reject':
participle = 'rejected'
ChangeState.changeRequestStatus(requestName,
'rejected',
wmstatUrl=self.wmstatWriteURL)
else:
participle = 'approved'
ChangeState.changeRequestStatus(requestName,
'assignment-approved',
wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return self.templatepage("Acknowledge",
participle=participle,
requests=requests)
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
requestType = request["RequestType"]
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
procString = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString():
procString = helper.getProcessingString()
dashboardActivity = helper.getDashboardActivity()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage("Assign",
requests=[request],
teams=teams,
assignments=assignments,
sites=self.sites,
phedexNodes=self.phedexNodes,
mergedLFNBases=self.allMergedLFNBases,
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra,
procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=[],
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents)
def putRequest(self, requestName=None, status=None, priority=None):
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchDBName', self.configDBName)
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return request
def handleApprove(self, **kwargs):
""" Handler for approving requests """
requests = self.requestNamesFromCheckboxes(kwargs)
particple = ""
for requestName in requests:
if kwargs["action"] == "Reject":
participle = "rejected"
ChangeState.changeRequestStatus(requestName, "rejected")
else:
participle = "approved"
ChangeState.changeRequestStatus(requestName, "assignment-approved")
priority = kwargs.get(requestName + ":priority", "")
if priority != "":
Utilities.changePriority(requestName, priority)
return self.templatepage("Acknowledge", participle=participle, requests=requests)
def index(self, all=0):
""" Main page """
# returns dict of name:id
allRequests = Utilities.requestsWithStatus('assignment-approved')
teams = ProdManagement.listTeams()
procVer = ""
acqEra = ""
procString = ""
dashboardActivity = None
badRequestNames = []
goodRequests = []
reqMergedBase = None
reqUnmergedBase = None
blockCloseMaxWaitTime = 66400
blockCloseMaxFiles = 500
blockCloseMaxEvents = 250000000
blockCloseMaxSize = 5000000000000
for request in allRequests:
# make sure there's a workload attached
try:
helper = Utilities.loadWorkload(request)
except Exception, ex:
logging.error("Assign error: %s " % str(ex))
badRequestNames.append(request["RequestName"])
else:
# get defaults from the first good one
if not goodRequests:
# forget it if it fails.
try:
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString() != None:
procString = helper.getProcessingString()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime(
)
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
dashboardActivity = helper.getDashboardActivity()
goodRequests.append(request)
except Exception, ex:
logging.error("Assign error: %s " % str(ex))
badRequestNames.append(request["RequestName"])
else:
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
#Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except:
#Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, types.StringTypes):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName,
'rejected',
wmstatUrl=self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(
400, "Must assign to one or more teams")
kwargs["Teams"] = teams
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(
requestName, team, wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
participle = kwargs['action'] + 'ed'
return self.templatepage("Acknowledge",
participle=participle,
requests=requestNames)
def _getConfigCache(self, requestName, processMethod):
try:
request = Utilities.requestDetails(requestName)
except Exception, ex:
msg = "Cannot find request %s, check logs." % requestName
logging.error("%s, reason: %s" % (msg, ex))
return msg
def handleApprove(self, **kwargs):
""" Handler for approving requests """
requests = self.requestNamesFromCheckboxes(kwargs)
particple = ''
for requestName in requests:
if kwargs['action'] == 'Reject':
participle = 'rejected'
ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl = self.wmstatWriteURL)
else:
participle = 'approved'
ChangeState.changeRequestStatus(requestName, 'assignment-approved', wmstatUrl = self.wmstatWriteURL)
priority = kwargs.get(requestName+':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
return self.templatepage("Acknowledge", participle=participle,
requests=requests)
def announceRequest(self, requestName, cascade=False):
"""
Announce a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and announce them too.
"""
requestsToAnnounce = [requestName]
if cascade == "True":
requestsToAnnounce.extend(
Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToAnnounce:
Utilities.changeStatus(requestName, 'announced',
self.wmstatWriteURL, self.acdcURL)
return
def getRequest(self, requestName=None):
""" If a request name is specified, return the details of the request.
Otherwise, return an overview of all requests """
if requestName == None:
return GetRequest.getRequests()
else:
return Utilities.requestDetails(requestName)
def putCampaign(self, campaign, request=None):
""" Adds a campaign if it doesn't already exist, and optionally
associates a request with it """
if request:
requestID = GetRequest.requestID(request)
if requestID:
result = Campaign.associateCampaign(campaign, requestID)
Utilities.associateCampaign(campaign=campaign,
requestName=request,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return result
else:
return False
else:
Campaign.addCampaign(campaign)
def putRequest(self, requestName=None, status=None, priority=None):
""" Checks the request n the body with one arg, and changes the status with kwargs """
request = None
if requestName:
request = self.findRequest(requestName)
if request == None:
""" Creates a new request, with a JSON-encoded schema that is sent in the
body of the request """
body = cherrypy.request.body.read()
schema = Utilities.unidecode(JsonWrapper.loads(body))
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
try:
request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName)
except RuntimeError, ex:
raise cherrypy.HTTPError(400, ex.message)
def announceRequest(self, requestName, cascade = False):
"""
Announce a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and announce them too.
"""
requestsToAnnounce = [requestName]
if cascade == "True":
requestsToAnnounce.extend(Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToAnnounce:
Utilities.changeStatus(requestName, 'announced',
self.wmstatWriteURL,
self.acdcURL)
return
def index(self):
""" Main web page for creating requests """
# Get the scram Architecture from the keys and the
# CMSSW versions from the values
self.scramArchs = SoftwareAdmin.listSoftware().keys()
versionLists = SoftwareAdmin.listSoftware().values()
self.versions = []
for l in versionLists:
for v in l:
if not v in self.versions:
self.versions.append(v)
self.versions.sort()
# see if this was configured with a hardcoded user. If not, take from the request header
requestor = self.requestor
if not requestor:
requestor = cherrypy.request.user["login"]
if not requestor:
return "No username found in your certificate"
if not requestor in Registration.listUsers():
return "User %s is not registered. Contact a ReqMgr administrator." % requestor
groups = GroupInfo.groupsForUser(requestor).keys()
if groups == []:
return "User " + requestor + " is not in any groups. Contact a ReqMgr administrator."
campaigns = Campaign.listCampaigns()
return self.templatepage("WebRequestSchema",
yuiroot=self.yuiroot,
requestor=requestor,
groups=groups,
versions=self.versions,
archs=self.scramArchs,
alldocs=Utilities.unidecode(self.allDocs()),
allcampaigns=campaigns,
defaultVersion=self.cmsswVersion,
defaultArch=self.defaultArch,
defaultSkimConfig=self.defaultSkimConfig)
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask(performance = False)
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
def putCampaign(self, campaign, request=None):
""" Adds a campaign if it doesn't already exist, and optionally
associates a request with it """
if request:
requestID = GetRequest.requestID(request)
if requestID:
result = Campaign.associateCampaign(campaign, requestID)
Utilities.associateCampaign(campaign = campaign,
requestName = request,
couchURL = self.couchUrl,
couchDBName = self.workloadDBName)
return result
else:
return False
else:
Campaign.addCampaign(campaign)
def _getConfigCache(self, requestName, processMethod):
try:
request = Utilities.requestDetails(requestName)
except Exception, ex:
msg = "Cannot find request %s, check logs." % requestName
logging.error("%s, reason: %s" % (msg, ex))
return msg
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask()
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
def index(self):
""" Main web page for creating requests """
# Get the scram Architecture from the keys and the
# CMSSW versions from the values
self.scramArchs = SoftwareAdmin.listSoftware().keys()
versionLists = SoftwareAdmin.listSoftware().values()
self.versions = []
for l in versionLists:
for v in l:
if not v in self.versions:
self.versions.append(v)
self.versions.sort()
# see if this was configured with a hardcoded user. If not, take from the request header
requestor = self.requestor
if not requestor:
requestor = cherrypy.request.user["login"]
if not requestor:
return "No username found in your certificate"
if not requestor in Registration.listUsers():
return "User %s is not registered. Contact a ReqMgr administrator." % requestor
groups = GroupInfo.groupsForUser(requestor).keys()
if groups == []:
return "User " + requestor + " is not in any groups. Contact a ReqMgr administrator."
campaigns = Campaign.listCampaigns()
return self.templatepage("WebRequestSchema", yuiroot=self.yuiroot,
requestor=requestor,
groups=groups,
versions=self.versions,
archs = self.scramArchs,
alldocs = Utilities.unidecode(self.allDocs()),
allcampaigns = campaigns,
defaultVersion=self.cmsswVersion,
defaultSkimConfig=self.defaultSkimConfig)
def index(self, all=0):
""" Main page """
# returns dict of name:id
allRequests = Utilities.requestsWithStatus("assignment-approved")
teams = ProdManagement.listTeams()
procVer = ""
acqEra = ""
procString = ""
dashboardActivity = None
badRequestNames = []
goodRequests = []
reqMergedBase = None
reqUnmergedBase = None
blockCloseMaxWaitTime = 66400
blockCloseMaxFiles = 500
blockCloseMaxEvents = 250000000
blockCloseMaxSize = 5000000000000
for request in allRequests:
# make sure there's a workload attached
try:
helper = Utilities.loadWorkload(request)
except Exception, ex:
logging.error("Assign error: %s " % str(ex))
badRequestNames.append(request["RequestName"])
else:
# get defaults from the first good one
if not goodRequests:
# forget it if it fails.
try:
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString() != None:
procString = helper.getProcessingString()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
dashboardActivity = helper.getDashboardActivity()
goodRequests.append(request)
except Exception, ex:
logging.error("Assign error: %s " % str(ex))
badRequestNames.append(request["RequestName"])
else:
def details(self, requestName):
""" A page showing the details for the requests """
self.validate(requestName)
try:
request = Utilities.requestDetails(requestName)
except AssertionError:
raise cherrypy.HTTPError(404, "Cannot load request %s" % requestName)
adminHtml = statusMenu(requestName, request['RequestStatus']) \
+ ' Priority ' + Utilities.priorityMenu(request)
return self.templatepage("Request", requestName=requestName,
detailsFields=self.detailsFields,
requestSchema=request,
docId=request.get('ProcConfigCacheID', None),
assignments=request['Assignments'],
adminHtml=adminHtml,
messages=request['RequestMessages'],
updateDictList=request['RequestUpdates'])
def closeOutRequest(self, requestName, cascade=False):
"""
Close out a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and close them out
too.
"""
requestsToCloseOut = [requestName]
if cascade == "True":
requestsToCloseOut.extend(
Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToCloseOut:
Utilities.changeStatus(requestName, 'closed-out',
self.wmstatWriteURL, self.acdcURL)
return
def getMostRecentOutputForPrepID(self, prepID):
"""Return the datasets produced by the most recently submitted request with this prep ID"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
# most recent will have the largest ID
requestID = max(requestIDs)
request = GetRequest.getRequest(requestID)
helper = Utilities.loadWorkload(request)
return helper.listOutputDatasets()
def closeOutRequest(self, requestName, cascade = False):
"""
Close out a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and close them out
too.
"""
requestsToCloseOut = [requestName]
if cascade == "True":
requestsToCloseOut.extend(Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToCloseOut:
Utilities.changeStatus(requestName, 'closed-out',
self.wmstatWriteURL,
self.acdcURL)
return
def details(self, requestName):
""" A page showing the details for the requests """
self.validate(requestName)
try:
request = Utilities.requestDetails(requestName)
except AssertionError:
raise cherrypy.HTTPError(404, "Cannot load request %s" % requestName)
adminHtml = statusMenu(requestName, request['RequestStatus']) \
+ ' Priority: ' + Utilities.priorityMenu(request)
return self.templatepage("Request", requestName=requestName,
detailsFields=self.detailsFields,
requestSchema=request,
docId=request.get('ConfigCacheID', None),
assignments=request['Assignments'],
adminHtml=adminHtml,
messages=request['RequestMessages'],
updateDictList=request['RequestUpdates'])
def testA_SiteWhitelist(self):
"""
_SiteWhitelist_
Test to see if we can get the siteWhitelist to work properly.
"""
secconfig = getattr(self.config, "SecurityModule")
cherrypy.server.environment = 'test'
cherrypy.tools.secmodv2 = NullAuth(secconfig)
self.config.UnitTests.views.active.rest.templates = 'templateDir'
self.config.UnitTests.views.active.rest.yuiroot = 'yuiroot'
self.config.UnitTests.views.active.rest.wildcardKeys = {
'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*',
'US*': '_US_'
}
from WMCore.HTTPFrontEnd.RequestManager.Assign import Assign
assign = Assign(config=self.config.UnitTests.views.active.rest,
noSiteDB=True)
siteList = [
'T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL', 'T2_US_UCSD', 'T2_US_UNL',
'T2_US_CIT'
]
self.assertEqual(assign.sites, [])
assign.sites.extend(siteList)
Utilities.addSiteWildcards(assign.wildcardKeys, assign.sites,
assign.wildcardSites)
for s in siteList:
self.assertTrue(s in assign.sites)
self.assertTrue('T1*' in assign.sites)
self.assertTrue('T2*' in assign.sites)
self.assertFalse('T3*' in assign.sites)
self.assertTrue('US*' in assign.sites)
self.assertEqual(assign.wildcardSites['T1*'],
['T1_US_FNAL', 'T1_CH_CERN', 'T1_UK_RAL'])
self.assertEqual(assign.wildcardSites['T2*'],
['T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
self.assertEqual(
assign.wildcardSites['US*'],
['T1_US_FNAL', 'T2_US_UCSD', 'T2_US_UNL', 'T2_US_CIT'])
def __init__(self, config, noSiteDB = False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.clipboardDB = config.clipboardDB
cleanUrl = Utilities.removePasswordFromUrl(self.couchUrl)
self.clipboardUrl = "%s/%s/_design/OpsClipboard/index.html" % (cleanUrl, self.clipboardDB)
self.opshold = config.opshold
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
if not noSiteDB:
self.sites = Utilities.sites(config.sitedb)
else:
self.sites = []
self.allMergedLFNBases = [
"/store/backfill/1", "/store/backfill/2",
"/store/data", "/store/mc", "/store/generator", "/store/relval"]
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.mergedLFNBases = {
"ReReco" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
"DataProcessing" : ["/store/backfill/1", "/store/backfill/2", "/store/data"],
"ReDigi" : ["/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc"],
"MonteCarlo" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"RelValMC" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"Resubmission" : ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data"],
"MonteCarloFromGEN" : ["/store/backfill/1", "/store/backfill/2", "/store/mc"],
"TaskChain": ["/store/backfill/1", "/store/backfill/2", "/store/mc", "/store/data", "/store/relval"],
"LHEStepZero": ["/store/backfill/1", "/store/backfill/2", "/store/generator"]}
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def makeSchema(self, **schema):
schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
schema.setdefault('CouchDBName', self.configDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
request = Utilities.makeRequest(decodedSchema, self.couchUrl, self.workloadDBName)
except RuntimeError, e:
raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
def makeSchema(self, **schema):
schema.setdefault('CouchURL',
Utilities.removePasswordFromUrl(self.couchUrl))
# wrong naming ... but it's all over the place, it's the config cache DB name
schema.setdefault('CouchDBName', self.configDBName)
schema.setdefault('CouchWorkloadDBName', self.workloadDBName)
decodedSchema = {}
for key in schema.keys():
try:
decodedSchema[key] = JsonWrapper.loads(schema[key])
except:
# We don't know what kind of exception we'll get, so ignore them all
# If it does except, it probably wasn't in JSON to begin with.
# Anything else should be caught by the parsers and the validation
decodedSchema[key] = schema[key]
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(decodedSchema, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, decodedSchema, self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
# catching here KeyError is just terrible
except (RuntimeError, KeyError, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % str(ex))
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
baseURL = cherrypy.request.base
raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' %
(baseURL, request['RequestName']))
def getOutputForPrepID(self, prepID):
"""Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
result = {}
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listOutputDatasets()
return result
def getOutputForPrepID(self, prepID):
"""Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
result = {}
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listOutputDatasets()
return result
def doAdmin(self, **kwargs):
""" format of kwargs is {'requestname:status' : 'approved', 'requestname:priority' : '2'} """
message = ""
for k, v in kwargs.iteritems():
if k.endswith(':status'):
requestName = k.split(':')[0]
self.validate(requestName)
status = v
priority = kwargs[requestName+':priority']
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
message += "Changed priority for %s to %s.\n" % (requestName, priority)
if status != "":
Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL)
message += "Changed status for %s to %s\n" % (requestName, status)
if status == "assigned":
# make a page to choose teams
raise cherrypy.HTTPRedirect('/reqmgr/assign/one/%s' % requestName)
return message + detailsBackLink(requestName)
def validateStats(self, index):
""" Check the values for the updates """
if 'stats' not in index:
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in ['input_lumis', 'input_num_files',
'input_events', 'total_jobs']:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
def modifyWorkload(self, requestName, workload,
CMSSWVersion=None, GlobalTag=None,
runWhitelist=None, runBlacklist=None,
blockWhitelist=None, blockBlacklist=None,
ScramArch=None):
""" handles the "Modify" button of the details page """
self.validate(requestName)
helper = WMWorkloadHelper()
helper.load(workload)
schema = helper.data.request.schema
message = ""
if runWhitelist != "" and runWhitelist != None:
l = Utilities.parseRunList(runWhitelist)
helper.setRunWhitelist(l)
schema.RunWhitelist = l
message += 'Changed runWhiteList to %s<br>' % l
if runBlacklist != "" and runBlacklist != None:
l = Utilities.parseRunList(runBlacklist)
helper.setRunBlacklist(l)
schema.RunBlacklist = l
message += 'Changed runBlackList to %s<br>' % l
if blockWhitelist != "" and blockWhitelist != None:
l = Utilities.parseBlockList(blockWhitelist)
helper.setBlockWhitelist(l)
schema.BlockWhitelist = l
message += 'Changed blockWhiteList to %s<br>' % l
if blockBlacklist != "" and blockBlacklist != None:
l = Utilities.parseBlockList(blockBlacklist)
helper.setBlockBlacklist(l)
schema.BlockBlacklist = l
message += 'Changed blockBlackList to %s<br>' % l
Utilities.saveWorkload(helper, workload)
return message + detailsBackLink(requestName)
def testD_AllSoftwareVersions(self):
"""
_AllSoftwareVersions_
This test is a bit weird because it just checks to make sure you're getting versions
with CMSSW in the name. I don't want to do specifics because software versions
change all the time.
"""
result = ReqMgrWebTools.allSoftwareVersions()
self.assertTrue(len(result) > 0)
for ver in result:
self.assertTrue('CMSSW' in ver)
def handleApprove(self, **kwargs):
""" Handler for approving requests """
requests = self.requestNamesFromCheckboxes(kwargs)
particple = ''
for requestName in requests:
if kwargs['action'] == 'Reject':
participle = 'rejected'
ChangeState.changeRequestStatus(requestName,
'rejected',
wmstatUrl=self.wmstatWriteURL)
else:
participle = 'approved'
ChangeState.changeRequestStatus(requestName,
'assignment-approved',
wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return self.templatepage("Acknowledge",
participle=participle,
requests=requests)
def getRequest(self, requestName=None):
""" If a request name is specified, return the details of the request.
Otherwise, return an overview of all requests """
if requestName == None:
result = GetRequest.getRequests()
else:
result = Utilities.requestDetails(requestName)
try:
teamNames = GetRequest.getAssignmentsByName(requestName)
result['teams'] = teamNames
except:
# Ignore errors, then we just don't have a team name
pass
return result
def getConfigIDs(self, prim, proc, tier):
"""
_getConfigIDs_
Get the ConfigIDs for the specified request
"""
result = {}
dataset = self.getDataset(prim, proc, tier)
requests = GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput", dataset)
for request in requests:
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listAllCMSSWConfigCacheIDs()
return result
