class Approve(BulkOperations):
""" Page for Physics group leaders to approve requests """
def __init__(self, config):
BulkOperations.__init__(self, config)
self.wmstatWriteURL = "%s/%s" % (config.couchUrl.rstrip('/'),
config.wmstatDBName)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def index(self):
""" Page for approving requests """
return self.draw(self.requests())
def requests(self):
""" Base list of the requests """
return Utilities.requestsWhichCouldLeadTo('assignment-approved')
def draw(self, requests):
return self.templatepage("BulkOperations",
operation="Approve",
actions=["Approve", "Reject"],
searchFields=["RequestName", "RequestType"],
requests=requests)
@cherrypy.expose
#@cherrypy.tools.secmodv2() security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleApprove(self, **kwargs):
""" Handler for approving requests """
requests = self.requestNamesFromCheckboxes(kwargs)
particple = ''
for requestName in requests:
if kwargs['action'] == 'Reject':
participle = 'rejected'
ChangeState.changeRequestStatus(requestName,
'rejected',
wmstatUrl=self.wmstatWriteURL)
else:
participle = 'approved'
ChangeState.changeRequestStatus(requestName,
'assignment-approved',
wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
return self.templatepage("Acknowledge",
participle=participle,
requests=requests)
return index
def validateStats(self, index):
""" Check the values for the updates """
if not index.has_key('stats'):
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in ['input_lummis', 'input_num_files',
'input_events', 'total_jobs']:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
# had no permission control before, security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups())
def deleteRequest(self, requestName):
"""
Deletes a request from the ReqMgr MySQL/Oracle database
and also from CoucDB.
"""
# 404 will be thrown automatically on a non-existing request
request = self.getRequest(requestName)
helper = Utilities.loadWorkload(request)
couchDocId = requestName
helper.deleteCouch(self.couchUrl, self.workloadDBName, couchDocId)
# #4289 - Request delete operation deletes the request from
# MySQL/Oracle but not from CouchDB, fix here
# Seangchan shall also fix here deleting such requests from WMStats (#4398)
class ReqMgrBrowser(WebAPI):
""" For browsing and modifying requests: Disabled use WMStats instead """
def __init__(self, config):
WebAPI.__init__(self, config)
# Take a guess
self.templatedir = config.templates
self.fields = ['RequestName', 'Group', 'Requestor', 'RequestType',
"RequestPriority", 'RequestStatus', 'Complete', 'Success']
self.calculatedFields = {'Written': 'percentWritten', 'Merged':'percentMerged',
'Complete':'percentComplete', 'Success' : 'percentSuccess'}
# entries in the table that show up as HTML links for that entry
self.linkedFields = {'Group': '../admin/group',
'Requestor': '../admin/user',
'RequestName': 'details'}
self.detailsFields = ['RequestName', 'RequestType', 'Requestor', 'CMSSWVersion',
'ScramArch', 'GlobalTag', 'RequestNumEvents',
'InputDataset', 'PrimaryDataset', 'AcquisitionEra', 'ProcessingVersion',
'RunWhitelist', 'RunBlacklist', 'BlockWhitelist', 'BlockBlacklist',
'RequestWorkflow', 'Scenario', 'Campaign', 'PrimaryDataset',
'Acquisition Era', 'Processing Version', 'Merged LFN Base', 'Unmerged LFN Base',
'Site Whitelist', 'Site Blacklist']
self.adminMode = True
self.adminFields = {}
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.yuiroot = config.yuiroot
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName)
cherrypy.engine.subscribe('start_thread', self.initThread)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def validate(self, v, name=''):
""" Checks if alphanumeric, tolerating spaces """
try:
WMCore.Lexicon.identifier(v)
except AssertionError as ex:
raise cherrypy.HTTPError(400, "Bad input: %s" % str(ex))
return v
@cherrypy.expose
@cherrypy.tools.secmodv2()
def search(self, value, field):
""" Search for a regular expression in a certain field of all requests """
filteredRequests = []
requests = GetRequest.getRequests()
for request in requests:
if request[field].find(value) != -1:
filteredRequests.append(request)
requests = filteredRequests
tableBody = self.drawRequests(requests)
return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot,
fields=self.fields, tableBody=tableBody)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def index(self):
requests = GetRequest.getRequests()
tableBody = self.drawRequests(requests)
#tableBody = []
return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot,
fields=self.fields, tableBody=tableBody)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def splitting(self, requestName):
"""
_splitting_
Retrieve the current values for splitting parameters for all tasks in
the spec. Format them in the manner that the splitting page expects
and pass them to the template.
"""
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
splittingDict = helper.listJobSplittingParametersByTask(performance = False)
taskNames = splittingDict.keys()
taskNames.sort()
splitInfo = []
for taskName in taskNames:
jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName])
splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"],
"splitParams": jsonSplittingParams,
"taskType": splittingDict[taskName]["type"],
"taskName": taskName})
return self.templatepage("Splitting", requestName = requestName,
taskInfo = splitInfo, taskNames = taskNames)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def handleSplittingPage(self, requestName, splittingTask, splittingAlgo,
**submittedParams):
"""
_handleSplittingPage_
Parse job splitting parameters sent from the splitting parameter update
page. Pull down the request and modify the new spec applying the
updated splitting parameters.
"""
splitParams = {}
if splittingAlgo == "FileBased":
splitParams["files_per_job"] = int(submittedParams["files_per_job"])
elif splittingAlgo == "TwoFileBased":
splitParams["files_per_job"] = int(submittedParams["two_files_per_job"])
elif splittingAlgo == "LumiBased":
splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"])
if str(submittedParams["halt_job_on_file_boundaries"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventAwareLumiBased":
splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"])
splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"])
if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True":
splitParams["halt_job_on_file_boundaries"] = True
else:
splitParams["halt_job_on_file_boundaries"] = False
elif splittingAlgo == "EventBased":
splitParams["events_per_job"] = int(submittedParams["events_per_job"])
if "events_per_lumi" in submittedParams:
splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"])
if "lheInputFiles" in submittedParams:
if str(submittedParams["lheInputFiles"]) == "True":
splitParams["lheInputFiles"] = True
else:
splitParams["lheInputFiles"] = False
elif splittingAlgo == "Harvest":
splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"])
elif 'Merg' in splittingTask:
for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']:
splitParams[field] = int(submittedParams[field])
if "include_parents" in submittedParams.keys():
if str(submittedParams["include_parents"]) == "True":
splitParams["include_parents"] = True
else:
splitParams["include_parents"] = False
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams))
helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams)
Utilities.saveWorkload(helper, request['RequestWorkflow'])
return "Successfully updated splitting parameters for " + splittingTask \
+ " " + detailsBackLink(requestName)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def details(self, requestName):
""" A page showing the details for the requests """
self.validate(requestName)
try:
request = Utilities.requestDetails(requestName)
except AssertionError:
raise cherrypy.HTTPError(404, "Cannot load request %s" % requestName)
adminHtml = statusMenu(requestName, request['RequestStatus']) \
+ ' Priority: ' + Utilities.priorityMenu(request)
return self.templatepage("Request", requestName=requestName,
detailsFields=self.detailsFields,
requestSchema=request,
docId=request.get('ConfigCacheID', None),
assignments=request['Assignments'],
adminHtml=adminHtml,
messages=request['RequestMessages'],
updateDictList=request['RequestUpdates'])
def _getConfigCache(self, requestName, processMethod):
try:
request = Utilities.requestDetails(requestName)
except Exception as ex:
msg = "Cannot find request %s, check logs." % requestName
logging.error("%s, reason: %s" % (msg, ex))
return msg
url = request.get("ConfigCacheUrl", None) or self.couchUrl
try:
configCache = ConfigCache(url, self.configDBName)
configDocId = request["ConfigCacheID"]
configCache.loadByID(configDocId)
except Exception as ex:
msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url)
logging.error("%s, reason: %s" % (msg, ex))
return msg
return getattr(configCache, processMethod)()
@cherrypy.expose
@cherrypy.tools.secmodv2()
def showOriginalConfig(self, requestName):
"""
Makes a link to the original text of the config document.
"""
self.validate(requestName)
return '<pre>' + self._getConfigCache(requestName, "getConfig") + '</pre>'
@cherrypy.expose
@cherrypy.tools.secmodv2()
def showTweakFile(self, requestName):
"""
Makes a link to the dump of the tweakfile.
"""
self.validate(requestName)
tweakString = self._getConfigCache(requestName, "getPSetTweaks")
return str(tweakString).replace('\n', '<br>')
@cherrypy.expose
@cherrypy.tools.secmodv2()
def showWorkload(self, requestName):
""" Displays the workload """
self.validate(requestName)
try:
request = GetRequest.getRequestByName(requestName)
except (Exception, RuntimeError) as ex:
raise cherrypy.HTTPError(400, "Invalid request. %s" % str(ex))
request = Utilities.prepareForTable(request)
helper = Utilities.loadWorkload(request)
workloadText = str(helper.data)
return cgi.escape(workloadText).replace("\n", "<br/>\n")
def drawRequests(self, requests):
""" Display all requests """
result = ""
for request in requests:
# see if this is slower
result += self.drawRequest(request)
return result
def drawRequest(self, request):
""" make a table row with information from this request """
html = '<tr>'
requestName = request['RequestName']
for field in self.fields:
# hanole any fields that have functions linked to them
entry = cgi.escape(str(request.get(field, '')))
if field in self.calculatedFields:
method = getattr(self, self.calculatedFields[field])
entry = method(request)
elif self.adminMode and field in self.adminFields:
method = getattr(self, self.adminFields[field])
entry = method(requestName, value)
elif field in self.linkedFields:
entry = linkedTableEntry(self.linkedFields[field], entry)
html += '<td>%s</td>' % entry
html += '</tr>\n'
return html
def percentWritten(self, request):
""" Finds the biggest percentage among all the updates """
maxPercent = 0
for update in request["RequestUpdates"]:
if "events_written" in update and request["RequestNumEvents"] != 0:
percent = update["events_written"] / request["RequestNumEvents"]
if percent > maxPercent:
maxPercent = percent
if "files_written" in update and request["RequestSizeFiles"] != 0:
percent = update["files_written"] / request["RequestSizeFiles"]
if percent > maxPercent:
maxPercent = percent
return "%i%%" % maxPercent
def percentMerged(self, request):
""" Finds the biggest percentage among all the updates """
maxPercent = 0
for update in request["RequestUpdates"]:
if "events_merged" in update and request["RequestNumEvents"] != 0:
percent = update["events_merged"] / request["RequestNumEvents"]
if percent > maxPercent:
maxPercent = percent
if "files_merged" in update and request["RequestSizeFiles"] != 0:
percent = update["files_merged"] / request["RequestSizeFiles"]
if percent > maxPercent:
maxPercent = percent
return "%i%%" % maxPercent
def percentComplete(self, request):
pct = request.get('percent_complete', 0)
return "%i%%" % pct
def percentSuccess(self, request):
pct = request.get('percent_success', 0)
return "%i%%" % pct
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups())
def doAdmin(self, **kwargs):
""" format of kwargs is {'requestname:status' : 'approved', 'requestname:priority' : '2'} """
message = ""
for k, v in kwargs.iteritems():
if k.endswith(':status'):
requestName = k.split(':')[0]
self.validate(requestName)
status = v
priority = kwargs[requestName+':priority']
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
message += "Changed priority for %s to %s.\n" % (requestName, priority)
if status != "":
Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL)
message += "Changed status for %s to %s\n" % (requestName, status)
if status == "assigned":
# make a page to choose teams
raise cherrypy.HTTPRedirect('/reqmgr/assign/one/%s' % requestName)
return message + detailsBackLink(requestName)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups())
# FIXME needs to check if authorized, or original user
def modifyWorkload(self, requestName, workload,
CMSSWVersion=None, GlobalTag=None,
runWhitelist=None, runBlacklist=None,
blockWhitelist=None, blockBlacklist=None,
ScramArch=None):
""" handles the "Modify" button of the details page """
self.validate(requestName)
helper = WMWorkloadHelper()
helper.load(workload)
schema = helper.data.request.schema
message = ""
if runWhitelist != "" and runWhitelist != None:
l = Utilities.parseRunList(runWhitelist)
helper.setRunWhitelist(l)
schema.RunWhitelist = l
message += 'Changed runWhiteList to %s<br>' % l
if runBlacklist != "" and runBlacklist != None:
l = Utilities.parseRunList(runBlacklist)
helper.setRunBlacklist(l)
schema.RunBlacklist = l
message += 'Changed runBlackList to %s<br>' % l
if blockWhitelist != "" and blockWhitelist != None:
l = Utilities.parseBlockList(blockWhitelist)
helper.setBlockWhitelist(l)
schema.BlockWhitelist = l
message += 'Changed blockWhiteList to %s<br>' % l
if blockBlacklist != "" and blockBlacklist != None:
l = Utilities.parseBlockList(blockBlacklist)
helper.setBlockBlacklist(l)
schema.BlockBlacklist = l
message += 'Changed blockBlackList to %s<br>' % l
Utilities.saveWorkload(helper, workload)
return message + detailsBackLink(requestName)
def group(self, groupName):
""" Web page of details about the group."""
self.validate(groupName)
users = GroupInfo.usersInGroup(groupName)
return self.templatepage("Group", group=groupName, users=users)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def users(self):
""" Lists all users. Should be paginated later """
allUsers = Registration.listUsers()
self.validate(allUsers)
return self.templatepage("Users", users=allUsers)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups())
def handleAddUser(self, user, email=None):
self.validate(user)
Registration.registerUser(user, email)
return "Added user %s" % user
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups())
def handleAddToGroup(self, user, group):
""" Adds a user to the group """
self.validate(user)
self.validate(group)
GroupManagement.addUserToGroup(user, group)
return "Added %s to %s " % (user, group)
@cherrypy.expose
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra,
procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=badRequestNames,
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents)
@cherrypy.expose
#@cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles) security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
#Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = JsonWrapper.loads(kwargs[key])
except:
#Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
class ReqMgrRESTModel(RESTModel):
""" The REST interface to the ReqMgr database. Documentation may
be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """
def __init__(self, config):
RESTModel.__init__(self, config)
self.couchUrl = config.couchUrl
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'),
config.wmstatDBName)
self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName)
self.security_params = {'roles': config.security_roles}
# Optional values for individual methods
self.reqPriorityMax = getattr(config, 'maxReqPriority', 100)
self._addMethod('GET',
'request',
self.getRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'assignment',
self.getAssignment,
args=['teamName', 'request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'user',
self.getUser,
args=['userName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'group',
self.getGroup,
args=['group', 'user'],
secured=True,
expires=0)
self._addMethod('GET',
'version',
self.getVersion,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'team',
self.getTeam,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'workQueue',
self.getWorkQueue,
args=['request', 'workQueue'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'message',
self.getMessage,
args=['request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'inputdataset',
self.getInputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'outputdataset',
self.getOutputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'campaign',
self.getCampaign,
args=['campaign'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('PUT',
'request',
self.putRequest,
args=['requestName', 'status', 'priority', 'stats'],
secured=True,
validation=[
self.isalnumExceptStats, self.reqPriority,
self.validateStats
])
self._addMethod('PUT',
'clone',
self.cloneRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum])
self._addMethod('PUT',
'assignment',
self.putAssignment,
args=['team', 'requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'user',
self.putUser,
args=['userName', 'email', 'dnName'],
secured=True,
security_params=self.security_params,
validation=[self.validateUser])
self._addMethod('PUT',
'group',
self.putGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'version',
self.putVersion,
args=['version', 'scramArch'],
secured=True,
security_params=self.security_params,
validation=[self.validateVersion])
self._addMethod('PUT',
'team',
self.putTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'workQueue',
self.putWorkQueue,
args=['request', 'url'],
secured=True,
security_params=self.security_params,
validation=[self.validatePutWorkQueue])
self._addMethod('PUT',
'message',
self.putMessage,
args=['request'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'campaign',
self.putCampaign,
args=['campaign', 'request'],
secured=True,
validation=[self.isalnum])
self._addMethod('POST',
'request',
self.postRequest,
args=[
'requestName', 'events_written', 'events_merged',
'files_written', 'files_merged', 'percent_written',
'percent_success', 'dataset'
],
secured=True,
validation=[self.validateUpdates])
self._addMethod('POST',
'closeout',
self.closeOutRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('POST',
'announce',
self.announceRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('DELETE',
'request',
self.deleteRequest,
args=['requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'user',
self.deleteUser,
args=['user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'group',
self.deleteGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'version',
self.deleteVersion,
args=['version', 'scramArch'],
secured=True,
validation=[self.validateVersion])
self._addMethod('DELETE',
'team',
self.deleteTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'campaign',
self.deleteCampaign,
args=['campaign'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('GET',
'requestnames',
self.getRequestNames,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'outputDatasetsByRequestName',
self.getOutputForRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'outputDatasetsByPrepID',
self.getOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'mostRecentOutputDatasetsByPrepID',
self.getMostRecentOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'configIDs',
self.getConfigIDs,
args=['prim', 'proc', 'tier'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'requestsByStatusAndTeam',
self.getRequestsByStatusAndTeam,
args=['teamName', 'status'],
secured=True,
validation=[self.isalnum],
expires=0)
cherrypy.engine.subscribe('start_thread', self.initThread)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def isalnum(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
for v in index.values():
WMCore.Lexicon.identifier(v)
return index
def isalnumExceptStats(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
if index.has_key('stats'):
return index
return self.isalnum(index)
def getDataset(self, prim, proc, tier):
""" If only prim exists, assume it's urlquoted.
If all three exists, assue it's /prim/proc/tier
"""
if not proc and not tier:
dataset = urllib.unquote(prim)
elif prim and proc and tier:
dataset = "/%s/%s/%s" % (prim, proc, tier)
WMCore.Lexicon.dataset(dataset)
return dataset
def intpriority(self, index):
""" Casts priority to an integer """
if index.has_key('priority'):
value = int(index['priority'])
if math.fabs(value) >= sys.maxint:
msg = "Invalid priority! Priority must have abs() less then MAXINT!"
raise cherrypy.HTTPError(400, msg)
index['priority'] = value
return index
def reqPriority(self, index):
"""
_reqPriority_
Sets request priority to an integer.
Also makes sure it's within a certain value.
"""
if not index.has_key('priority'):
return index
index = self.intpriority(index=index)
value = index['priority']
if math.fabs(
value) > self.reqPriorityMax and not Utilities.privileged():
msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax
raise cherrypy.HTTPError(400, msg)
return index
def validateUser(self, index):
assert index['userName'].isalnum()
assert '@' in index['email']
assert index['email'].replace('@', '').replace('.', '').isalnum()
if 'dnName' in index:
assert index['dnName'].replace(' ', '').isalnum()
return index
def validateVersion(self, index):
""" Make sure it's a legitimate CMSSW version format """
WMCore.Lexicon.cmsswversion(index['version'])
return index
def getRequest(self, requestName=None):
""" If a request name is specified, return the details of the request.
Otherwise, return an overview of all requests """
if requestName == None:
result = GetRequest.getRequests()
else:
result = Utilities.requestDetails(requestName)
try:
teamNames = GetRequest.getAssignmentsByName(requestName)
result['teams'] = teamNames
except:
# Ignore errors, then we just don't have a team name
pass
return result
def getRequestNames(self):
# 2013-02-13 is wrong anyway (e.g. Campaign is not working)
# should be removed
""" return all the request names in RequestManager as list """
#TODO this could me combined with getRequest
return GetRequest.getOverview()
def getOutputForRequest(self, requestName):
"""Return the datasets produced by this request."""
return Utilities.getOutputForRequest(requestName)
def getOutputForPrepID(self, prepID):
"""Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
result = {}
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listOutputDatasets()
return result
def getMostRecentOutputForPrepID(self, prepID):
"""Return the datasets produced by the most recently submitted request with this prep ID"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
# most recent will have the largest ID
requestIDs.sort()
requestIDs.reverse()
request = None
# Go through each request in order from largest to smallest
# looking for the first non-failed/non-canceled request
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
rejectList = ['aborted', 'failed', 'rejected', 'epic-failed']
requestStatus = request.get("RequestStatus", 'aborted').lower()
if requestStatus not in rejectList:
break
if request != None:
helper = Utilities.loadWorkload(request)
return helper.listOutputDatasets()
else:
return []
def getAssignment(self, teamName=None, request=None):
""" If a team name is passed in, get all assignments for that team.
If a request is passed in, return a list of teams the request is
assigned to
"""
# better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve?
#requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName)
# Maybe now assigned to team is same as assigned to ProdMgr
result = []
if teamName != None:
requestIDs = ListRequests.listRequestsByTeam(teamName,
"assigned").values()
requests = [GetRequest.getRequest(reqID) for reqID in requestIDs]
# put highest priorities first
requests.sort(key=lambda r: r['RequestPriority'], reverse=True)
# return list of tuples, since we need sorting
result = [[req['RequestName'], req['RequestWorkflow']]
for req in requests]
elif request != None:
result = GetRequest.getAssignmentsByName(request)
return result
def getRequestsByStatusAndTeam(self, teamName, status):
""" Get a list of request names with the given team and status.
"""
requestNames = ListRequests.listRequestsByTeam(teamName, status).keys()
return requestNames
def getUser(self, userName=None, group=None):
""" No args returns a list of all users. Group returns groups this user is in. Username
returs a JSON with information about the user """
if userName != None:
if not Registration.isRegistered(userName):
raise cherrypy.HTTPError(404, "Cannot find user")
result = {}
result['groups'] = GroupInfo.groupsForUser(userName).keys()
result['requests'] = UserRequests.listRequests(userName).keys()
result.update(Registration.userInfo(userName))
return result
elif group != None:
GroupInfo.usersInGroup(group)
else:
return Registration.listUsers()
def getGroup(self, group=None, user=None):
""" No args lists all groups, one args returns JSON with users."""
if group != None:
result = {}
result['users'] = GroupInfo.usersInGroup(group)
return result
elif user != None:
return GroupInfo.groupsForUser(user).keys()
else:
return GroupInfo.listGroups()
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
archList = SoftwareAdmin.listSoftware()
result = []
for arch in archList.keys():
for version in archList[arch]:
if not version in result:
result.append(version)
return result
def getTeam(self):
""" Returns a list of all teams registered with ReqMgr """
return ProdManagement.listTeams()
def getWorkQueue(self, request=None, workQueue=None):
""" If a request is passed in, return the URl of the workqueue.
If a workqueue is passed in, return all requests acquired by it """
if workQueue != None:
return ProdMgrRetrieve.findAssignedRequests(workQueue)
if request != None:
return ProdManagement.getProdMgr(request)
# return all the workqueue ulr
return GetRequest.getGlobalQueues()
def getMessage(self, request):
""" Returns a list of messages attached to this request """
return ChangeState.getMessages(request)
def getInputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this input dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput",
dataset)
def getOutputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this output dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput",
dataset)
def getCampaign(self, campaign=None):
""" returns a list of all campaigns if no argument, and a list of
all requests in a campaign if there is an argument """
if campaign == None:
return Campaign.listCampaigns()
else:
return Campaign.listRequestsByCampaign(campaign)
def putWorkQueue(self, request, url):
""" Registers the request as "acquired" by the workqueue with the given URL """
Utilities.changeStatus(request, "acquired", self.wmstatWriteURL,
self.acdcURL)
return ProdManagement.associateProdMgr(request, urllib.unquote(url))
def validatePutWorkQueue(self, index):
assert index['request'].replace('_',
'').replace('-',
'').replace('.',
'').isalnum()
assert index['url'].startswith('http')
return index
# had no permission control before, security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def putRequest(self,
requestName=None,
status=None,
priority=None,
stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception, ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = (
"Creating request failed, unsupported input arg: %s: %s"
% (deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor',
cherrypy.request.user["login"])
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
self.error("Create request failed, reason: %s" % ex)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
self.error("Create request failed, reason: %s" % ex)
if hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName=requestName,
stats=stats,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return request
class ReqMgrRESTModel(RESTModel):
""" The REST interface to the ReqMgr database. Documentation may
be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """
def __init__(self, config):
RESTModel.__init__(self, config)
self.couchUrl = config.couchUrl
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'),
config.wmstatDBName)
self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName)
self.security_params = {'roles': config.security_roles}
# Optional values for individual methods
self.reqPriorityMax = getattr(config, 'maxReqPriority', 100)
self._addMethod('GET',
'request',
self.getRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'assignment',
self.getAssignment,
args=['teamName', 'request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'user',
self.getUser,
args=['userName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'group',
self.getGroup,
args=['group', 'user'],
secured=True,
expires=0)
self._addMethod('GET',
'version',
self.getVersion,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'team',
self.getTeam,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'workQueue',
self.getWorkQueue,
args=['request', 'workQueue'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'message',
self.getMessage,
args=['request'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'inputdataset',
self.getInputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'outputdataset',
self.getOutputDataset,
args=['prim', 'proc', 'tier'],
secured=True)
self._addMethod('GET',
'campaign',
self.getCampaign,
args=['campaign'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('PUT',
'request',
self.putRequest,
args=['requestName', 'status', 'priority', 'stats'],
secured=True,
validation=[
self.isalnumExceptStats, self.reqPriority,
self.validateStats
])
self._addMethod('PUT',
'clone',
self.cloneRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum])
self._addMethod('PUT',
'assignment',
self.putAssignment,
args=['team', 'requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'user',
self.putUser,
args=['userName', 'email', 'dnName'],
secured=True,
security_params=self.security_params,
validation=[self.validateUser])
self._addMethod('PUT',
'group',
self.putGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'version',
self.putVersion,
args=['version', 'scramArch'],
secured=True,
security_params=self.security_params,
validation=[self.validateVersion])
self._addMethod('PUT',
'team',
self.putTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'workQueue',
self.putWorkQueue,
args=['request', 'url'],
secured=True,
security_params=self.security_params,
validation=[self.validatePutWorkQueue])
self._addMethod('PUT',
'message',
self.putMessage,
args=['request'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('PUT',
'campaign',
self.putCampaign,
args=['campaign', 'request'],
secured=True,
validation=[self.isalnum])
self._addMethod('POST',
'request',
self.postRequest,
args=[
'requestName', 'events_written', 'events_merged',
'files_written', 'files_merged', 'percent_written',
'percent_success', 'dataset'
],
secured=True,
validation=[self.validateUpdates])
self._addMethod('POST',
'closeout',
self.closeOutRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('POST',
'announce',
self.announceRequest,
args=['requestName', 'cascade'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('DELETE',
'request',
self.deleteRequest,
args=['requestName'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'user',
self.deleteUser,
args=['user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'group',
self.deleteGroup,
args=['group', 'user'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'version',
self.deleteVersion,
args=['version', 'scramArch'],
secured=True,
validation=[self.validateVersion])
self._addMethod('DELETE',
'team',
self.deleteTeam,
args=['team'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('DELETE',
'campaign',
self.deleteCampaign,
args=['campaign'],
secured=True,
security_params=self.security_params,
validation=[self.isalnum])
self._addMethod('GET',
'requestnames',
self.getRequestNames,
args=[],
secured=True,
expires=0)
self._addMethod('GET',
'outputDatasetsByRequestName',
self.getOutputForRequest,
args=['requestName'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'outputDatasetsByPrepID',
self.getOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'mostRecentOutputDatasetsByPrepID',
self.getMostRecentOutputForPrepID,
args=['prepID'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'configIDs',
self.getConfigIDs,
args=['prim', 'proc', 'tier'],
secured=True,
validation=[self.isalnum],
expires=0)
self._addMethod('GET',
'requestsByStatusAndTeam',
self.getRequestsByStatusAndTeam,
args=['teamName', 'status'],
secured=True,
validation=[self.isalnum],
expires=0)
cherrypy.engine.subscribe('start_thread', self.initThread)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def isalnum(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
for v in index.values():
WMCore.Lexicon.identifier(v)
return index
def isalnumExceptStats(self, index):
""" Validates that all input is alphanumeric,
with spaces and underscores tolerated"""
if 'stats' in index:
return index
return self.isalnum(index)
def getDataset(self, prim, proc, tier):
""" If only prim exists, assume it's urlquoted.
If all three exists, assue it's /prim/proc/tier
"""
if not proc and not tier:
dataset = urllib.unquote(prim)
elif prim and proc and tier:
dataset = "/%s/%s/%s" % (prim, proc, tier)
WMCore.Lexicon.dataset(dataset)
return dataset
def intpriority(self, index):
""" Casts priority to an integer """
if 'priority' in index:
value = int(index['priority'])
if math.fabs(value) >= sys.maxsize:
msg = "Invalid priority! Priority must have abs() less then MAXINT!"
raise cherrypy.HTTPError(400, msg)
index['priority'] = value
return index
def reqPriority(self, index):
"""
_reqPriority_
Sets request priority to an integer.
Also makes sure it's within a certain value.
"""
if 'priority' not in index:
return index
index = self.intpriority(index=index)
value = index['priority']
if math.fabs(
value) > self.reqPriorityMax and not Utilities.privileged():
msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax
raise cherrypy.HTTPError(400, msg)
return index
def validateUser(self, index):
assert index['userName'].isalnum()
assert '@' in index['email']
assert index['email'].replace('@', '').replace('.', '').isalnum()
if 'dnName' in index:
assert index['dnName'].replace(' ', '').isalnum()
return index
def validateVersion(self, index):
""" Make sure it's a legitimate CMSSW version format """
WMCore.Lexicon.cmsswversion(index['version'])
return index
def getRequest(self, requestName=None):
""" If a request name is specified, return the details of the request.
Otherwise, return an overview of all requests """
if requestName == None:
result = GetRequest.getRequests()
else:
result = Utilities.requestDetails(requestName)
try:
teamNames = GetRequest.getAssignmentsByName(requestName)
result['teams'] = teamNames
except:
# Ignore errors, then we just don't have a team name
pass
return result
def getRequestNames(self):
# 2013-02-13 is wrong anyway (e.g. Campaign is not working)
# should be removed
""" return all the request names in RequestManager as list """
#TODO this could me combined with getRequest
return GetRequest.getOverview()
def getOutputForRequest(self, requestName):
"""Return the datasets produced by this request."""
return Utilities.getOutputForRequest(requestName)
def getOutputForPrepID(self, prepID):
"""Return the datasets produced by this prep ID. in a dict of requestName:dataset list"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
result = {}
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listOutputDatasets()
return result
def getMostRecentOutputForPrepID(self, prepID):
"""Return the datasets produced by the most recently submitted request with this prep ID"""
requestIDs = GetRequest.getRequestByPrepID(prepID)
# most recent will have the largest ID
requestIDs.sort()
requestIDs.reverse()
request = None
# Go through each request in order from largest to smallest
# looking for the first non-failed/non-canceled request
for requestID in requestIDs:
request = GetRequest.getRequest(requestID)
rejectList = ['aborted', 'failed', 'rejected', 'epic-failed']
requestStatus = request.get("RequestStatus", 'aborted').lower()
if requestStatus not in rejectList:
break
if request != None:
helper = Utilities.loadWorkload(request)
return helper.listOutputDatasets()
else:
return []
def getAssignment(self, teamName=None, request=None):
""" If a team name is passed in, get all assignments for that team.
If a request is passed in, return a list of teams the request is
assigned to
"""
# better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve?
#requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName)
# Maybe now assigned to team is same as assigned to ProdMgr
result = []
if teamName != None:
requestIDs = ListRequests.listRequestsByTeam(teamName,
"assigned").values()
requests = [GetRequest.getRequest(reqID) for reqID in requestIDs]
# put highest priorities first
requests.sort(key=lambda r: r['RequestPriority'], reverse=True)
# return list of tuples, since we need sorting
result = [[req['RequestName'], req['RequestWorkflow']]
for req in requests]
elif request != None:
result = GetRequest.getAssignmentsByName(request)
return result
def getRequestsByStatusAndTeam(self, teamName, status):
""" Get a list of request names with the given team and status.
"""
requestNames = ListRequests.listRequestsByTeam(teamName, status).keys()
return requestNames
def getUser(self, userName=None, group=None):
""" No args returns a list of all users. Group returns groups this user is in. Username
returs a JSON with information about the user """
if userName != None:
if not Registration.isRegistered(userName):
raise cherrypy.HTTPError(404, "Cannot find user")
result = {}
result['groups'] = GroupInfo.groupsForUser(userName).keys()
result['requests'] = UserRequests.listRequests(userName).keys()
result.update(Registration.userInfo(userName))
return result
elif group != None:
GroupInfo.usersInGroup(group)
else:
return Registration.listUsers()
def getGroup(self, group=None, user=None):
""" No args lists all groups, one args returns JSON with users."""
if group != None:
result = {}
result['users'] = GroupInfo.usersInGroup(group)
return result
elif user != None:
return GroupInfo.groupsForUser(user).keys()
else:
return GroupInfo.listGroups()
def getVersion(self):
""" Returns a list of all CMSSW versions registered with ReqMgr """
archList = SoftwareAdmin.listSoftware()
result = []
for arch in archList.keys():
for version in archList[arch]:
if not version in result:
result.append(version)
return result
def getTeam(self):
""" Returns a list of all teams registered with ReqMgr """
return ProdManagement.listTeams()
def getWorkQueue(self, request=None, workQueue=None):
""" If a request is passed in, return the URl of the workqueue.
If a workqueue is passed in, return all requests acquired by it """
if workQueue != None:
return ProdMgrRetrieve.findAssignedRequests(workQueue)
if request != None:
return ProdManagement.getProdMgr(request)
# return all the workqueue ulr
return GetRequest.getGlobalQueues()
def getMessage(self, request):
""" Returns a list of messages attached to this request """
return ChangeState.getMessages(request)
def getInputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this input dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput",
dataset)
def getOutputDataset(self, prim, proc=None, tier=None):
""" returns a list of requests with this output dataset
Input can either be a single urlquoted dataset, or a
/prim/proc/tier"""
dataset = self.getDataset(prim, proc, tier)
return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput",
dataset)
def getCampaign(self, campaign=None):
""" returns a list of all campaigns if no argument, and a list of
all requests in a campaign if there is an argument """
if campaign == None:
return Campaign.listCampaigns()
else:
return Campaign.listRequestsByCampaign(campaign)
def putWorkQueue(self, request, url):
""" Registers the request as "acquired" by the workqueue with the given URL """
Utilities.changeStatus(request, "acquired", self.wmstatWriteURL,
self.acdcURL)
return ProdManagement.associateProdMgr(request, urllib.unquote(url))
def validatePutWorkQueue(self, index):
assert index['request'].replace('_',
'').replace('-',
'').replace('.',
'').isalnum()
assert index['url'].startswith('http')
return index
# had no permission control before, security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def putRequest(self,
requestName=None,
status=None,
priority=None,
stats=None):
request = None
if requestName:
try:
request = self.getRequest(requestName)
except Exception as ex:
# request presumably doesn't exist
request = None
if request == None:
# Create a new request, with a JSON-encoded schema that is
# sent in the body of the HTTP request
body = cherrypy.request.body.read()
reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body))
# check for forbidden input arguments:
for deprec in deprecatedRequestArgs:
if deprec in reqInputArgs:
msg = (
"Creating request failed, unsupported input arg: %s: %s"
% (deprec, reqInputArgs[deprec]))
self.error(msg)
raise cherrypy.HTTPError(400, msg)
reqInputArgs.setdefault(
'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl))
reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName)
# wrong naming ... but it's all over the place, it's the config cache DB name
reqInputArgs.setdefault('CouchDBName', self.configDBName)
reqInputArgs.setdefault('Requestor',
cherrypy.request.user["login"])
try:
self.info(
"Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: "
"'%s' ..." %
(reqInputArgs, self.workloadDBName,
Utilities.removePasswordFromUrl(self.wmstatWriteURL)))
request = Utilities.makeRequest(self, reqInputArgs,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL)
except cherrypy.HTTPError as ex:
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
# Assume that this is a valid HTTPError
raise ex
except (WMException, Exception) as ex:
# TODO problem not to expose logs to the client
# e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message
msg = traceback.format_exc()
self.error("Create request failed, reason: %s" % msg)
if hasattr(ex, "message"):
if hasattr(ex.message, '__call__'):
detail = ex.message()
else:
detail = str(ex)
elif hasattr(ex, "name"):
detail = ex.name
else:
detail = "check logs."
msg = "Create request failed, %s" % detail
raise cherrypy.HTTPError(400, msg)
self.info("Request '%s' created." % request['RequestName'])
# see if status & priority need to be upgraded
if status != None:
# forbid assignment here
if status == 'assigned':
raise cherrypy.HTTPError(
403,
"Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>"
)
try:
Utilities.changeStatus(requestName, status,
self.wmstatWriteURL, self.acdcURL)
except RuntimeError as ex:
# ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002
if status != 'announced' and status != 'closed-out':
self.error("RuntimeError while changeStatus: reason: %s" %
ex)
raise cherrypy.HTTPError(
403, "Failed to change status: %s" % str(ex))
if priority != None:
Utilities.changePriority(requestName, priority,
self.wmstatWriteURL)
if stats != None:
Utilities.updateRequestStats(requestName=requestName,
stats=stats,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return request
def cloneRequest(self, requestName):
"""
Input assumes an existing request, checks that.
The original existing request is not touched.
A new request is generated.
The cloned request has a newly generated RequestName, new timestamp,
RequestDate, however -everything- else is copied from the original request.
Addition: since Edgar changed his mind, he no longer
wants this cloned request be in the 'new' state but put
straight into 'assignment-approved' state.
"""
request = None
if requestName:
self.info("Cloning request: request name: '%s'" % requestName)
requestOrigDict = self.getRequest(requestName)
if requestOrigDict:
self.info("Request found, cloning ...")
newReqSchema = Utilities.getNewRequestSchema(requestOrigDict)
# since we cloned the request, all the attributes
# manipulation in Utilities.makeRequest() should not be necessary
# the cloned request shall be identical but following arguments
toRemove = ["RequestName", "RequestDate", "timeStamp"]
toRemove.extend(deprecatedRequestArgs)
for remove in toRemove:
try:
del requestOrigDict[remove]
except KeyError:
pass
newReqSchema.update(requestOrigDict) # clone
# problem that priority wasn't preserved went down from here:
# via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory
request = Utilities.buildWorkloadAndCheckIn(
self,
newReqSchema,
self.couchUrl,
self.workloadDBName,
self.wmstatWriteURL,
clone=True)
# change the clone request state as desired
Utilities.changeStatus(newReqSchema["RequestName"],
"assignment-approved",
self.wmstatWriteURL, self.acdcURL)
return request
else:
msg = "Request '%s' not found." % requestName
self.warning(msg)
raise cherrypy.HTTPError(404, msg)
else:
msg = "Received empty request name: '%s', exit." % requestName
self.warn(msg)
raise cherrypy.HTTPError(400, msg)
def putAssignment(self, team, requestName):
""" Assigns this request to this team """
# see if it's already assigned
team = urllib.unquote(team)
if not team in ProdManagement.listTeams():
raise cherrypy.HTTPError(404, "Cannot find this team")
requestNamesAndIDs = ListRequests.listRequestsByTeam(team)
if requestName in requestNamesAndIDs.keys():
raise cherrypy.HTTPError(400, "Already assigned to this team")
return ChangeState.assignRequest(requestName, team)
def putUser(self, userName, email, dnName=None):
""" Needs to be passed an e-mail address, maybe dnName """
if Registration.isRegistered(userName):
return "User already exists"
result = Registration.registerUser(userName, email, dnName)
def putGroup(self, group, user=None):
""" Creates a group, or if a user is passed, adds that user to the group """
if (user != None):
# assume group exists and add user to it
return GroupManagement.addUserToGroup(user, group)
if GroupInfo.groupExists(group):
return "Group already exists"
GroupManagement.addGroup(group)
def putVersion(self, version, scramArch=None):
""" Registers a new CMSSW version with ReqMgr """
return SoftwareAdmin.addSoftware(version, scramArch=scramArch)
def putTeam(self, team):
""" Registers a team with ReqMgr """
return ProdManagement.addTeam(urllib.unquote(team))
def putMessage(self, request):
""" Attaches a message to this request """
message = JsonWrapper.loads(cherrypy.request.body.read())
result = ChangeState.putMessage(request, message)
return result
def putCampaign(self, campaign, request=None):
""" Adds a campaign if it doesn't already exist, and optionally
associates a request with it """
if request:
requestID = GetRequest.requestID(request)
if requestID:
result = Campaign.associateCampaign(campaign, requestID)
Utilities.associateCampaign(campaign=campaign,
requestName=request,
couchURL=self.couchUrl,
couchDBName=self.workloadDBName)
return result
else:
return False
else:
Campaign.addCampaign(campaign)
# def postRequest(self, requestName, events_written=None, events_merged=None,
# files_written=None, files_merged = None, dataset=None):
def postRequest(self, requestName, **kwargs):
"""
Add a progress update to the request Id provided, params can
optionally contain:
- *events_written* Int
- *events_merged* Int
- *files_written* Int
- *files_merged* int
- *percent_success* float
- *percent_complete float
- *dataset* string (dataset name)
"""
return ChangeState.updateRequest(requestName, kwargs)
def closeOutRequest(self, requestName, cascade=False):
"""
Close out a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and close them out
too.
"""
requestsToCloseOut = [requestName]
if cascade == "True":
requestsToCloseOut.extend(
Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToCloseOut:
Utilities.changeStatus(requestName, 'closed-out',
self.wmstatWriteURL, self.acdcURL)
return
def announceRequest(self, requestName, cascade=False):
"""
Announce a request, if the cascade option is given
then it will search for any Resubmission requests
for which the given request is a parent and announce them too.
"""
requestsToAnnounce = [requestName]
if cascade == "True":
requestsToAnnounce.extend(
Utilities.retrieveResubmissionChildren(requestName,
self.couchUrl,
self.workloadDBName))
for requestName in requestsToAnnounce:
Utilities.changeStatus(requestName, 'announced',
self.wmstatWriteURL, self.acdcURL)
return
def validateUpdates(self, index):
""" Check the values for the updates """
for k in [
'events_written', 'events_merged', 'files_written',
'files_merged'
]:
if k in index:
index[k] = int(index[k])
for k in ['percent_success', 'percent_complete']:
if k in index:
index[k] = float(index[k])
return index
def validateStats(self, index):
""" Check the values for the updates """
if 'stats' not in index:
return index
index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats']))
for k in [
'input_lummis', 'input_num_files', 'input_events', 'total_jobs'
]:
if k in index['stats']:
index['stats'][k] = int(index['stats'][k])
return index
# had no permission control before, security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def deleteRequest(self, requestName):
"""
Deletes a request from the ReqMgr MySQL/Oracle database
and also from CoucDB.
"""
# 404 will be thrown automatically on a non-existing request
request = self.getRequest(requestName)
helper = Utilities.loadWorkload(request)
couchDocId = requestName
helper.deleteCouch(self.couchUrl, self.workloadDBName, couchDocId)
# #4289 - Request delete operation deletes the request from
# MySQL/Oracle but not from CouchDB, fix here
# Seangchan shall also fix here deleting such requests from WMStats (#4398)
# returns None ...
response = RequestAdmin.deleteRequest(requestName)
return response
def deleteUser(self, user):
""" Deletes a user, as well as deleting his requests and removing
him from all groups """
if user in self.getUser():
requests = json.loads(self.getUser(user))['requests']
for request in requests:
self.deleteRequest(request)
for group in GroupInfo.groupsForUser(user).keys():
GroupManagement.removeUserFromGroup(user, group)
return UserManagement.deleteUser(user)
def deleteGroup(self, group, user=None):
""" If no user is sent, delete the group. Otherwise, delete the user from the group """
if user == None:
return GroupManagement.deleteGroup(group)
else:
return GroupManagement.removeUserFromGroup(user, group)
def deleteVersion(self, version, scramArch):
""" Un-register this software version with ReqMgr """
SoftwareAdmin.removeSoftware(version, scramArch)
def deleteTeam(self, team):
""" Delete this team from ReqMgr """
ProdManagement.removeTeam(urllib.unquote(team))
def deleteCampaign(self, campaign):
return Campaign.deleteCampaign(campaign)
def getConfigIDs(self, prim, proc, tier):
"""
_getConfigIDs_
Get the ConfigIDs for the specified request
"""
result = {}
dataset = self.getDataset(prim, proc, tier)
requests = GetRequest.getRequestsByCriteria(
"Datasets.GetRequestByInput", dataset)
for request in requests:
requestName = request["RequestName"]
helper = Utilities.loadWorkload(request)
result[requestName] = helper.listAllCMSSWConfigCacheIDs()
return result
class Assign(WebAPI):
""" Used by data ops to assign requests to processing sites"""
def __init__(self, config, noSiteDB=False):
"""
_init_
Note, noSiteDB added for TESTING PURPOSED ONLY!
"""
WebAPI.__init__(self, config)
ReqMgrAuth.assign_roles = config.security_roles
# Take a guess
self.templatedir = config.templates
self.couchUrl = config.couchUrl
self.configDBName = config.configDBName
self.workloadDBName = config.workloadDBName
self.configDBName = config.configDBName
self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName)
if not noSiteDB:
try:
# Download a list of all the sites from SiteDB, uses v2 API.
sitedb = SiteDBJSON()
self.sites = sitedb.getAllCMSNames()
self.sites.sort()
self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True)
self.phedexNodes.sort()
except Exception as ex:
msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex
cherrypy.log(msg)
raise
else:
self.sites = []
# store result lfn base with all Physics group
storeResultLFNBase = ["/store/results/analysisops",
"/store/results/b_physics",
"/store/results/b_tagging",
"/store/results/b2g",
"/store/results/e_gamma_ecal",
"/store/results/ewk",
"/store/results/exotica",
"/store/results/forward",
"/store/results/heavy_ions",
"/store/results/higgs",
"/store/results/jets_met_hcal",
"/store/results/muon",
"/store/results/qcd",
"/store/results/susy",
"/store/results/tau_pflow",
"/store/results/top",
"/store/results/tracker_dpg",
"/store/results/tracker_pog",
"/store/results/trigger"]
# yet 0.9.40 had also another self.mergedLFNBases which was differentiating
# list of mergedLFNBases based on type of request, removed and all bases
# will be displayed regardless of the request type (discussion with Edgar)
self.allMergedLFNBases = [
"/store/backfill/1",
"/store/backfill/2",
"/store/data",
"/store/mc",
"/store/generator",
"/store/relval",
"/store/hidata",
"/store/himc"]
self.allMergedLFNBases.extend(storeResultLFNBase)
self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"]
self.yuiroot = config.yuiroot
cherrypy.engine.subscribe('start_thread', self.initThread)
self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*',
'T2*': 'T2_*',
'T3*': 'T3_*'})
self.wildcardSites = {}
Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
# myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def validate(self, v, name=""):
"""
_validate_
Checks different fields with different Lexicon methods,
if not the field name is not known then apply the identifier check
"""
# Make sure the value is a string, otherwise the Lexicon complains
strValue = str(v)
try:
if name == "ProcessingVersion":
WMCore.Lexicon.procversion(strValue)
elif name == "AcquisitionEra":
WMCore.Lexicon.acqname(strValue)
elif name == "ProcessingString":
WMCore.Lexicon.procstring(strValue)
else:
WMCore.Lexicon.identifier(strValue)
except AssertionError as ex:
raise cherrypy.HTTPError(400, "Bad input: %s" % str(ex))
return v
def validateDatatier(self, datatier, dbsUrl):
"""
_validateDatatier_
Provided a list of datatiers extracted from the outputDatasets, checks
whether they all exist in DBS already.
"""
dbsTiers = DBSReader.listDatatiers(dbsUrl)
badTiers = list(set(datatier) - set(dbsTiers))
if badTiers:
raise cherrypy.HTTPError(400, "Bad datatier(s): %s not available in DBS." % badTiers)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles)
def one(self, requestName):
""" Assign a single request """
self.validate(requestName)
request = GetRequest.getRequestByName(requestName)
request = Utilities.prepareForTable(request)
# get assignments
teams = ProdManagement.listTeams()
assignments = GetRequest.getAssignmentsByName(requestName)
# might be a list, or a dict team:priority
if isinstance(assignments, dict):
assignments = assignments.keys()
procVer = ""
acqEra = ""
procString = ""
helper = Utilities.loadWorkload(request)
if helper.getAcquisitionEra() != None:
acqEra = helper.getAcquisitionEra()
if helper.getProcessingVersion() != None:
procVer = helper.getProcessingVersion()
if helper.getProcessingString():
procString = helper.getProcessingString()
dashboardActivity = helper.getDashboardActivity()
blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
blockCloseMaxSize = helper.getBlockCloseMaxSize()
(reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
return self.templatepage("Assign", requests=[request], teams=teams,
assignments=assignments, sites=self.sites,
phedexNodes=self.phedexNodes,
mergedLFNBases=self.allMergedLFNBases,
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra, procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=[],
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles)
def index(self, all=0):
""" Main page """
# returns dict of name:id
allRequests = Utilities.requestsWithStatus('assignment-approved')
teams = ProdManagement.listTeams()
procVer = ""
acqEra = ""
procString = ""
dashboardActivity = None
badRequestNames = []
goodRequests = allRequests
reqMergedBase = None
reqUnmergedBase = None
blockCloseMaxWaitTime = 66400
blockCloseMaxFiles = 500
blockCloseMaxEvents = 250000000
blockCloseMaxSize = 5000000000000
# for request in allRequests:
# # make sure there's a workload attached
# try:
# helper = Utilities.loadWorkload(request)
# except Exception, ex:
# logging.error("Assign error: %s " % str(ex))
# badRequestNames.append(request["RequestName"])
# else:
# # get defaults from the first good one
# if not goodRequests:
# # forget it if it fails.
# try:
# if helper.getAcquisitionEra() != None:
# acqEra = helper.getAcquisitionEra()
# if helper.getProcessingVersion() != None:
# procVer = helper.getProcessingVersion()
# if helper.getProcessingString() != None:
# procString = helper.getProcessingString()
# blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime()
# blockCloseMaxFiles = helper.getBlockCloseMaxFiles()
# blockCloseMaxEvents = helper.getBlockCloseMaxEvents()
# blockCloseMaxSize = helper.getBlockCloseMaxSize()
# (reqMergedBase, reqUnmergedBase) = helper.getLFNBases()
# dashboardActivity = helper.getDashboardActivity()
# goodRequests.append(request)
# except Exception, ex:
# logging.error("Assign error: %s " % str(ex))
# badRequestNames.append(request["RequestName"])
# else:
# goodRequests.append(request)
return self.templatepage("Assign", all=all, requests=goodRequests, teams=teams,
assignments=[], sites=self.sites,
phedexNodes=self.phedexNodes,
mergedLFNBases=self.allMergedLFNBases,
reqMergedBase=reqMergedBase,
unmergedLFNBases=self.allUnmergedLFNBases,
reqUnmergedBase=reqUnmergedBase,
acqEra=acqEra, procVer=procVer,
procString=procString,
dashboardActivity=dashboardActivity,
badRequests=badRequestNames,
blockCloseMaxWaitTime=blockCloseMaxWaitTime,
blockCloseMaxFiles=blockCloseMaxFiles,
blockCloseMaxSize=blockCloseMaxSize,
blockCloseMaxEvents=blockCloseMaxEvents)
@cherrypy.expose
# @cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles) security issue fix
@cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups())
def handleAssignmentPage(self, **kwargs):
""" handler for the main page """
# Accept Json encoded strings
decodedArgs = {}
for key in kwargs.keys():
try:
decodedArgs[key] = json.loads(kwargs[key])
except Exception:
# Probably wasn't JSON
decodedArgs[key] = kwargs[key]
kwargs = decodedArgs
# handle the checkboxes
teams = []
requestNames = []
for key, value in kwargs.iteritems():
if isinstance(value, basestring):
kwargs[key] = value.strip()
if key.startswith("Team"):
teams.append(key[4:])
if key.startswith("checkbox"):
requestName = key[8:]
self.validate(requestName)
requestNames.append(key[8:])
for requestName in requestNames:
if kwargs['action'] == 'Reject':
ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl=self.wmstatWriteURL)
else:
assignments = GetRequest.getAssignmentsByName(requestName)
if teams == [] and assignments == []:
raise cherrypy.HTTPError(400, "Must assign to one or more teams")
kwargs["Teams"] = teams
self.assignWorkload(requestName, kwargs)
for team in teams:
if not team in assignments:
ChangeState.assignRequest(requestName, team, wmstatUrl=self.wmstatWriteURL)
priority = kwargs.get(requestName + ':priority', '')
if priority != '':
Utilities.changePriority(requestName, priority, self.wmstatWriteURL)
participle = kwargs['action'] + 'ed'
return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def assignWorkload(self, requestName, kwargs):
""" Make all the necessary changes in the Workload to reflect the new assignment """
request = GetRequest.getRequestByName(requestName)
helper = Utilities.loadWorkload(request)
try:
helper.validateArgumentForAssignment(kwargs)
except WMSpecFactoryException as ex:
raise cherrypy.HTTPError(400, str(ex.message()))
except Exception:
msg = traceback.format_exc()
raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg)
# Validate the different parts of the processed dataset
processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"),
"ProcessingString": kwargs.get("ProcessingString"),
"ProcessingVersion": kwargs.get("ProcessingVersion")
}
for field, values in processedDatasetParts.iteritems():
if field in kwargs and isinstance(kwargs[field], dict):
for value in kwargs[field].values():
self.validate(value, field)
else:
self.validate(kwargs.get(field, values), field)
# Set white list and black list
whiteList = kwargs.get("SiteWhitelist", [])
blackList = kwargs.get("SiteBlacklist", [])
if not isinstance(whiteList, list):
whiteList = [whiteList]
if not isinstance(blackList, list):
blackList = [blackList]
helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList,
wildcardDict=self.wildcardSites)
res = set(whiteList) & set(blackList)
if len(res):
raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res))
helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None))
helper.setProcessingString(kwargs.get("ProcessingString", None))
helper.setProcessingVersion(kwargs.get("ProcessingVersion", None))
# Now verify the output datasets
datatier = []
outputDatasets = helper.listOutputDatasets()
for dataset in outputDatasets:
tokens = dataset.split("/")
procds = tokens[2]
datatier.append(tokens[3])
try:
WMCore.Lexicon.procdataset(procds)
except AssertionError as ex:
raise cherrypy.HTTPError(400,
"Bad output dataset name, check the processed dataset.\n %s" %
str(ex))
# Verify whether the output datatiers are available in DBS
self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl())
# FIXME not validated
helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"])
helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)),
int(kwargs.get("MaxMergeSize", 4294967296)),
int(kwargs.get("MaxMergeEvents", 50000)))
helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None),
kwargs.get("MaxVSize", None),
kwargs.get("SoftTimeout", None),
kwargs.get("GracePeriod", None))
# Check whether we should check location for the data
helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)),
pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False)))
helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False)))
# Set phedex subscription information
custodialList = kwargs.get("CustodialSites", [])
nonCustodialList = kwargs.get("NonCustodialSites", [])
autoApproveList = kwargs.get("AutoApproveSubscriptionSites", [])
subscriptionPriority = kwargs.get("SubscriptionPriority", "Low")
custodialType = kwargs.get("CustodialSubType", "Replica")
nonCustodialType = kwargs.get("NonCustodialSubType", "Replica")
helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites,
custodialSites=custodialList,
nonCustodialSites=nonCustodialList,
autoApproveSites=autoApproveList,
custodialSubType=custodialType,
nonCustodialSubType=nonCustodialType,
custodialGroup=kwargs.get("CustodialGroup", "DataOps"),
nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"),
priority=subscriptionPriority,
deleteFromSource=kwargs.get("DeleteFromSource", False))
# Block closing information
blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime()))
blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles()))
blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents()))
blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize()))
helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles,
blockCloseMaxEvents, blockCloseMaxSize)
helper.setMemory(kwargs.get("Memory"))
helper.setCores(kwargs.get("Multicore"))
helper.setDashboardActivity(kwargs.get("Dashboard", ""))
helper.setTaskProperties(kwargs)
Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
# update AcquisitionEra in the Couch document (#4380)
# request object returned above from Oracle doesn't have information Couch
# database
reqDetails = Utilities.requestDetails(request["RequestName"])
couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"])
couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest",
fields={"AcquisitionEra": reqDetails["AcquisitionEra"],
"ProcessingVersion": reqDetails["ProcessingVersion"],
"CustodialSites": custodialList,
"NonCustodialSites": nonCustodialList,
"AutoApproveSubscriptionSites": autoApproveList,
"SubscriptionPriority": subscriptionPriority,
"CustodialSubType": custodialType,
"NonCustodialSubType": nonCustodialType,
"CustodialGroup": kwargs.get("CustodialGroup", "DataOps"),
"NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"),
"DeleteFromSource": kwargs.get("DeleteFromSource", False),
"Teams": kwargs["Teams"],
"OutputDatasets": outputDatasets,
"SiteWhitelist": whiteList,
"SiteBlacklist": blackList,
"MergedLFNBase": kwargs["MergedLFNBase"],
"UnmergedLFNBase": kwargs["UnmergedLFNBase"],
"Dashboard": kwargs.get("Dashboard", ""),
"TrustSitelists": kwargs.get("TrustSitelists", False),
"TrustPUSitelists": kwargs.get("TrustPUSitelists", False),
"AllowOpportunistic": kwargs.get("AllowOpportunistic", False)},
useBody=True)
class Admin(WebAPI):
""" Handles administration functions for ReqMgr """
def __init__(self, config):
WebAPI.__init__(self, config)
# Take a guess
self.templatedir = config.templates
self.htmldir = config.html
cherrypy.engine.subscribe('start_thread', self.initThread)
def initThread(self, thread_index):
""" The ReqMgr expects the DBI to be contained in the Thread """
myThread = threading.currentThread()
#myThread = cherrypy.thread_data
# Get it from the DBFormatter superclass
myThread.dbi = self.dbi
def validate(self, v, name=''):
""" Checks if alphanumeric, tolerating spaces """
if isinstance(v, list):
for entry in v:
self.validate(entry)
else:
try:
WMCore.Lexicon.identifier(v)
except AssertionError:
raise cherrypy.HTTPError(400, "Bad input %s" % name)
return v
@cherrypy.expose
@cherrypy.tools.secmodv2()
def index(self):
""" Main web page """
return Utilities.serveFile('text/html', self.htmldir, "Admin.html")
@cherrypy.expose
@cherrypy.tools.secmodv2()
def user(self, userName):
""" Web page of details about the user, and sets user priority """
self.validate(userName)
groups = GroupInfo.groupsForUser(userName).keys()
requests = UserRequests.listRequests(userName).keys()
priority = UserManagement.getPriority(userName)
allGroups = GroupInfo.listGroups()
self.validate(groups)
self.validate(requests)
self.validate(allGroups)
return self.templatepage("User",
user=userName,
groups=groups,
allGroups=allGroups,
requests=requests,
priority=priority)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleUserPriority(self, user, userPriority):
""" Handles setting user priority """
self.validate(user)
UserManagement.setPriority(user, userPriority)
return "Updated user %s priority to %s" % (user, userPriority)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def group(self, groupName):
""" Web page of details about the user, and sets user priority """
self.validate(groupName)
users = GroupInfo.usersInGroup(groupName)
priority = GroupManagement.getPriority(groupName)
return self.templatepage("Group",
group=groupName,
users=users,
priority=priority)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleGroupPriority(self, group, groupPriority):
""" Handles setting group priority """
self.validate(group)
GroupManagement.setPriority(group, groupPriority)
return "Updated group %s priority to %s" % (group, groupPriority)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def users(self):
""" Lists all users. Should be paginated later """
allUsers = Registration.listUsers()
self.validate(allUsers)
return self.templatepage("Users", users=allUsers)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAddUser(self, user, email=None):
""" Handles setting user priority """
self.validate(user)
Registration.registerUser(user, email)
return "Added user %s" % user
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAddToGroup(self, user, group):
""" Adds a user to the group """
self.validate(user)
self.validate(group)
GroupManagement.addUserToGroup(user, group)
return "Added %s to %s " % (user, group)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def groups(self):
""" Lists all users. Should be paginated later """
allGroups = GroupInfo.listGroups()
self.validate(allGroups)
return self.templatepage("Groups", groups=allGroups)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAddGroup(self, group):
""" Handles adding a group """
self.validate(group)
GroupManagement.addGroup(group)
return "Added group %s " % group
@cherrypy.expose
@cherrypy.tools.secmodv2()
def teams(self):
""" Lists all teams """
teams = ProdManagement.listTeams().keys()
self.validate(teams)
return self.templatepage("Teams", teams=teams)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def team(self, teamName):
""" Details for a team """
self.validate(teamName)
assignments = ListRequests.listRequestsByTeam(teamName)
if assignments == None:
assignments = []
else:
assignments = assignments.keys()
self.validate(assignments)
return self.templatepage("Team", team=teamName, requests=assignments)
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAddTeam(self, team):
""" Handles a request to add a team """
self.validate(team)
ProdManagement.addTeam(team)
return "Added team %s" % team
@cherrypy.expose
@cherrypy.tools.secmodv2()
def versions(self):
""" Lists all versions """
archList = SoftwareAdmin.listSoftware()
versions = []
for versionList in archList.values():
for version in versionList:
if not version in versions:
versions.append(version)
versions.sort()
for version in versions:
WMCore.Lexicon.cmsswversion(version)
return self.templatepage("Versions", versions=versions)
@cherrypy.expose
@cherrypy.tools.secmodv2()
def scramArchs(self):
"""
_scramArchs_
List all scramArchs in the DB
Prelim for putting this in the template pages
"""
return SoftwareAdmin.listSoftware().keys()
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAddVersion(self, version):
""" Registers a version """
WMCore.Lexicon.cmsswversion(version)
SoftwareAdmin.updateSoftware(version)
return "Added version %s" % version
@cherrypy.expose
@cherrypy.tools.secmodv2(role=Utilities.security_roles(),
group=Utilities.security_groups())
def handleAllVersions(self):
""" Registers all versions in the TC """
Utilities.updateScramArchsAndCMSSWVersions()
return "Updated versions to current standard"
