class Approve(BulkOperations): """ Page for Physics group leaders to approve requests """ def __init__(self, config): BulkOperations.__init__(self, config) self.wmstatWriteURL = "%s/%s" % (config.couchUrl.rstrip('/'), config.wmstatDBName) @cherrypy.expose @cherrypy.tools.secmodv2() def index(self): """ Page for approving requests """ return self.draw(self.requests()) def requests(self): """ Base list of the requests """ return Utilities.requestsWhichCouldLeadTo('assignment-approved') def draw(self, requests): return self.templatepage("BulkOperations", operation="Approve", actions=["Approve", "Reject"], searchFields=["RequestName", "RequestType"], requests=requests) @cherrypy.expose #@cherrypy.tools.secmodv2() security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleApprove(self, **kwargs): """ Handler for approving requests """ requests = self.requestNamesFromCheckboxes(kwargs) particple = '' for requestName in requests: if kwargs['action'] == 'Reject': participle = 'rejected' ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl=self.wmstatWriteURL) else: participle = 'approved' ChangeState.changeRequestStatus(requestName, 'assignment-approved', wmstatUrl=self.wmstatWriteURL) priority = kwargs.get(requestName + ':priority', '') if priority != '': Utilities.changePriority(requestName, priority, self.wmstatWriteURL) return self.templatepage("Acknowledge", participle=participle, requests=requests)
return index def validateStats(self, index): """ Check the values for the updates """ if not index.has_key('stats'): return index index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats'])) for k in ['input_lummis', 'input_num_files', 'input_events', 'total_jobs']: if k in index['stats']: index['stats'][k] = int(index['stats'][k]) return index # had no permission control before, security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups()) def deleteRequest(self, requestName): """ Deletes a request from the ReqMgr MySQL/Oracle database and also from CoucDB. """ # 404 will be thrown automatically on a non-existing request request = self.getRequest(requestName) helper = Utilities.loadWorkload(request) couchDocId = requestName helper.deleteCouch(self.couchUrl, self.workloadDBName, couchDocId) # #4289 - Request delete operation deletes the request from # MySQL/Oracle but not from CouchDB, fix here # Seangchan shall also fix here deleting such requests from WMStats (#4398)
class ReqMgrBrowser(WebAPI): """ For browsing and modifying requests: Disabled use WMStats instead """ def __init__(self, config): WebAPI.__init__(self, config) # Take a guess self.templatedir = config.templates self.fields = ['RequestName', 'Group', 'Requestor', 'RequestType', "RequestPriority", 'RequestStatus', 'Complete', 'Success'] self.calculatedFields = {'Written': 'percentWritten', 'Merged':'percentMerged', 'Complete':'percentComplete', 'Success' : 'percentSuccess'} # entries in the table that show up as HTML links for that entry self.linkedFields = {'Group': '../admin/group', 'Requestor': '../admin/user', 'RequestName': 'details'} self.detailsFields = ['RequestName', 'RequestType', 'Requestor', 'CMSSWVersion', 'ScramArch', 'GlobalTag', 'RequestNumEvents', 'InputDataset', 'PrimaryDataset', 'AcquisitionEra', 'ProcessingVersion', 'RunWhitelist', 'RunBlacklist', 'BlockWhitelist', 'BlockBlacklist', 'RequestWorkflow', 'Scenario', 'Campaign', 'PrimaryDataset', 'Acquisition Era', 'Processing Version', 'Merged LFN Base', 'Unmerged LFN Base', 'Site Whitelist', 'Site Blacklist'] self.adminMode = True self.adminFields = {} self.couchUrl = config.couchUrl self.configDBName = config.configDBName self.workloadDBName = config.workloadDBName self.yuiroot = config.yuiroot self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName) cherrypy.engine.subscribe('start_thread', self.initThread) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() #myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def validate(self, v, name=''): """ Checks if alphanumeric, tolerating spaces """ try: WMCore.Lexicon.identifier(v) except AssertionError as ex: raise cherrypy.HTTPError(400, "Bad input: %s" % str(ex)) return v @cherrypy.expose @cherrypy.tools.secmodv2() def search(self, value, field): """ Search for a regular expression in a certain field of all requests """ filteredRequests = [] requests = GetRequest.getRequests() for request in requests: if request[field].find(value) != -1: filteredRequests.append(request) requests = filteredRequests tableBody = self.drawRequests(requests) return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot, fields=self.fields, tableBody=tableBody) @cherrypy.expose @cherrypy.tools.secmodv2() def index(self): requests = GetRequest.getRequests() tableBody = self.drawRequests(requests) #tableBody = [] return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot, fields=self.fields, tableBody=tableBody) @cherrypy.expose @cherrypy.tools.secmodv2() def splitting(self, requestName): """ _splitting_ Retrieve the current values for splitting parameters for all tasks in the spec. Format them in the manner that the splitting page expects and pass them to the template. """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) splittingDict = helper.listJobSplittingParametersByTask(performance = False) taskNames = splittingDict.keys() taskNames.sort() splitInfo = [] for taskName in taskNames: jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName]) splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"], "splitParams": jsonSplittingParams, "taskType": splittingDict[taskName]["type"], "taskName": taskName}) return self.templatepage("Splitting", requestName = requestName, taskInfo = splitInfo, taskNames = taskNames) @cherrypy.expose @cherrypy.tools.secmodv2() def handleSplittingPage(self, requestName, splittingTask, splittingAlgo, **submittedParams): """ _handleSplittingPage_ Parse job splitting parameters sent from the splitting parameter update page. Pull down the request and modify the new spec applying the updated splitting parameters. """ splitParams = {} if splittingAlgo == "FileBased": splitParams["files_per_job"] = int(submittedParams["files_per_job"]) elif splittingAlgo == "TwoFileBased": splitParams["files_per_job"] = int(submittedParams["two_files_per_job"]) elif splittingAlgo == "LumiBased": splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"]) if str(submittedParams["halt_job_on_file_boundaries"]) == "True": splitParams["halt_job_on_file_boundaries"] = True else: splitParams["halt_job_on_file_boundaries"] = False elif splittingAlgo == "EventAwareLumiBased": splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"]) splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"]) if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True": splitParams["halt_job_on_file_boundaries"] = True else: splitParams["halt_job_on_file_boundaries"] = False elif splittingAlgo == "EventBased": splitParams["events_per_job"] = int(submittedParams["events_per_job"]) if "events_per_lumi" in submittedParams: splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"]) if "lheInputFiles" in submittedParams: if str(submittedParams["lheInputFiles"]) == "True": splitParams["lheInputFiles"] = True else: splitParams["lheInputFiles"] = False elif splittingAlgo == "Harvest": splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"]) elif 'Merg' in splittingTask: for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']: splitParams[field] = int(submittedParams[field]) if "include_parents" in submittedParams.keys(): if str(submittedParams["include_parents"]) == "True": splitParams["include_parents"] = True else: splitParams["include_parents"] = False self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams)) helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams) Utilities.saveWorkload(helper, request['RequestWorkflow']) return "Successfully updated splitting parameters for " + splittingTask \ + " " + detailsBackLink(requestName) @cherrypy.expose @cherrypy.tools.secmodv2() def details(self, requestName): """ A page showing the details for the requests """ self.validate(requestName) try: request = Utilities.requestDetails(requestName) except AssertionError: raise cherrypy.HTTPError(404, "Cannot load request %s" % requestName) adminHtml = statusMenu(requestName, request['RequestStatus']) \ + ' Priority: ' + Utilities.priorityMenu(request) return self.templatepage("Request", requestName=requestName, detailsFields=self.detailsFields, requestSchema=request, docId=request.get('ConfigCacheID', None), assignments=request['Assignments'], adminHtml=adminHtml, messages=request['RequestMessages'], updateDictList=request['RequestUpdates']) def _getConfigCache(self, requestName, processMethod): try: request = Utilities.requestDetails(requestName) except Exception as ex: msg = "Cannot find request %s, check logs." % requestName logging.error("%s, reason: %s" % (msg, ex)) return msg url = request.get("ConfigCacheUrl", None) or self.couchUrl try: configCache = ConfigCache(url, self.configDBName) configDocId = request["ConfigCacheID"] configCache.loadByID(configDocId) except Exception as ex: msg = "Cannot find ConfigCache document %s on %s." % (configDocId, url) logging.error("%s, reason: %s" % (msg, ex)) return msg return getattr(configCache, processMethod)() @cherrypy.expose @cherrypy.tools.secmodv2() def showOriginalConfig(self, requestName): """ Makes a link to the original text of the config document. """ self.validate(requestName) return '<pre>' + self._getConfigCache(requestName, "getConfig") + '</pre>' @cherrypy.expose @cherrypy.tools.secmodv2() def showTweakFile(self, requestName): """ Makes a link to the dump of the tweakfile. """ self.validate(requestName) tweakString = self._getConfigCache(requestName, "getPSetTweaks") return str(tweakString).replace('\n', '<br>') @cherrypy.expose @cherrypy.tools.secmodv2() def showWorkload(self, requestName): """ Displays the workload """ self.validate(requestName) try: request = GetRequest.getRequestByName(requestName) except (Exception, RuntimeError) as ex: raise cherrypy.HTTPError(400, "Invalid request. %s" % str(ex)) request = Utilities.prepareForTable(request) helper = Utilities.loadWorkload(request) workloadText = str(helper.data) return cgi.escape(workloadText).replace("\n", "<br/>\n") def drawRequests(self, requests): """ Display all requests """ result = "" for request in requests: # see if this is slower result += self.drawRequest(request) return result def drawRequest(self, request): """ make a table row with information from this request """ html = '<tr>' requestName = request['RequestName'] for field in self.fields: # hanole any fields that have functions linked to them entry = cgi.escape(str(request.get(field, ''))) if field in self.calculatedFields: method = getattr(self, self.calculatedFields[field]) entry = method(request) elif self.adminMode and field in self.adminFields: method = getattr(self, self.adminFields[field]) entry = method(requestName, value) elif field in self.linkedFields: entry = linkedTableEntry(self.linkedFields[field], entry) html += '<td>%s</td>' % entry html += '</tr>\n' return html def percentWritten(self, request): """ Finds the biggest percentage among all the updates """ maxPercent = 0 for update in request["RequestUpdates"]: if "events_written" in update and request["RequestNumEvents"] != 0: percent = update["events_written"] / request["RequestNumEvents"] if percent > maxPercent: maxPercent = percent if "files_written" in update and request["RequestSizeFiles"] != 0: percent = update["files_written"] / request["RequestSizeFiles"] if percent > maxPercent: maxPercent = percent return "%i%%" % maxPercent def percentMerged(self, request): """ Finds the biggest percentage among all the updates """ maxPercent = 0 for update in request["RequestUpdates"]: if "events_merged" in update and request["RequestNumEvents"] != 0: percent = update["events_merged"] / request["RequestNumEvents"] if percent > maxPercent: maxPercent = percent if "files_merged" in update and request["RequestSizeFiles"] != 0: percent = update["files_merged"] / request["RequestSizeFiles"] if percent > maxPercent: maxPercent = percent return "%i%%" % maxPercent def percentComplete(self, request): pct = request.get('percent_complete', 0) return "%i%%" % pct def percentSuccess(self, request): pct = request.get('percent_success', 0) return "%i%%" % pct @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups()) def doAdmin(self, **kwargs): """ format of kwargs is {'requestname:status' : 'approved', 'requestname:priority' : '2'} """ message = "" for k, v in kwargs.iteritems(): if k.endswith(':status'): requestName = k.split(':')[0] self.validate(requestName) status = v priority = kwargs[requestName+':priority'] if priority != '': Utilities.changePriority(requestName, priority, self.wmstatWriteURL) message += "Changed priority for %s to %s.\n" % (requestName, priority) if status != "": Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL) message += "Changed status for %s to %s\n" % (requestName, status) if status == "assigned": # make a page to choose teams raise cherrypy.HTTPRedirect('/reqmgr/assign/one/%s' % requestName) return message + detailsBackLink(requestName) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups()) # FIXME needs to check if authorized, or original user def modifyWorkload(self, requestName, workload, CMSSWVersion=None, GlobalTag=None, runWhitelist=None, runBlacklist=None, blockWhitelist=None, blockBlacklist=None, ScramArch=None): """ handles the "Modify" button of the details page """ self.validate(requestName) helper = WMWorkloadHelper() helper.load(workload) schema = helper.data.request.schema message = "" if runWhitelist != "" and runWhitelist != None: l = Utilities.parseRunList(runWhitelist) helper.setRunWhitelist(l) schema.RunWhitelist = l message += 'Changed runWhiteList to %s<br>' % l if runBlacklist != "" and runBlacklist != None: l = Utilities.parseRunList(runBlacklist) helper.setRunBlacklist(l) schema.RunBlacklist = l message += 'Changed runBlackList to %s<br>' % l if blockWhitelist != "" and blockWhitelist != None: l = Utilities.parseBlockList(blockWhitelist) helper.setBlockWhitelist(l) schema.BlockWhitelist = l message += 'Changed blockWhiteList to %s<br>' % l if blockBlacklist != "" and blockBlacklist != None: l = Utilities.parseBlockList(blockBlacklist) helper.setBlockBlacklist(l) schema.BlockBlacklist = l message += 'Changed blockBlackList to %s<br>' % l Utilities.saveWorkload(helper, workload) return message + detailsBackLink(requestName)
def group(self, groupName): """ Web page of details about the group.""" self.validate(groupName) users = GroupInfo.usersInGroup(groupName) return self.templatepage("Group", group=groupName, users=users) @cherrypy.expose @cherrypy.tools.secmodv2() def users(self): """ Lists all users. Should be paginated later """ allUsers = Registration.listUsers() self.validate(allUsers) return self.templatepage("Users", users=allUsers) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups()) def handleAddUser(self, user, email=None): self.validate(user) Registration.registerUser(user, email) return "Added user %s" % user @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group = Utilities.security_groups()) def handleAddToGroup(self, user, group): """ Adds a user to the group """ self.validate(user) self.validate(group) GroupManagement.addUserToGroup(user, group) return "Added %s to %s " % (user, group) @cherrypy.expose
unmergedLFNBases=self.allUnmergedLFNBases, reqUnmergedBase=reqUnmergedBase, acqEra=acqEra, procVer=procVer, procString=procString, dashboardActivity=dashboardActivity, badRequests=badRequestNames, blockCloseMaxWaitTime=blockCloseMaxWaitTime, blockCloseMaxFiles=blockCloseMaxFiles, blockCloseMaxSize=blockCloseMaxSize, blockCloseMaxEvents=blockCloseMaxEvents) @cherrypy.expose #@cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles) security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAssignmentPage(self, **kwargs): """ handler for the main page """ #Accept Json encoded strings decodedArgs = {} for key in kwargs.keys(): try: decodedArgs[key] = JsonWrapper.loads(kwargs[key]) except: #Probably wasn't JSON decodedArgs[key] = kwargs[key] kwargs = decodedArgs # handle the checkboxes teams = [] requestNames = [] for key, value in kwargs.iteritems():
class ReqMgrRESTModel(RESTModel): """ The REST interface to the ReqMgr database. Documentation may be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """ def __init__(self, config): RESTModel.__init__(self, config) self.couchUrl = config.couchUrl self.workloadDBName = config.workloadDBName self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName) self.security_params = {'roles': config.security_roles} # Optional values for individual methods self.reqPriorityMax = getattr(config, 'maxReqPriority', 100) self._addMethod('GET', 'request', self.getRequest, args=['requestName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'assignment', self.getAssignment, args=['teamName', 'request'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'user', self.getUser, args=['userName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'group', self.getGroup, args=['group', 'user'], secured=True, expires=0) self._addMethod('GET', 'version', self.getVersion, args=[], secured=True, expires=0) self._addMethod('GET', 'team', self.getTeam, args=[], secured=True, expires=0) self._addMethod('GET', 'workQueue', self.getWorkQueue, args=['request', 'workQueue'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'message', self.getMessage, args=['request'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'inputdataset', self.getInputDataset, args=['prim', 'proc', 'tier'], secured=True) self._addMethod('GET', 'outputdataset', self.getOutputDataset, args=['prim', 'proc', 'tier'], secured=True) self._addMethod('GET', 'campaign', self.getCampaign, args=['campaign'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('PUT', 'request', self.putRequest, args=['requestName', 'status', 'priority', 'stats'], secured=True, validation=[ self.isalnumExceptStats, self.reqPriority, self.validateStats ]) self._addMethod('PUT', 'clone', self.cloneRequest, args=['requestName'], secured=True, validation=[self.isalnum]) self._addMethod('PUT', 'assignment', self.putAssignment, args=['team', 'requestName'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'user', self.putUser, args=['userName', 'email', 'dnName'], secured=True, security_params=self.security_params, validation=[self.validateUser]) self._addMethod('PUT', 'group', self.putGroup, args=['group', 'user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'version', self.putVersion, args=['version', 'scramArch'], secured=True, security_params=self.security_params, validation=[self.validateVersion]) self._addMethod('PUT', 'team', self.putTeam, args=['team'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'workQueue', self.putWorkQueue, args=['request', 'url'], secured=True, security_params=self.security_params, validation=[self.validatePutWorkQueue]) self._addMethod('PUT', 'message', self.putMessage, args=['request'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'campaign', self.putCampaign, args=['campaign', 'request'], secured=True, validation=[self.isalnum]) self._addMethod('POST', 'request', self.postRequest, args=[ 'requestName', 'events_written', 'events_merged', 'files_written', 'files_merged', 'percent_written', 'percent_success', 'dataset' ], secured=True, validation=[self.validateUpdates]) self._addMethod('POST', 'closeout', self.closeOutRequest, args=['requestName', 'cascade'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('POST', 'announce', self.announceRequest, args=['requestName', 'cascade'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('DELETE', 'request', self.deleteRequest, args=['requestName'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'user', self.deleteUser, args=['user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'group', self.deleteGroup, args=['group', 'user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'version', self.deleteVersion, args=['version', 'scramArch'], secured=True, validation=[self.validateVersion]) self._addMethod('DELETE', 'team', self.deleteTeam, args=['team'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'campaign', self.deleteCampaign, args=['campaign'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('GET', 'requestnames', self.getRequestNames, args=[], secured=True, expires=0) self._addMethod('GET', 'outputDatasetsByRequestName', self.getOutputForRequest, args=['requestName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'outputDatasetsByPrepID', self.getOutputForPrepID, args=['prepID'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'mostRecentOutputDatasetsByPrepID', self.getMostRecentOutputForPrepID, args=['prepID'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'configIDs', self.getConfigIDs, args=['prim', 'proc', 'tier'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'requestsByStatusAndTeam', self.getRequestsByStatusAndTeam, args=['teamName', 'status'], secured=True, validation=[self.isalnum], expires=0) cherrypy.engine.subscribe('start_thread', self.initThread) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() #myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def isalnum(self, index): """ Validates that all input is alphanumeric, with spaces and underscores tolerated""" for v in index.values(): WMCore.Lexicon.identifier(v) return index def isalnumExceptStats(self, index): """ Validates that all input is alphanumeric, with spaces and underscores tolerated""" if index.has_key('stats'): return index return self.isalnum(index) def getDataset(self, prim, proc, tier): """ If only prim exists, assume it's urlquoted. If all three exists, assue it's /prim/proc/tier """ if not proc and not tier: dataset = urllib.unquote(prim) elif prim and proc and tier: dataset = "/%s/%s/%s" % (prim, proc, tier) WMCore.Lexicon.dataset(dataset) return dataset def intpriority(self, index): """ Casts priority to an integer """ if index.has_key('priority'): value = int(index['priority']) if math.fabs(value) >= sys.maxint: msg = "Invalid priority! Priority must have abs() less then MAXINT!" raise cherrypy.HTTPError(400, msg) index['priority'] = value return index def reqPriority(self, index): """ _reqPriority_ Sets request priority to an integer. Also makes sure it's within a certain value. """ if not index.has_key('priority'): return index index = self.intpriority(index=index) value = index['priority'] if math.fabs( value) > self.reqPriorityMax and not Utilities.privileged(): msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax raise cherrypy.HTTPError(400, msg) return index def validateUser(self, index): assert index['userName'].isalnum() assert '@' in index['email'] assert index['email'].replace('@', '').replace('.', '').isalnum() if 'dnName' in index: assert index['dnName'].replace(' ', '').isalnum() return index def validateVersion(self, index): """ Make sure it's a legitimate CMSSW version format """ WMCore.Lexicon.cmsswversion(index['version']) return index def getRequest(self, requestName=None): """ If a request name is specified, return the details of the request. Otherwise, return an overview of all requests """ if requestName == None: result = GetRequest.getRequests() else: result = Utilities.requestDetails(requestName) try: teamNames = GetRequest.getAssignmentsByName(requestName) result['teams'] = teamNames except: # Ignore errors, then we just don't have a team name pass return result def getRequestNames(self): # 2013-02-13 is wrong anyway (e.g. Campaign is not working) # should be removed """ return all the request names in RequestManager as list """ #TODO this could me combined with getRequest return GetRequest.getOverview() def getOutputForRequest(self, requestName): """Return the datasets produced by this request.""" return Utilities.getOutputForRequest(requestName) def getOutputForPrepID(self, prepID): """Return the datasets produced by this prep ID. in a dict of requestName:dataset list""" requestIDs = GetRequest.getRequestByPrepID(prepID) result = {} for requestID in requestIDs: request = GetRequest.getRequest(requestID) requestName = request["RequestName"] helper = Utilities.loadWorkload(request) result[requestName] = helper.listOutputDatasets() return result def getMostRecentOutputForPrepID(self, prepID): """Return the datasets produced by the most recently submitted request with this prep ID""" requestIDs = GetRequest.getRequestByPrepID(prepID) # most recent will have the largest ID requestIDs.sort() requestIDs.reverse() request = None # Go through each request in order from largest to smallest # looking for the first non-failed/non-canceled request for requestID in requestIDs: request = GetRequest.getRequest(requestID) rejectList = ['aborted', 'failed', 'rejected', 'epic-failed'] requestStatus = request.get("RequestStatus", 'aborted').lower() if requestStatus not in rejectList: break if request != None: helper = Utilities.loadWorkload(request) return helper.listOutputDatasets() else: return [] def getAssignment(self, teamName=None, request=None): """ If a team name is passed in, get all assignments for that team. If a request is passed in, return a list of teams the request is assigned to """ # better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve? #requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName) # Maybe now assigned to team is same as assigned to ProdMgr result = [] if teamName != None: requestIDs = ListRequests.listRequestsByTeam(teamName, "assigned").values() requests = [GetRequest.getRequest(reqID) for reqID in requestIDs] # put highest priorities first requests.sort(key=lambda r: r['RequestPriority'], reverse=True) # return list of tuples, since we need sorting result = [[req['RequestName'], req['RequestWorkflow']] for req in requests] elif request != None: result = GetRequest.getAssignmentsByName(request) return result def getRequestsByStatusAndTeam(self, teamName, status): """ Get a list of request names with the given team and status. """ requestNames = ListRequests.listRequestsByTeam(teamName, status).keys() return requestNames def getUser(self, userName=None, group=None): """ No args returns a list of all users. Group returns groups this user is in. Username returs a JSON with information about the user """ if userName != None: if not Registration.isRegistered(userName): raise cherrypy.HTTPError(404, "Cannot find user") result = {} result['groups'] = GroupInfo.groupsForUser(userName).keys() result['requests'] = UserRequests.listRequests(userName).keys() result.update(Registration.userInfo(userName)) return result elif group != None: GroupInfo.usersInGroup(group) else: return Registration.listUsers() def getGroup(self, group=None, user=None): """ No args lists all groups, one args returns JSON with users.""" if group != None: result = {} result['users'] = GroupInfo.usersInGroup(group) return result elif user != None: return GroupInfo.groupsForUser(user).keys() else: return GroupInfo.listGroups() def getVersion(self): """ Returns a list of all CMSSW versions registered with ReqMgr """ archList = SoftwareAdmin.listSoftware() result = [] for arch in archList.keys(): for version in archList[arch]: if not version in result: result.append(version) return result def getTeam(self): """ Returns a list of all teams registered with ReqMgr """ return ProdManagement.listTeams() def getWorkQueue(self, request=None, workQueue=None): """ If a request is passed in, return the URl of the workqueue. If a workqueue is passed in, return all requests acquired by it """ if workQueue != None: return ProdMgrRetrieve.findAssignedRequests(workQueue) if request != None: return ProdManagement.getProdMgr(request) # return all the workqueue ulr return GetRequest.getGlobalQueues() def getMessage(self, request): """ Returns a list of messages attached to this request """ return ChangeState.getMessages(request) def getInputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this input dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput", dataset) def getOutputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this output dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput", dataset) def getCampaign(self, campaign=None): """ returns a list of all campaigns if no argument, and a list of all requests in a campaign if there is an argument """ if campaign == None: return Campaign.listCampaigns() else: return Campaign.listRequestsByCampaign(campaign) def putWorkQueue(self, request, url): """ Registers the request as "acquired" by the workqueue with the given URL """ Utilities.changeStatus(request, "acquired", self.wmstatWriteURL, self.acdcURL) return ProdManagement.associateProdMgr(request, urllib.unquote(url)) def validatePutWorkQueue(self, index): assert index['request'].replace('_', '').replace('-', '').replace('.', '').isalnum() assert index['url'].startswith('http') return index # had no permission control before, security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def putRequest(self, requestName=None, status=None, priority=None, stats=None): request = None if requestName: try: request = self.getRequest(requestName) except Exception, ex: # request presumably doesn't exist request = None if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) # check for forbidden input arguments: for deprec in deprecatedRequestArgs: if deprec in reqInputArgs: msg = ( "Creating request failed, unsupported input arg: %s: %s" % (deprec, reqInputArgs[deprec])) self.error(msg) raise cherrypy.HTTPError(400, msg) reqInputArgs.setdefault( 'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName) # wrong naming ... but it's all over the place, it's the config cache DB name reqInputArgs.setdefault('CouchDBName', self.configDBName) reqInputArgs.setdefault('Requestor', cherrypy.request.user["login"]) try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: self.error("Create request failed, reason: %s" % ex) # Assume that this is a valid HTTPError raise ex except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError( 403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>" ) try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError( 403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) if stats != None: Utilities.updateRequestStats(requestName=requestName, stats=stats, couchURL=self.couchUrl, couchDBName=self.workloadDBName) return request
class ReqMgrRESTModel(RESTModel): """ The REST interface to the ReqMgr database. Documentation may be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """ def __init__(self, config): RESTModel.__init__(self, config) self.couchUrl = config.couchUrl self.workloadDBName = config.workloadDBName self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName) self.security_params = {'roles': config.security_roles} # Optional values for individual methods self.reqPriorityMax = getattr(config, 'maxReqPriority', 100) self._addMethod('GET', 'request', self.getRequest, args=['requestName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'assignment', self.getAssignment, args=['teamName', 'request'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'user', self.getUser, args=['userName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'group', self.getGroup, args=['group', 'user'], secured=True, expires=0) self._addMethod('GET', 'version', self.getVersion, args=[], secured=True, expires=0) self._addMethod('GET', 'team', self.getTeam, args=[], secured=True, expires=0) self._addMethod('GET', 'workQueue', self.getWorkQueue, args=['request', 'workQueue'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'message', self.getMessage, args=['request'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'inputdataset', self.getInputDataset, args=['prim', 'proc', 'tier'], secured=True) self._addMethod('GET', 'outputdataset', self.getOutputDataset, args=['prim', 'proc', 'tier'], secured=True) self._addMethod('GET', 'campaign', self.getCampaign, args=['campaign'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('PUT', 'request', self.putRequest, args=['requestName', 'status', 'priority', 'stats'], secured=True, validation=[ self.isalnumExceptStats, self.reqPriority, self.validateStats ]) self._addMethod('PUT', 'clone', self.cloneRequest, args=['requestName'], secured=True, validation=[self.isalnum]) self._addMethod('PUT', 'assignment', self.putAssignment, args=['team', 'requestName'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'user', self.putUser, args=['userName', 'email', 'dnName'], secured=True, security_params=self.security_params, validation=[self.validateUser]) self._addMethod('PUT', 'group', self.putGroup, args=['group', 'user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'version', self.putVersion, args=['version', 'scramArch'], secured=True, security_params=self.security_params, validation=[self.validateVersion]) self._addMethod('PUT', 'team', self.putTeam, args=['team'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'workQueue', self.putWorkQueue, args=['request', 'url'], secured=True, security_params=self.security_params, validation=[self.validatePutWorkQueue]) self._addMethod('PUT', 'message', self.putMessage, args=['request'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'campaign', self.putCampaign, args=['campaign', 'request'], secured=True, validation=[self.isalnum]) self._addMethod('POST', 'request', self.postRequest, args=[ 'requestName', 'events_written', 'events_merged', 'files_written', 'files_merged', 'percent_written', 'percent_success', 'dataset' ], secured=True, validation=[self.validateUpdates]) self._addMethod('POST', 'closeout', self.closeOutRequest, args=['requestName', 'cascade'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('POST', 'announce', self.announceRequest, args=['requestName', 'cascade'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('DELETE', 'request', self.deleteRequest, args=['requestName'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'user', self.deleteUser, args=['user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'group', self.deleteGroup, args=['group', 'user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'version', self.deleteVersion, args=['version', 'scramArch'], secured=True, validation=[self.validateVersion]) self._addMethod('DELETE', 'team', self.deleteTeam, args=['team'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'campaign', self.deleteCampaign, args=['campaign'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('GET', 'requestnames', self.getRequestNames, args=[], secured=True, expires=0) self._addMethod('GET', 'outputDatasetsByRequestName', self.getOutputForRequest, args=['requestName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'outputDatasetsByPrepID', self.getOutputForPrepID, args=['prepID'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'mostRecentOutputDatasetsByPrepID', self.getMostRecentOutputForPrepID, args=['prepID'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'configIDs', self.getConfigIDs, args=['prim', 'proc', 'tier'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'requestsByStatusAndTeam', self.getRequestsByStatusAndTeam, args=['teamName', 'status'], secured=True, validation=[self.isalnum], expires=0) cherrypy.engine.subscribe('start_thread', self.initThread) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() #myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def isalnum(self, index): """ Validates that all input is alphanumeric, with spaces and underscores tolerated""" for v in index.values(): WMCore.Lexicon.identifier(v) return index def isalnumExceptStats(self, index): """ Validates that all input is alphanumeric, with spaces and underscores tolerated""" if 'stats' in index: return index return self.isalnum(index) def getDataset(self, prim, proc, tier): """ If only prim exists, assume it's urlquoted. If all three exists, assue it's /prim/proc/tier """ if not proc and not tier: dataset = urllib.unquote(prim) elif prim and proc and tier: dataset = "/%s/%s/%s" % (prim, proc, tier) WMCore.Lexicon.dataset(dataset) return dataset def intpriority(self, index): """ Casts priority to an integer """ if 'priority' in index: value = int(index['priority']) if math.fabs(value) >= sys.maxsize: msg = "Invalid priority! Priority must have abs() less then MAXINT!" raise cherrypy.HTTPError(400, msg) index['priority'] = value return index def reqPriority(self, index): """ _reqPriority_ Sets request priority to an integer. Also makes sure it's within a certain value. """ if 'priority' not in index: return index index = self.intpriority(index=index) value = index['priority'] if math.fabs( value) > self.reqPriorityMax and not Utilities.privileged(): msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax raise cherrypy.HTTPError(400, msg) return index def validateUser(self, index): assert index['userName'].isalnum() assert '@' in index['email'] assert index['email'].replace('@', '').replace('.', '').isalnum() if 'dnName' in index: assert index['dnName'].replace(' ', '').isalnum() return index def validateVersion(self, index): """ Make sure it's a legitimate CMSSW version format """ WMCore.Lexicon.cmsswversion(index['version']) return index def getRequest(self, requestName=None): """ If a request name is specified, return the details of the request. Otherwise, return an overview of all requests """ if requestName == None: result = GetRequest.getRequests() else: result = Utilities.requestDetails(requestName) try: teamNames = GetRequest.getAssignmentsByName(requestName) result['teams'] = teamNames except: # Ignore errors, then we just don't have a team name pass return result def getRequestNames(self): # 2013-02-13 is wrong anyway (e.g. Campaign is not working) # should be removed """ return all the request names in RequestManager as list """ #TODO this could me combined with getRequest return GetRequest.getOverview() def getOutputForRequest(self, requestName): """Return the datasets produced by this request.""" return Utilities.getOutputForRequest(requestName) def getOutputForPrepID(self, prepID): """Return the datasets produced by this prep ID. in a dict of requestName:dataset list""" requestIDs = GetRequest.getRequestByPrepID(prepID) result = {} for requestID in requestIDs: request = GetRequest.getRequest(requestID) requestName = request["RequestName"] helper = Utilities.loadWorkload(request) result[requestName] = helper.listOutputDatasets() return result def getMostRecentOutputForPrepID(self, prepID): """Return the datasets produced by the most recently submitted request with this prep ID""" requestIDs = GetRequest.getRequestByPrepID(prepID) # most recent will have the largest ID requestIDs.sort() requestIDs.reverse() request = None # Go through each request in order from largest to smallest # looking for the first non-failed/non-canceled request for requestID in requestIDs: request = GetRequest.getRequest(requestID) rejectList = ['aborted', 'failed', 'rejected', 'epic-failed'] requestStatus = request.get("RequestStatus", 'aborted').lower() if requestStatus not in rejectList: break if request != None: helper = Utilities.loadWorkload(request) return helper.listOutputDatasets() else: return [] def getAssignment(self, teamName=None, request=None): """ If a team name is passed in, get all assignments for that team. If a request is passed in, return a list of teams the request is assigned to """ # better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve? #requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName) # Maybe now assigned to team is same as assigned to ProdMgr result = [] if teamName != None: requestIDs = ListRequests.listRequestsByTeam(teamName, "assigned").values() requests = [GetRequest.getRequest(reqID) for reqID in requestIDs] # put highest priorities first requests.sort(key=lambda r: r['RequestPriority'], reverse=True) # return list of tuples, since we need sorting result = [[req['RequestName'], req['RequestWorkflow']] for req in requests] elif request != None: result = GetRequest.getAssignmentsByName(request) return result def getRequestsByStatusAndTeam(self, teamName, status): """ Get a list of request names with the given team and status. """ requestNames = ListRequests.listRequestsByTeam(teamName, status).keys() return requestNames def getUser(self, userName=None, group=None): """ No args returns a list of all users. Group returns groups this user is in. Username returs a JSON with information about the user """ if userName != None: if not Registration.isRegistered(userName): raise cherrypy.HTTPError(404, "Cannot find user") result = {} result['groups'] = GroupInfo.groupsForUser(userName).keys() result['requests'] = UserRequests.listRequests(userName).keys() result.update(Registration.userInfo(userName)) return result elif group != None: GroupInfo.usersInGroup(group) else: return Registration.listUsers() def getGroup(self, group=None, user=None): """ No args lists all groups, one args returns JSON with users.""" if group != None: result = {} result['users'] = GroupInfo.usersInGroup(group) return result elif user != None: return GroupInfo.groupsForUser(user).keys() else: return GroupInfo.listGroups() def getVersion(self): """ Returns a list of all CMSSW versions registered with ReqMgr """ archList = SoftwareAdmin.listSoftware() result = [] for arch in archList.keys(): for version in archList[arch]: if not version in result: result.append(version) return result def getTeam(self): """ Returns a list of all teams registered with ReqMgr """ return ProdManagement.listTeams() def getWorkQueue(self, request=None, workQueue=None): """ If a request is passed in, return the URl of the workqueue. If a workqueue is passed in, return all requests acquired by it """ if workQueue != None: return ProdMgrRetrieve.findAssignedRequests(workQueue) if request != None: return ProdManagement.getProdMgr(request) # return all the workqueue ulr return GetRequest.getGlobalQueues() def getMessage(self, request): """ Returns a list of messages attached to this request """ return ChangeState.getMessages(request) def getInputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this input dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput", dataset) def getOutputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this output dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput", dataset) def getCampaign(self, campaign=None): """ returns a list of all campaigns if no argument, and a list of all requests in a campaign if there is an argument """ if campaign == None: return Campaign.listCampaigns() else: return Campaign.listRequestsByCampaign(campaign) def putWorkQueue(self, request, url): """ Registers the request as "acquired" by the workqueue with the given URL """ Utilities.changeStatus(request, "acquired", self.wmstatWriteURL, self.acdcURL) return ProdManagement.associateProdMgr(request, urllib.unquote(url)) def validatePutWorkQueue(self, index): assert index['request'].replace('_', '').replace('-', '').replace('.', '').isalnum() assert index['url'].startswith('http') return index # had no permission control before, security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def putRequest(self, requestName=None, status=None, priority=None, stats=None): request = None if requestName: try: request = self.getRequest(requestName) except Exception as ex: # request presumably doesn't exist request = None if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) # check for forbidden input arguments: for deprec in deprecatedRequestArgs: if deprec in reqInputArgs: msg = ( "Creating request failed, unsupported input arg: %s: %s" % (deprec, reqInputArgs[deprec])) self.error(msg) raise cherrypy.HTTPError(400, msg) reqInputArgs.setdefault( 'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName) # wrong naming ... but it's all over the place, it's the config cache DB name reqInputArgs.setdefault('CouchDBName', self.configDBName) reqInputArgs.setdefault('Requestor', cherrypy.request.user["login"]) try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: msg = traceback.format_exc() self.error("Create request failed, reason: %s" % msg) # Assume that this is a valid HTTPError raise ex except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message msg = traceback.format_exc() self.error("Create request failed, reason: %s" % msg) if hasattr(ex, "message"): if hasattr(ex.message, '__call__'): detail = ex.message() else: detail = str(ex) elif hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError( 403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>" ) try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError( 403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) if stats != None: Utilities.updateRequestStats(requestName=requestName, stats=stats, couchURL=self.couchUrl, couchDBName=self.workloadDBName) return request def cloneRequest(self, requestName): """ Input assumes an existing request, checks that. The original existing request is not touched. A new request is generated. The cloned request has a newly generated RequestName, new timestamp, RequestDate, however -everything- else is copied from the original request. Addition: since Edgar changed his mind, he no longer wants this cloned request be in the 'new' state but put straight into 'assignment-approved' state. """ request = None if requestName: self.info("Cloning request: request name: '%s'" % requestName) requestOrigDict = self.getRequest(requestName) if requestOrigDict: self.info("Request found, cloning ...") newReqSchema = Utilities.getNewRequestSchema(requestOrigDict) # since we cloned the request, all the attributes # manipulation in Utilities.makeRequest() should not be necessary # the cloned request shall be identical but following arguments toRemove = ["RequestName", "RequestDate", "timeStamp"] toRemove.extend(deprecatedRequestArgs) for remove in toRemove: try: del requestOrigDict[remove] except KeyError: pass newReqSchema.update(requestOrigDict) # clone # problem that priority wasn't preserved went down from here: # via CheckIn.checkIn() -> MakeRequest.createRequest() -> Request.New.py factory request = Utilities.buildWorkloadAndCheckIn( self, newReqSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL, clone=True) # change the clone request state as desired Utilities.changeStatus(newReqSchema["RequestName"], "assignment-approved", self.wmstatWriteURL, self.acdcURL) return request else: msg = "Request '%s' not found." % requestName self.warning(msg) raise cherrypy.HTTPError(404, msg) else: msg = "Received empty request name: '%s', exit." % requestName self.warn(msg) raise cherrypy.HTTPError(400, msg) def putAssignment(self, team, requestName): """ Assigns this request to this team """ # see if it's already assigned team = urllib.unquote(team) if not team in ProdManagement.listTeams(): raise cherrypy.HTTPError(404, "Cannot find this team") requestNamesAndIDs = ListRequests.listRequestsByTeam(team) if requestName in requestNamesAndIDs.keys(): raise cherrypy.HTTPError(400, "Already assigned to this team") return ChangeState.assignRequest(requestName, team) def putUser(self, userName, email, dnName=None): """ Needs to be passed an e-mail address, maybe dnName """ if Registration.isRegistered(userName): return "User already exists" result = Registration.registerUser(userName, email, dnName) def putGroup(self, group, user=None): """ Creates a group, or if a user is passed, adds that user to the group """ if (user != None): # assume group exists and add user to it return GroupManagement.addUserToGroup(user, group) if GroupInfo.groupExists(group): return "Group already exists" GroupManagement.addGroup(group) def putVersion(self, version, scramArch=None): """ Registers a new CMSSW version with ReqMgr """ return SoftwareAdmin.addSoftware(version, scramArch=scramArch) def putTeam(self, team): """ Registers a team with ReqMgr """ return ProdManagement.addTeam(urllib.unquote(team)) def putMessage(self, request): """ Attaches a message to this request """ message = JsonWrapper.loads(cherrypy.request.body.read()) result = ChangeState.putMessage(request, message) return result def putCampaign(self, campaign, request=None): """ Adds a campaign if it doesn't already exist, and optionally associates a request with it """ if request: requestID = GetRequest.requestID(request) if requestID: result = Campaign.associateCampaign(campaign, requestID) Utilities.associateCampaign(campaign=campaign, requestName=request, couchURL=self.couchUrl, couchDBName=self.workloadDBName) return result else: return False else: Campaign.addCampaign(campaign) # def postRequest(self, requestName, events_written=None, events_merged=None, # files_written=None, files_merged = None, dataset=None): def postRequest(self, requestName, **kwargs): """ Add a progress update to the request Id provided, params can optionally contain: - *events_written* Int - *events_merged* Int - *files_written* Int - *files_merged* int - *percent_success* float - *percent_complete float - *dataset* string (dataset name) """ return ChangeState.updateRequest(requestName, kwargs) def closeOutRequest(self, requestName, cascade=False): """ Close out a request, if the cascade option is given then it will search for any Resubmission requests for which the given request is a parent and close them out too. """ requestsToCloseOut = [requestName] if cascade == "True": requestsToCloseOut.extend( Utilities.retrieveResubmissionChildren(requestName, self.couchUrl, self.workloadDBName)) for requestName in requestsToCloseOut: Utilities.changeStatus(requestName, 'closed-out', self.wmstatWriteURL, self.acdcURL) return def announceRequest(self, requestName, cascade=False): """ Announce a request, if the cascade option is given then it will search for any Resubmission requests for which the given request is a parent and announce them too. """ requestsToAnnounce = [requestName] if cascade == "True": requestsToAnnounce.extend( Utilities.retrieveResubmissionChildren(requestName, self.couchUrl, self.workloadDBName)) for requestName in requestsToAnnounce: Utilities.changeStatus(requestName, 'announced', self.wmstatWriteURL, self.acdcURL) return def validateUpdates(self, index): """ Check the values for the updates """ for k in [ 'events_written', 'events_merged', 'files_written', 'files_merged' ]: if k in index: index[k] = int(index[k]) for k in ['percent_success', 'percent_complete']: if k in index: index[k] = float(index[k]) return index def validateStats(self, index): """ Check the values for the updates """ if 'stats' not in index: return index index['stats'] = Utilities.unidecode(JsonWrapper.loads(index['stats'])) for k in [ 'input_lummis', 'input_num_files', 'input_events', 'total_jobs' ]: if k in index['stats']: index['stats'][k] = int(index['stats'][k]) return index # had no permission control before, security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def deleteRequest(self, requestName): """ Deletes a request from the ReqMgr MySQL/Oracle database and also from CoucDB. """ # 404 will be thrown automatically on a non-existing request request = self.getRequest(requestName) helper = Utilities.loadWorkload(request) couchDocId = requestName helper.deleteCouch(self.couchUrl, self.workloadDBName, couchDocId) # #4289 - Request delete operation deletes the request from # MySQL/Oracle but not from CouchDB, fix here # Seangchan shall also fix here deleting such requests from WMStats (#4398) # returns None ... response = RequestAdmin.deleteRequest(requestName) return response def deleteUser(self, user): """ Deletes a user, as well as deleting his requests and removing him from all groups """ if user in self.getUser(): requests = json.loads(self.getUser(user))['requests'] for request in requests: self.deleteRequest(request) for group in GroupInfo.groupsForUser(user).keys(): GroupManagement.removeUserFromGroup(user, group) return UserManagement.deleteUser(user) def deleteGroup(self, group, user=None): """ If no user is sent, delete the group. Otherwise, delete the user from the group """ if user == None: return GroupManagement.deleteGroup(group) else: return GroupManagement.removeUserFromGroup(user, group) def deleteVersion(self, version, scramArch): """ Un-register this software version with ReqMgr """ SoftwareAdmin.removeSoftware(version, scramArch) def deleteTeam(self, team): """ Delete this team from ReqMgr """ ProdManagement.removeTeam(urllib.unquote(team)) def deleteCampaign(self, campaign): return Campaign.deleteCampaign(campaign) def getConfigIDs(self, prim, proc, tier): """ _getConfigIDs_ Get the ConfigIDs for the specified request """ result = {} dataset = self.getDataset(prim, proc, tier) requests = GetRequest.getRequestsByCriteria( "Datasets.GetRequestByInput", dataset) for request in requests: requestName = request["RequestName"] helper = Utilities.loadWorkload(request) result[requestName] = helper.listAllCMSSWConfigCacheIDs() return result
class Assign(WebAPI): """ Used by data ops to assign requests to processing sites""" def __init__(self, config, noSiteDB=False): """ _init_ Note, noSiteDB added for TESTING PURPOSED ONLY! """ WebAPI.__init__(self, config) ReqMgrAuth.assign_roles = config.security_roles # Take a guess self.templatedir = config.templates self.couchUrl = config.couchUrl self.configDBName = config.configDBName self.workloadDBName = config.workloadDBName self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) if not noSiteDB: try: # Download a list of all the sites from SiteDB, uses v2 API. sitedb = SiteDBJSON() self.sites = sitedb.getAllCMSNames() self.sites.sort() self.phedexNodes = sitedb.getAllPhEDExNodeNames(excludeBuffer=True) self.phedexNodes.sort() except Exception as ex: msg = "ERROR: Could not retrieve sites from SiteDB, reason: %s" % ex cherrypy.log(msg) raise else: self.sites = [] # store result lfn base with all Physics group storeResultLFNBase = ["/store/results/analysisops", "/store/results/b_physics", "/store/results/b_tagging", "/store/results/b2g", "/store/results/e_gamma_ecal", "/store/results/ewk", "/store/results/exotica", "/store/results/forward", "/store/results/heavy_ions", "/store/results/higgs", "/store/results/jets_met_hcal", "/store/results/muon", "/store/results/qcd", "/store/results/susy", "/store/results/tau_pflow", "/store/results/top", "/store/results/tracker_dpg", "/store/results/tracker_pog", "/store/results/trigger"] # yet 0.9.40 had also another self.mergedLFNBases which was differentiating # list of mergedLFNBases based on type of request, removed and all bases # will be displayed regardless of the request type (discussion with Edgar) self.allMergedLFNBases = [ "/store/backfill/1", "/store/backfill/2", "/store/data", "/store/mc", "/store/generator", "/store/relval", "/store/hidata", "/store/himc"] self.allMergedLFNBases.extend(storeResultLFNBase) self.allUnmergedLFNBases = ["/store/unmerged", "/store/temp"] self.yuiroot = config.yuiroot cherrypy.engine.subscribe('start_thread', self.initThread) self.wildcardKeys = getattr(config, 'wildcardKeys', {'T1*': 'T1_*', 'T2*': 'T2_*', 'T3*': 'T3_*'}) self.wildcardSites = {} Utilities.addSiteWildcards(self.wildcardKeys, self.sites, self.wildcardSites) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() # myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def validate(self, v, name=""): """ _validate_ Checks different fields with different Lexicon methods, if not the field name is not known then apply the identifier check """ # Make sure the value is a string, otherwise the Lexicon complains strValue = str(v) try: if name == "ProcessingVersion": WMCore.Lexicon.procversion(strValue) elif name == "AcquisitionEra": WMCore.Lexicon.acqname(strValue) elif name == "ProcessingString": WMCore.Lexicon.procstring(strValue) else: WMCore.Lexicon.identifier(strValue) except AssertionError as ex: raise cherrypy.HTTPError(400, "Bad input: %s" % str(ex)) return v def validateDatatier(self, datatier, dbsUrl): """ _validateDatatier_ Provided a list of datatiers extracted from the outputDatasets, checks whether they all exist in DBS already. """ dbsTiers = DBSReader.listDatatiers(dbsUrl) badTiers = list(set(datatier) - set(dbsTiers)) if badTiers: raise cherrypy.HTTPError(400, "Bad datatier(s): %s not available in DBS." % badTiers) @cherrypy.expose @cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles) def one(self, requestName): """ Assign a single request """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) request = Utilities.prepareForTable(request) # get assignments teams = ProdManagement.listTeams() assignments = GetRequest.getAssignmentsByName(requestName) # might be a list, or a dict team:priority if isinstance(assignments, dict): assignments = assignments.keys() procVer = "" acqEra = "" procString = "" helper = Utilities.loadWorkload(request) if helper.getAcquisitionEra() != None: acqEra = helper.getAcquisitionEra() if helper.getProcessingVersion() != None: procVer = helper.getProcessingVersion() if helper.getProcessingString(): procString = helper.getProcessingString() dashboardActivity = helper.getDashboardActivity() blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime() blockCloseMaxFiles = helper.getBlockCloseMaxFiles() blockCloseMaxEvents = helper.getBlockCloseMaxEvents() blockCloseMaxSize = helper.getBlockCloseMaxSize() (reqMergedBase, reqUnmergedBase) = helper.getLFNBases() return self.templatepage("Assign", requests=[request], teams=teams, assignments=assignments, sites=self.sites, phedexNodes=self.phedexNodes, mergedLFNBases=self.allMergedLFNBases, reqMergedBase=reqMergedBase, unmergedLFNBases=self.allUnmergedLFNBases, reqUnmergedBase=reqUnmergedBase, acqEra=acqEra, procVer=procVer, procString=procString, dashboardActivity=dashboardActivity, badRequests=[], blockCloseMaxWaitTime=blockCloseMaxWaitTime, blockCloseMaxFiles=blockCloseMaxFiles, blockCloseMaxSize=blockCloseMaxSize, blockCloseMaxEvents=blockCloseMaxEvents) @cherrypy.expose @cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles) def index(self, all=0): """ Main page """ # returns dict of name:id allRequests = Utilities.requestsWithStatus('assignment-approved') teams = ProdManagement.listTeams() procVer = "" acqEra = "" procString = "" dashboardActivity = None badRequestNames = [] goodRequests = allRequests reqMergedBase = None reqUnmergedBase = None blockCloseMaxWaitTime = 66400 blockCloseMaxFiles = 500 blockCloseMaxEvents = 250000000 blockCloseMaxSize = 5000000000000 # for request in allRequests: # # make sure there's a workload attached # try: # helper = Utilities.loadWorkload(request) # except Exception, ex: # logging.error("Assign error: %s " % str(ex)) # badRequestNames.append(request["RequestName"]) # else: # # get defaults from the first good one # if not goodRequests: # # forget it if it fails. # try: # if helper.getAcquisitionEra() != None: # acqEra = helper.getAcquisitionEra() # if helper.getProcessingVersion() != None: # procVer = helper.getProcessingVersion() # if helper.getProcessingString() != None: # procString = helper.getProcessingString() # blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime() # blockCloseMaxFiles = helper.getBlockCloseMaxFiles() # blockCloseMaxEvents = helper.getBlockCloseMaxEvents() # blockCloseMaxSize = helper.getBlockCloseMaxSize() # (reqMergedBase, reqUnmergedBase) = helper.getLFNBases() # dashboardActivity = helper.getDashboardActivity() # goodRequests.append(request) # except Exception, ex: # logging.error("Assign error: %s " % str(ex)) # badRequestNames.append(request["RequestName"]) # else: # goodRequests.append(request) return self.templatepage("Assign", all=all, requests=goodRequests, teams=teams, assignments=[], sites=self.sites, phedexNodes=self.phedexNodes, mergedLFNBases=self.allMergedLFNBases, reqMergedBase=reqMergedBase, unmergedLFNBases=self.allUnmergedLFNBases, reqUnmergedBase=reqUnmergedBase, acqEra=acqEra, procVer=procVer, procString=procString, dashboardActivity=dashboardActivity, badRequests=badRequestNames, blockCloseMaxWaitTime=blockCloseMaxWaitTime, blockCloseMaxFiles=blockCloseMaxFiles, blockCloseMaxSize=blockCloseMaxSize, blockCloseMaxEvents=blockCloseMaxEvents) @cherrypy.expose # @cherrypy.tools.secmodv2(role=ReqMgrAuth.assign_roles) security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAssignmentPage(self, **kwargs): """ handler for the main page """ # Accept Json encoded strings decodedArgs = {} for key in kwargs.keys(): try: decodedArgs[key] = json.loads(kwargs[key]) except Exception: # Probably wasn't JSON decodedArgs[key] = kwargs[key] kwargs = decodedArgs # handle the checkboxes teams = [] requestNames = [] for key, value in kwargs.iteritems(): if isinstance(value, basestring): kwargs[key] = value.strip() if key.startswith("Team"): teams.append(key[4:]) if key.startswith("checkbox"): requestName = key[8:] self.validate(requestName) requestNames.append(key[8:]) for requestName in requestNames: if kwargs['action'] == 'Reject': ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl=self.wmstatWriteURL) else: assignments = GetRequest.getAssignmentsByName(requestName) if teams == [] and assignments == []: raise cherrypy.HTTPError(400, "Must assign to one or more teams") kwargs["Teams"] = teams self.assignWorkload(requestName, kwargs) for team in teams: if not team in assignments: ChangeState.assignRequest(requestName, team, wmstatUrl=self.wmstatWriteURL) priority = kwargs.get(requestName + ':priority', '') if priority != '': Utilities.changePriority(requestName, priority, self.wmstatWriteURL) participle = kwargs['action'] + 'ed' return self.templatepage("Acknowledge", participle=participle, requests=requestNames) def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) try: helper.validateArgumentForAssignment(kwargs) except WMSpecFactoryException as ex: raise cherrypy.HTTPError(400, str(ex.message())) except Exception: msg = traceback.format_exc() raise cherrypy.HTTPError(400, "Unhandled error: %s" % msg) # Validate the different parts of the processed dataset processedDatasetParts = {"AcquisitionEra": kwargs.get("AcquisitionEra"), "ProcessingString": kwargs.get("ProcessingString"), "ProcessingVersion": kwargs.get("ProcessingVersion") } for field, values in processedDatasetParts.iteritems(): if field in kwargs and isinstance(kwargs[field], dict): for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs.get(field, values), field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) if not isinstance(whiteList, list): whiteList = [whiteList] if not isinstance(blackList, list): blackList = [blackList] helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) res = set(whiteList) & set(blackList) if len(res): raise cherrypy.HTTPError(400, "White and blacklist the same site is not allowed %s" % list(res)) helper.setAcquisitionEra(kwargs.get("AcquisitionEra", None)) helper.setProcessingString(kwargs.get("ProcessingString", None)) helper.setProcessingVersion(kwargs.get("ProcessingVersion", None)) # Now verify the output datasets datatier = [] outputDatasets = helper.listOutputDatasets() for dataset in outputDatasets: tokens = dataset.split("/") procds = tokens[2] datatier.append(tokens[3]) try: WMCore.Lexicon.procdataset(procds) except AssertionError as ex: raise cherrypy.HTTPError(400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex)) # Verify whether the output datatiers are available in DBS self.validateDatatier(datatier, dbsUrl=helper.getDbsUrl()) # FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring(kwargs.get("MaxRSS", None), kwargs.get("MaxVSize", None), kwargs.get("SoftTimeout", None), kwargs.get("GracePeriod", None)) # Check whether we should check location for the data helper.setTrustLocationFlag(inputFlag=strToBool(kwargs.get("TrustSitelists", False)), pileupFlag=strToBool(kwargs.get("TrustPUSitelists", False))) helper.setAllowOpportunistic(allowOpport=strToBool(kwargs.get("AllowOpportunistic", False))) # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) autoApproveList = kwargs.get("AutoApproveSubscriptionSites", []) subscriptionPriority = kwargs.get("SubscriptionPriority", "Low") custodialType = kwargs.get("CustodialSubType", "Replica") nonCustodialType = kwargs.get("NonCustodialSubType", "Replica") helper.setSubscriptionInformationWildCards(wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, custodialSubType=custodialType, nonCustodialSubType=nonCustodialType, custodialGroup=kwargs.get("CustodialGroup", "DataOps"), nonCustodialGroup=kwargs.get("NonCustodialGroup", "DataOps"), priority=subscriptionPriority, deleteFromSource=kwargs.get("DeleteFromSource", False)) # Block closing information blockCloseMaxWaitTime = int(kwargs.get("BlockCloseMaxWaitTime", helper.getBlockCloseMaxWaitTime())) blockCloseMaxFiles = int(kwargs.get("BlockCloseMaxFiles", helper.getBlockCloseMaxFiles())) blockCloseMaxEvents = int(kwargs.get("BlockCloseMaxEvents", helper.getBlockCloseMaxEvents())) blockCloseMaxSize = int(kwargs.get("BlockCloseMaxSize", helper.getBlockCloseMaxSize())) helper.setBlockCloseSettings(blockCloseMaxWaitTime, blockCloseMaxFiles, blockCloseMaxEvents, blockCloseMaxSize) helper.setMemory(kwargs.get("Memory")) helper.setCores(kwargs.get("Multicore")) helper.setDashboardActivity(kwargs.get("Dashboard", "")) helper.setTaskProperties(kwargs) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL) # update AcquisitionEra in the Couch document (#4380) # request object returned above from Oracle doesn't have information Couch # database reqDetails = Utilities.requestDetails(request["RequestName"]) couchDb = Database(reqDetails["CouchWorkloadDBName"], reqDetails["CouchURL"]) couchDb.updateDocument(request["RequestName"], "ReqMgr", "updaterequest", fields={"AcquisitionEra": reqDetails["AcquisitionEra"], "ProcessingVersion": reqDetails["ProcessingVersion"], "CustodialSites": custodialList, "NonCustodialSites": nonCustodialList, "AutoApproveSubscriptionSites": autoApproveList, "SubscriptionPriority": subscriptionPriority, "CustodialSubType": custodialType, "NonCustodialSubType": nonCustodialType, "CustodialGroup": kwargs.get("CustodialGroup", "DataOps"), "NonCustodialGroup": kwargs.get("NonCustodialGroup", "DataOps"), "DeleteFromSource": kwargs.get("DeleteFromSource", False), "Teams": kwargs["Teams"], "OutputDatasets": outputDatasets, "SiteWhitelist": whiteList, "SiteBlacklist": blackList, "MergedLFNBase": kwargs["MergedLFNBase"], "UnmergedLFNBase": kwargs["UnmergedLFNBase"], "Dashboard": kwargs.get("Dashboard", ""), "TrustSitelists": kwargs.get("TrustSitelists", False), "TrustPUSitelists": kwargs.get("TrustPUSitelists", False), "AllowOpportunistic": kwargs.get("AllowOpportunistic", False)}, useBody=True)
class Admin(WebAPI): """ Handles administration functions for ReqMgr """ def __init__(self, config): WebAPI.__init__(self, config) # Take a guess self.templatedir = config.templates self.htmldir = config.html cherrypy.engine.subscribe('start_thread', self.initThread) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() #myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def validate(self, v, name=''): """ Checks if alphanumeric, tolerating spaces """ if isinstance(v, list): for entry in v: self.validate(entry) else: try: WMCore.Lexicon.identifier(v) except AssertionError: raise cherrypy.HTTPError(400, "Bad input %s" % name) return v @cherrypy.expose @cherrypy.tools.secmodv2() def index(self): """ Main web page """ return Utilities.serveFile('text/html', self.htmldir, "Admin.html") @cherrypy.expose @cherrypy.tools.secmodv2() def user(self, userName): """ Web page of details about the user, and sets user priority """ self.validate(userName) groups = GroupInfo.groupsForUser(userName).keys() requests = UserRequests.listRequests(userName).keys() priority = UserManagement.getPriority(userName) allGroups = GroupInfo.listGroups() self.validate(groups) self.validate(requests) self.validate(allGroups) return self.templatepage("User", user=userName, groups=groups, allGroups=allGroups, requests=requests, priority=priority) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleUserPriority(self, user, userPriority): """ Handles setting user priority """ self.validate(user) UserManagement.setPriority(user, userPriority) return "Updated user %s priority to %s" % (user, userPriority) @cherrypy.expose @cherrypy.tools.secmodv2() def group(self, groupName): """ Web page of details about the user, and sets user priority """ self.validate(groupName) users = GroupInfo.usersInGroup(groupName) priority = GroupManagement.getPriority(groupName) return self.templatepage("Group", group=groupName, users=users, priority=priority) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleGroupPriority(self, group, groupPriority): """ Handles setting group priority """ self.validate(group) GroupManagement.setPriority(group, groupPriority) return "Updated group %s priority to %s" % (group, groupPriority) @cherrypy.expose @cherrypy.tools.secmodv2() def users(self): """ Lists all users. Should be paginated later """ allUsers = Registration.listUsers() self.validate(allUsers) return self.templatepage("Users", users=allUsers) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAddUser(self, user, email=None): """ Handles setting user priority """ self.validate(user) Registration.registerUser(user, email) return "Added user %s" % user @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAddToGroup(self, user, group): """ Adds a user to the group """ self.validate(user) self.validate(group) GroupManagement.addUserToGroup(user, group) return "Added %s to %s " % (user, group) @cherrypy.expose @cherrypy.tools.secmodv2() def groups(self): """ Lists all users. Should be paginated later """ allGroups = GroupInfo.listGroups() self.validate(allGroups) return self.templatepage("Groups", groups=allGroups) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAddGroup(self, group): """ Handles adding a group """ self.validate(group) GroupManagement.addGroup(group) return "Added group %s " % group @cherrypy.expose @cherrypy.tools.secmodv2() def teams(self): """ Lists all teams """ teams = ProdManagement.listTeams().keys() self.validate(teams) return self.templatepage("Teams", teams=teams) @cherrypy.expose @cherrypy.tools.secmodv2() def team(self, teamName): """ Details for a team """ self.validate(teamName) assignments = ListRequests.listRequestsByTeam(teamName) if assignments == None: assignments = [] else: assignments = assignments.keys() self.validate(assignments) return self.templatepage("Team", team=teamName, requests=assignments) @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAddTeam(self, team): """ Handles a request to add a team """ self.validate(team) ProdManagement.addTeam(team) return "Added team %s" % team @cherrypy.expose @cherrypy.tools.secmodv2() def versions(self): """ Lists all versions """ archList = SoftwareAdmin.listSoftware() versions = [] for versionList in archList.values(): for version in versionList: if not version in versions: versions.append(version) versions.sort() for version in versions: WMCore.Lexicon.cmsswversion(version) return self.templatepage("Versions", versions=versions) @cherrypy.expose @cherrypy.tools.secmodv2() def scramArchs(self): """ _scramArchs_ List all scramArchs in the DB Prelim for putting this in the template pages """ return SoftwareAdmin.listSoftware().keys() @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAddVersion(self, version): """ Registers a version """ WMCore.Lexicon.cmsswversion(version) SoftwareAdmin.updateSoftware(version) return "Added version %s" % version @cherrypy.expose @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def handleAllVersions(self): """ Registers all versions in the TC """ Utilities.updateScramArchsAndCMSSWVersions() return "Updated versions to current standard"