def makeSchema(self, **schema): schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) schema.setdefault('CouchDBName', self.configDBName) try: request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName) except RuntimeError, e: raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
def makeSchema(self, **schema): schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) schema.setdefault('CouchDBName', self.configDBName) decodedSchema = {} for key in schema.keys(): try: decodedSchema[key] = JsonWrapper.loads(schema[key]) except: # We don't know what kind of exception we'll get, so ignore them all # If it does except, it probably wasn't in JSON to begin with. # Anything else should be caught by the parsers and the validation decodedSchema[key] = schema[key] try: self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (decodedSchema, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) # catching here KeyError is just terrible except (RuntimeError, KeyError, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) baseURL = cherrypy.request.base raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
def putRequest(self, requestName=None, status=None, priority=None): request = None if requestName: request = self.findRequest(requestName) if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) reqInputArgs.setdefault( 'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchDBName', self.configDBName) try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: self.error("Create request failed, reason: %s" % ex) # Assume that this is a valid HTTPError raise except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError( 403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>" ) try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError( 403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) return request
def putRequest(self, requestName=None, status=None, priority=None): """ Checks the request n the body with one arg, and changes the status with kwargs """ request = None if requestName: request = self.findRequest(requestName) if request == None: """ Creates a new request, with a JSON-encoded schema that is sent in the body of the request """ body = cherrypy.request.body.read() schema = Utilities.unidecode(JsonWrapper.loads(body)) schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) schema.setdefault('CouchDBName', self.configDBName) try: request = Utilities.makeRequest(schema, self.couchUrl, self.workloadDBName) except RuntimeError, ex: raise cherrypy.HTTPError(400, ex.message)
def putRequest(self, requestName=None, status=None, priority=None): request = None if requestName: request = self.findRequest(requestName) if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) reqInputArgs.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchDBName', self.configDBName) try: self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: self.error("Create request failed, reason: %s" % ex) # Assume that this is a valid HTTPError raise except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>") try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) return request
def makeSchema(self, **schema): schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) schema.setdefault('CouchDBName', self.configDBName) decodedSchema = {} for key in schema.keys(): try: decodedSchema[key] = JsonWrapper.loads(schema[key]) except: # We don't know what kind of exception we'll get, so ignore them all # If it does except, it probably wasn't in JSON to begin with. # Anything else should be caught by the parsers and the validation decodedSchema[key] = schema[key] try: request = Utilities.makeRequest(decodedSchema, self.couchUrl, self.workloadDBName) except RuntimeError, e: raise cherrypy.HTTPError(400, "Error creating request: %s" % e)
def makeSchema(self, **schema): schema.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) # wrong naming ... but it's all over the place, it's the config cache DB name schema.setdefault('CouchDBName', self.configDBName) schema.setdefault('CouchWorkloadDBName', self.workloadDBName) decodedSchema = {} for key in schema.keys(): try: decodedSchema[key] = JsonWrapper.loads(schema[key]) except: # We don't know what kind of exception we'll get, so ignore them all # If it does except, it probably wasn't in JSON to begin with. # Anything else should be caught by the parsers and the validation decodedSchema[key] = schema[key] try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (decodedSchema, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, decodedSchema, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) # catching here KeyError is just terrible except (RuntimeError, KeyError, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % str(ex)) if hasattr(ex, "message"): if hasattr(ex.message, '__call__'): detail = ex.message() else: detail = str(ex) elif hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) baseURL = cherrypy.request.base raise cherrypy.HTTPRedirect('%s/reqmgr/view/details/%s' % (baseURL, request['RequestName']))
def putRequest(self, requestName=None, status=None, priority=None, stats=None): request = None if requestName: try: request = self.getRequest(requestName) except Exception as ex: # request presumably doesn't exist request = None if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) # check for forbidden input arguments: for deprec in deprecatedRequestArgs: if deprec in reqInputArgs: msg = ("Creating request failed, unsupported input arg: %s: %s" % (deprec, reqInputArgs[deprec])) self.error(msg) raise cherrypy.HTTPError(400, msg) reqInputArgs.setdefault('CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName) # wrong naming ... but it's all over the place, it's the config cache DB name reqInputArgs.setdefault('CouchDBName', self.configDBName) reqInputArgs.setdefault('Requestor', cherrypy.request.user["login"]) try: self.info("Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: msg = traceback.format_exc() self.error("Create request failed, reason: %s" % msg) # Assume that this is a valid HTTPError raise ex except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message msg = traceback.format_exc() self.error("Create request failed, reason: %s" % msg) if hasattr(ex, "message"): if hasattr(ex.message, '__call__'): detail = ex.message() else: detail = str(ex) elif hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError(403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>") try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError(403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) if stats != None: Utilities.updateRequestStats(requestName = requestName, stats = stats, couchURL = self.couchUrl, couchDBName = self.workloadDBName) return request
class ReqMgrRESTModel(RESTModel): """ The REST interface to the ReqMgr database. Documentation may be found at https://twiki.cern.ch/twiki/bin/viewauth/CMS/ReqMgrSystemDesign """ def __init__(self, config): RESTModel.__init__(self, config) self.couchUrl = config.couchUrl self.workloadDBName = config.workloadDBName self.configDBName = config.configDBName self.wmstatWriteURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.wmstatDBName) self.acdcURL = "%s/%s" % (self.couchUrl.rstrip('/'), config.acdcDBName) self.security_params = {'roles': config.security_roles} # Optional values for individual methods self.reqPriorityMax = getattr(config, 'maxReqPriority', 100) self._addMethod('GET', 'request', self.getRequest, args=['requestName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'assignment', self.getAssignment, args=['teamName', 'request'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'user', self.getUser, args=['userName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'group', self.getGroup, args=['group', 'user'], secured=True, expires=0) self._addMethod('GET', 'version', self.getVersion, args=[], secured=True, expires=0) self._addMethod('GET', 'team', self.getTeam, args=[], secured=True, expires=0) self._addMethod('GET', 'workQueue', self.getWorkQueue, args=['request', 'workQueue'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'message', self.getMessage, args=['request'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'inputdataset', self.getInputDataset, args=['prim', 'proc', 'tier'], secured=True) self._addMethod('GET', 'outputdataset', self.getOutputDataset, args=['prim', 'proc', 'tier'], secured=True) self._addMethod('GET', 'campaign', self.getCampaign, args=['campaign'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('PUT', 'request', self.putRequest, args=['requestName', 'status', 'priority', 'stats'], secured=True, validation=[ self.isalnumExceptStats, self.reqPriority, self.validateStats ]) self._addMethod('PUT', 'clone', self.cloneRequest, args=['requestName'], secured=True, validation=[self.isalnum]) self._addMethod('PUT', 'assignment', self.putAssignment, args=['team', 'requestName'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'user', self.putUser, args=['userName', 'email', 'dnName'], secured=True, security_params=self.security_params, validation=[self.validateUser]) self._addMethod('PUT', 'group', self.putGroup, args=['group', 'user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'version', self.putVersion, args=['version', 'scramArch'], secured=True, security_params=self.security_params, validation=[self.validateVersion]) self._addMethod('PUT', 'team', self.putTeam, args=['team'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'workQueue', self.putWorkQueue, args=['request', 'url'], secured=True, security_params=self.security_params, validation=[self.validatePutWorkQueue]) self._addMethod('PUT', 'message', self.putMessage, args=['request'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('PUT', 'campaign', self.putCampaign, args=['campaign', 'request'], secured=True, validation=[self.isalnum]) self._addMethod('POST', 'request', self.postRequest, args=[ 'requestName', 'events_written', 'events_merged', 'files_written', 'files_merged', 'percent_written', 'percent_success', 'dataset' ], secured=True, validation=[self.validateUpdates]) self._addMethod('POST', 'closeout', self.closeOutRequest, args=['requestName', 'cascade'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('POST', 'announce', self.announceRequest, args=['requestName', 'cascade'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('DELETE', 'request', self.deleteRequest, args=['requestName'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'user', self.deleteUser, args=['user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'group', self.deleteGroup, args=['group', 'user'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'version', self.deleteVersion, args=['version', 'scramArch'], secured=True, validation=[self.validateVersion]) self._addMethod('DELETE', 'team', self.deleteTeam, args=['team'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('DELETE', 'campaign', self.deleteCampaign, args=['campaign'], secured=True, security_params=self.security_params, validation=[self.isalnum]) self._addMethod('GET', 'requestnames', self.getRequestNames, args=[], secured=True, expires=0) self._addMethod('GET', 'outputDatasetsByRequestName', self.getOutputForRequest, args=['requestName'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'outputDatasetsByPrepID', self.getOutputForPrepID, args=['prepID'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'mostRecentOutputDatasetsByPrepID', self.getMostRecentOutputForPrepID, args=['prepID'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'configIDs', self.getConfigIDs, args=['prim', 'proc', 'tier'], secured=True, validation=[self.isalnum], expires=0) self._addMethod('GET', 'requestsByStatusAndTeam', self.getRequestsByStatusAndTeam, args=['teamName', 'status'], secured=True, validation=[self.isalnum], expires=0) cherrypy.engine.subscribe('start_thread', self.initThread) def initThread(self, thread_index): """ The ReqMgr expects the DBI to be contained in the Thread """ myThread = threading.currentThread() #myThread = cherrypy.thread_data # Get it from the DBFormatter superclass myThread.dbi = self.dbi def isalnum(self, index): """ Validates that all input is alphanumeric, with spaces and underscores tolerated""" for v in index.values(): WMCore.Lexicon.identifier(v) return index def isalnumExceptStats(self, index): """ Validates that all input is alphanumeric, with spaces and underscores tolerated""" if index.has_key('stats'): return index return self.isalnum(index) def getDataset(self, prim, proc, tier): """ If only prim exists, assume it's urlquoted. If all three exists, assue it's /prim/proc/tier """ if not proc and not tier: dataset = urllib.unquote(prim) elif prim and proc and tier: dataset = "/%s/%s/%s" % (prim, proc, tier) WMCore.Lexicon.dataset(dataset) return dataset def intpriority(self, index): """ Casts priority to an integer """ if index.has_key('priority'): value = int(index['priority']) if math.fabs(value) >= sys.maxint: msg = "Invalid priority! Priority must have abs() less then MAXINT!" raise cherrypy.HTTPError(400, msg) index['priority'] = value return index def reqPriority(self, index): """ _reqPriority_ Sets request priority to an integer. Also makes sure it's within a certain value. """ if not index.has_key('priority'): return index index = self.intpriority(index=index) value = index['priority'] if math.fabs( value) > self.reqPriorityMax and not Utilities.privileged(): msg = "Invalid requestPriority! Request priority must have abs() less then %i!" % self.reqPriorityMax raise cherrypy.HTTPError(400, msg) return index def validateUser(self, index): assert index['userName'].isalnum() assert '@' in index['email'] assert index['email'].replace('@', '').replace('.', '').isalnum() if 'dnName' in index: assert index['dnName'].replace(' ', '').isalnum() return index def validateVersion(self, index): """ Make sure it's a legitimate CMSSW version format """ WMCore.Lexicon.cmsswversion(index['version']) return index def getRequest(self, requestName=None): """ If a request name is specified, return the details of the request. Otherwise, return an overview of all requests """ if requestName == None: result = GetRequest.getRequests() else: result = Utilities.requestDetails(requestName) try: teamNames = GetRequest.getAssignmentsByName(requestName) result['teams'] = teamNames except: # Ignore errors, then we just don't have a team name pass return result def getRequestNames(self): # 2013-02-13 is wrong anyway (e.g. Campaign is not working) # should be removed """ return all the request names in RequestManager as list """ #TODO this could me combined with getRequest return GetRequest.getOverview() def getOutputForRequest(self, requestName): """Return the datasets produced by this request.""" return Utilities.getOutputForRequest(requestName) def getOutputForPrepID(self, prepID): """Return the datasets produced by this prep ID. in a dict of requestName:dataset list""" requestIDs = GetRequest.getRequestByPrepID(prepID) result = {} for requestID in requestIDs: request = GetRequest.getRequest(requestID) requestName = request["RequestName"] helper = Utilities.loadWorkload(request) result[requestName] = helper.listOutputDatasets() return result def getMostRecentOutputForPrepID(self, prepID): """Return the datasets produced by the most recently submitted request with this prep ID""" requestIDs = GetRequest.getRequestByPrepID(prepID) # most recent will have the largest ID requestIDs.sort() requestIDs.reverse() request = None # Go through each request in order from largest to smallest # looking for the first non-failed/non-canceled request for requestID in requestIDs: request = GetRequest.getRequest(requestID) rejectList = ['aborted', 'failed', 'rejected', 'epic-failed'] requestStatus = request.get("RequestStatus", 'aborted').lower() if requestStatus not in rejectList: break if request != None: helper = Utilities.loadWorkload(request) return helper.listOutputDatasets() else: return [] def getAssignment(self, teamName=None, request=None): """ If a team name is passed in, get all assignments for that team. If a request is passed in, return a list of teams the request is assigned to """ # better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve? #requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName) # Maybe now assigned to team is same as assigned to ProdMgr result = [] if teamName != None: requestIDs = ListRequests.listRequestsByTeam(teamName, "assigned").values() requests = [GetRequest.getRequest(reqID) for reqID in requestIDs] # put highest priorities first requests.sort(key=lambda r: r['RequestPriority'], reverse=True) # return list of tuples, since we need sorting result = [[req['RequestName'], req['RequestWorkflow']] for req in requests] elif request != None: result = GetRequest.getAssignmentsByName(request) return result def getRequestsByStatusAndTeam(self, teamName, status): """ Get a list of request names with the given team and status. """ requestNames = ListRequests.listRequestsByTeam(teamName, status).keys() return requestNames def getUser(self, userName=None, group=None): """ No args returns a list of all users. Group returns groups this user is in. Username returs a JSON with information about the user """ if userName != None: if not Registration.isRegistered(userName): raise cherrypy.HTTPError(404, "Cannot find user") result = {} result['groups'] = GroupInfo.groupsForUser(userName).keys() result['requests'] = UserRequests.listRequests(userName).keys() result.update(Registration.userInfo(userName)) return result elif group != None: GroupInfo.usersInGroup(group) else: return Registration.listUsers() def getGroup(self, group=None, user=None): """ No args lists all groups, one args returns JSON with users.""" if group != None: result = {} result['users'] = GroupInfo.usersInGroup(group) return result elif user != None: return GroupInfo.groupsForUser(user).keys() else: return GroupInfo.listGroups() def getVersion(self): """ Returns a list of all CMSSW versions registered with ReqMgr """ archList = SoftwareAdmin.listSoftware() result = [] for arch in archList.keys(): for version in archList[arch]: if not version in result: result.append(version) return result def getTeam(self): """ Returns a list of all teams registered with ReqMgr """ return ProdManagement.listTeams() def getWorkQueue(self, request=None, workQueue=None): """ If a request is passed in, return the URl of the workqueue. If a workqueue is passed in, return all requests acquired by it """ if workQueue != None: return ProdMgrRetrieve.findAssignedRequests(workQueue) if request != None: return ProdManagement.getProdMgr(request) # return all the workqueue ulr return GetRequest.getGlobalQueues() def getMessage(self, request): """ Returns a list of messages attached to this request """ return ChangeState.getMessages(request) def getInputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this input dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput", dataset) def getOutputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this output dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput", dataset) def getCampaign(self, campaign=None): """ returns a list of all campaigns if no argument, and a list of all requests in a campaign if there is an argument """ if campaign == None: return Campaign.listCampaigns() else: return Campaign.listRequestsByCampaign(campaign) def putWorkQueue(self, request, url): """ Registers the request as "acquired" by the workqueue with the given URL """ Utilities.changeStatus(request, "acquired", self.wmstatWriteURL, self.acdcURL) return ProdManagement.associateProdMgr(request, urllib.unquote(url)) def validatePutWorkQueue(self, index): assert index['request'].replace('_', '').replace('-', '').replace('.', '').isalnum() assert index['url'].startswith('http') return index # had no permission control before, security issue fix @cherrypy.tools.secmodv2(role=Utilities.security_roles(), group=Utilities.security_groups()) def putRequest(self, requestName=None, status=None, priority=None, stats=None): request = None if requestName: try: request = self.getRequest(requestName) except Exception, ex: # request presumably doesn't exist request = None if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) # check for forbidden input arguments: for deprec in deprecatedRequestArgs: if deprec in reqInputArgs: msg = ( "Creating request failed, unsupported input arg: %s: %s" % (deprec, reqInputArgs[deprec])) self.error(msg) raise cherrypy.HTTPError(400, msg) reqInputArgs.setdefault( 'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName) # wrong naming ... but it's all over the place, it's the config cache DB name reqInputArgs.setdefault('CouchDBName', self.configDBName) reqInputArgs.setdefault('Requestor', cherrypy.request.user["login"]) try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: self.error("Create request failed, reason: %s" % ex) # Assume that this is a valid HTTPError raise ex except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message self.error("Create request failed, reason: %s" % ex) if hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError( 403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>" ) try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError( 403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) if stats != None: Utilities.updateRequestStats(requestName=requestName, stats=stats, couchURL=self.couchUrl, couchDBName=self.workloadDBName) return request
def putRequest(self, requestName=None, status=None, priority=None, stats=None): request = None if requestName: try: request = self.getRequest(requestName) except Exception as ex: # request presumably doesn't exist request = None if request == None: # Create a new request, with a JSON-encoded schema that is # sent in the body of the HTTP request body = cherrypy.request.body.read() reqInputArgs = Utilities.unidecode(JsonWrapper.loads(body)) # check for forbidden input arguments: for deprec in deprecatedRequestArgs: if deprec in reqInputArgs: msg = ( "Creating request failed, unsupported input arg: %s: %s" % (deprec, reqInputArgs[deprec])) self.error(msg) raise cherrypy.HTTPError(400, msg) reqInputArgs.setdefault( 'CouchURL', Utilities.removePasswordFromUrl(self.couchUrl)) reqInputArgs.setdefault('CouchWorkloadDBName', self.workloadDBName) # wrong naming ... but it's all over the place, it's the config cache DB name reqInputArgs.setdefault('CouchDBName', self.configDBName) reqInputArgs.setdefault('Requestor', cherrypy.request.user["login"]) try: self.info( "Creating a request for: '%s'\n\tworkloadDB: '%s'\n\twmstatUrl: " "'%s' ..." % (reqInputArgs, self.workloadDBName, Utilities.removePasswordFromUrl(self.wmstatWriteURL))) request = Utilities.makeRequest(self, reqInputArgs, self.couchUrl, self.workloadDBName, self.wmstatWriteURL) except cherrypy.HTTPError as ex: msg = traceback.format_exc() self.error("Create request failed, reason: %s" % msg) # Assume that this is a valid HTTPError raise ex except (WMException, Exception) as ex: # TODO problem not to expose logs to the client # e.g. on ConfigCacheID not found, the entire CouchDB traceback is sent in ex_message msg = traceback.format_exc() self.error("Create request failed, reason: %s" % msg) if hasattr(ex, "message"): if hasattr(ex.message, '__call__'): detail = ex.message() else: detail = str(ex) elif hasattr(ex, "name"): detail = ex.name else: detail = "check logs." msg = "Create request failed, %s" % detail raise cherrypy.HTTPError(400, msg) self.info("Request '%s' created." % request['RequestName']) # see if status & priority need to be upgraded if status != None: # forbid assignment here if status == 'assigned': raise cherrypy.HTTPError( 403, "Cannot change status without a team. Please use PUT /reqmgr/reqMgr/assignment/<team>/<requestName>" ) try: Utilities.changeStatus(requestName, status, self.wmstatWriteURL, self.acdcURL) except RuntimeError as ex: # ignore some of these errors: https://svnweb.cern.ch/trac/CMSDMWM/ticket/2002 if status != 'announced' and status != 'closed-out': self.error("RuntimeError while changeStatus: reason: %s" % ex) raise cherrypy.HTTPError( 403, "Failed to change status: %s" % str(ex)) if priority != None: Utilities.changePriority(requestName, priority, self.wmstatWriteURL) if stats != None: Utilities.updateRequestStats(requestName=requestName, stats=stats, couchURL=self.couchUrl, couchDBName=self.workloadDBName) return request