def one(self, requestName): """ Assign a single request """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) request = Utilities.prepareForTable(request) requestType = request["RequestType"] # get assignments teams = ProdManagement.listTeams() assignments = GetRequest.getAssignmentsByName(requestName) # might be a list, or a dict team:priority if isinstance(assignments, dict): assignments = assignments.keys() procVer = "" acqEra = "" helper = Utilities.loadWorkload(request) if helper.getAcquisitionEra() != None: acqEra = helper.getAcquisitionEra() if helper.getProcessingVersion() != None: procVer = helper.getProcessingVersion() dashboardActivity = helper.getDashboardActivity() (reqMergedBase, reqUnmergedBase) = helper.getLFNBases() return self.templatepage("Assign", requests=[request], teams=teams, assignments=assignments, sites=self.sites, mergedLFNBases=self.mergedLFNBases[requestType], reqMergedBase=reqMergedBase, unmergedLFNBases=self.allUnmergedLFNBases, reqUnmergedBase=reqUnmergedBase, acqEra = acqEra, procVer = procVer, dashboardActivity=dashboardActivity, badRequests=[])
def getRequest(requestId): """ _getRequest_ Get the full details of the request to be passed to the PM. Includes - All basic request information from DB - Teams the request has been assigned to - Priority per assigned team - URL to download the workflow spec from TODO: Add download URL for the workflow spec. Follow up with FVL what needs to be added here """ request = GetRequest.getRequest(requestId) assigned = GetRequest.getRequestAssignments(requestId) for team in assigned: team['TeamPriority'] += request['RequestPriority'] request['Assignments'] = assigned return request
def getMostRecentOutputForPrepID(self, prepID): """Return the datasets produced by the most recently submitted request with this prep ID""" requestIDs = GetRequest.getRequestByPrepID(prepID) # most recent will have the largest ID requestID = max(requestIDs) request = GetRequest.getRequest(requestID) helper = Utilities.loadWorkload(request) return helper.listOutputDatasets()
def getOutputForPrepID(self, prepID): """Return the datasets produced by this prep ID. in a dict of requestName:dataset list""" requestIDs = GetRequest.getRequestByPrepID(prepID) result = {} for requestID in requestIDs: request = GetRequest.getRequest(requestID) requestName = request["RequestName"] helper = Utilities.loadWorkload(request) result[requestName] = helper.listOutputDatasets() return result
def one(self, requestName): """ Assign a single request """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) request = Utilities.prepareForTable(request) # get assignments teams = ProdManagement.listTeams() assignments = GetRequest.getAssignmentsByName(requestName) # might be a list, or a dict team:priority if isinstance(assignments, dict): assignments = assignments.keys() procVer = "" acqEra = "" procString = "" helper = Utilities.loadWorkload(request) if helper.getAcquisitionEra() != None: acqEra = helper.getAcquisitionEra() if helper.getProcessingVersion() != None: procVer = helper.getProcessingVersion() if helper.getProcessingString(): procString = helper.getProcessingString() dashboardActivity = helper.getDashboardActivity() blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime() blockCloseMaxFiles = helper.getBlockCloseMaxFiles() blockCloseMaxEvents = helper.getBlockCloseMaxEvents() blockCloseMaxSize = helper.getBlockCloseMaxSize() (reqMergedBase, reqUnmergedBase) = helper.getLFNBases() return self.templatepage( "Assign", requests=[request], teams=teams, assignments=assignments, sites=self.sites, phedexNodes=self.phedexNodes, mergedLFNBases=self.allMergedLFNBases, reqMergedBase=reqMergedBase, unmergedLFNBases=self.allUnmergedLFNBases, reqUnmergedBase=reqUnmergedBase, acqEra=acqEra, procVer=procVer, procString=procString, dashboardActivity=dashboardActivity, badRequests=[], blockCloseMaxWaitTime=blockCloseMaxWaitTime, blockCloseMaxFiles=blockCloseMaxFiles, blockCloseMaxSize=blockCloseMaxSize, blockCloseMaxEvents=blockCloseMaxEvents, )
def getRequest(self, requestName=None): """ If a request name is specified, return the details of the request. Otherwise, return an overview of all requests """ if requestName == None: result = GetRequest.getRequests() else: result = Utilities.requestDetails(requestName) try: teamNames = GetRequest.getAssignmentsByName(requestName) result['teams'] = teamNames except: # Ignore errors, then we just don't have a team name pass return result
def one(self, requestName): """ Assign a single request """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) request = Utilities.prepareForTable(request) requestType = request["RequestType"] # get assignments teams = ProdManagement.listTeams() assignments = GetRequest.getAssignmentsByName(requestName) # might be a list, or a dict team:priority if isinstance(assignments, dict): assignments = assignments.keys() procVer = "" acqEra = "" procString = "" helper = Utilities.loadWorkload(request) if helper.getAcquisitionEra() != None: acqEra = helper.getAcquisitionEra() if helper.getProcessingVersion() != None: procVer = helper.getProcessingVersion() if helper.getProcessingString(): procString = helper.getProcessingString() dashboardActivity = helper.getDashboardActivity() blockCloseMaxWaitTime = helper.getBlockCloseMaxWaitTime() blockCloseMaxFiles = helper.getBlockCloseMaxFiles() blockCloseMaxEvents = helper.getBlockCloseMaxEvents() blockCloseMaxSize = helper.getBlockCloseMaxSize() (reqMergedBase, reqUnmergedBase) = helper.getLFNBases() return self.templatepage("Assign", requests=[request], teams=teams, assignments=assignments, sites=self.sites, phedexNodes=self.phedexNodes, mergedLFNBases=self.allMergedLFNBases, reqMergedBase=reqMergedBase, unmergedLFNBases=self.allUnmergedLFNBases, reqUnmergedBase=reqUnmergedBase, acqEra=acqEra, procVer=procVer, procString=procString, dashboardActivity=dashboardActivity, badRequests=[], blockCloseMaxWaitTime=blockCloseMaxWaitTime, blockCloseMaxFiles=blockCloseMaxFiles, blockCloseMaxSize=blockCloseMaxSize, blockCloseMaxEvents=blockCloseMaxEvents)
def splitting(self, requestName): """ _splitting_ Retrieve the current values for splitting parameters for all tasks in the spec. Format them in the manner that the splitting page expects and pass them to the template. """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) splittingDict = helper.listJobSplittingParametersByTask(performance = False) taskNames = splittingDict.keys() taskNames.sort() splitInfo = [] for taskName in taskNames: jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName]) splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"], "splitParams": jsonSplittingParams, "taskType": splittingDict[taskName]["type"], "taskName": taskName}) return self.templatepage("Splitting", requestName = requestName, taskInfo = splitInfo, taskNames = taskNames)
def changeStatus(requestName, status, wmstatUrl, acdcUrl): """ Changes the status for this request """ request = GetRequest.getRequestByName(requestName) if not status in RequestStatus.StatusList: raise RuntimeError, "Bad status code " + status if not request.has_key('RequestStatus'): raise RuntimeError, "Cannot find status for request " + requestName oldStatus = request['RequestStatus'] if not status in RequestStatus.NextStatus[oldStatus]: raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % ( oldStatus, status, RequestStatus.NextStatus[oldStatus]) if status == 'aborted': # delete from the workqueue if not privileged() and not ownsRequest(request): raise cherrypy.HTTPError(403, "You are not allowed to abort this request") elif not privileged(): raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request") # delete from the workqueue if it's been assigned to one if oldStatus in ["acquired", "running", "running-closed", "running-open"]: abortRequest(requestName) else: raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus) if status == 'announced': # cleanup acdc database, if possible if acdcUrl: url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl) acdcService = CouchService(url = url, database = database) acdcService.removeFilesetsByCollectionName(requestName) # finally, perform the transition, have to do it in both Oracle and CouchDB # and in WMStats ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def index(self): requests = GetRequest.getRequests() tableBody = self.drawRequests(requests) return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot, fields=self.fields, tableBody=tableBody)
def requestDetails(requestName): """ Adds details from the Couch document as well as the database """ WMCore.Lexicon.identifier(requestName) request = GetRequest.getRequestDetails(requestName) helper = loadWorkload(request) schema = helper.data.request.schema.dictionary_whole_tree_() # take the stuff from the DB preferentially schema.update(request) task = helper.getTopLevelTask()[0] schema['Site Whitelist'] = task.siteWhitelist() schema['Site Blacklist'] = task.siteBlacklist() schema['MergedLFNBase'] = str(helper.getMergedLFNBase()) schema['UnmergedLFNBase'] = str(helper.getUnmergedLFNBase()) schema['Campaign'] = str(helper.getCampaign()) schema['AcquisitionEra'] = str(helper.getAcquisitionEra()) if schema['SoftwareVersions'] == ['DEPRECATED']: schema['SoftwareVersions'] = helper.getCMSSWVersions() # Check in the CouchWorkloadDBName if not present schema.setdefault("CouchWorkloadDBName", "reqmgr_workload_cache") # https://github.com/dmwm/WMCore/issues/4588 schema["SubscriptionInformation"] = helper.getSubscriptionInformation() return schema
def changeStatus(requestName, status, wmstatUrl, acdcUrl): """ Changes the status for this request """ request = GetRequest.getRequestByName(requestName) if not status in RequestStatus.StatusList: raise RuntimeError, "Bad status code " + status if not request.has_key('RequestStatus'): raise RuntimeError, "Cannot find status for request " + requestName oldStatus = request['RequestStatus'] if not status in RequestStatus.NextStatus[oldStatus]: raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % ( oldStatus, status, RequestStatus.NextStatus[oldStatus]) if status == 'aborted' or status == 'force-complete': # delete from the workqueue if not privileged() and not ownsRequest(request): raise cherrypy.HTTPError(403, "You are not allowed to %s this request" % status) elif not privileged(): raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request") # delete from the workqueue if it's been assigned to one if status in RequestStatus.NextStatus[oldStatus]: abortRequest(requestName) else: raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus) if status == 'announced': # cleanup acdc database, if possible if acdcUrl: url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl) acdcService = CouchService(url = url, database = database) acdcService.removeFilesetsByCollectionName(requestName) # finally, perform the transition, have to do it in both Oracle and CouchDB # and in WMStats ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def splitting(self, requestName): """ _splitting_ Retrieve the current values for splitting parameters for all tasks in the spec. Format them in the manner that the splitting page expects and pass them to the template. """ self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) splittingDict = helper.listJobSplittingParametersByTask() taskNames = splittingDict.keys() taskNames.sort() splitInfo = [] for taskName in taskNames: jsonSplittingParams = JsonWrapper.dumps(splittingDict[taskName]) splitInfo.append({"splitAlgo": splittingDict[taskName]["algorithm"], "splitParams": jsonSplittingParams, "taskType": splittingDict[taskName]["type"], "taskName": taskName}) return self.templatepage("Splitting", requestName = requestName, taskInfo = splitInfo, taskNames = taskNames)
def changeStatus(requestName, status, wmstatUrl): """ Changes the status for this request """ request = GetRequest.getRequestByName(requestName) if not status in RequestStatus.StatusList: raise RuntimeError, "Bad status code " + status if not request.has_key('RequestStatus'): raise RuntimeError, "Cannot find status for request " + requestName oldStatus = request['RequestStatus'] if not status in RequestStatus.NextStatus[oldStatus]: raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % ( oldStatus, status, RequestStatus.NextStatus[oldStatus]) if status == 'aborted': # delete from the workqueue if not privileged() and not ownsRequest(request): raise cherrypy.HTTPError( 403, "You are not allowed to abort this request") elif not privileged(): raise cherrypy.HTTPError( 403, "You are not allowed to change the state for this request") # delete from the workqueue if it's been assigned to one if oldStatus in ["acquired", "running"]: abortRequest(requestName) else: raise cherrypy.HTTPError( 400, "You cannot abort a request in state %s" % oldStatus) #FIXME needs logic about who is allowed to do which transition ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def getOutputForRequest(requestName): """Return the datasets produced by this request.""" request = GetRequest.getRequestByName(requestName) if not request: return [] helper = loadWorkload(request) return helper.listOutputDatasets()
def getRequest(self, requestName=None): """ If a request name is specified, return the details of the request. Otherwise, return an overview of all requests """ if requestName == None: return GetRequest.getRequests() else: return Utilities.requestDetails(requestName)
def index(self): """ Main web page """ print cherrypy.request.user requests = GetRequest.getRequests() tableBody = self.drawRequests(requests) return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot, fields=self.fields, tableBody=tableBody)
def _raiseCheckInError(request, ex, msg): """ Private function called only from this module. Always from the except exception block - the traceback is put in the local log file. """ requestName = request['RequestName'] msg += '\n' + str(ex) msg += "\nUnable to check in new request %s" % requestName reqId = GetRequest.requestID(requestName) # make absolutely sure you're deleting the right one oldReqId = request['RequestID'] if reqId: # make absolutely sure you're deleting the right one oldReqId = request['RequestID'] if oldReqId != reqId: raise RequestCheckInError( "Bad state deleting request %s/%s. Please contact a ReqMgr administrator" % (oldReqId / reqId)) else: RequestAdmin.deleteRequest(requestName) # get information about the last exception trace = traceback.format_exception(*sys.exc_info()) traceString = ''.join(trace) logging.error("%s\n%s" % (msg, traceString)) raise RequestCheckInError(msg)
def changeStatus(requestName, status, wmstatUrl): """ Changes the status for this request """ request = GetRequest.getRequestByName(requestName) if not status in RequestStatus.StatusList: raise RuntimeError, "Bad status code " + status if not request.has_key('RequestStatus'): raise RuntimeError, "Cannot find status for request " + requestName oldStatus = request['RequestStatus'] if not status in RequestStatus.NextStatus[oldStatus]: raise RuntimeError, "Cannot change status from %s to %s. Allowed values are %s" % ( oldStatus, status, RequestStatus.NextStatus[oldStatus]) if status == 'aborted': # delete from the workqueue if not privileged() and not ownsRequest(request): raise cherrypy.HTTPError(403, "You are not allowed to abort this request") elif not privileged(): raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request") # delete from the workqueue if it's been assigned to one if oldStatus in ["acquired", "running"]: abortRequest(requestName) else: raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus) #FIXME needs logic about who is allowed to do which transition ChangeState.changeRequestStatus(requestName, status, wmstatUrl = wmstatUrl) return
def _raiseCheckInError(request, ex, msg): """ Private function called only from this module. Always from the except exception block - the traceback is put in the local log file. """ requestName = request['RequestName'] msg +='\n' + str(ex) msg += "\nUnable to check in new request %s" % requestName reqId = GetRequest.requestID(requestName) # make absolutely sure you're deleting the right one oldReqId = request['RequestID'] if reqId: # make absolutely sure you're deleting the right one oldReqId = request['RequestID'] if oldReqId != reqId: raise RequestCheckInError("Bad state deleting request %s/%s. Please contact a ReqMgr administrator" % (oldReqId/ reqId)) else: RequestAdmin.deleteRequest(reqId) # get information about the last exception trace = traceback.format_exception(*sys.exc_info()) traceString = ''.join(trace) logging.error("%s\n%s" % (msg, traceString)) raise RequestCheckInError(msg)
def changeStatus(requestName, status, wmstatUrl, acdcUrl): """ Changes the status for this request """ request = GetRequest.getRequestByName(requestName) if not status in RequestStatus.StatusList: raise RuntimeError("Bad status code " + status) if 'RequestStatus' not in request: raise RuntimeError("Cannot find status for request " + requestName) oldStatus = request['RequestStatus'] if not status in RequestStatus.NextStatus[oldStatus]: raise RuntimeError("Cannot change status from %s to %s. Allowed values are %s" % ( oldStatus, status, RequestStatus.NextStatus[oldStatus])) if status == 'aborted' or status == 'force-complete': # delete from the workqueue if not privileged() and not ownsRequest(request): raise cherrypy.HTTPError(403, "You are not allowed to %s this request" % status) elif not privileged(): raise cherrypy.HTTPError(403, "You are not allowed to change the state for this request") # delete from the workqueue if it's been assigned to one if status in RequestStatus.NextStatus[oldStatus]: abortRequest(requestName) else: raise cherrypy.HTTPError(400, "You cannot abort a request in state %s" % oldStatus) # ACDC database is cleaned by separate thread # if status == 'announced': # # cleanup acdc database, if possible # if acdcUrl: # url, database = WMCore.Lexicon.splitCouchServiceURL(acdcUrl) # acdcService = CouchService(url = url, database = database) # acdcService.removeFilesetsByCollectionName(requestName) # finally, perform the transition, have to do it in both Oracle and CouchDB # and in WMStats ChangeState.changeRequestStatus(requestName, status, wmstatUrl=wmstatUrl)
def requestDetails(requestName): """ Adds details from the Couch document as well as the database """ WMCore.Lexicon.identifier(requestName) request = GetRequest.getRequestDetails(requestName) helper = loadWorkload(request) schema = helper.data.request.schema.dictionary_whole_tree_() # take the stuff from the DB preferentially schema.update(request) task = helper.getTopLevelTask()[0] schema['Site Whitelist'] = task.siteWhitelist() schema['Site Blacklist'] = task.siteBlacklist() schema['MergedLFNBase'] = str(helper.getMergedLFNBase()) schema['UnmergedLFNBase'] = str(helper.getUnmergedLFNBase()) schema['Campaign'] = str(helper.getCampaign()) schema['AcquisitionEra'] = str(helper.getAcquisitionEra()) schema["ProcessingString"] = str(helper.getProcessingString()) schema["ProcessingVersion"] = int(helper.getProcessingVersion()) if schema['SoftwareVersions'] == ['DEPRECATED']: schema['SoftwareVersions'] = helper.getCMSSWVersions() # Check in the CouchWorkloadDBName if not present schema.setdefault("CouchWorkloadDBName", "reqmgr_workload_cache") # https://github.com/dmwm/WMCore/issues/4588 schema["SubscriptionInformation"] = helper.getSubscriptionInformation() return schema
def makeWorkload(self, schema): originalRequest = GetRequest.getRequestByName(schema['OriginalRequestName']) helper = loadWorkload(originalRequest) helper.truncate(schema["RequestName"], schema["InitialTaskPath"], schema["ACDCServer"], schema["ACDCDatabase"]) return helper.data
def getOutputDataset(self, prim, proc=None, tier=None): """ returns a list of requests with this output dataset Input can either be a single urlquoted dataset, or a /prim/proc/tier""" dataset = self.getDataset(prim, proc, tier) return GetRequest.getRequestsByCriteria("Datasets.GetRequestByOutput", dataset)
def makeWorkload(self, schema): originalRequest = GetRequest.getRequestByName( schema['OriginalRequestName']) helper = loadWorkload(originalRequest) helper.truncate(schema["RequestName"], schema["InitialTaskPath"], schema["ACDCServer"], schema["ACDCDatabase"]) return helper.data
def requestsWithStatus(status): requestIds = theseIds = ListRequests.listRequestsByStatus(status).values() requests = [] for requestId in requestIds: request = GetRequest.getRequest(requestId) request = prepareForTable(request) requests.append(request) return requests
def changePriority(requestName, priority): """ Changes the priority that's stored in the workload """ # fill in all details request = GetRequest.getRequestByName(requestName) ChangeState.changeRequestPriority(requestName, priority) helper = loadWorkload(request) helper.data.request.priority = int(priority) saveWorkload(helper, request['RequestWorkflow'])
def __call__(self, workloadName, arguments): requestName = arguments['OriginalRequestName'] originalRequest = GetRequest.getRequestByName(requestName) helper = loadWorkload(originalRequest) helper.truncate(arguments["RequestName"], arguments["InitialTaskPath"], arguments["ACDCServer"], arguments["ACDCDatabase"]) return helper
def __call__(self, workloadName, arguments): requestName = arguments['OriginalRequestName'] originalRequest = GetRequest.getRequestByName(requestName) helper = loadWorkload(originalRequest) helper.truncate(arguments["RequestName"], arguments["InitialTaskPath"], arguments["ACDCServer"], arguments["ACDCDatabase"], arguments.get("CollectionName")) helper.ignoreOutputModules(arguments.get("IgnoredOutputModules", [])) return helper
def getWorkQueue(self, request=None, workQueue=None): """ If a request is passed in, return the URl of the workqueue. If a workqueue is passed in, return all requests acquired by it """ if workQueue != None: return ProdMgrRetrieve.findAssignedRequests(workQueue) if request != None: return ProdManagement.getProdMgr(request) # return all the workqueue ulr return GetRequest.getGlobalQueues()
def getAssignment(self, teamName=None, request=None): """ If a team name is passed in, get all assignments for that team. If a request is passed in, return a list of teams the request is assigned to """ # better to use ReqMgr/RequestDB/Interface/ProdSystem/ProdMgrRetrieve? #requestIDs = ProdMgrRetrieve.findAssignedRequests(teamName) # Maybe now assigned to team is same as assigned to ProdMgr result = [] if teamName != None: requestIDs = ListRequests.listRequestsByTeam(teamName, "assigned").values() requests = [GetRequest.getRequest(reqID) for reqID in requestIDs] # put highest priorities first requests.sort(key=lambda r: r['RequestPriority'], reverse=True) # return list of tuples, since we need sorting result = [[req['RequestName'], req['RequestWorkflow']] for req in requests] elif request != None: result = GetRequest.getAssignmentsByName(request) return result
def handleSplittingPage(self, requestName, splittingTask, splittingAlgo, **submittedParams): """ _handleSplittingPage_ Parse job splitting parameters sent from the splitting parameter update page. Pull down the request and modify the new spec applying the updated splitting parameters. """ splitParams = {} if splittingAlgo == "FileBased": splitParams["files_per_job"] = int(submittedParams["files_per_job"]) elif splittingAlgo == "TwoFileBased": splitParams["files_per_job"] = int(submittedParams["two_files_per_job"]) elif splittingAlgo == "LumiBased": splitParams["lumis_per_job"] = int(submittedParams["lumis_per_job"]) if str(submittedParams["halt_job_on_file_boundaries"]) == "True": splitParams["halt_job_on_file_boundaries"] = True else: splitParams["halt_job_on_file_boundaries"] = False elif splittingAlgo == "EventAwareLumiBased": splitParams["events_per_job"] = int(submittedParams["avg_events_per_job"]) splitParams["max_events_per_lumi"] = int(submittedParams["max_events_per_lumi"]) if str(submittedParams["halt_job_on_file_boundaries_event_aware"]) == "True": splitParams["halt_job_on_file_boundaries"] = True else: splitParams["halt_job_on_file_boundaries"] = False elif splittingAlgo == "EventBased": splitParams["events_per_job"] = int(submittedParams["events_per_job"]) if "events_per_lumi" in submittedParams: splitParams["events_per_lumi"] = int(submittedParams["events_per_lumi"]) if "lheInputFiles" in submittedParams: if str(submittedParams["lheInputFiles"]) == "True": splitParams["lheInputFiles"] = True else: splitParams["lheInputFiles"] = False elif splittingAlgo == "Harvest": splitParams["periodic_harvest_interval"] = int(submittedParams["periodic_harvest_interval"]) elif 'Merg' in splittingTask: for field in ['min_merge_size', 'max_merge_size', 'max_merge_events', 'max_wait_time']: splitParams[field] = int(submittedParams[field]) if "include_parents" in submittedParams.keys(): if str(submittedParams["include_parents"]) == "True": splitParams["include_parents"] = True else: splitParams["include_parents"] = False self.validate(requestName) request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) logging.info("SetSplitting " + requestName + splittingTask + splittingAlgo + str(splitParams)) helper.setJobSplittingParameters(splittingTask, splittingAlgo, splitParams) Utilities.saveWorkload(helper, request['RequestWorkflow']) return "Successfully updated splitting parameters for " + splittingTask \ + " " + detailsBackLink(requestName)
def changePriority(requestName, priority, wmstatUrl = None): """ Changes the priority that's stored in the workload """ # fill in all details request = GetRequest.getRequestByName(requestName) groupPriority = request.get('ReqMgrGroupBasePriority', 0) userPriority = request.get('ReqMgrRequestorBasePriority', 0) ChangeState.changeRequestPriority(requestName, priority) helper = loadWorkload(request) totalPriority = int(priority) + int(userPriority) + int(groupPriority) helper.data.request.priority = totalPriority saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
def search(self, value, field): """ Search for a regular expression in a certain field of all requests """ filteredRequests = [] requests = GetRequest.getRequests() for request in requests: if request[field].find(value) != -1: filteredRequests.append(request) requests = filteredRequests tableBody = self.drawRequests(requests) return self.templatepage("ReqMgrBrowser", yuiroot=self.yuiroot, fields=self.fields, tableBody=tableBody)
def putCampaign(self, campaign, request=None): """ Adds a campaign if it doesn't already exist, and optionally associates a request with it """ if request: requestID = GetRequest.requestID(request) if requestID: return Campaign.associateCampaign(campaign, requestID) else: return False else: Campaign.addCampaign(campaign)
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) for field in ["AcquisitionEra", "ProcessingVersion"]: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) #FIXME not validated helper.setLFNBase(kwargs["MergedLFNBase"], kwargs["UnmergedLFNBase"]) helper.setMergeParameters(int(kwargs.get("MinMergeSize", 2147483648)), int(kwargs.get("MaxMergeSize", 4294967296)), int(kwargs.get("MaxMergeEvents", 50000))) helper.setupPerformanceMonitoring( int(kwargs.get("maxRSS", 2411724)), int(kwargs.get("maxVSize", 2411724)), int(kwargs.get("SoftTimeout", 167000)), int(kwargs.get("GracePeriod", 300))) # Check whether we should check location for the data if "useSiteListAsLocation" in kwargs: helper.setLocationDataSourceFlag() # Set phedex subscription information custodialList = kwargs.get("CustodialSites", []) nonCustodialList = kwargs.get("NonCustodialSites", []) if "AutoApprove" in kwargs: autoApproveList = nonCustodialList else: autoApproveList = [] priority = kwargs.get("Priority", "Low") if priority not in ["Low", "Normal", "High"]: raise cherrypy.HTTPError(400, "Invalid subscription priority") helper.setSubscriptionInformationWildCards( wildcardDict=self.wildcardSites, custodialSites=custodialList, nonCustodialSites=nonCustodialList, autoApproveSites=autoApproveList, priority=priority) helper.setDashboardActivity(kwargs.get("dashboard", "")) Utilities.saveWorkload(helper, request['RequestWorkflow'], self.wmstatWriteURL)
def changePriority(requestName, priority, wmstatUrl=None): """ Changes the priority that's stored in the workload """ # fill in all details request = GetRequest.getRequestByName(requestName) groupPriority = request.get('ReqMgrGroupBasePriority', 0) userPriority = request.get('ReqMgrRequestorBasePriority', 0) ChangeState.changeRequestPriority(requestName, priority) helper = loadWorkload(request) totalPriority = int(priority) + int(userPriority) + int(groupPriority) helper.data.request.priority = totalPriority saveWorkload(helper, request['RequestWorkflow'], wmstatUrl)
def getMostRecentOutputForPrepID(self, prepID): """Return the datasets produced by the most recently submitted request with this prep ID""" requestIDs = sorted(GetRequest.getRequestByPrepID(prepID)) # most recent will have the largest ID requestIDs.reverse() request = None # Go through each request in order from largest to smallest # looking for the first non-failed/non-canceled request for requestID in requestIDs: request = GetRequest.getRequest(requestID) rejectList = ['aborted', 'failed', 'rejected', 'epic-failed'] requestStatus = request.get("RequestStatus", 'aborted').lower() if requestStatus not in rejectList: break if request != None: helper = Utilities.loadWorkload(request) return helper.listOutputDatasets() else: return []
def requestDetails(requestName): """ Adds details from the Couch document as well as the database """ WMCore.Lexicon.identifier(requestName) request = GetRequest.getRequestDetails(requestName) helper = loadWorkload(request) schema = helper.data.request.schema.dictionary_() # take the stuff from the DB preferentially schema.update(request) task = helper.getTopLevelTask()[0] schema['Site Whitelist'] = task.siteWhitelist() schema['Site Blacklist'] = task.siteBlacklist() return schema
def handleAssignmentPage(self, **kwargs): """ handler for the main page """ #Accept Json encoded strings decodedArgs = {} for key in kwargs.keys(): try: decodedArgs[key] = JsonWrapper.loads(kwargs[key]) except: #Probably wasn't JSON decodedArgs[key] = kwargs[key] kwargs = decodedArgs # handle the checkboxes teams = [] requestNames = [] for key, value in kwargs.iteritems(): if isinstance(value, types.StringTypes): kwargs[key] = value.strip() if key.startswith("Team"): teams.append(key[4:]) if key.startswith("checkbox"): requestName = key[8:] self.validate(requestName) requestNames.append(key[8:]) for requestName in requestNames: if kwargs['action'] == 'Reject': ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl = self.wmstatWriteURL) else: assignments = GetRequest.getAssignmentsByName(requestName) if teams == [] and assignments == []: raise cherrypy.HTTPError(400, "Must assign to one or more teams") self.assignWorkload(requestName, kwargs) for team in teams: if not team in assignments: ChangeState.assignRequest(requestName, team, wmstatUrl = self.wmstatWriteURL) priority = kwargs.get(requestName+':priority', '') if priority != '': Utilities.changePriority(requestName, priority, self.wmstatWriteURL) participle=kwargs['action']+'ed' if self.opshold and kwargs['action'] == 'Assign': participle='put into "ops-hold" state (see <a href="%s">OpsClipboard</a>)' % self.clipboardUrl # this, previously used, call made all requests injected into OpsClipboard to # have campaign_id null since the call doesn't propagate request's # CampaignName at all, AcquisitionEra remains default null and probably # a bunch of other things is wrong too #requests = [GetRequest.getRequestByName(requestName) for requestName in requestNames] requests = [Utilities.requestDetails(requestName) for requestName in requestNames] OpsClipboard.inject(self.couchUrl, self.clipboardDB, *requests) for request in requestNames: ChangeState.changeRequestStatus(requestName, 'ops-hold', wmstatUrl = self.wmstatWriteURL) return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def associateCampaign(campaign, requestName, couchURL, couchDBName): """ _associateCampaign_ Associate a campaign and a request inside the workloadSpec This is done by loading the workloadSpec from couch, modifying the campaign, and saving it again. """ WMCore.Lexicon.identifier(requestName) request = GetRequest.getRequestDetails(requestName) helper = loadWorkload(request) helper.setCampaign(campaign=campaign) helper.saveCouch(couchUrl=couchURL, couchDBName=couchDBName)
def showWorkload(self, requestName): """ Displays the workload """ self.validate(requestName) try: request = GetRequest.getRequestByName(requestName) except (Exception, RuntimeError) as ex: raise cherrypy.HTTPError(400, "Invalid request. %s" % str(ex)) request = Utilities.prepareForTable(request) helper = Utilities.loadWorkload(request) workloadText = str(helper.data) return cgi.escape(workloadText).replace("\n", "<br/>\n")
def showWorkload(self, requestName): """ Displays the workload """ self.validate(requestName) try: request = GetRequest.getRequestByName(requestName) except (Exception, RuntimeError) as ex: raise cherrypy.HTTPError(400, "Invalid request.") request = Utilities.prepareForTable(request) helper = Utilities.loadWorkload(request) workloadText = str(helper.data) return cgi.escape(workloadText).replace("\n", "<br/>\n")
def getConfigIDs(self, prim, proc, tier): """ _getConfigIDs_ Get the ConfigIDs for the specified request """ result = {} dataset = self.getDataset(prim, proc, tier) requests = GetRequest.getRequestsByCriteria("Datasets.GetRequestByInput", dataset) for request in requests: requestName = request["RequestName"] helper = Utilities.loadWorkload(request) result[requestName] = helper.listAllCMSSWConfigCacheIDs() return result
def handleAssignmentPage(self, **kwargs): """ handler for the main page """ #Accept Json encoded strings decodedArgs = {} for key in kwargs.keys(): try: decodedArgs[key] = JsonWrapper.loads(kwargs[key]) except: #Probably wasn't JSON decodedArgs[key] = kwargs[key] kwargs = decodedArgs # handle the checkboxes teams = [] requestNames = [] for key, value in kwargs.iteritems(): if isinstance(value, types.StringTypes): kwargs[key] = value.strip() if key.startswith("Team"): teams.append(key[4:]) if key.startswith("checkbox"): requestName = key[8:] self.validate(requestName) requestNames.append(key[8:]) for requestName in requestNames: if kwargs['action'] == 'Reject': ChangeState.changeRequestStatus(requestName, 'rejected', wmstatUrl=self.wmstatWriteURL) else: assignments = GetRequest.getAssignmentsByName(requestName) if teams == [] and assignments == []: raise cherrypy.HTTPError( 400, "Must assign to one or more teams") kwargs["Teams"] = teams self.assignWorkload(requestName, kwargs) for team in teams: if not team in assignments: ChangeState.assignRequest( requestName, team, wmstatUrl=self.wmstatWriteURL) priority = kwargs.get(requestName + ':priority', '') if priority != '': Utilities.changePriority(requestName, priority, self.wmstatWriteURL) participle = kwargs['action'] + 'ed' return self.templatepage("Acknowledge", participle=participle, requests=requestNames)
def putCampaign(self, campaign, request=None): """ Adds a campaign if it doesn't already exist, and optionally associates a request with it """ if request: requestID = GetRequest.requestID(request) if requestID: result = Campaign.associateCampaign(campaign, requestID) Utilities.associateCampaign(campaign=campaign, requestName=request, couchURL=self.couchUrl, couchDBName=self.workloadDBName) return result else: return False else: Campaign.addCampaign(campaign)
def testE_CheckStatusChanges(self): """ _CheckStatusChanges_ Check status changes for a single request. See whether we can move the request through the proper chain. Figure out what happens when we fail. """ myThread = threading.currentThread() userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) configID = self.createConfig() schema["ConfigCacheID"] = configID schema["CouchDBName"] = self.couchDBName schema["CouchURL"] = os.environ.get("COUCHURL") result = self.jsonSender.put('request/testRequest', schema) self.assertEqual(result[1], 200) requestName = result[0]['RequestName'] # There should only be one request in the DB result = GetRequest.requestID(requestName=requestName) self.assertEqual(result, 1) result = self.jsonSender.get('request/%s' % requestName) self.assertEqual(result[0]['Group'], groupName) self.assertEqual(result[0]['Requestor'], userName) # Let's see what we can do in terms of setting status self.changeStatusAndCheck(requestName=requestName, statusName='new') # Let's try an illegal status change, just for the hell of it raises = False try: self.jsonSender.put('request/%s?status=negotiating' % requestName) except HTTPException, ex: raises = True self.assertEqual(ex.status, 403) self.assertTrue('Failed to change status' in ex.result) self.assertFalse(requestName in ex.result)
def requestDetails(requestName): """ Adds details from the Couch document as well as the database """ WMCore.Lexicon.identifier(requestName) request = GetRequest.getRequestDetails(requestName) helper = loadWorkload(request) schema = helper.data.request.schema.dictionary_whole_tree_() # take the stuff from the DB preferentially schema.update(request) task = helper.getTopLevelTask()[0] schema['Site Whitelist'] = task.siteWhitelist() schema['Site Blacklist'] = task.siteBlacklist() schema['MergedLFNBase'] = str(helper.getMergedLFNBase()) schema['UnmergedLFNBase'] = str(helper.getUnmergedLFNBase()) schema['Campaign'] = str(helper.getCampaign()) schema['AcquisitionEra'] = str(helper.getAcquisitionEra()) if schema['SoftwareVersions'] == ['DEPRECATED']: schema['SoftwareVersions'] = helper.getCMSSWVersions() return schema
def assignWorkload(self, requestName, kwargs): """ Make all the necessary changes in the Workload to reflect the new assignment """ request = GetRequest.getRequestByName(requestName) helper = Utilities.loadWorkload(request) #Validate the different parts of the processed dataset processedDatasetParts = ["AcquisitionEra", "ProcessingVersion"] if kwargs.get("ProcessingString", None): processedDatasetParts.append("ProcessingString") for field in processedDatasetParts: if type(kwargs[field]) == dict: for value in kwargs[field].values(): self.validate(value, field) else: self.validate(kwargs[field], field) # Set white list and black list whiteList = kwargs.get("SiteWhitelist", []) blackList = kwargs.get("SiteBlacklist", []) if type(whiteList) != list: whiteList = [whiteList] if type(blackList) != list: blackList = [blackList] helper.setSiteWildcardsLists(siteWhitelist=whiteList, siteBlacklist=blackList, wildcardDict=self.wildcardSites) # Set ProcessingVersion and AcquisitionEra, which could be json encoded dicts helper.setProcessingVersion(kwargs["ProcessingVersion"]) helper.setAcquisitionEra(kwargs["AcquisitionEra"]) helper.setProcessingString(kwargs.get("ProcessingString", None)) # Now verify the output datasets outputDatasets = helper.listOutputDatasets() for dataset in outputDatasets: tokens = dataset.split("/") procds = tokens[2] try: WMCore.Lexicon.procdataset(procds) except AssertionError, ex: raise cherrypy.HTTPError( 400, "Bad output dataset name, check the processed dataset.\n %s" % str(ex))