def loadWorkload(self, inputWorkload): """ If workload is sane, then use it """ if inputWorkload == None: self.workload = None if isinstance(inputWorkload, WMWorkload): self.workload = WMWorkloadHelper(inputWorkload) return if isinstance(inputWorkload, WMWorkloadHelper): self.workload = inputWorkload return if not os.path.exists(inputWorkload): raise Exception('Could not find %s in local file system' % (str(inputWorkload))) testWorkload = WMWorkloadHelper(WMWorkload("workload")) testWorkload.load(inputWorkload) self.workload = testWorkload return
def loadWorkload(request): """ Returns a WMWorkloadHelper for the workload contained in the request """ url = request['RequestWorkflow'] helper = WMWorkloadHelper() try: WMCore.Lexicon.couchurl(url) except Exception: raise cherrypy.HTTPError(400, "Invalid workload "+urllib.quote(url)) helper = WMWorkloadHelper() try: helper.load(url) except Exception: raise cherrypy.HTTPError(404, "Cannot find workload "+removePasswordFromUrl(url)) return helper
def buildWorkload(self, arguments): """ _buildWorkload_ Build a resubmission workload from a previous workload, it loads the workload and truncates it. """ helper = WMWorkloadHelper() # where to find the original spec file originalRequestURL = "%s/%s" % (arguments['CouchURL'], arguments['CouchWorkloadDBName']) helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName) helper.truncate(self.workloadName, self.initialTaskPath, self.acdcServer, self.acdcDatabase, self.collectionName) helper.ignoreOutputModules(self.ignoredOutputModules) # override a couple of parameters, if provided by user if 'Memory' in arguments: helper.setMemory(arguments['Memory']) if 'Campaign' in arguments and not helper.getCampaign(): helper.setCampaign(arguments["Campaign"]) if 'RequestPriority' in arguments: helper.setPriority(arguments["RequestPriority"]) if 'TimePerEvent' in arguments: for task in helper.taskIterator(): task.setJobResourceInformation( timePerEvent=arguments["TimePerEvent"]) return helper
def buildWorkload(self, arguments): """ _buildWorkload_ Build a resubmission workload from a previous workload, it loads the workload and truncates it. """ helper = WMWorkloadHelper() # where to find the original spec file originalRequestURL = "%s/%s" % (arguments['CouchURL'], arguments['CouchWorkloadDBName']) helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName) helper.truncate(self.workloadName, self.initialTaskPath, self.acdcServer, self.acdcDatabase, self.collectionName) helper.ignoreOutputModules(self.ignoredOutputModules) # override a couple of parameters, if provided by user # Note that if it was provided by the user, then it's already part of the arguments too if "RequestPriority" in self.userArgs: helper.setPriority(arguments["RequestPriority"]) if "Memory" in self.userArgs: helper.setMemory(arguments["Memory"]) if "Multicore" in self.userArgs or "EventStreams" in self.userArgs: self.setCoresAndStreams(helper, arguments) if "TimePerEvent" in self.userArgs: helper.setTimePerEvent(arguments.get("TimePerEvent")) return helper
def retrieveSchema(workflowName): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() helper.load(specURL) schema = {} # for (key, value) in helper.data.request.schema.dictionary_().iteritems(): for (key, value ) in helper.data.request.schema.dictionary_whole_tree_().iteritems(): if key == 'ProdConfigCacheID': schema['ConfigCacheID'] = value elif key == 'ProcConfigCacheID': schema['ConfigCacheID'] = value elif key == 'RequestSizeEvents': schema['RequestNumEvents'] = value elif value != None: schema[key] = value # print "Retrieved schema:\n", schema schema['Requestor'] = 'anlevin' schema['Group'] = 'DATAOPS' # schema['Task1']['SplittingArguments'] = {'lumis_per_job': 5} # schema['Memory'] = 1394 # schema['Task1']['KeepOutput'] = True # schema['RequestString'] = 'TEST_DELETE_THIS_' # schema['BlockWhitelist'] = ['/MinimumBias/Run2012D-HLTPhysics-Tier1PromptSkim-v1/RAW-RECO#ce668e80-26a2-11e2-80e7-00155dffff9d'] # schema['Task1']['BlockBlacklist'] = ['/DoubleMu/Run2011A-ZMu-08Nov2011-v1/RAW-RECO#93c53d22-25b2-11e1-8c62-003048f02c8a'] # schema['Task1']['RunWhitelist'] = [208307] # del schema['Task2']['MCPileup'] return schema
def buildWorkload(self, originalRequestURL): """ _buildWorkload_ Build a resubmission workload from a previous workload, it loads the workload and truncates it. """ #TODO remove the dependency on reqmgr1 if originalRequestURL == None: # reqmgr1 call (Due to reqmgr2 dependency imports here from WMCore.HTTPFrontEnd.RequestManager.ReqMgrWebTools import loadWorkload from WMCore.RequestManager.RequestDB.Interface.Request.GetRequest import getRequestByName originalRequest = getRequestByName(self.originalRequestName) helper = loadWorkload(originalRequest) else: # reqmgr2 call helper = WMWorkloadHelper() helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName) helper.truncate(self.workloadName, self.initialTaskPath, self.acdcServer, self.acdcDatabase, self.collectionName) helper.ignoreOutputModules(self.ignoredOutputModules) return helper
def getWMWorkload(self): """ _getWMTask_ Get the WMTask for this space TODO: Refactor to getWMWorkload method """ if self.workload != None: return self.workload try: import WMSandbox except ImportError as ex: msg = "Error importing WMSandbox module" msg += str(ex) raise RuntimeError(msg) wmsandboxLoc = inspect.getsourcefile(WMSandbox) workloadPcl = wmsandboxLoc.replace("__init__.py","WMWorkload.pkl") with open(workloadPcl, 'rb') as handle: wmWorkload = pickle.load(handle) self.workload = WMWorkloadHelper(wmWorkload) return
def buildWorkload(self, arguments): """ _buildWorkload_ Build a resubmission workload from a previous workload, it loads the workload and truncates it. """ helper = WMWorkloadHelper() # where to find the original spec file originalRequestURL = "%s/%s" % (arguments['CouchURL'], arguments['CouchWorkloadDBName']) helper.loadSpecFromCouch(originalRequestURL, self.originalRequestName) helper.truncate(self.workloadName, self.initialTaskPath, self.acdcServer, self.acdcDatabase, self.collectionName) helper.ignoreOutputModules(self.ignoredOutputModules) # override a couple of parameters, if provided by user if 'RequestPriority' in arguments: helper.setPriority(arguments["RequestPriority"]) if arguments['OriginalRequestType'] != 'TaskChain' or isinstance(arguments['Memory'], dict): helper.setMemory(arguments['Memory']) if arguments['OriginalRequestType'] != 'TaskChain' or isinstance(arguments.get('TimePerEvent'), dict): helper.setTimePerEvent(arguments.get("TimePerEvent")) return helper
def modifyWorkload(self, requestName, workload, CMSSWVersion=None, GlobalTag=None, runWhitelist=None, runBlacklist=None, blockWhitelist=None, blockBlacklist=None, ScramArch=None): """ handles the "Modify" button of the details page """ self.validate(requestName) helper = WMWorkloadHelper() helper.load(workload) schema = helper.data.request.schema message = "" if runWhitelist != "" and runWhitelist != None: l = Utilities.parseRunList(runWhitelist) helper.setRunWhitelist(l) schema.RunWhitelist = l message += 'Changed runWhiteList to %s<br>' % l if runBlacklist != "" and runBlacklist != None: l = Utilities.parseRunList(runBlacklist) helper.setRunBlacklist(l) schema.RunBlacklist = l message += 'Changed runBlackList to %s<br>' % l if blockWhitelist != "" and blockWhitelist != None: l = Utilities.parseBlockList(blockWhitelist) helper.setBlockWhitelist(l) schema.BlockWhitelist = l message += 'Changed blockWhiteList to %s<br>' % l if blockBlacklist != "" and blockBlacklist != None: l = Utilities.parseBlockList(blockBlacklist) helper.setBlockBlacklist(l) schema.BlockBlacklist = l message += 'Changed blockBlackList to %s<br>' % l Utilities.saveWorkload(helper, workload) return message + detailsBackLink(requestName)
def get(self, name, web_form=False): """ getting job splitting algorithm. :arg str name: name to appear in the result message. :returns: row with response, here 1 item list with message. """ helper = WMWorkloadHelper() try: helper.loadSpecFromCouch(self.reqdb_url, name) except Exception: raise cherrypy.HTTPError(404, "Cannot find workload: %s" % name) splittingDict = helper.listJobSplittingParametersByTask( performance=False) taskNames = sorted(splittingDict.keys()) splitInfo = [] for taskName in taskNames: splitInfo.append({ "splitAlgo": splittingDict[taskName]["algorithm"], "splitParams": splittingDict[taskName], "taskType": splittingDict[taskName]["type"], "taskName": taskName }) if web_form: splitInfo = create_web_splitting_format(splitInfo) return splitInfo
def retrieveSchema(workflowName, user, group): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() #print " retrieving original workflow...", helper.load(specURL) #print "done." schema = {} #for (key, value) in helper.data.request.schema.dictionary_whole_tree_().iteritems(): for (key, value) in helper.data.request.schema.dictionary_().iteritems(): #print key, value if key == 'ProcConfigCacheID': schema['ConfigCacheID'] = value elif key == 'RequestSizeEvents': schema['RequestSizeEvents'] = value #schema['RequestNumEvents'] = int(value) elif key == 'Requestor': schema['Requestor'] = user elif key == 'Group': schema['Group'] = group #elif key=='ProdJobSplitArgs': # schema['ProdJobSplitArgs']={'events_per_job': 500000, 'events_per_lumi': 300, 'lheInputFiles': True} elif value != None: schema[key] = value return schema
def queueWork(self, wmspecUrl, request=None, team=None): """ Take and queue work from a WMSpec. If request name is provided but doesn't match WMSpec name an error is raised. If team is provided work will only be available to queue's belonging to that team. Duplicate specs will be ignored. """ self.logger.info('queueWork() begin queueing "%s"' % wmspecUrl) wmspec = WMWorkloadHelper() wmspec.load(wmspecUrl) if request: # validate request name try: Lexicon.requestName(request) except Exception, ex: # can throw many errors e.g. AttributeError, AssertionError etc. error = WorkQueueWMSpecError( wmspec, "Request name validation error: %s" % str(ex)) raise error if request != wmspec.name(): raise WorkQueueWMSpecError( wmspec, 'Request & workflow name mismatch %s vs %s' % (request, wmspec.name()))
def retrieveSchema(url, workflowName, user, group): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() #print " retrieving original workflow...", helper.load(specURL) #print "done." schema = {} #for (key, value) in helper.data.request.schema.dictionary_whole_tree_().iteritems(): for (key, value) in helper.data.request.schema.dictionary_().iteritems(): #print key, value if key == 'ProcConfigCacheID': schema['ConfigCacheID'] = value elif key == 'RequestSizeEvents': schema['RequestSizeEvents'] = value #schema['RequestNumEvents'] = int(value) elif key == 'Requestor': schema['Requestor'] = user elif key == 'Group': schema['Group'] = group elif key == 'RequestNumEvents': schema['RequestNumEvents'] = getFinalRequestedNumEvents( url, workflowName) elif key == 'FirstLumi': schema['FirstLumi'] = getMaxLumi(url, workflowName) * 2 elif key == 'FirstEvent': schema['FirstEvent'] = deprecated.dbsTest.getInputEvents( url, workflowName) * 2 elif key == 'RequestString': schema['RequestString'] = 'ACDC_' + value elif value != None: schema[key] = value return schema
def getMasterName(startDir, wmWorkload = None, workflow = None): """ Gets a universal name for the jobGroup directory Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN) """ if wmWorkload != None: workload = wmWorkload.name() elif not os.path.exists(workflow.spec): msg = "Could not find Workflow spec %s: " % (workflow.spec) msg += "Cannot create work area without spec!" logging.error(msg) raise CreateWorkAreaException(msg) else: wmWorkload = WMWorkloadHelper(WMWorkload("workload")) wmWorkload.load(workflow.spec) workload = wmWorkload.name() task = workflow.task if task.startswith("/" + workload + "/"): task = task[len(workload) + 2:] return (os.path.join(startDir, workload), os.path.join(startDir, workload, task))
def getMasterName(self): """ Gets a universal name for the jobGroup directory Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN) """ if self.wmWorkload != None: workload = self.wmWorkload.name() elif not os.path.exists(self.workflow.spec): logging.error( "Could not find Workflow spec %s; labeling jobs by job ID only!" % (self.workflow.spec)) return os.path.join(self.startDir, self.jobGroup.uid), os.path.join( self.startDir, self.jobGroup.uid) else: wmWorkload = WMWorkloadHelper(WMWorkload("workload")) wmWorkload.load(self.workflow.spec) workload = wmWorkload.name() task = self.workflow.task if task.startswith("/" + workload + "/"): task = task[len(workload) + 2:] return os.path.join(self.startDir, workload), os.path.join(self.startDir, workload, task)
def retrieveSchema(workflowName): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() helper.load(specURL) paramBlacklist = [ 'AcquisitionEra', 'BlockCloseMaxEvents', 'BlockCloseMaxFiles', 'BlockCloseMaxSize', 'BlockCloseMaxWaitTime', 'CouchURL', 'CouchWorkloadDBName', 'CustodialGroup', 'CustodialSubType', 'Dashboard', 'GracePeriod', 'Group', 'HardTimeout', 'InitialPriority', 'inputMode', 'MaxMergeEvents', 'MaxMergeSize', 'MaxRSS', 'MaxVSize', 'MergedLFNBase', 'MinMergeSize', 'NonCustodialGroup', 'NonCustodialSubType', 'ProcessingString', 'OutputDatasets', 'ReqMgr2Only', 'Requestor', 'RequestDate' 'RequestorDN', 'RequestName', 'RequestStatus', 'RequestTransition', 'RequestWorkflow', 'SiteWhitelist', 'SoftTimeout', 'SoftwareVersions', 'SubscriptionPriority', 'Team', 'timeStamp' ] schema = {} for (key, value ) in helper.data.request.schema.dictionary_whole_tree_().iteritems(): if not value or key in paramBlacklist: continue else: schema[key] = value schema['Requestor'] = 'amaltaro' schema['Group'] = 'DATAOPS' return schema
def getDataFromSpecFile(specFile): workload = WMWorkloadHelper() workload.load(specFile) campaign = workload.getCampaign() result = {"Campaign": campaign} for task in workload.taskIterator(): result[task.getPathName()] = task.getPrepID() return result
def validate_request_update_args(request_args, config, reqmgr_db_service, param): """ param and safe structure is RESTArgs structure: named tuple RESTArgs(args=[], kwargs={}) validate post request 1. read data from body 2. validate the permission (authentication) 3. validate state transition (against previous state from couchdb) 2. validate using workload validation 3. convert data from body to arguments (spec instance, argument with default setting) TODO: rasie right kind of error with clear message """ request_name = request_args["RequestName"] # this need to be deleted for validation del request_args["RequestName"] couchurl = '%s/%s' % (config.couch_host, config.couch_reqmgr_db) workload = WMWorkloadHelper() # param structure is RESTArgs structure. workload.loadSpecFromCouch(couchurl, request_name) # first validate the permission by status and request type. # if the status is not set only ReqMgr Admin can change the the values # TODO for each step, assigned, approved, announce find out what other values # can be set request_args["RequestType"] = workload.requestType() permission = getWritePermission(request_args) authz_match(permission['role'], permission['group']) del request_args["RequestType"] #validate the status if "RequestStatus" in request_args: validate_state_transition(reqmgr_db_service, request_name, request_args["RequestStatus"]) # delete request_args since it is not part of spec argument sand validation args_without_status = {} args_without_status.update(request_args) del args_without_status["RequestStatus"] else: args_without_status = request_args if len(args_without_status) > 0 and not workqueue_stat_validation( args_without_status): # validate the arguments against the spec argumentSpecdefinition #TODO: currently only assigned status allows any update other then Status update workload.validateArgumentForAssignment(args_without_status) # to update request_args with type conversion request_args.update(args_without_status) return workload, request_args
def getSplitting(requestName): reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/' reqmgr = RequestManager(dict={'endpoint': reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) params = workload.getTopLevelTask()[0].jobSplittingParameters() algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm() return params['events_per_job']
def loadWorkload(self, requestName): """ _loadWorkload_ Load the workload from couch after we've saved it there. """ workload = WMWorkloadHelper() url = '%s/%s/%s/spec' % (os.environ['COUCHURL'], self.couchDBName, requestName) workload.load(url) return workload
def validate_request_update_args(request_args, config, reqmgr_db_service, param): """ param and safe structure is RESTArgs structure: named tuple RESTArgs(args=[], kwargs={}) validate post/put request 1. read data from body 2. validate the permission (authentication) 3. validate state transition (against previous state from couchdb) 2. validate using workload validation 3. convert data from body to arguments (spec instance, argument with default setting) TODO: raise right kind of error with clear message """ # this needs to be deleted for validation request_name = request_args.pop("RequestName") couchurl = '%s/%s' % (config.couch_host, config.couch_reqmgr_db) workload = WMWorkloadHelper() workload.loadSpecFromCouch(couchurl, request_name) # first validate the permission by status and request type. # if the status is not set only ReqMgr Admin can change the values # TODO for each step, assigned, approved, announce find out what other values # can be set request_args["RequestType"] = workload.requestType() permission = getWritePermission(request_args) authz_match(permission['role'], permission['group']) del request_args["RequestType"] # validate the status if "RequestStatus" in request_args: validate_state_transition(reqmgr_db_service, request_name, request_args["RequestStatus"]) if request_args["RequestStatus"] in STATES_ALLOW_ONLY_STATE_TRANSITION: # if state change doesn't allow other transition nothing else to validate args_only_status = {} args_only_status["RequestStatus"] = request_args["RequestStatus"] args_only_status["cascade"] = request_args.get("cascade", False) return workload, args_only_status elif request_args["RequestStatus"] == 'assigned': workload.validateArgumentForAssignment(request_args) # TODO: fetch it from the assignment arg definition if 'RequestPriority' in request_args: request_args['RequestPriority'] = int(request_args['RequestPriority']) if (lambda x: (x >= 0 and x < 1e6))(request_args['RequestPriority']) is False: raise InvalidSpecParameterValue( "RequestPriority must be an integer between 0 and 1e6") return workload, request_args
def loadOverride(self, specPath): """ _loadOverride_ Loads the spec (if not in the cache) and gets the location where the files should be injected to. If not possible to load the spec, raise an exception. """ helper = WMWorkloadHelper() helper.load(specPath) overrideSite = helper.getPhEDExInjectionOverride() self.specCache[specPath] = overrideSite return overrideSite
def loadWorkload(): """ _loadWorkload_ Load the Workload from the WMSandbox Area """ sandboxLoc = locateWMSandbox() workloadPcl = "%s/WMWorkload.pkl" % sandboxLoc with open(workloadPcl, 'r') as handle: wmWorkload = pickle.load(handle) return WMWorkloadHelper(wmWorkload)
def getElementsForSplitting(self): """Returns the elements from the inbox that need to be split, if WorkflowName specified only return elements to split for that workflow""" elements = self.getInboxElements(status='Negotiating') specs = {} # cache as may have multiple elements for same spec for ele in elements: if ele['RequestName'] not in specs: wmspec = WMWorkloadHelper() wmspec.load(self.parentCouchUrlWithAuth + "/%s/spec" % ele['RequestName']) specs[ele['RequestName']] = wmspec ele['WMSpec'] = specs[ele['RequestName']] del specs return elements
def retrieveSchema( workflowName, reqmgrCouchURL="https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"): """ Creates the cloned specs for the original request Updates parameters """ from WMCore.WMSpec.WMWorkload import WMWorkloadHelper specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() helper.load(specURL) return helper
def main(): start = time.time() # blockName = match['Inputs'].keys()[0] blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055" # acdcInfo = match['ACDC'] acdcInfo = {"database": "acdcserver", "fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0", "collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222", "server": "https://cmsweb.cern.ch/couchdb"} acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) splitedBlockName = ACDCBlock.splitBlockName(blockName) print("Splitted block name: %s" % splitedBlockName) fileLists = acdc.getChunkFiles(acdcInfo['collection'], acdcInfo['fileset'], splitedBlockName['Offset'], splitedBlockName['NumOfFiles']) print("Retrieved %d unique files from the ACDCServer" % len(fileLists)) block = {} block["Files"] = fileLists wantedLumis = set([252052, 240646]) for f in fileLists: for run in f['runs']: maskDict = run.json() lumisSet = set(maskDict['Lumis'].keys()) if wantedLumis.intersection(lumisSet): print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet))) # with open("chunkfiles.json", 'w') as fo: # json.dump(block, fo) end = time.time() print("Spent %s secs running so far" % (end - start)) sys.exit(1) ### Now doing the WMBSHelper stuff reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache" requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376" wmspec = WMWorkloadHelper() wmspec.loadSpecFromCouch(reqUrl, requestName) taskName = "HIG-RunIIFall17DRPremix-00788_0" mask = None cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache" # wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir']) wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir) sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def retrieveSchema(workflowName): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() helper.load(specURL) schema = {} # for (key, value) in helper.data.request.schema.dictionary_().iteritems(): for (key, value ) in helper.data.request.schema.dictionary_whole_tree_().iteritems(): if key == 'ProdConfigCacheID': schema['ConfigCacheID'] = value elif key == 'ProcConfigCacheID': schema['ConfigCacheID'] = value elif key == 'RequestSizeEvents': schema['RequestNumEvents'] = value elif key == 'ProcessingString' and value == {}: continue elif key == 'AcquisitionEra' and value == {}: continue elif key == 'SkimConfigs' and not value: continue elif value != None: schema[key] = value # print "Retrieved schema:\n", schema ### FOR DEBUG request = deepcopy(schema) request['Requestor'] = 'anlevin' request['Group'] = 'DATAOPS' ### Now changing the parameters according to HG1309 x = 1 while x <= schema['TaskChain']: task = 'Task' + str(x) for (key, value) in schema[task].iteritems(): if key == "SplittingAlgorithm": request[task]['SplittingAlgo'] = value del request[task]['SplittingAlgorithm'] elif key == "SplittingArguments": for (k2, v2) in schema[task][key].iteritems(): if k2 == "lumis_per_job": request[task]["LumisPerJob"] = v2 elif k2 == "events_per_job": request[task]["EventsPerJob"] = v2 del request[task]['SplittingArguments'] x += 1 #request['Memory'] = 3900 #del request['SiteWhitelist'] #request['RequestString'] = 'TEST_ANDREW_TEST' # request['Task1']['BlockBlacklist'] = ['/DoubleMu/Run2011A-ZMu-08Nov2011-v1/RAW-RECO#93c53d22-25b2-11e1-8c62-003048f02c8a'] #request['Task1']['BlockWhitelist'] = ['/RelValQCD_Pt_80_170_BCtoE_8TeV/CMSSW_6_2_0_pre8-PRE_ST62_V8-v3/GEN-SIM#d99587e0-625e-11e3-ad0f-00221959e7c0','/RelValQCD_Pt_80_170_BCtoE_8TeV/CMSSW_6_2_0_pre8-PRE_ST62_V8-v3/GEN-SIM#8689de68-606d-11e3-ad0f-00221959e7c0'] return request
def getRequestInformationAndWorkload(requestName, reqmgrUrl, centralRequestDBURL): """ _getRequestInformationAndWorkload_ Retrieve the request information for assignment and the full pickled workload. """ wfDBReader = RequestDBReader(centralRequestDBURL, couchapp="ReqMgr") result = wfDBReader.getRequestByNames(requestName, True) workloadDB = Database(result[requestName]['CouchWorkloadDBName'], result[requestName]['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) return workload, result[requestName]
def getRequestInformationAndWorkload(requestName, reqmgrUrl): """ _getRequestInformationAndWorkload_ Retrieve the request information for assignment and the full pickled workload. """ reqmgr = RequestManager(dict={'endpoint': reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) return workload, result
def retrieveSchema(workflowName, newBlockBlacklist): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() helper.load(specURL) schema = {} for (key, value) in helper.data.request.schema.dictionary_().iteritems(): #print key if key == 'ProdConfigCacheID': schema['ProdConfigCacheID'] = value elif value != None: schema[key] = value schema["BlockBlacklist"] = "[" + newBlockBlacklist + "]" return schema