def getSplitting(requestName): reqmgrUrl='https://cmsweb.cern.ch/reqmgr/reqMgr/' reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) params = workload.getTopLevelTask()[0].jobSplittingParameters() algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm() return params['events_per_job']
def getSplitting(requestName): reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/' reqmgr = RequestManager(dict={'endpoint': reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) params = workload.getTopLevelTask()[0].jobSplittingParameters() algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm() return params['events_per_job']
def main(): demPolicy = Block() reqmgr = RequestManager(dict = {'endpoint' : 'https://cmsweb.cern.ch/reqmgr/reqMgr'}) result = reqmgr.getRequest('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207') workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207', 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) x,y = demPolicy(wmspec = workload, task = workload.getTopLevelTask()[0]) print x print y
def createResubmitSpec(self, serverUrl, couchDB): """ _createResubmitSpec_ Create a bogus resubmit workload. """ self.site = "cmssrm.fnal.gov" workload = WMWorkloadHelper(WMWorkload("TestWorkload")) reco = workload.newTask("reco") workload.setOwnerDetails(name = "evansde77", group = "DMWM") # first task uses the input dataset reco.addInputDataset(primary = "PRIMARY", processed = "processed-v1", tier = "TIER1") reco.data.input.splitting.algorithm = "File" reco.setTaskType("Processing") cmsRunReco = reco.makeStep("cmsRun1") cmsRunReco.setStepType("CMSSW") reco.applyTemplates() cmsRunRecoHelper = cmsRunReco.getTypeHelper() cmsRunRecoHelper.addOutputModule("outputRECO", primaryDataset = "PRIMARY", processedDataset = "processed-v2", dataTier = "TIER2", lfnBase = "/store/dunkindonuts", mergedLFNBase = "/store/kfc") dcs = DataCollectionService(url = serverUrl, database = couchDB) def getJob(workload): job = Job() job["task"] = workload.getTask("reco").getPathName() job["workflow"] = workload.name() job["location"] = self.site job["owner"] = "evansde77" job["group"] = "DMWM" return job testFileA = WMFile(lfn = makeUUID(), size = 1024, events = 1024) testFileA.setLocation([self.site]) testFileA.addRun(Run(1, 1, 2)) testFileB = WMFile(lfn = makeUUID(), size = 1024, events = 1024) testFileB.setLocation([self.site]) testFileB.addRun(Run(1, 3, 4)) testJobA = getJob(workload) testJobA.addFile(testFileA) testJobA.addFile(testFileB) dcs.failedJobs([testJobA]) topLevelTask = workload.getTopLevelTask()[0] workload.truncate("Resubmit_TestWorkload", topLevelTask.getPathName(), serverUrl, couchDB) return workload
def retrieveSchema(workflowName): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() print " retrieving original workflow...", helper.load(specURL) print "done." schema = {} for (key, value) in helper.data.request.schema.dictionary_().iteritems(): if value != None: schema[key] = value schema["Requestor"] = "linacre" schema["Group"] = "DATAOPS" del schema["RequestName"] del schema["CouchDBName"] del schema["CouchURL"] assign = {} assign["unmergedLFNBase"] = helper.data.properties.unmergedLFNBase assign["mergedLFNBase"] = helper.data.properties.mergedLFNBase assign["processingVersion"] = helper.data.properties.processingVersion assign["dashboardActivity"] = helper.data.properties.dashboardActivity assign["acquisitionEra"] = helper.data.properties.acquisitionEra topLevelTask = helper.getTopLevelTask()[0] assign["SiteWhitelist"] = topLevelTask.siteWhitelist() mergeTask = None for mergeTask in topLevelTask.childTaskIterator(): if mergeTask.taskType() == "Merge": if mergeTask.getPathName().find("DQM") == -1: break assign["MinMergeSize"] = mergeTask.jobSplittingParameters( )["min_merge_size"] assign["MaxMergeSize"] = mergeTask.jobSplittingParameters( )["max_merge_size"] assign["MaxMergeEvents"] = mergeTask.jobSplittingParameters().get( "max_merge_events", 50000) return (schema, assign)
def main(): toCheckList = '/home/dballest/Dev-Workspace/dev-scripts/data/upgrade-vocms85-613.txt' handle = open(toCheckList, 'r') timePerJobFile = open('/home/dballest/Dev-Workspace/dev-scripts/data/upgrade-vocms85-tpj.data', 'w') eventsPerLumiFile = open('/home/dballest/Dev-Workspace/dev-scripts/data/upgrade-vocms85-epl.data', 'w') lumisPerMergeFile = open('/home/dballest/Dev-Workspace/dev-scripts/data/upgrade-vocms85-lpm.data', 'w') count = 0 for request in handle: z = WMWorkloadHelper() z.load('https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache/%s/spec' % request.strip()) schema = z.data.request.schema requesType = schema.RequestType if requesType != 'MonteCarlo': continue timePerEvent = schema.TimePerEvent sizePerEvent = schema.SizePerEvent events = schema.RequestNumEvents eff = getattr(schema, "FilterEfficiency", 1.0) topTask = z.getTopLevelTask()[0] eventsPerJob = topTask.data.input.splitting.events_per_job if eff < 1.0: count += 1 for childTask in topTask.childTaskIterator(): if childTask.data.taskType == 'Merge': mergeSizeLimit = childTask.data.input.splitting.max_merge_size mergeEventLimit = childTask.data.input.splitting.max_merge_events break sizeOfZeroEvent = 131091.0 sizePerLumi = eventsPerJob*sizePerEvent*eff lumisPerMergedBySize = mergeSizeLimit/(sizePerLumi + sizeOfZeroEvent) lumisPerMergedByEvent = mergeEventLimit/(eventsPerJob*eff) timePerJobFile.write("%f\n" % (timePerEvent * eventsPerJob)) eventsPerLumiFile.write("%f\n" % (eventsPerJob*eff)) lumisPerMergeFile.write("%f\n" % min(lumisPerMergedBySize, lumisPerMergedByEvent)) print count handle.close() timePerJobFile.close() eventsPerLumiFile.close() lumisPerMergeFile.close()
def retrieveSchema(workflowName): specURL = os.path.join(reqmgrCouchURL, workflowName, "spec") helper = WMWorkloadHelper() print " retrieving original workflow...", helper.load(specURL) print "done." schema = {} for (key, value) in helper.data.request.schema.dictionary_().iteritems(): if value != None: schema[key] = value schema["Requestor"] = "linacre" schema["Group"] = "DATAOPS" del schema["RequestName"] del schema["CouchDBName"] del schema["CouchURL"] assign = {} assign["unmergedLFNBase"] = helper.data.properties.unmergedLFNBase assign["mergedLFNBase"] = helper.data.properties.mergedLFNBase assign["processingVersion"] = helper.data.properties.processingVersion assign["dashboardActivity"] = helper.data.properties.dashboardActivity assign["acquisitionEra"] = helper.data.properties.acquisitionEra topLevelTask = helper.getTopLevelTask()[0] assign["SiteWhitelist"] = topLevelTask.siteWhitelist() mergeTask = None for mergeTask in topLevelTask.childTaskIterator(): if mergeTask.taskType() == "Merge": if mergeTask.getPathName().find("DQM") == -1: break assign["MinMergeSize"] = mergeTask.jobSplittingParameters()["min_merge_size"] assign["MaxMergeSize"] = mergeTask.jobSplittingParameters()["max_merge_size"] assign["MaxMergeEvents"] = mergeTask.jobSplittingParameters().get("max_merge_events", 50000) return (schema, assign)