def loadWorkload(self, inputWorkload): """ If workload is sane, then use it """ if inputWorkload == None: self.workload = None if isinstance(inputWorkload, WMWorkload): self.workload = WMWorkloadHelper(inputWorkload) return if isinstance(inputWorkload, WMWorkloadHelper): self.workload = inputWorkload return if not os.path.exists(inputWorkload): raise Exception('Could not find %s in local file system' % (str(inputWorkload))) testWorkload = WMWorkloadHelper(WMWorkload("workload")) testWorkload.load(inputWorkload) self.workload = testWorkload return
def getMasterName(self): """ Gets a universal name for the jobGroup directory Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN) """ if self.wmWorkload != None: workload = self.wmWorkload.name() elif not os.path.exists(self.workflow.spec): logging.error( "Could not find Workflow spec %s; labeling jobs by job ID only!" % (self.workflow.spec)) return os.path.join(self.startDir, self.jobGroup.uid), os.path.join( self.startDir, self.jobGroup.uid) else: wmWorkload = WMWorkloadHelper(WMWorkload("workload")) wmWorkload.load(self.workflow.spec) workload = wmWorkload.name() task = self.workflow.task if task.startswith("/" + workload + "/"): task = task[len(workload) + 2:] return os.path.join(self.startDir, workload), os.path.join(self.startDir, workload, task)
def getMasterName(startDir, wmWorkload = None, workflow = None): """ Gets a universal name for the jobGroup directory Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN) """ if wmWorkload != None: workload = wmWorkload.name() elif not os.path.exists(workflow.spec): msg = "Could not find Workflow spec %s: " % (workflow.spec) msg += "Cannot create work area without spec!" logging.error(msg) raise CreateWorkAreaException(msg) else: wmWorkload = WMWorkloadHelper(WMWorkload("workload")) wmWorkload.load(workflow.spec) workload = wmWorkload.name() task = workflow.task if task.startswith("/" + workload + "/"): task = task[len(workload) + 2:] return (os.path.join(startDir, workload), os.path.join(startDir, workload, task))
def retrieveWMSpec(workflow=None, wmWorkloadURL=None): """ _retrieveWMSpec_ Given a subscription, this function loads the WMSpec associated with that workload """ if not wmWorkloadURL and workflow: wmWorkloadURL = workflow.spec if not wmWorkloadURL or not os.path.isfile(wmWorkloadURL): logging.error("WMWorkloadURL %s is empty", wmWorkloadURL) return None wmWorkload = WMWorkloadHelper(WMWorkload("workload")) wmWorkload.load(wmWorkloadURL) return wmWorkload
def createTestWMSpec(self): """ _createTestWMSpec_ Create a WMSpec that has a processing, merge, cleanup and skims tasks that can be used by the subscription creation test. """ testWorkload = WMWorkloadHelper(WMWorkload("TestWorkload")) testWorkload.setDashboardActivity("TestReReco") testWorkload.setSpecUrl("/path/to/workload") testWorkload.setOwnerDetails("sfoulkes", "DMWM", {'dn': 'MyDN'}) procTask = testWorkload.newTask("ProcessingTask") procTask.setTaskType("Processing") procTask.setSplittingAlgorithm("FileBased", files_per_job=1) procTaskCMSSW = procTask.makeStep("cmsRun1") procTaskCMSSW.setStepType("CMSSW") procTaskCMSSWHelper = procTaskCMSSW.getTypeHelper() procTask.setTaskType("Processing") procTask.setSiteWhitelist(["site1"]) procTask.setSiteBlacklist(["site2"]) procTask.applyTemplates() procTaskCMSSWHelper.addOutputModule("OutputA", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) mergeTask = procTask.addTask("MergeTask") mergeTask.setInputReference(procTaskCMSSW, outputModule="OutputA") mergeTask.setTaskType("Merge") mergeTask.setSplittingAlgorithm("WMBSMergeBySize", min_merge_size=1, max_merge_size=2, max_merge_events=3) mergeTaskCMSSW = mergeTask.makeStep("cmsRun1") mergeTaskCMSSW.setStepType("CMSSW") mergeTaskCMSSWHelper = mergeTaskCMSSW.getTypeHelper() mergeTask.setTaskType("Merge") mergeTask.applyTemplates() mergeTaskCMSSWHelper.addOutputModule("Merged", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) cleanupTask = procTask.addTask("CleanupTask") cleanupTask.setInputReference(procTaskCMSSW, outputModule="OutputA") cleanupTask.setTaskType("Merge") cleanupTask.setSplittingAlgorithm("SiblingProcessingBased", files_per_job=50) cleanupTaskCMSSW = cleanupTask.makeStep("cmsRun1") cleanupTaskCMSSW.setStepType("CMSSW") cleanupTaskCMSSWHelper = cleanupTaskCMSSW.getTypeHelper() cleanupTask.setTaskType("Cleanup") cleanupTask.applyTemplates() skimTask = mergeTask.addTask("SkimTask") skimTask.setTaskType("Skim") skimTask.setInputReference(mergeTaskCMSSW, outputModule="Merged") skimTask.setSplittingAlgorithm("FileBased", files_per_job=1, include_parents=True) skimTaskCMSSW = skimTask.makeStep("cmsRun1") skimTaskCMSSW.setStepType("CMSSW") skimTaskCMSSWHelper = skimTaskCMSSW.getTypeHelper() skimTask.setTaskType("Skim") skimTask.applyTemplates() skimTaskCMSSWHelper.addOutputModule("SkimOutputA", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) skimTaskCMSSWHelper.addOutputModule("SkimOutputB", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) return testWorkload