def loadWorkload(self, inputWorkload):
"""
If workload is sane, then use it
"""
if inputWorkload == None:
self.workload = None
if isinstance(inputWorkload, WMWorkload):
self.workload = WMWorkloadHelper(inputWorkload)
return
if isinstance(inputWorkload, WMWorkloadHelper):
self.workload = inputWorkload
return
if not os.path.exists(inputWorkload):
raise Exception('Could not find %s in local file system' %
(str(inputWorkload)))
testWorkload = WMWorkloadHelper(WMWorkload("workload"))
testWorkload.load(inputWorkload)
self.workload = testWorkload
return
def getMasterName(self):
"""
Gets a universal name for the jobGroup directory
Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN)
"""
if self.wmWorkload != None:
workload = self.wmWorkload.name()
elif not os.path.exists(self.workflow.spec):
logging.error(
"Could not find Workflow spec %s; labeling jobs by job ID only!"
% (self.workflow.spec))
return os.path.join(self.startDir,
self.jobGroup.uid), os.path.join(
self.startDir, self.jobGroup.uid)
else:
wmWorkload = WMWorkloadHelper(WMWorkload("workload"))
wmWorkload.load(self.workflow.spec)
workload = wmWorkload.name()
task = self.workflow.task
if task.startswith("/" + workload + "/"):
task = task[len(workload) + 2:]
return os.path.join(self.startDir,
workload), os.path.join(self.startDir, workload,
task)
def getMasterName(startDir, wmWorkload = None, workflow = None):
"""
Gets a universal name for the jobGroup directory
Return the uid as the name if none available (THIS SHOULD NEVER HAPPEN)
"""
if wmWorkload != None:
workload = wmWorkload.name()
elif not os.path.exists(workflow.spec):
msg = "Could not find Workflow spec %s: " % (workflow.spec)
msg += "Cannot create work area without spec!"
logging.error(msg)
raise CreateWorkAreaException(msg)
else:
wmWorkload = WMWorkloadHelper(WMWorkload("workload"))
wmWorkload.load(workflow.spec)
workload = wmWorkload.name()
task = workflow.task
if task.startswith("/" + workload + "/"):
task = task[len(workload) + 2:]
return (os.path.join(startDir, workload),
os.path.join(startDir, workload, task))
def retrieveWMSpec(workflow=None, wmWorkloadURL=None):
"""
_retrieveWMSpec_
Given a subscription, this function loads the WMSpec associated with that workload
"""
if not wmWorkloadURL and workflow:
wmWorkloadURL = workflow.spec
if not wmWorkloadURL or not os.path.isfile(wmWorkloadURL):
logging.error("WMWorkloadURL %s is empty", wmWorkloadURL)
return None
wmWorkload = WMWorkloadHelper(WMWorkload("workload"))
wmWorkload.load(wmWorkloadURL)
return wmWorkload
def createTestWMSpec(self):
"""
_createTestWMSpec_
Create a WMSpec that has a processing, merge, cleanup and skims tasks that
can be used by the subscription creation test.
"""
testWorkload = WMWorkloadHelper(WMWorkload("TestWorkload"))
testWorkload.setDashboardActivity("TestReReco")
testWorkload.setSpecUrl("/path/to/workload")
testWorkload.setOwnerDetails("sfoulkes", "DMWM", {'dn': 'MyDN'})
procTask = testWorkload.newTask("ProcessingTask")
procTask.setTaskType("Processing")
procTask.setSplittingAlgorithm("FileBased", files_per_job=1)
procTaskCMSSW = procTask.makeStep("cmsRun1")
procTaskCMSSW.setStepType("CMSSW")
procTaskCMSSWHelper = procTaskCMSSW.getTypeHelper()
procTask.setTaskType("Processing")
procTask.setSiteWhitelist(["site1"])
procTask.setSiteBlacklist(["site2"])
procTask.applyTemplates()
procTaskCMSSWHelper.addOutputModule("OutputA",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
mergeTask = procTask.addTask("MergeTask")
mergeTask.setInputReference(procTaskCMSSW, outputModule="OutputA")
mergeTask.setTaskType("Merge")
mergeTask.setSplittingAlgorithm("WMBSMergeBySize",
min_merge_size=1,
max_merge_size=2,
max_merge_events=3)
mergeTaskCMSSW = mergeTask.makeStep("cmsRun1")
mergeTaskCMSSW.setStepType("CMSSW")
mergeTaskCMSSWHelper = mergeTaskCMSSW.getTypeHelper()
mergeTask.setTaskType("Merge")
mergeTask.applyTemplates()
mergeTaskCMSSWHelper.addOutputModule("Merged",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
cleanupTask = procTask.addTask("CleanupTask")
cleanupTask.setInputReference(procTaskCMSSW, outputModule="OutputA")
cleanupTask.setTaskType("Merge")
cleanupTask.setSplittingAlgorithm("SiblingProcessingBased",
files_per_job=50)
cleanupTaskCMSSW = cleanupTask.makeStep("cmsRun1")
cleanupTaskCMSSW.setStepType("CMSSW")
cleanupTaskCMSSWHelper = cleanupTaskCMSSW.getTypeHelper()
cleanupTask.setTaskType("Cleanup")
cleanupTask.applyTemplates()
skimTask = mergeTask.addTask("SkimTask")
skimTask.setTaskType("Skim")
skimTask.setInputReference(mergeTaskCMSSW, outputModule="Merged")
skimTask.setSplittingAlgorithm("FileBased",
files_per_job=1,
include_parents=True)
skimTaskCMSSW = skimTask.makeStep("cmsRun1")
skimTaskCMSSW.setStepType("CMSSW")
skimTaskCMSSWHelper = skimTaskCMSSW.getTypeHelper()
skimTask.setTaskType("Skim")
skimTask.applyTemplates()
skimTaskCMSSWHelper.addOutputModule("SkimOutputA",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
skimTaskCMSSWHelper.addOutputModule("SkimOutputB",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
return testWorkload
