def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
def setUp(self):
"""bootstrap tests"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
#self.testInit.setDatabaseConnection()
#self.testInit.setSchema(customModules = ["WMCore.WMBS"],
# useDefault = False)
self.testInit.setupCouch("wmcore-workloadsummary", "WorkloadSummary")
self.workload1 = newWorkload("WorkloadSummaryTest1")
self.workload2 = newWorkload("WorkloadSummaryTest2")
def createWorkload(self):
"""
_createWorkload_
Create a new workload.
"""
ownerProps = {
'dn': self.owner_dn,
'vogroup': self.owner_vogroup,
'vorole': self.owner_vorole
}
workload = newWorkload(self.workloadName)
workload.setOwnerDetails(self.owner, self.group, ownerProps)
workload.setStartPolicy("DatasetBlock",
SliceType="NumberOfFiles",
SliceSize=1)
workload.setEndPolicy("SingleShot")
workload.setAcquisitionEra(acquisitionEras=self.acquisitionEra)
workload.setProcessingVersion(
processingVersions=self.processingVersion)
workload.setProcessingString(processingStrings=self.processingString)
workload.setValidStatus(validStatus=self.validStatus)
workload.setLumiList(lumiLists=self.lumiList)
workload.setPriority(self.priority)
workload.setPrepID(self.prepID)
return workload
def createTestWorkload(self):
"""
_createTestWorkload_
"""
workload = newWorkload("ACDCTest")
reco = workload.newTask("reco")
skim1 = reco.addTask("skim1")
workload.setOwnerDetails(name="evansde77", group="DMWM")
# first task uses the input dataset
reco.addInputDataset(name='/PRIMARY/processed-v1/TIERONE',
primary="PRIMARY",
processed="processed-v1",
tier="TIERONE")
cmsRunReco = reco.makeStep("cmsRun1")
cmsRunReco.setStepType("CMSSW")
reco.applyTemplates()
cmsRunRecoHelper = cmsRunReco.getTypeHelper()
cmsRunRecoHelper.addOutputModule("outputRECO",
primaryDataset="PRIMARY",
processedDataset="processed-v2",
dataTier="TIERTWO",
lfnBase="/store/dunkindonuts",
mergedLFNBase="/store/kfc")
# second step uses an input reference
cmsRunSkim = skim1.makeStep("cmsRun2")
cmsRunSkim.setStepType("CMSSW")
skim1.applyTemplates()
skim1.setInputReference(cmsRunReco, outputModule="outputRECO")
return workload
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def createTestWorkload(self):
"""
_createTestWorkload_
"""
workload = newWorkload("ACDCTest")
reco = workload.newTask("reco")
skim1 = reco.addTask("skim1")
workload.setOwnerDetails(name="evansde77", group="DMWM")
# first task uses the input dataset
reco.addInputDataset(name='/PRIMARY/processed-v1/TIERONE',
primary="PRIMARY", processed="processed-v1", tier="TIERONE")
cmsRunReco = reco.makeStep("cmsRun1")
cmsRunReco.setStepType("CMSSW")
reco.applyTemplates()
cmsRunRecoHelper = cmsRunReco.getTypeHelper()
cmsRunRecoHelper.addOutputModule("outputRECO",
primaryDataset="PRIMARY",
processedDataset="processed-v2",
dataTier="TIERTWO",
lfnBase="/store/dunkindonuts",
mergedLFNBase="/store/kfc")
# second step uses an input reference
cmsRunSkim = skim1.makeStep("cmsRun2")
cmsRunSkim.setStepType("CMSSW")
skim1.applyTemplates()
skim1.setInputReference(cmsRunReco, outputModule="outputRECO")
return workload
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception as ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443")
helper.addOutputModule(
"outputModule1", primaryDataset="Primary",
processedDataset='Processed',
dataTier='Tier',
lfnBase="/store/unmerged/whatever"
)
def testA(self):
"""
instantiate & apply template
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
# first up, create the template
try:
template = CMSSWTemplate()
except Exception as ex:
msg = "Failed to instantiate the CMSSW Step Template"
msg += str(ex)
self.fail(msg)
# now apply it to the step
try:
template(step)
except Exception as ex:
msg = "Failed to apply template to step"
msg += str(ex)
self.fail(msg)
# TODO: Check the step has the appropriate attributes expected for CMSSW
self.assertTrue(
hasattr(step, "application"))
self.assertTrue(
hasattr(step.application, "setup"))
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testA(self):
"""
instantiate & apply template
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
# first up, create the template
try:
template = CMSSWTemplate()
except Exception as ex:
msg = "Failed to instantiate the CMSSW Step Template"
msg += str(ex)
self.fail(msg)
# now apply it to the step
try:
template(step)
except Exception as ex:
msg = "Failed to apply template to step"
msg += str(ex)
self.fail(msg)
# TODO: Check the step has the appropriate attributes expected for CMSSW
self.assertTrue(
hasattr(step, "application"))
self.assertTrue(
hasattr(step.application, "setup"))
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception as ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
helper.cmsswSetup("CMSSW_X_Y_Z", scramArch="slc5_ia32_gcc443")
helper.addOutputModule(
"outputModule1", primaryDataset="Primary",
processedDataset='Processed',
dataTier='Tier',
lfnBase="/store/unmerged/whatever"
)
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW']
self.stateMap = {'Merge' : [],
'Processing Done' : []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.wmstatsWriter.insertGenericRequest({'_id' : workflowName})
self.wmstatsWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name = 'TestStreamerFileset')
topFileset.create()
options = {'spec' : specPath, 'owner' : 'ItsAMeMario',
'name' : workflowName, 'wfType' : 'tier0'}
topLevelWorkflow = Workflow(task = '/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task = '/%s/Repack/%s' % (workflowName, task), **options)
mergeWorkflow.create()
unmergedFileset = Fileset(name = 'TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(task = '/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name = 'TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def setupRepackWorkflow(self):
"""
_setupRepackWorkflow_
Populate WMBS with a repack-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Repack_Run481516_StreamZ'
mergeTasks = ['RepackMergewrite_QuadElectron_RAW', 'RepackMergewrite_TriPhoton_RAW',
'RepackMergewrite_SingleNeutrino_RAW']
self.stateMap = {'Merge': [],
'Processing Done': []}
self.orderedStates = ['Merge', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest({'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
repackTask = workload.newTask('Repack')
for task in mergeTasks:
repackTask.addTask(task)
repackTask.addTask('RepackCleanupUnmergedwrite_QuadElectron_RAW')
specPath = os.path.join(self.testDir, 'Repack.pkl')
workload.save(specPath)
# Populate WMBS
topFileset = Fileset(name='TestStreamerFileset')
topFileset.create()
options = {'spec': specPath, 'owner': 'ItsAMeMario',
'name': workflowName, 'wfType': 'tier0'}
topLevelWorkflow = Workflow(task='/%s/Repack' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(topFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topFileset)
for task in mergeTasks:
mergeWorkflow = Workflow(task='/%s/Repack/%s' % (workflowName, task), **options)
mergeWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFileset%s' % task)
unmergedFileset.create()
mergeSub = Subscription(unmergedFileset, mergeWorkflow)
mergeSub.create()
self.stateMap['Processing Done'].append(unmergedFileset)
cleanupWorkflow = Workflow(task='/Repack_Run481516_StreamZ/Repack/RepackCleanupUnmergedwrite_QuadElectron_RAW',
**options)
cleanupWorkflow.create()
unmergedFileset = Fileset(name='TestUnmergedFilesetToCleanup')
unmergedFileset.create()
cleanupSub = Subscription(unmergedFileset, cleanupWorkflow)
cleanupSub.create()
return
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK',
'insert fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'],
"failed"), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus("not_exist_schema",
"assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(
self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'),
'OK', 'update fail')
self.assertEquals(
self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {
'total_jobs': 100,
'input_events': 1000,
'input_lumis': 1234,
'input_num_file': 5
}
self.assertEquals(
self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'],
totalStats), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.insertTotalStats("not_exist_schema",
totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK',
'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
def createWorkload(name="BasicProcessing"):
workload = newWorkload(name)
workload.setOwner("DMWMTest")
workload.setStartPolicy('DatasetBlock')
workload.setEndPolicy('SingleShot')
workload.setCampaign("TestCampaign")
workload.setPrepID("TestPrepID")
# //
# // set up the production task
#//
rereco = workload.newTask("ReReco1")
rereco.setTaskType("Processing")
rerecoCmssw = rereco.makeStep("cmsRun1")
rerecoCmssw.setStepType("CMSSW")
skimStageOut = rerecoCmssw.addStep("stageOut1")
skimStageOut.setStepType("StageOut")
skimLogArch = rerecoCmssw.addStep("logArch1")
skimLogArch.setStepType("LogArchive")
rereco.applyTemplates()
rereco.setSplittingAlgorithm("FileBased", files_per_job=1)
rereco.addInputDataset(
primary = "Cosmics",
processed = "CRAFT09-PromptReco-v1",
tier = "RECO",
dbsurl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader")
# //
# // rereco cmssw step
#//
#
# TODO: Anywhere helper.data is accessed means we need a method added to the
# type based helper class to provide a clear API.
rerecoCmsswHelper = rerecoCmssw.getTypeHelper()
rerecoCmsswHelper.cmsswSetup(
"CMSSW_3_1_2",
softwareEnvironment=" . /uscmst1/prod/sw/cms/bashrc prod"
)
rerecoCmsswHelper.setDataProcessingConfig(
"cosmics", "promptReco", globalTag="GLOBAL::BALLS",
writeTiers=['RECO'])
rerecoCmsswHelper.addOutputModule(
"outputRECO", primaryDataset="Primary",
processedDataset="Processed",
dataTier="RECO")
#Add a stageOut step
skimStageOutHelper = skimStageOut.getTypeHelper()
skimLogArchHelper = skimLogArch.getTypeHelper()
rereco.addGenerator("BasicNaming")
return workload
def createWorkload(name="BasicProcessing"):
workload = newWorkload(name)
workload.setOwner("DMWMTest")
workload.setStartPolicy('DatasetBlock')
workload.setEndPolicy('SingleShot')
workload.setCampaign("TestCampaign")
workload.setPrepID("TestPrepID")
# //
# // set up the production task
#//
rereco = workload.newTask("ReReco1")
rereco.setTaskType("Processing")
rerecoCmssw = rereco.makeStep("cmsRun1")
rerecoCmssw.setStepType("CMSSW")
skimStageOut = rerecoCmssw.addStep("stageOut1")
skimStageOut.setStepType("StageOut")
skimLogArch = rerecoCmssw.addStep("logArch1")
skimLogArch.setStepType("LogArchive")
rereco.applyTemplates()
rereco.setSplittingAlgorithm("FileBased", files_per_job=1)
rereco.addInputDataset(
primary = "Cosmics",
processed = "CRAFT09-PromptReco-v1",
tier = "RECO",
dbsurl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader")
# //
# // rereco cmssw step
#//
#
# TODO: Anywhere helper.data is accessed means we need a method added to the
# type based helper class to provide a clear API.
rerecoCmsswHelper = rerecoCmssw.getTypeHelper()
rerecoCmsswHelper.cmsswSetup(
"CMSSW_3_1_2",
softwareEnvironment=" . /uscmst1/prod/sw/cms/bashrc prod"
)
rerecoCmsswHelper.setDataProcessingConfig(
"cosmics", "promptReco", globalTag="GLOBAL::BALLS",
writeTiers=['RECO'])
rerecoCmsswHelper.addOutputModule(
"outputRECO", primaryDataset="Primary",
processedDataset="Processed",
dataTier="RECO")
#Add a stageOut step
skimStageOutHelper = skimStageOut.getTypeHelper()
skimLogArchHelper = skimLogArch.getTypeHelper()
rereco.addGenerator("BasicNaming")
return workload
def newWorkload(self):
"""
_newWorkload_
Create a new workload.
"""
workload = newWorkload(self.workloadName)
workload.setOwner(self.owner)
#workload.data.properties.acquisitionEra = self.acquisitionEra
return workload
def createWorkload(self):
"""
_createWorkload_
Create a new workload.
"""
workload = newWorkload("TestWorkload")
workload.setOwner("*****@*****.**")
workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1)
workload.setEndPolicy("SingleShot")
workload.setAcquisitionEra("WMAgentCommissioning10")
return workload
def createWorkload(self):
"""
_createWorkload_
Create a new workload.
"""
workload = newWorkload("TestWorkload")
workload.setOwner("*****@*****.**")
workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1)
workload.setEndPolicy("SingleShot")
workload.setAcquisitionEra("WMAgentCommissioning10")
return workload
def oneTaskTwoStep():
workload = newWorkload("OneTaskTwoStepTest")
task1 = workload.newTask("FirstTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
task1.applyTemplates()
return workload
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.workflowsByStatus(["failed"])
self.assertEquals(requests, [schema[0]['RequestName']])
def oneTaskTwoStep():
workload = newWorkload("OneTaskTwoStepTest")
workload.setDbsUrl("https://cmsweb.cern.ch/dbs/prod/global/DBSReader")
task1 = workload.newTask("FirstTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
task1.applyTemplates()
return workload
def oneTaskTwoStep():
workload = newWorkload("OneTaskTwoStepTest")
task1 = workload.newTask("FirstTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
task1.applyTemplates()
return workload
def createWorkload(self):
"""
_createWorkload_
Create a new workload.
"""
ownerProps = {'dn': self.owner_dn, 'vogroup': self.owner_vogroup, 'vorole': self.owner_vorole}
workload = newWorkload(self.workloadName)
workload.setOwnerDetails(self.owner, self.group, ownerProps)
workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1)
workload.setEndPolicy("SingleShot")
workload.setAcquisitionEra(acquisitionEras = self.acquisitionEra)
workload.setProcessingVersion(processingVersions = self.processingVersion)
workload.setValidStatus(validStatus = self.validStatus)
return workload
def testMulticoreSettings(self):
"""
test multicore related methods
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getNumberOfCores(), 1)
helper.setNumberOfCores(8)
self.assertEqual(helper.getNumberOfCores(), 8)
def testMulticoreSettings(self):
"""
test multicore related methods
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getNumberOfCores(), 1)
helper.setNumberOfCores(8)
self.assertEqual(helper.getNumberOfCores(), 8)
def testA(self):
"""
instantiate & apply template
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
# first up, create the template
try:
template = CMSSWTemplate()
except Exception, ex:
msg = "Failed to instantiate the CMSSW Step Template"
msg += str(ex)
self.fail(msg)
def testA(self):
"""
instantiate & apply template
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
# first up, create the template
try:
template = CMSSWTemplate()
except Exception, ex:
msg = "Failed to instantiate the CMSSW Step Template"
msg += str(ex)
self.fail(msg)
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception, ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
def testB(self):
"""
try using the helper API to set and get information in the step
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = step = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
try:
helper = template.helper(step)
except Exception, ex:
msg = "Failure to create CMSSW Step Helper"
msg += str(ex)
self.fail(msg)
def testChainedProcessing(self):
"""
check the chained processing set up is correct
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
step = task.makeStep("TemplateTest")
template = CMSSWTemplate()
helper = template.helper(step.data)
inputStepName = "some_inputStepName"
inputOutputModule = "some_inputOutputModule"
helper.setupChainedProcessing(inputStepName, inputOutputModule)
self.assertEqual(helper.data.input.chainedProcessing, True)
self.assertEqual(helper.data.input.inputStepName, "some_inputStepName")
self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def testChainedProcessing(self):
"""
check the chained processing set up is correct
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
step = task.makeStep("TemplateTest")
template = CMSSWTemplate()
helper = template.helper(step.data)
inputStepName = "some_inputStepName"
inputOutputModule = "some_inputOutputModule"
helper.setupChainedProcessing(inputStepName, inputOutputModule)
self.assertEqual(helper.data.input.chainedProcessing, True)
self.assertEqual(helper.data.input.inputStepName, "some_inputStepName")
self.assertEqual(helper.data.input.inputOutputModule, "some_inputOutputModule")
def oneTaskFourStep():
workload = newWorkload("OneTaskFourStepTest")
task1 = workload.newTask("FirstTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
step3 = step1.addStep("cmsRun2")
step3.setStepType("CMSSW")
step4 = step3.addStep("stageOut2")
step4.setStepType("StageOut")
return workload
def oneTaskFourStep():
workload = newWorkload("OneTaskFourStepTest")
workload.setDbsUrl("https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader")
task1 = workload.newTask("FirstTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
step3 = step1.addStep("cmsRun2")
step3.setStepType("CMSSW")
step4 = step3.addStep("stageOut2")
step4.setStepType("StageOut")
return workload
def oneTaskFourStep():
workload = newWorkload("OneTaskFourStepTest")
task1 = workload.newTask("FirstTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
step3 = step1.addStep("cmsRun2")
step3.setStepType("CMSSW")
step4 = step3.addStep("stageOut2")
step4.setStepType("StageOut")
return workload
def twoTaskTree():
workload = newWorkload("TwoTaskTree")
task1 = workload.newTask("FirstTask")
task2 = task1.addTask("SecondTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
step3 = task2.makeStep("cmsRun2")
step3.setStepType("CMSSW")
step4 = step3.addStep("stageOut2")
step4.setStepType("StageOut")
return workload
def twoTaskTree():
workload = newWorkload("TwoTaskTree")
workload.setDbsUrl("https://cmsweb.cern.ch/dbs/prod/global/DBSReader")
task1 = workload.newTask("FirstTask")
task2 = task1.addTask("SecondTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
step3 = task2.makeStep("cmsRun2")
step3.setStepType("CMSSW")
step4 = step3.addStep("stageOut2")
step4.setStepType("StageOut")
return workload
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def createWorkload(self):
"""
_createWorkload_
Create a new workload.
"""
ownerProps = {'dn': self.owner_dn, 'vogroup': self.owner_vogroup, 'vorole': self.owner_vorole}
workload = newWorkload(self.workloadName)
workload.setOwnerDetails(self.owner, self.group, ownerProps)
workload.setStartPolicy("DatasetBlock", SliceType = "NumberOfFiles", SliceSize = 1)
workload.setEndPolicy("SingleShot")
workload.setAcquisitionEra(acquisitionEras = self.acquisitionEra)
workload.setProcessingVersion(processingVersions = self.processingVersion)
workload.setProcessingString(processingStrings = self.processingString)
workload.setValidStatus(validStatus = self.validStatus)
workload.setLumiList(lumiLists = self.lumiList)
workload.setPriority(self.priority)
workload.setCampaign(self.campaign)
workload.setRequestType(self.requestType)
workload.setPrepID(self.prepID)
return workload
def twoTaskTree():
workload = newWorkload("TwoTaskTree")
task1 = workload.newTask("FirstTask")
task2 = task1.addTask("SecondTask")
step1 = task1.makeStep("cmsRun1")
step1.setStepType("CMSSW")
step2 = step1.addStep("stageOut1")
step2.setStepType("StageOut")
step3 = task2.makeStep("cmsRun2")
step3.setStepType("CMSSW")
step4 = step3.addStep("stageOut2")
step4.setStepType("StageOut")
return workload
def testGPUSettings(self):
"""
Test GPU methods at CMSSW template level
"""
workload = newWorkload("UnitTests")
task = workload.newTask("CMSSWTemplate")
stepHelper = task.makeStep("TemplateTest")
step = stepHelper.data
template = CMSSWTemplate()
template(step)
helper = template.helper(step)
self.assertEqual(helper.getGPURequired(), "forbidden")
self.assertIsNone(helper.getGPURequirements())
helper.setGPUSettings("optional", "test 1 2 3")
self.assertEqual(helper.getGPURequired(), "optional")
self.assertItemsEqual(helper.getGPURequirements(), "test 1 2 3")
helper.setGPUSettings("required", {"key1": "value1", "key2": "value2"})
self.assertEqual(helper.getGPURequired(), "required")
self.assertItemsEqual(helper.getGPURequirements(), {
"key1": "value1",
"key2": "value2"
})
def createWorkload(name="MultiTaskProduction"):
workload = newWorkload("MultiTaskProduction")
workload.setOwner("DMWMTest")
workload.setStartPolicy('MonteCarlo')
workload.setEndPolicy('SingleShot')
# //
# // set up the production task
#//
production = workload.newTask("Production1")
production.addProduction(totalevents = 1000)
prodCmssw = production.makeStep("cmsRun1")
prodCmssw.setStepType("CMSSW")
prodStageOut = prodCmssw.addStep("stageOut1")
prodStageOut.setStepType("StageOut")
production.applyTemplates()
production.setSiteWhitelist(["T2_XX_SiteA"])
# //
# // set up the merge task
#//
merge = production.addTask("Merge")
mergeCmssw = merge.makeStep("cmsRun1")
mergeCmssw.setStepType("CMSSW")
mergeStageOut = mergeCmssw.addStep("stageOut1")
mergeStageOut.setStepType("StageOut")
merge.applyTemplates()
# //
# // populate the details of the production tasks
#//
# //
# // production cmssw step
#//
#
# TODO: Anywhere helper.data is accessed means we need a method added to the
# type based helper class to provide a clear API.
prodCmsswHelper = prodCmssw.getTypeHelper()
prodCmsswHelper.data.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
prodCmsswHelper.data.application.setup.softwareEnvironment = " . /uscmst1/prod/sw/cms/bashrc prod"
prodCmsswHelper.data.application.configuration.configCacheUrl = "http://whatever"
prodCmsswHelper.addOutputModule("writeData", primaryDataset = "Primary",
processedDataset = "Processed",
dataTier = "TIER")
#print prodCmsswHelper.data
# //
# // production stage out step
#//
prodStageOutHelper = prodStageOut.getTypeHelper()
# //
# // merge cmssw step
#//
# point it at the input step from the previous task
merge.setInputReference(prodCmssw, outputModule = "writeData")
# //
# // populate the details of the merge tasks
#//
# print workload.data
production = workload.newTask("Production2")
production.addProduction(totalevents = 2000)
prodCmssw = production.makeStep("cmsRun1")
prodCmssw.setStepType("CMSSW")
prodStageOut = prodCmssw.addStep("stageOut1")
prodStageOut.setStepType("StageOut")
production.applyTemplates()
production.setSiteWhitelist(["T2_XX_SiteA"])
# //
# // set up the merge task
#//
merge = production.addTask("Merge")
mergeCmssw = merge.makeStep("cmsRun1")
mergeCmssw.setStepType("CMSSW")
mergeStageOut = mergeCmssw.addStep("stageOut1")
mergeStageOut.setStepType("StageOut")
merge.applyTemplates()
# //
# // populate the details of the production tasks
#//
# //
# // production cmssw step
#//
#
# TODO: Anywhere helper.data is accessed means we need a method added to the
# type based helper class to provide a clear API.
prodCmsswHelper = prodCmssw.getTypeHelper()
prodCmsswHelper.data.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
prodCmsswHelper.data.application.setup.softwareEnvironment = " . /uscmst1/prod/sw/cms/bashrc prod"
prodCmsswHelper.data.application.configuration.configCacheUrl = "http://whatever"
prodCmsswHelper.addOutputModule("writeData", primaryDataset = "Primary",
processedDataset = "Processed",
dataTier = "TIER")
#print prodCmsswHelper.data
# //
# // production stage out step
#//
prodStageOutHelper = prodStageOut.getTypeHelper()
# //
# // merge cmssw step
#//
# point it at the input step from the previous task
merge.setInputReference(prodCmssw, outputModule = "writeData")
return workload
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
ret = scram("python -c '%s'" % pythonScript, logName=subprocess.PIPE, runtimeDir=os.getcwd())
if ret > 0:
msg = scram.diagnostic()
handleException("FAILED", EC_CMSRunWrapper, 'Error getting output modules from the pset.\n\tScram Env %s\n\tCommand:%s' % (msg, pythonScript))
mintime()
sys.exit(EC_CMSRunWrapper)
return literal_eval(scram.stdout)
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter' : 0, 'workflow' : 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [('%s/TweakPSet.py --location=%s '+
'--inputFile=\'%s\' '+
'--runAndLumis=\'%s\' '+
'--firstEvent=%s '+
'--lastEvent=%s '+
'--firstLumi=%s '+
'--firstRun=%s '+
'--seeding=%s '+
'--lheInputFiles=%s '+
'--oneEventMode=%s ' +
'--eventsPerLumi=%s') %
(os.getcwd(), os.getcwd(),
opts.inputFile,
opts.runAndLumis,
opts.firstEvent,
opts.lastEvent,
opts.firstLumi,
opts.firstRun,
opts.seeding,
opts.lheInputFiles,
opts.oneEventMode,
opts.eventsPerLumi)]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.reqDBWriter.insertGenericRequest(schema[0])
self.assertEquals(result[0]['ok'], True, 'insert fail')
result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'],
"failed")
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestStatus("not_exist_schema",
"assigned")
self.assertEquals(result, 'Error: document not found')
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], {"Teams": ['teamA']})
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
"not_exist_schema", {"Teams": ['teamA']})
self.assertEquals(result, 'Error: document not found')
totalStats = {
'TotalEstimatedJobs': 100,
'TotalInputEvents': 1000,
'TotalInputLumis': 1234,
'TotalInputFiles': 5
}
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
"not_exist_schema", totalStats)
self.assertEquals(result, 'Error: document not found')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
properties = {
"RequestPriority": spec1.priority(),
'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec1.listOutputDatasets()
}
result = self.reqDBWriter.updateRequestProperty(
spec1.name(), properties)
self.assertEquals(result, 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
properties = {
"RequestPriority": spec2.priority(),
'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec2.listOutputDatasets()
}
result = self.reqDBWriter.updateRequestProperty(
spec2.name(), properties)
self.assertEquals(result, 'Error: document not found')
requests = self.wmstatsReader.getRequestByStatus(["failed"],
jobInfoFlag=False,
legacyFormat=True)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestByStatus(["failed"])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestSummaryWithJobInfo(
schema[0]['RequestName'])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
_Tier1ReRecoWorkload_
"""
import os, pickle, sys, shutil
from WMCore.WMSpec.WMWorkload import newWorkload
from WMCore.WMSpec.WMStep import makeWMStep
from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper
DBSURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
# //
# // Set up the basic workload task and step structure
#//
workload = newWorkload("MultiTaskReReco")
workload.setOwner("WMTest")
workload.setStartPolicy('DatasetBlock')
workload.setEndPolicy('SingleShot')
# //
# // set up the production task
#//
rereco = workload.newTask("ReReco1")
rerecoCmssw = rereco.makeStep("cmsRun1")
rerecoCmssw.setStepType("CMSSW")
skimStageOut = rerecoCmssw.addStep("stageOut1")
skimStageOut.setStepType("StageOut")
skimLogArch = rerecoCmssw.addStep("logArch1")
skimLogArch.setStepType("LogArchive")
rereco.applyTemplates()
_BasicProductionWorkload_
Sample/Tester for a simple production workflow and associated merge.
Production task produces a single output dataset,
Merge task is used to merge that dataset
"""
from WMCore.WMSpec.WMWorkload import newWorkload
from WMCore.WMSpec.WMStep import makeWMStep
from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper
# //
# // Set up the basic workload task and step structure
#//
workload = newWorkload("MultiTaskProduction")
workload.setOwner("WMTest")
workload.setStartPolicy('MonteCarlo', **{'SliceType': 'NumberOfEvents', 'SliceSize': 100})
workload.setEndPolicy('SingleShot')
# //
# // set up the production task
#//
production = workload.newTask("Production1")
production.addProduction(totalEvents = 1000)
production.setTaskType("Merge")
prodCmssw = production.makeStep("cmsRun1")
prodCmssw.setStepType("CMSSW")
prodStageOut = prodCmssw.addStep("stageOut1")
prodStageOut.setStepType("StageOut")
production.applyTemplates()
Sample/Tester for a production workflow that writes several datasets
and associated merges for each dataset.
Production task produces three output datasets,
Merge tasks are used to merge those datasets
"""
from WMCore.WMSpec.WMWorkload import newWorkload
from WMCore.WMSpec.WMStep import makeWMStep
from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper
# //
# // Set up the basic workload task and step structure
# //
workload = newWorkload("MultiOutputs")
workload.setOwner("WMTest")
workload.setStartPolicy("MonteCarlo")
workload.setEndPolicy("SingleShot")
# //
# // set up the production task
# //
production = workload.newTask("Production")
# WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction"
production.setTaskType("Merge")
production.addProduction(totalEvents=1000)
prodCmssw = production.makeStep("cmsRun1")
prodCmssw.setStepType("CMSSW")
prodStageOut = prodCmssw.addStep("stageOut1")
prodStageOut.setStepType("StageOut")
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1')
os.mkdir(self.cmsstepdir)
open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close()
open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir)
realstep = StageOutTemplate.StageOutStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join(self.testDir, 'stepdir')
os.mkdir(self.stepDir)
builder = StageOutBuilder.StageOut()
builder(step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('StageOut')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
'w').close()
shutil.copyfile(
os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
Sample/Tester for a production workflow that writes several datasets
and associated merges for each dataset.
Production task produces three output datasets,
Merge tasks are used to merge those datasets
"""
from WMCore.WMSpec.WMWorkload import newWorkload
from WMCore.WMSpec.WMStep import makeWMStep
from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper
# //
# // Set up the basic workload task and step structure
#//
workload = newWorkload("MultiOutputs")
workload.setOwner("WMTest")
workload.setStartPolicy('MonteCarlo')
workload.setEndPolicy('SingleShot')
workload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
# //
# // set up the production task
#//
production = workload.newTask("Production")
#WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction"
production.setTaskType("Merge")
production.addProduction(totalEvents=1000)
prodCmssw = production.makeStep("cmsRun1")
prodCmssw.setStepType("CMSSW")
prodStageOut = prodCmssw.addStep("stageOut1")
#!/usr/bin/env python
"""
_Tier1ReRecoWorkload_
"""
from WMCore.WMSpec.WMWorkload import newWorkload
DBSURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
# //
# // Set up the basic workload task and step structure
#//
workload = newWorkload("MultiTaskReReco")
workload.setOwner("WMTest")
workload.setStartPolicy('DatasetBlock')
workload.setEndPolicy('SingleShot')
# //
# // set up the production task
#//
rereco = workload.newTask("ReReco1")
rerecoCmssw = rereco.makeStep("cmsRun1")
rerecoCmssw.setStepType("CMSSW")
skimStageOut = rerecoCmssw.addStep("stageOut1")
skimStageOut.setStepType("StageOut")
skimLogArch = rerecoCmssw.addStep("logArch1")
skimLogArch.setStepType("LogArchive")
rereco.applyTemplates()
rereco.setSplittingAlgorithm("FileBased", files_per_job = 1)
def setupExpressWorkflow(self):
"""
_setupExpressWorkflow_
Populate WMBS with a express-like workflow,
every subscription must be unfinished at first
"""
workflowName = 'Express_Run481516_StreamZFast'
secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT',
'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM',
'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT',
'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO']
alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd'
dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged'
self.stateMap = {'Merge': [],
'Harvesting': [],
'Processing Done': []}
self.orderedStates = ['Merge', 'Harvesting', 'Processing Done']
# Populate WMStats
self.requestDBWriter.insertGenericRequest({'RequestName': workflowName})
self.requestDBWriter.updateRequestStatus(workflowName, 'Closed')
# Create a wmspec in disk
workload = newWorkload(workflowName)
expressTask = workload.newTask('Express')
for task in secondLevelTasks:
secondLevelTask = expressTask.addTask(task)
if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO':
secondLevelTask.addTask(alcaHarvestTask)
elif task == 'ExpressMergewrite_StreamZFast_DQM':
secondLevelTask.addTask(dqmHarvestTask)
specPath = os.path.join(self.testDir, 'Express.pkl')
workload.save(specPath)
# Populate WMBS
sharedFileset = Fileset(name='TestFileset')
sharedFileset.create()
sharedFileset.markOpen(False)
options = {'spec': specPath, 'owner': 'ItsAMeMario',
'name': workflowName, 'wfType': 'tier0'}
topLevelWorkflow = Workflow(task='/%s/Express' % workflowName,
**options)
topLevelWorkflow.create()
topLevelSub = Subscription(sharedFileset, topLevelWorkflow)
topLevelSub.create()
self.stateMap['Merge'].append(topLevelSub)
for task in [x for x in secondLevelTasks if not x.count('CleanupUnmerged')]:
secondLevelWorkflow = Workflow(task='/%s/Express/%s' % (workflowName, task), **options)
secondLevelWorkflow.create()
mergeSub = Subscription(sharedFileset, secondLevelWorkflow)
mergeSub.create()
self.stateMap['Harvesting'].append(mergeSub)
for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask),
('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]:
harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' % (workflowName, parent, child),
**options)
harvestingWorkflow.create()
harvestingSub = Subscription(sharedFileset, harvestingWorkflow)
harvestingSub.create()
self.stateMap['Processing Done'].append(harvestingSub)
return
Sample/Tester for a production workflow that writes several datasets
and associated merges for each dataset.
Production task produces three output datasets,
Merge tasks are used to merge those datasets
"""
from WMCore.WMSpec.WMWorkload import newWorkload
from WMCore.WMSpec.WMStep import makeWMStep
from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper
# //
# // Set up the basic workload task and step structure
#//
workload = newWorkload("MultiProduction")
workload.setOwner("WMTest")
workload.setStartPolicy('MonteCarlo')
workload.setEndPolicy('SingleShot')
# //
# // set up the production task
#//
production = workload.newTask("Production")
#WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction"
production.setTaskType("Merge")
production.addProduction(totalEvents = 1000)
prodCmssw = production.makeStep("cmsRun1")
prodCmssw.setStepType("CMSSW")
prodStageOut = prodCmssw.addStep("stageOut1")
prodStageOut.setStepType("StageOut")
_BasicProductionWorkload_
Sample/Tester for a simple production workflow and associated merge.
Production task produces a single output dataset,
Merge task is used to merge that dataset
"""
from WMCore.WMSpec.WMWorkload import newWorkload
from WMCore.WMSpec.WMStep import makeWMStep
from WMCore.WMSpec.Steps.StepFactory import getStepTypeHelper
# //
# // Set up the basic workload task and step structure
#//
workload = newWorkload("BasicProduction")
workload.setOwner("WMTest")
workload.setStartPolicy('MonteCarlo', **{
'SliceType': 'NumberOfEvents',
'SliceSize': 100
})
workload.setEndPolicy('SingleShot')
# //
# // set up the production task
#//
production = workload.newTask("Production")
production.addProduction(totalevents=1000)
#WARNING: this is arbitrary task type (wmbs schema only supprot "Processing", "Merge", "Harvest") - maybe add "MCProduction"
production.setTaskType("Merge")
production.setSplittingAlgorithm("EventBased", events_per_job=100)
prodCmssw = production.makeStep("cmsRun1")
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.reqDBWriter.insertGenericRequest(schema[0])
self.assertEquals(result[0]['ok'], True, 'insert fail')
result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed")
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned")
self.assertEquals(result,'Error: document not found')
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], {"Teams": ['teamA']})
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty("not_exist_schema", {"Teams": ['teamA']})
self.assertEquals(result, 'Error: document not found')
totalStats = {'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5}
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty("not_exist_schema", totalStats)
self.assertEquals(result, 'Error: document not found')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
properties = {"RequestPriority": spec1.priority(),
'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec1.listOutputDatasets()}
result = self.reqDBWriter.updateRequestProperty(spec1.name(), properties)
self.assertEquals(result, 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
properties = {"RequestPriority": spec2.priority(),
'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec2.listOutputDatasets()}
result = self.reqDBWriter.updateRequestProperty(spec2.name(), properties)
self.assertEquals(result, 'Error: document not found')
requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False, legacyFormat = True)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestByStatus(["failed"])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestSummaryWithJobInfo(schema[0]['RequestName'])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR):
ret = scram("python -c '%s'" % pythonScript,
runtimeDir=os.getcwd())
if ret > 0:
msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic(
)
handleException("FAILED", EC_CMSRunWrapper, msg)
mintime()
sys.exit(EC_CMSRunWrapper)
output = literal_eval(scram.getStdout())
return output
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter': 0, 'workflow': 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [
('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' +
'--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' +
'--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' +
'--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' +
'--maxRuntime=%s') %
(os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis,
opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun,
opts.seeding, opts.lheInputFiles, opts.oneEventMode,
opts.eventsPerLumi, opts.maxRuntime)
]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw
def makeWorkload(self, schema):
workload = newWorkload(schema['RequestName']).data
return workload
