def createWorkload(self):
"""
Create a workload in order to test things
"""
workload = rerecoWorkload("Tier1ReReco", getTestArguments())
rereco = workload.getTask("DataProcessing")
return workload
def createWorkload(self):
"""
Create a workload in order to test things
"""
workload = rerecoWorkload("Tier1ReReco", getTestArguments())
rereco = workload.getTask("DataProcessing")
return workload
def getRequestSchema():
schema = getTestArguments()
schema.update(RequestName = "TestReReco",
RequestType = "ReReco",
CmsPath = "/uscmst1/prod/sw/cms",
CouchURL = None,
CouchDBName = None,
Group = "PeopleLikeMe",
InputDataset = '/PRIM/PROC/TIER',
Requestor = "me"
)
return schema
def createReRecoSpec(self, specName, returnType="spec", splitter = None,
inputDataset = None, dbsUrl = None, **additionalArgs):
# update args, then reset them
args = getTestArguments()
args.update(additionalArgs)
spec = TestReRecoWorkload(specName, args)
if inputDataset != None:
spec.taskIterator().next().data.input.dataset.primary = inputDataset
if dbsUrl != None:
print dbsUrl
spec.taskIterator().next().data.input.dataset.dbsurl = dbsUrl
return self._selectReturnType(spec, returnType, splitter)
def setUp(self):
"""
_setUp_
Setup some reasonable defaults for the ReReco workflow.
"""
self.unmergedLFNBase = "/store/backfill/2/unmerged"
self.mergedLFNBase = "/store/backfill/2"
self.processingVersion = "v1"
self.cmsswVersion = "CMSSW_3_4_2_patch1"
self.acquisitionEra = "WMAgentCommissioining10"
self.primaryDataset = "MinimumBias"
self.workload = rerecoWorkload("Tier1ReReco", getTestArguments())
return
def setUp(self):
"""
_setUp_
Setup some reasonable defaults for the ReReco workflow.
"""
self.unmergedLFNBase = "/store/backfill/2/unmerged"
self.mergedLFNBase = "/store/backfill/2"
self.processingVersion = "v1"
self.cmsswVersion = "CMSSW_3_4_2_patch1"
self.acquisitionEra = "WMAgentCommissioining10"
self.primaryDataset = "MinimumBias"
self.workload = rerecoWorkload("Tier1ReReco", getTestArguments())
return
def getMergeACDCSpec(self, splittingAlgo="ParentlessMergeBySize", splittingArgs={}):
"""
_getMergeACDCSpec_
Get a ACDC spec for the merge task of a ReReco workload
"""
Tier1ReRecoWorkload = rerecoWorkload(self.workflowName, getTestArguments())
Tier1ReRecoWorkload.truncate(
"ACDC_%s" % self.workflowName,
"/%s/DataProcessing/DataProcessingMergeRECOoutput" % self.workflowName,
self.couchUrl,
self.acdcDBName,
)
Tier1ReRecoWorkload.setJobSplittingParameters(
"/ACDC_%s/DataProcessingMergeRECOoutput" % self.workflowName, splittingAlgo, splittingArgs
)
return Tier1ReRecoWorkload
def getProcessingACDCSpec(
self, splittingAlgo="LumiBased", splittingArgs={"lumis_per_job": 8}, setLocationFlag=False
):
"""
_getProcessingACDCSpec_
Get a ACDC spec for the processing task of a ReReco workload
"""
Tier1ReRecoWorkload = rerecoWorkload(self.workflowName, getTestArguments())
Tier1ReRecoWorkload.truncate(
"ACDC_%s" % self.workflowName, "/%s/DataProcessing" % self.workflowName, self.couchUrl, self.acdcDBName
)
Tier1ReRecoWorkload.setJobSplittingParameters(
"/ACDC_%s/DataProcessing" % self.workflowName, splittingAlgo, splittingArgs
)
if setLocationFlag:
Tier1ReRecoWorkload.setLocationDataSourceFlag()
Tier1ReRecoWorkload.setSiteWhitelist(self.siteWhitelist)
return Tier1ReRecoWorkload
import os
import shutil
import tempfile
from WMCore.WMSpec.StdSpecs.ReReco \
import rerecoWorkload as TestReRecoWorkload, getTestArguments
from Samples.TestMonteCarloWorkload \
import monteCarloWorkload as TestMCWorkload, getMCArgs
from Samples.BasicProductionWorkload \
import createWorkload as BasicProductionWorkload
from Samples.BasicProcessingWorkload \
import createWorkload as BasicProcessingWorkload
#from Samples.MultiTaskProcessingWorkload import createWorkload as MultiTaskProcessingWorkload
#from Samples.MultiTaskProductionWorkload import createWorkload as MultiTaskProductionWorkload
rerecoArgs = getTestArguments()
mcArgs = getMCArgs()
class WMSpecGenerator(object):
def __init__(self, dirLocation=None):
if not dirLocation:
dirLocation = tempfile.mkdtemp()
if not os.path.exists(dirLocation):
os.makedirs(dirLocation)
self.dir = dirLocation
def _save(self, spec):
specFile = os.path.join(self.dir, spec.name() + ".spec")
# Basic production Spec
spec.setSpecUrl(specFile)
from WMCore.WMBS.Fileset import Fileset
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile
from WMCore.DataStructs.Run import Run
from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments
from DBSAPI.dbsApi import DbsApi
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
# The default arguments are set in:
# WMCORE/src/python/WMCore/WMSpec/StdSpecs/ReReco.py
arguments = getTestArguments()
arguments["StdJobSplitAlgo"] = "FileBased"
arguments["StdJobSplitArgs"] = {"files_per_job": 1}
arguments["SkimJobSplitAlgo"] = "FileBased"
arguments["SkimJobSplitArgs"] = {"files_per_job": 1, "include_parents": True}
if len(sys.argv) != 2:
print "Usage:"
print "./injectReRecoWorkflow.py PROCESSING_VERSION"
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
connectToDB()
workloadName = "ReReco-%s" % arguments["ProcessingVersion"]
"""
import unittest
from WMCore.WorkQueue.Policy.Start.Dataset import Dataset
from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments
from WMCore.Services.EmulatorSwitch import EmulatorHelper
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
from WMCore.Services.DBS.DBSReader import DBSReader
from WMCore_t.WMSpec_t.samples.MultiTaskProcessingWorkload \
import workload as MultiTaskProcessingWorkload
from WMCore.WorkQueue.WorkQueueExceptions import *
from WMCore_t.WorkQueue_t.WorkQueue_t import getFirstTask
from WMQuality.Emulators.DataBlockGenerator import Globals
rerecoArgs = getTestArguments()
parentProcArgs = getTestArguments()
parentProcArgs.update(IncludeParents = "True")
class DatasetTestCase(unittest.TestCase):
splitArgs = dict(SliceType = 'NumberOfFiles', SliceSize = 5)
def testTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow"""
Tier1ReRecoWorkload = rerecoWorkload('ReRecoWorkload', rerecoArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
from WMCore.WMBS.Fileset import Fileset
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMComponent.DBSBuffer.Database.Interface.DBSBufferFile import DBSBufferFile
from WMCore.DataStructs.Run import Run
from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments
from DBSAPI.dbsApi import DbsApi
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
# The default arguments are set in:
# WMCORE/src/python/WMCore/WMSpec/StdSpecs/ReReco.py
arguments = getTestArguments()
arguments["StdJobSplitAlgo"] = "FileBased"
arguments["StdJobSplitArgs"] = {"files_per_job": 1}
arguments["SkimJobSplitAlgo"] = "FileBased"
arguments["SkimJobSplitArgs"] = {"files_per_job": 1, "include_parents": True}
if len(sys.argv) != 2:
print "Usage:"
print "./injectReRecoWorkflow.py PROCESSING_VERSION"
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
connectToDB()
workloadName = "ReReco-%s" % arguments["ProcessingVersion"]
"""
WorkQueue.Policy.Start.Dataset tests
"""
import unittest
from WMCore.WorkQueue.Policy.Start.Dataset import Dataset
from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments
from WMCore.Services.EmulatorSwitch import EmulatorHelper
from WMCore.Services.DBS.DBSReader import DBSReader
from WMCore_t.WMSpec_t.samples.MultiTaskProcessingWorkload \
import workload as MultiTaskProcessingWorkload
from WMCore.WorkQueue.WorkQueueExceptions import *
from WMCore_t.WorkQueue_t.WorkQueue_t import getFirstTask
from WMQuality.Emulators.DataBlockGenerator import Globals
rerecoArgs = getTestArguments()
parentProcArgs = getTestArguments()
parentProcArgs.update(IncludeParents="True")
class DatasetTestCase(unittest.TestCase):
splitArgs = dict(SliceType='NumberOfFiles', SliceSize=5)
def setUp(self):
EmulatorHelper.setEmulators(phedex=True,
dbs=True,
siteDB=True,
requestMgr=False)
def tearDown(self):
import os
import shutil
import tempfile
from WMCore.WMSpec.StdSpecs.ReReco \
import rerecoWorkload as TestReRecoWorkload, getTestArguments
from Samples.TestMonteCarloWorkload \
import monteCarloWorkload as TestMCWorkload, getMCArgs
from Samples.BasicProductionWorkload \
import createWorkload as BasicProductionWorkload
from Samples.BasicProcessingWorkload \
import createWorkload as BasicProcessingWorkload
#from Samples.MultiTaskProcessingWorkload import createWorkload as MultiTaskProcessingWorkload
#from Samples.MultiTaskProductionWorkload import createWorkload as MultiTaskProductionWorkload
rerecoArgs = getTestArguments()
mcArgs = getMCArgs()
class WMSpecGenerator(object):
def __init__(self, dirLocation=None):
if not dirLocation:
dirLocation = tempfile.mkdtemp()
if not os.path.exists(dirLocation):
os.makedirs(dirLocation)
self.dir = dirLocation
def _save(self, spec):
specFile = os.path.join(self.dir, spec.name() + ".spec")
# Basic production Spec
spec.setSpecUrl(specFile)