def createWorkload(self): """ Create a workload in order to test things """ workload = rerecoWorkload("Tier1ReReco", getTestArguments()) rereco = workload.getTask("DataProcessing") return workload
def getRequestSchema(): schema = getTestArguments() schema.update(RequestName = "TestReReco", RequestType = "ReReco", CmsPath = "/uscmst1/prod/sw/cms", CouchURL = None, CouchDBName = None, Group = "PeopleLikeMe", InputDataset = '/PRIM/PROC/TIER', Requestor = "me" ) return schema
def createReRecoSpec(self, specName, returnType="spec", splitter = None, inputDataset = None, dbsUrl = None, **additionalArgs): # update args, then reset them args = getTestArguments() args.update(additionalArgs) spec = TestReRecoWorkload(specName, args) if inputDataset != None: spec.taskIterator().next().data.input.dataset.primary = inputDataset if dbsUrl != None: print dbsUrl spec.taskIterator().next().data.input.dataset.dbsurl = dbsUrl return self._selectReturnType(spec, returnType, splitter)
def setUp(self): """ _setUp_ Setup some reasonable defaults for the ReReco workflow. """ self.unmergedLFNBase = "/store/backfill/2/unmerged" self.mergedLFNBase = "/store/backfill/2" self.processingVersion = "v1" self.cmsswVersion = "CMSSW_3_4_2_patch1" self.acquisitionEra = "WMAgentCommissioining10" self.primaryDataset = "MinimumBias" self.workload = rerecoWorkload("Tier1ReReco", getTestArguments()) return
def getMergeACDCSpec(self, splittingAlgo="ParentlessMergeBySize", splittingArgs={}): """ _getMergeACDCSpec_ Get a ACDC spec for the merge task of a ReReco workload """ Tier1ReRecoWorkload = rerecoWorkload(self.workflowName, getTestArguments()) Tier1ReRecoWorkload.truncate( "ACDC_%s" % self.workflowName, "/%s/DataProcessing/DataProcessingMergeRECOoutput" % self.workflowName, self.couchUrl, self.acdcDBName, ) Tier1ReRecoWorkload.setJobSplittingParameters( "/ACDC_%s/DataProcessingMergeRECOoutput" % self.workflowName, splittingAlgo, splittingArgs ) return Tier1ReRecoWorkload
def getProcessingACDCSpec( self, splittingAlgo="LumiBased", splittingArgs={"lumis_per_job": 8}, setLocationFlag=False ): """ _getProcessingACDCSpec_ Get a ACDC spec for the processing task of a ReReco workload """ Tier1ReRecoWorkload = rerecoWorkload(self.workflowName, getTestArguments()) Tier1ReRecoWorkload.truncate( "ACDC_%s" % self.workflowName, "/%s/DataProcessing" % self.workflowName, self.couchUrl, self.acdcDBName ) Tier1ReRecoWorkload.setJobSplittingParameters( "/ACDC_%s/DataProcessing" % self.workflowName, splittingAlgo, splittingArgs ) if setLocationFlag: Tier1ReRecoWorkload.setLocationDataSourceFlag() Tier1ReRecoWorkload.setSiteWhitelist(self.siteWhitelist) return Tier1ReRecoWorkload
import os import shutil import tempfile from WMCore.WMSpec.StdSpecs.ReReco \ import rerecoWorkload as TestReRecoWorkload, getTestArguments from Samples.TestMonteCarloWorkload \ import monteCarloWorkload as TestMCWorkload, getMCArgs from Samples.BasicProductionWorkload \ import createWorkload as BasicProductionWorkload from Samples.BasicProcessingWorkload \ import createWorkload as BasicProcessingWorkload #from Samples.MultiTaskProcessingWorkload import createWorkload as MultiTaskProcessingWorkload #from Samples.MultiTaskProductionWorkload import createWorkload as MultiTaskProductionWorkload rerecoArgs = getTestArguments() mcArgs = getMCArgs() class WMSpecGenerator(object): def __init__(self, dirLocation=None): if not dirLocation: dirLocation = tempfile.mkdtemp() if not os.path.exists(dirLocation): os.makedirs(dirLocation) self.dir = dirLocation def _save(self, spec): specFile = os.path.join(self.dir, spec.name() + ".spec") # Basic production Spec spec.setSpecUrl(specFile)
from WMCore.WMBS.Fileset import Fileset from WMCore.WMBS.Subscription import Subscription from WMCore.WMBS.Workflow import Workflow from WMComponent.DBS3Buffer.DBSBufferFile import DBSBufferFile from WMCore.DataStructs.Run import Run from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments from DBSAPI.dbsApi import DbsApi from WMCore.WMSpec.Makers.TaskMaker import TaskMaker from WMCore.WorkQueue.WMBSHelper import WMBSHelper # The default arguments are set in: # WMCORE/src/python/WMCore/WMSpec/StdSpecs/ReReco.py arguments = getTestArguments() arguments["StdJobSplitAlgo"] = "FileBased" arguments["StdJobSplitArgs"] = {"files_per_job": 1} arguments["SkimJobSplitAlgo"] = "FileBased" arguments["SkimJobSplitArgs"] = {"files_per_job": 1, "include_parents": True} if len(sys.argv) != 2: print "Usage:" print "./injectReRecoWorkflow.py PROCESSING_VERSION" sys.exit(1) else: arguments["ProcessingVersion"] = sys.argv[1] connectToDB() workloadName = "ReReco-%s" % arguments["ProcessingVersion"]
""" import unittest from WMCore.WorkQueue.Policy.Start.Dataset import Dataset from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments from WMCore.Services.EmulatorSwitch import EmulatorHelper EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True, requestMgr = False) from WMCore.Services.DBS.DBSReader import DBSReader from WMCore_t.WMSpec_t.samples.MultiTaskProcessingWorkload \ import workload as MultiTaskProcessingWorkload from WMCore.WorkQueue.WorkQueueExceptions import * from WMCore_t.WorkQueue_t.WorkQueue_t import getFirstTask from WMQuality.Emulators.DataBlockGenerator import Globals rerecoArgs = getTestArguments() parentProcArgs = getTestArguments() parentProcArgs.update(IncludeParents = "True") class DatasetTestCase(unittest.TestCase): splitArgs = dict(SliceType = 'NumberOfFiles', SliceSize = 5) def testTier1ReRecoWorkload(self): """Tier1 Re-reco workflow""" Tier1ReRecoWorkload = rerecoWorkload('ReRecoWorkload', rerecoArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
from WMCore.WMBS.Fileset import Fileset from WMCore.WMBS.Subscription import Subscription from WMCore.WMBS.Workflow import Workflow from WMComponent.DBSBuffer.Database.Interface.DBSBufferFile import DBSBufferFile from WMCore.DataStructs.Run import Run from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments from DBSAPI.dbsApi import DbsApi from WMCore.WMSpec.Makers.TaskMaker import TaskMaker from WMCore.WorkQueue.WMBSHelper import WMBSHelper # The default arguments are set in: # WMCORE/src/python/WMCore/WMSpec/StdSpecs/ReReco.py arguments = getTestArguments() arguments["StdJobSplitAlgo"] = "FileBased" arguments["StdJobSplitArgs"] = {"files_per_job": 1} arguments["SkimJobSplitAlgo"] = "FileBased" arguments["SkimJobSplitArgs"] = {"files_per_job": 1, "include_parents": True} if len(sys.argv) != 2: print "Usage:" print "./injectReRecoWorkflow.py PROCESSING_VERSION" sys.exit(1) else: arguments["ProcessingVersion"] = sys.argv[1] connectToDB() workloadName = "ReReco-%s" % arguments["ProcessingVersion"]
""" WorkQueue.Policy.Start.Dataset tests """ import unittest from WMCore.WorkQueue.Policy.Start.Dataset import Dataset from WMCore.WMSpec.StdSpecs.ReReco import rerecoWorkload, getTestArguments from WMCore.Services.EmulatorSwitch import EmulatorHelper from WMCore.Services.DBS.DBSReader import DBSReader from WMCore_t.WMSpec_t.samples.MultiTaskProcessingWorkload \ import workload as MultiTaskProcessingWorkload from WMCore.WorkQueue.WorkQueueExceptions import * from WMCore_t.WorkQueue_t.WorkQueue_t import getFirstTask from WMQuality.Emulators.DataBlockGenerator import Globals rerecoArgs = getTestArguments() parentProcArgs = getTestArguments() parentProcArgs.update(IncludeParents="True") class DatasetTestCase(unittest.TestCase): splitArgs = dict(SliceType='NumberOfFiles', SliceSize=5) def setUp(self): EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True, requestMgr=False) def tearDown(self):