def createTestSubscription(self, nFiles, nSites=1, closeFileset=False): """ _createTestSubscription_ Create a set of test subscriptions for testing purposes. """ if nSites > self.nSites: nSites = self.nSites testFileset = Fileset(name="TestFileset") testFileset.create() # Create a testWorkflow testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") testWorkflow.create() # Create the files for each site for s in range(nSites): for i in range(nFiles): newFile = File(makeUUID(), size=1024, events=100, locations=set(["site%i.cern.ch" % s])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription( fileset=testFileset, workflow=testWorkflow, split_algo="MinFileBased", type="Processing" ) testSubscription.create() # Close the fileset if closeFileset: testFileset.markOpen(isOpen=False) return testSubscription
def createFileCollection(self, name, nSubs, nFiles, workflowURL='test', site=None): """ _createFileCollection_ Create a collection of files for splitting into jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=workflowURL, owner="mnorman", name=name, task="/TestWorkload/ReReco") testWorkflow.create() for sub in range(nSubs): nameStr = '%s-%i' % (name, sub) testFileset = Fileset(name=nameStr) testFileset.create() for f in range(nFiles): # pick a random site if not site: tmpSite = 'se.%s' % (random.choice(self.sites)) else: tmpSite = 'se.%s' % (site) testFile = File(lfn="/lfn/%s/%i" % (nameStr, f), size=1024, events=10) testFile.setLocation(tmpSite) testFile.create() testFileset.addFile(testFile) testFileset.commit() testFileset.markOpen(isOpen=0) testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() return
def createTestSubscription(self, nFiles, nSites=1, closeFileset=False): """ _createTestSubscription_ Create a set of test subscriptions for testing purposes. """ if nSites > self.nSites: nSites = self.nSites testFileset = Fileset(name="TestFileset") testFileset.create() # Create a testWorkflow testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") testWorkflow.create() # Create the files for each site for s in range(nSites): for i in range(nFiles): newFile = File(makeUUID(), size=1024, events=100, locations=set(["T2_CH_CERN_%i" % s])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, split_algo="MinFileBased", type="Processing") testSubscription.create() # Close the fileset if closeFileset: testFileset.markOpen(isOpen=False) return testSubscription
def pollForClosable(self): """ _pollForClosable_ Search WMBS for filesets that can be closed and mark them as closed. """ myThread = threading.currentThread() myThread.transaction.begin() closableFilesetDAO = self.daoFactory(classname="Fileset.ListClosable") closableFilesets = closableFilesetDAO.execute() for closableFileset in closableFilesets: openFileset = Fileset(id=closableFileset) openFileset.load() logging.debug("Closing fileset %s" % openFileset.name) openFileset.markOpen(False) myThread.transaction.commit()
def pollForClosable(self): """ _pollForClosable_ Search WMBS for filesets that can be closed and mark them as closed. """ myThread = threading.currentThread() myThread.transaction.begin() closableFilesetDAO = self.daoFactory(classname="Fileset.ListClosable") closableFilesets = closableFilesetDAO.execute() for closableFileset in closableFilesets: openFileset = Fileset(id=closableFileset) openFileset.load() logging.debug("Closing fileset %s", openFileset.name) openFileset.markOpen(False) myThread.transaction.commit()
def createFileCollection(self, name, nSubs, nFiles, workflowURL = 'test', site = None): """ _createFileCollection_ Create a collection of files for splitting into jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec = workflowURL, owner = "mnorman", name = name, task="/TestWorkload/ReReco") testWorkflow.create() for sub in range(nSubs): nameStr = '%s-%i' % (name, sub) testFileset = Fileset(name = nameStr) testFileset.create() for f in range(nFiles): # pick a random site if not site: tmpSite = 'se.%s' % (random.choice(self.sites)) else: tmpSite = 'se.%s' % (site) testFile = File(lfn = "/lfn/%s/%i" % (nameStr, f), size = 1024, events = 10) testFile.setLocation(tmpSite) testFile.create() testFileset.addFile(testFile) testFileset.commit() testFileset.markOpen(isOpen = 0) testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased") testSubscription.create() return
def createGiantJobSet(self, name, config, nSubs = 10, nJobs = 10, nFiles = 1, spec = "spec.xml"): """ Creates a massive set of jobs """ jobList = [] for i in range(0, nSubs): # Make a bunch of subscriptions localName = '%s-%i' % (name, i) testWorkflow = Workflow(spec = spec, owner = self.OWNERDN, name = localName, task="Test", owner_vogroup="", owner_vorole="") testWorkflow.create() testWMBSFileset = Fileset(name = localName) testWMBSFileset.create() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() filesToComplete = [] for j in range(0, nJobs): # Create jobs for each subscription testFileA = File(lfn = "%s-%i-lfnA" % (localName, j) , size = 1024, events = 10) testFileA.addRun(Run(10, *[11,12,13,14,15,16,17,18,19,20, 21,22,23,24,25,26,27,28,29,30, 31,32,33,34,35,36,37,38,39,40])) testFileA.setLocation('malpaquet') testFileA.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.commit() filesToComplete.append(testFileA) testJob = Job(name = '%s-%i' % (localName, j)) testJob.addFile(testFileA) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJobGroup.add(testJob) jobList.append(testJob) for k in range(0, nFiles): # Create output files testFile = File(lfn = "%s-%i-output" % (localName, k) , size = 1024, events = 10) testFile.addRun(Run(10, *[12312])) testFile.setLocation('malpaquet') testFile.create() testJobGroup.output.addFile(testFile) testJobGroup.output.commit() testJobGroup.commit() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'success', 'complete') changer.propagate(testJobGroup.jobs, 'cleanout', 'success') testWMBSFileset.markOpen(0) testSubscription.completeFiles(filesToComplete) return jobList
class ConditionTest(unittest.TestCase): """ _ExpressTest_ Test for Express job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["T0.WMBS"]) self.splitterFactory = SplitterFactory(package = "T0.JobSplitting") myThread = threading.currentThread() daoFactory = DAOFactory(package = "T0.WMBS", logger = logging, dbinterface = myThread.dbi) wmbsDaoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) myThread.dbi.processData("""INSERT INTO wmbs_location (id, site_name, state) VALUES (1, 'SomeSite', 1) """, transaction = False) myThread.dbi.processData("""INSERT INTO wmbs_location_senames (location, se_name) VALUES (1, 'SomeSE') """, transaction = False) insertRunDAO = daoFactory(classname = "RunConfig.InsertRun") insertRunDAO.execute(binds = { 'RUN' : 1, 'TIME' : int(time.time()), 'HLTKEY' : "someHLTKey" }, transaction = False) insertLumiDAO = daoFactory(classname = "RunConfig.InsertLumiSection") insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 1 }, transaction = False) insertStreamDAO = daoFactory(classname = "RunConfig.InsertStream") insertStreamDAO.execute(binds = { 'STREAM' : "Express" }, transaction = False) insertStreamFilesetDAO = daoFactory(classname = "RunConfig.InsertStreamFileset") insertStreamFilesetDAO.execute(1, "Express", "TestFileset1") insertStreamerDAO = daoFactory(classname = "RunConfig.InsertStreamer") insertStreamerDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 1, 'STREAM' : "Express", 'TIME' : int(time.time()), 'LFN' : "/streamer", 'FILESIZE' : 0, 'EVENTS' : 0 }, transaction = False) insertPromptCalibrationDAO = daoFactory(classname = "RunConfig.InsertPromptCalibration") insertPromptCalibrationDAO.execute( { 'RUN' : 1, 'STREAM' : "Express" }, transaction = False) self.fileset1 = Fileset(name = "TestFileset1") self.fileset1.create() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Condition", type = "Condition") self.subscription1.create() # set parentage chain and sqlite fileset alcaRecoFile = File("/alcareco", size = 0, events = 0) alcaRecoFile.addRun(Run(1, *[1])) alcaRecoFile.setLocation("SomeSE", immediateSave = False) alcaRecoFile.create() alcaPromptFile = File("/alcaprompt", size = 0, events = 0) alcaPromptFile.addRun(Run(1, *[1])) alcaPromptFile.setLocation("SomeSE", immediateSave = False) alcaPromptFile.create() sqliteFile = File("/sqlite", size = 0, events = 0) sqliteFile.create() self.fileset1.addFile(sqliteFile) self.fileset1.commit() results = myThread.dbi.processData("""SELECT lfn FROM wmbs_file_details """, transaction = False)[0].fetchall() setParentageDAO = wmbsDaoFactory(classname = "Files.SetParentage") setParentageDAO.execute(binds = [ { 'parent' : "/streamer", 'child' : "/alcareco" }, { 'parent' : "/alcareco", 'child' : "/alcaprompt" }, { 'parent' : "/alcaprompt", 'child' : "/sqlite" } ], transaction = False) # default split parameters self.splitArgs = {} self.splitArgs['runNumber'] = 1 self.splitArgs['streamName'] = "Express" return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() return def isPromptCalibFinished(self): """ _isPromptCalibFinished_ """ myThread = threading.currentThread() result = myThread.dbi.processData("""SELECT finished FROM prompt_calib """, transaction = False)[0].fetchall()[0][0] return result def countPromptCalibFiles(self): """ _deleteSplitLumis_ """ myThread = threading.currentThread() result = myThread.dbi.processData("""SELECT COUNT(*) FROM prompt_calib_file """, transaction = False)[0].fetchall()[0][0] return result def test00(self): """ _test00_ Make sure the job splitter behaves correctly. Just make sure the job splitter does nothing when the fileset is open and populates t0ast data structures when it's closed. In the later case all input files should be marked as acquired without creating a job as well. """ mySplitArgs = self.splitArgs.copy() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.assertEqual(self.isPromptCalibFinished(), 0, "ERROR: prompt_calib should not be finished") self.assertEqual(self.countPromptCalibFiles(), 0, "ERROR: there should be no prompt_calib_file") jobGroups = jobFactory(**mySplitArgs) self.assertEqual(self.isPromptCalibFinished(), 0, "ERROR: prompt_calib should not be finished") self.assertEqual(self.countPromptCalibFiles(), 1, "ERROR: there should be one prompt_calib_file") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.assertEqual(self.isPromptCalibFinished(), 1, "ERROR: prompt_calib should be finished") self.assertEqual(self.countPromptCalibFiles(), 1, "ERROR: there should be one prompt_calib_file") return
def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() # FIXME: Let workflow put in values if spec is missing them workflow = Workflow( spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"], dn=self.wmSpec.getOwner().get("dn", "unknown"), group=self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"), name=self.wmSpec.name(), task=task.getPathName(), wfType=self.wmSpec.getDashboardActivity(), alternativeFilesetClose=alternativeFilesetClose, priority=self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset=fileset, workflow=workflow, split_algo=task.jobSplittingAlgorithm(), type=task.getPrimarySubType()) subscription.create() ### FIXME: I'm pretty sure we can improve how we handle this site white/black list for site in task.siteWhitelist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": True }]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": False }]) if self.topLevelSubscription is None: self.topLevelSubscription = subscription logging.info("Top level subscription %s created for %s", subscription["id"], self.wmSpec.name()) else: logging.info("Child subscription %s created for %s", subscription["id"], self.wmSpec.name()) outputModules = task.getOutputModulesForTask() ignoredOutputModules = task.getIgnoredOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): if outputModuleName in ignoredOutputModules: msg = "%s has %s as IgnoredOutputModule, skipping fileset creation." logging.info(msg, task.getPathName(), outputModuleName) continue dataTier = getattr(getattr(outputModule, outputModuleName), "dataTier", '') filesetName = self.outputFilesetName(task, outputModuleName, dataTier) outputFileset = Fileset(filesetName) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: childDatatier = getattr(childTask.data.input, 'dataTier', '') if childTask.taskType() in [ "Cleanup", "Merge" ] and childDatatier != dataTier: continue elif childTask.taskType( ) == "Merge" and childDatatier == dataTier: filesetName = self.outputFilesetName( childTask, "Merged", dataTier) mergedOutputFileset = Fileset(filesetName) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr( getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset is not None: self.mergeOutputMapping[ mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS( childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset is None: workflow.addOutput(outputModuleName + dataTier, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName + dataTier, outputFileset, mergedOutputFileset) return
def __call__(self, parameters): """ Perform the work required with the given parameters """ DefaultSlave.__call__(self, parameters) # Handle the message message = self.messageArgs # Lock on the running feeders list myThread = threading.currentThread() myThread.runningFeedersLock.acquire() # Create empty fileset if fileset.name doesn't exist filesetName = message["dataset"] feederType = message["FeederType"] fileType = message["FileType"] startRun = message["StartRun"] logging.debug("Dataset " + filesetName + " arrived") fileset = Fileset(name = filesetName+':'\ +feederType+':'+fileType+':'+startRun) # Check if the fileset is already there if fileset.exists() == False: # Empty fileset creation fileset.create() fileset.setLastUpdate(0) logging.info("Fileset %s whith id %s is added" \ %(fileset.name, str(fileset.id))) # Get feeder type feederType = message["FeederType"] # Check if there is a running feeder if myThread.runningFeeders.has_key(feederType): logging.info("HAVE FEEDER " + feederType + " RUNNING") logging.info(myThread.runningFeeders[feederType]) else: logging.info("NO FEEDER " + feederType + " RUNNING") # Check if we have a feeder in DB if self.queries.checkFeeder(feederType): # Have feeder, get info logging.info("Getting Feeder from DB") feederId = self.queries.getFeederId(feederType) logging.info(feederId) myThread.runningFeeders[feederType] = feederId else: # Create feeder logging.info("Adding Feeder to DB") self.queries.addFeeder(feederType, "StatePath") feederId = self.queries.getFeederId(feederType) logging.info(feederId) myThread.runningFeeders[feederType] = feederId # Fileset/Feeder association self.queries.addFilesetToManage(fileset.id, \ myThread.runningFeeders[feederType]) logging.info("Fileset %s is added to feeder %s" %(fileset.id, \ myThread.runningFeeders[feederType])) else: # If fileset already exist a new subscription # will be created for its workflow logging.info("Fileset exists: Subscription will be created for it") # Open it if close fileset.load() if fileset.open == False: fileset.markOpen(True) logging.info("Getting Feeder from DB") feederId = self.queries.getFeederId(feederType) logging.info(feederId) myThread.runningFeeders[feederType] = feederId self.queries.addFilesetToManage(fileset.id, \ myThread.runningFeeders[feederType]) logging.info("Fileset %s is added to feeder %s" %(fileset.id, \ myThread.runningFeeders[feederType])) myThread.runningFeedersLock.release() myThread.msgService.finish()
def createGiantJobSet(self, name, config, nSubs=10, nJobs=10, nFiles=1, spec="spec.xml"): """ Creates a massive set of jobs """ jobList = [] for i in range(0, nSubs): # Make a bunch of subscriptions localName = '%s-%i' % (name, i) testWorkflow = Workflow(spec=spec, owner=self.OWNERDN, name=localName, task="Test", owner_vogroup="", owner_vorole="") testWorkflow.create() testWMBSFileset = Fileset(name=localName) testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() filesToComplete = [] for j in range(0, nJobs): # Create jobs for each subscription testFileA = File(lfn="%s-%i-lfnA" % (localName, j), size=1024, events=10) testFileA.addRun( Run( 10, *[ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 ])) testFileA.setLocation('malpaquet') testFileA.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.commit() filesToComplete.append(testFileA) testJob = Job(name='%s-%i' % (localName, j)) testJob.addFile(testFileA) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJobGroup.add(testJob) jobList.append(testJob) for k in range(0, nFiles): # Create output files testFile = File(lfn="%s-%i-output" % (localName, k), size=1024, events=10) testFile.addRun(Run(10, *[12312])) testFile.setLocation('malpaquet') testFile.create() testJobGroup.output.addFile(testFile) testJobGroup.output.commit() testJobGroup.commit() changer = ChangeState(config) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'success', 'complete') changer.propagate(testJobGroup.jobs, 'cleanout', 'success') testWMBSFileset.markOpen(0) testSubscription.completeFiles(filesToComplete) return jobList
class WMBSHelper(WMConnectionBase): """ _WMBSHelper_ Interface between the WorkQueue and WMBS. """ def __init__(self, wmSpec, taskName, blockName=None, mask=None, cachepath='.'): """ _init_ Initialize DAOs and other things needed. """ self.block = blockName self.mask = mask self.wmSpec = wmSpec self.topLevelTask = wmSpec.getTask(taskName) self.cachepath = cachepath self.isDBS = True self.topLevelFileset = None self.topLevelSubscription = None self.topLevelTaskDBSBufferId = None self.mergeOutputMapping = {} # Initiate the pieces you need to run your own DAOs WMConnectionBase.__init__(self, "WMCore.WMBS") myThread = threading.currentThread() self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) # DAOs from WMBS for file commit self.setParentage = self.daofactory(classname="Files.SetParentage") self.setFileRunLumi = self.daofactory(classname="Files.AddRunLumi") self.setFileLocation = self.daofactory( classname="Files.SetLocationForWorkQueue") self.setFileAddChecksum = self.daofactory( classname="Files.AddChecksumByLFN") self.addFileAction = self.daofactory(classname="Files.Add") self.addToFileset = self.daofactory(classname="Files.AddDupsToFileset") self.getLocations = self.daofactory(classname="Locations.ListSites") self.getLocationInfo = self.daofactory( classname="Locations.GetSiteInfo") # DAOs from DBSBuffer self.dbsCreateFiles = self.dbsDaoFactory( classname="DBSBufferFiles.Add") self.dbsSetLocation = self.dbsDaoFactory( classname="DBSBufferFiles.SetLocationByLFN") self.dbsInsertLocation = self.dbsDaoFactory( classname="DBSBufferFiles.AddLocation") self.dbsSetChecksum = self.dbsDaoFactory( classname="DBSBufferFiles.AddChecksumByLFN") self.dbsInsertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow") # Added for file creation bookkeeping self.dbsFilesToCreate = [] self.addedLocations = [] self.wmbsFilesToCreate = [] self.insertedBogusDataset = -1 return def createSandbox(self): """Create the runtime sandbox""" sandboxCreator = SandboxCreator() sandboxCreator.makeSandbox(self.cachepath, self.wmSpec) def createTopLevelFileset(self, topLevelFilesetName=None): """ _createTopLevelFileset_ Create the top level fileset for the workflow. If the name of the top level fileset is not given create one. """ if topLevelFilesetName is None: filesetName = ( "%s-%s" % (self.wmSpec.name(), self.wmSpec.getTopLevelTask()[0].name())) if self.block: filesetName += "-%s" % self.block if self.mask: from hashlib import md5 mask_string = ",".join( ["%s=%s" % (x, self.mask[x]) for x in sorted(self.mask)]) filesetName += "-%s" % md5(mask_string).hexdigest() else: filesetName = topLevelFilesetName self.topLevelFileset = Fileset(filesetName) self.topLevelFileset.create() return def outputFilesetName(self, task, outputModuleName): """ _outputFilesetName_ Generate an output fileset name for the given task and output module. """ if task.taskType() == "Merge": outputFilesetName = "%s/merged-%s" % (task.getPathName(), outputModuleName) else: outputFilesetName = "%s/unmerged-%s" % (task.getPathName(), outputModuleName) return outputFilesetName def createSubscription(self, task, fileset, alternativeFilesetClose=False): """ _createSubscription_ Create subscriptions in the database. This includes workflows in WMBS and DBSBuffer, output maps, datasets and phedex subscriptions, and filesets for each task below and including the given task. """ sub = self._createSubscriptionsInWMBS(task, fileset, alternativeFilesetClose) self._createWorkflowsInDBSBuffer() self._createDatasetSubscriptionsInDBSBuffer() return sub def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() # FIXME: Let workflow put in values if spec is missing them workflow = Workflow( spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"], dn=self.wmSpec.getOwner().get("dn", "unknown"), group=self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"), name=self.wmSpec.name(), task=task.getPathName(), wfType=self.wmSpec.getDashboardActivity(), alternativeFilesetClose=alternativeFilesetClose, priority=self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset=fileset, workflow=workflow, split_algo=task.jobSplittingAlgorithm(), type=task.getPrimarySubType()) if subscription.exists(): subscription.load() msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)" self.logger.info(msg % (subscription['id'], task.getPathName())) else: subscription.create() for site in task.siteWhitelist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": True }]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": False }]) if self.topLevelSubscription is None: self.topLevelSubscription = subscription logging.info("Top level subscription created: %s", subscription["id"]) else: logging.info("Child subscription created: %s", subscription["id"]) outputModules = task.getOutputModulesForTask() ignoredOutputModules = task.getIgnoredOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): if outputModuleName in ignoredOutputModules: logging.info( "IgnoredOutputModule set for %s, skipping fileset creation.", outputModuleName) continue outputFileset = Fileset( self.outputFilesetName(task, outputModuleName)) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: if childTask.taskType() == "Merge": mergedOutputFileset = Fileset( self.outputFilesetName(childTask, "Merged")) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr( getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset != None: self.mergeOutputMapping[ mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS( childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset is None: workflow.addOutput(outputModuleName, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName, outputFileset, mergedOutputFileset) return self.topLevelSubscription def addMCFakeFile(self): """Add a fake file for wmbs to run production over""" needed = [ 'FirstEvent', 'FirstLumi', 'FirstRun', 'LastEvent', 'LastLumi', 'LastRun' ] for key in needed: if self.mask and self.mask.get(key) is None: msg = 'Invalid value "%s" for %s' % (self.mask.get(key), key) raise WorkQueueWMBSException(msg) locations = set() for site in self.getLocations.execute( conn=self.getDBConn(), transaction=self.existingTransaction()): try: siteInfo = self.getLocationInfo.execute( site, conn=self.getDBConn(), transaction=self.existingTransaction()) if not siteInfo: self.logger.info( 'Skipping MonteCarlo injection to site "%s" as unknown to wmbs' % site) continue locations.add(siteInfo[0]['pnn']) except Exception as ex: self.logger.error( 'Error getting storage element for "%s": %s' % (site, str(ex))) if not locations: msg = 'No locations to inject Monte Carlo work to, unable to proceed' raise WorkQueueWMBSException(msg) mcFakeFileName = ("MCFakeFile-%s" % self.topLevelFileset.name).encode( 'ascii', 'ignore') wmbsFile = File( lfn=mcFakeFileName, first_event=self.mask['FirstEvent'], last_event=self.mask['LastEvent'], events=self.mask['LastEvent'] - self.mask['FirstEvent'] + 1, # inclusive range locations=locations, merged=False, # merged causes dbs parentage relation ) if self.mask: lumis = range(self.mask['FirstLumi'], self.mask['LastLumi'] + 1) # inclusive range wmbsFile.addRun(Run(self.mask['FirstRun'], *lumis)) # assume run number static else: wmbsFile.addRun(Run(1, 1)) wmbsFile['inFileset'] = True # file is not a parent logging.info("WMBS File: %s on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations']) self.wmbsFilesToCreate.append(wmbsFile) totalFiles = self.topLevelFileset.addFilesToWMBSInBulk( self.wmbsFilesToCreate, self.wmSpec.name(), isDBS=self.isDBS) self.topLevelFileset.markOpen(False) return totalFiles def createSubscriptionAndAddFiles(self, block): """ _createSubscriptionAndAddFiles_ Create the subscription and add files at one time to put everything in one transaction. """ self.beginTransaction() self.createTopLevelFileset() try: sub = self.createSubscription(self.topLevelTask, self.topLevelFileset) except Exception as ex: myThread = threading.currentThread() myThread.transaction.rollback() msg = traceback.format_exc() logging.error("Failed to create subscription %s", msg) raise ex if block != None: logging.info('"%s" Injecting block %s (%d files) into wmbs', self.wmSpec.name(), self.block, len(block['Files'])) addedFiles = self.addFiles(block) # For MC case else: logging.info( '"%s" Injecting production %s:%s:%s - %s:%s:%s (run:lumi:event) into wmbs', self.wmSpec.name(), self.mask['FirstRun'], self.mask['FirstLumi'], self.mask['FirstEvent'], self.mask['LastRun'], self.mask['LastLumi'], self.mask['LastEvent']) addedFiles = self.addMCFakeFile() self.commitTransaction(existingTransaction=False) return sub, addedFiles def addFiles(self, block): """ _addFiles_ create wmbs files from given dbs block. as well as run lumi update """ if self.topLevelTask.getInputACDC(): self.isDBS = False for acdcFile in self.validFiles(block['Files']): self._addACDCFileToWMBSFile(acdcFile) else: self.isDBS = True for dbsFile in self.validFiles(block['Files']): self._addDBSFileToWMBSFile(dbsFile, block['PhEDExNodeNames']) # Add files to WMBS totalFiles = self.topLevelFileset.addFilesToWMBSInBulk( self.wmbsFilesToCreate, self.wmSpec.name(), isDBS=self.isDBS) # Add files to DBSBuffer self._createFilesInDBSBuffer() self.topLevelFileset.markOpen(block.get('IsOpen', False)) return totalFiles def getMergeOutputMapping(self): """ _getMergeOutputMapping_ retrieves the relationship between primary dataset and merge output fileset ids for all merge tasks created """ return self.mergeOutputMapping def _createWorkflowsInDBSBuffer(self): """ _createWorkflowsInDBSBuffer_ Register workflow information and settings in dbsbuffer for all tasks that will potentially produce any output in this spec. """ for task in self.wmSpec.listOutputProducingTasks(): workflow_id = self.dbsInsertWorkflow.execute( self.wmSpec.name(), task, self.wmSpec.getBlockCloseMaxWaitTime(), self.wmSpec.getBlockCloseMaxFiles(), self.wmSpec.getBlockCloseMaxEvents(), self.wmSpec.getBlockCloseMaxSize(), conn=self.getDBConn(), transaction=self.existingTransaction()) if task == self.topLevelTask.getPathName(): self.topLevelTaskDBSBufferId = workflow_id def _createDatasetSubscriptionsInDBSBuffer(self): """ _createDatasetSubscriptionsInDBSBuffer_ Insert the subscriptions defined in the workload for the output datasets with the different options. """ subInfo = self.wmSpec.getSubscriptionInformation() for dataset in subInfo: dbsDataset = DBSBufferDataset(path=dataset) dbsDataset.create() dbsDataset.addSubscription(subInfo[dataset]) return def _createFilesInDBSBuffer(self): """ _createFilesInDBSBuffer_ It does the actual job of creating things in DBSBuffer """ if len(self.dbsFilesToCreate) == 0: # Whoops, nothing to do! return dbsFileTuples = [] dbsFileLoc = [] dbsCksumBinds = [] locationsToAdd = [] selfChecksums = None # The first thing we need to do is add the datasetAlgo # Assume all files in a pass come from one datasetAlgo? if self.insertedBogusDataset == -1: self.insertedBogusDataset = self.dbsFilesToCreate[ 0].insertDatasetAlgo() for dbsFile in self.dbsFilesToCreate: # Append a tuple in the format specified by DBSBufferFiles.Add # Also run insertDatasetAlgo lfn = dbsFile['lfn'] selfChecksums = dbsFile['checksums'] newTuple = (lfn, dbsFile['size'], dbsFile['events'], self.insertedBogusDataset, dbsFile['status'], self.topLevelTaskDBSBufferId) if newTuple not in dbsFileTuples: dbsFileTuples.append(newTuple) if len(dbsFile['newlocations']) < 1: msg = '' msg += "File created without any locations!\n" msg += "File lfn: %s\n" % (lfn) msg += "Rejecting this group of files in DBS!\n" logging.error(msg) raise WorkQueueWMBSException(msg) for jobLocation in dbsFile['newlocations']: if jobLocation not in self.addedLocations: # If we don't have it, try and add it locationsToAdd.append(jobLocation) self.addedLocations.append(jobLocation) dbsFileLoc.append({'lfn': lfn, 'pnn': jobLocation}) if selfChecksums: # If we have checksums we have to create a bind # For each different checksum for entry in selfChecksums.keys(): dbsCksumBinds.append({ 'lfn': lfn, 'cksum': selfChecksums[entry], 'cktype': entry }) for jobLocation in locationsToAdd: self.dbsInsertLocation.execute( siteName=jobLocation, conn=self.getDBConn(), transaction=self.existingTransaction()) self.dbsCreateFiles.execute(files=dbsFileTuples, conn=self.getDBConn(), transaction=self.existingTransaction()) self.dbsSetLocation.execute(binds=dbsFileLoc, conn=self.getDBConn(), transaction=self.existingTransaction()) if len(dbsCksumBinds) > 0: self.dbsSetChecksum.execute(bulkList=dbsCksumBinds, conn=self.getDBConn(), transaction=self.existingTransaction()) # Now that we've created those files, clear the list self.dbsFilesToCreate = [] return def _addToDBSBuffer(self, dbsFile, checksums, locations): """ This step is just for increase the performance for Accountant doesn't neccessary to check the parentage """ dbsBuffer = DBSBufferFile(lfn=dbsFile["LogicalFileName"], size=dbsFile["FileSize"], events=dbsFile["NumberOfEvents"], checksums=checksums, locations=locations, status="GLOBAL") dbsBuffer.setDatasetPath('bogus') dbsBuffer.setAlgorithm(appName="cmsRun", appVer="Unknown", appFam="Unknown", psetHash="Unknown", configContent="Unknown") if not dbsBuffer.exists(): self.dbsFilesToCreate.append(dbsBuffer) # dbsBuffer.create() return def _addDBSFileToWMBSFile(self, dbsFile, storageElements, inFileset=True): """ There are two assumptions made to make this method behave properly, 1. DBS returns only one level of ParentList. If DBS returns multiple level of parentage, it will be still get handled. However that might not be what we wanted. In that case, restrict to one level. 2. Assumes parents files are in the same location as child files. This is not True in general case, but workquue should only select work only where child and parent files are in the same location """ wmbsParents = [] dbsFile.setdefault("ParentList", []) for parent in dbsFile["ParentList"]: wmbsParents.append( self._addDBSFileToWMBSFile(parent, storageElements, inFileset=False)) checksums = {} if dbsFile.get('Checksum'): checksums['cksum'] = dbsFile['Checksum'] if dbsFile.get('Adler32'): checksums['adler32'] = dbsFile['Adler32'] wmbsFile = File( lfn=dbsFile["LogicalFileName"], size=dbsFile["FileSize"], events=dbsFile["NumberOfEvents"], checksums=checksums, # TODO: need to get list of parent lfn parents=wmbsParents, locations=set(storageElements)) for lumi in dbsFile['LumiList']: if isinstance(lumi['LumiSectionNumber'], list): run = Run(lumi['RunNumber'], *lumi['LumiSectionNumber']) else: run = Run(lumi['RunNumber'], lumi['LumiSectionNumber']) wmbsFile.addRun(run) self._addToDBSBuffer(dbsFile, checksums, storageElements) logging.info("WMBS File: %s\n on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations']) wmbsFile['inFileset'] = bool(inFileset) self.wmbsFilesToCreate.append(wmbsFile) return wmbsFile def _convertACDCFileToDBSFile(self, acdcFile): """ convert ACDCFiles to dbs file format """ dbsFile = {} dbsFile["LogicalFileName"] = acdcFile["lfn"] dbsFile["FileSize"] = acdcFile["size"] dbsFile["NumberOfEvents"] = acdcFile["events"] return dbsFile def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True): """ adds the ACDC files into WMBS database """ wmbsParents = [] for parent in acdcFile["parents"]: parent = self._addACDCFileToWMBSFile(DatastructFile( lfn=parent, locations=acdcFile["locations"]), inFileset=False) wmbsParents.append(parent) # pass empty check sum since it won't be updated to dbs anyway checksums = {} wmbsFile = File(lfn=str(acdcFile["lfn"]), size=acdcFile["size"], events=acdcFile["events"], first_event=acdcFile.get('first_event', 0), last_event=acdcFile.get('last_event', 0), checksums=checksums, parents=wmbsParents, locations=acdcFile["locations"], merged=acdcFile.get('merged', True)) ## TODO need to get the lumi lists for run in acdcFile['runs']: wmbsFile.addRun(run) dbsFile = self._convertACDCFileToDBSFile(acdcFile) self._addToDBSBuffer(dbsFile, checksums, acdcFile["locations"]) logging.info("WMBS File: %s\n on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations']) wmbsFile['inFileset'] = bool(inFileset) self.wmbsFilesToCreate.append(wmbsFile) return wmbsFile def validFiles(self, files): """ Apply lumi mask and or run white/black list and return files which have one or more of the requested lumis """ runWhiteList = self.topLevelTask.inputRunWhitelist() runBlackList = self.topLevelTask.inputRunBlacklist() lumiMask = self.topLevelTask.getLumiMask() blackMask = None if lumiMask: # We have a lumiMask, so use it and modify with run white/black list if runWhiteList: lumiMask.selectRuns(runWhiteList) if runBlackList: lumiMask.removeRuns(runBlackList) elif runWhiteList: # We have a run whitelist, subtract off blacklist lumiMask = LumiList(runs=runWhiteList) if runBlackList: # We only have a blacklist, so make a black mask out of it instead lumiMask.removeRuns(runBlackList) else: lumiMask = None if runBlackList: blackMask = LumiList(runs=runBlackList) results = [] for f in files: if isinstance(f, basestring) or "LumiList" not in f: results.append(f) continue # Create a LumiList from the WMBS info runLumis = {} for x in f['LumiList']: if x['RunNumber'] in runLumis: runLumis[x['RunNumber']].extend(x['LumiSectionNumber']) else: runLumis[x['RunNumber']] = x['LumiSectionNumber'] fileLumiList = LumiList(runsAndLumis=runLumis) if lumiMask: if fileLumiList & lumiMask: # At least one lumi from file is in lumiMask results.append(f) elif blackMask: if fileLumiList - blackMask: # At least one lumi from file is not in blackMask results.append(f) else: # There is effectively no mask results.append(f) return results
def setupExpressWorkflow(self): """ _setupExpressWorkflow_ Populate WMBS with a express-like workflow, every subscription must be unfinished at first """ workflowName = 'Express_Run481516_StreamZFast' secondLevelTasks = [ 'ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT', 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM', 'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT', 'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO' ] alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd' dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged' self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []} self.orderedStates = ['Merge', 'Harvesting', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest( {'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) expressTask = workload.newTask('Express') for task in secondLevelTasks: secondLevelTask = expressTask.addTask(task) if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO': secondLevelTask.addTask(alcaHarvestTask) elif task == 'ExpressMergewrite_StreamZFast_DQM': secondLevelTask.addTask(dqmHarvestTask) specPath = os.path.join(self.testDir, 'Express.pkl') workload.save(specPath) # Populate WMBS sharedFileset = Fileset(name='TestFileset') sharedFileset.create() sharedFileset.markOpen(False) options = { 'spec': specPath, 'owner': 'ItsAMeMario', 'name': workflowName, 'wfType': 'tier0' } topLevelWorkflow = Workflow(task='/%s/Express' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(sharedFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topLevelSub) for task in [ x for x in secondLevelTasks if not x.count('CleanupUnmerged') ]: secondLevelWorkflow = Workflow(task='/%s/Express/%s' % (workflowName, task), **options) secondLevelWorkflow.create() mergeSub = Subscription(sharedFileset, secondLevelWorkflow) mergeSub.create() self.stateMap['Harvesting'].append(mergeSub) for (parent, child) in [ ('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask), ('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask) ]: harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' % (workflowName, parent, child), **options) harvestingWorkflow.create() harvestingSub = Subscription(sharedFileset, harvestingWorkflow) harvestingSub.create() self.stateMap['Processing Done'].append(harvestingSub) return
class RepackMergeTest(unittest.TestCase): """ _RepackMergeTest_ Test for RepackMerge job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "T0.WMBS"]) self.splitterFactory = SplitterFactory(package = "T0.JobSplitting") myThread = threading.currentThread() daoFactory = DAOFactory(package = "T0.WMBS", logger = logging, dbinterface = myThread.dbi) wmbsDaoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) myThread.dbi.processData("""INSERT INTO wmbs_location (id, site_name, state, state_time) VALUES (1, 'SomeSite', 1, 1) """, transaction = False) myThread.dbi.processData("""INSERT INTO wmbs_pnns (id, pnn) VALUES (2, 'SomePNN') """, transaction = False) myThread.dbi.processData("""INSERT INTO wmbs_location_pnns (location, pnn) VALUES (1, 2) """, transaction = False) insertRunDAO = daoFactory(classname = "RunConfig.InsertRun") insertRunDAO.execute(binds = { 'RUN' : 1, 'HLTKEY' : "someHLTKey" }, transaction = False) insertLumiDAO = daoFactory(classname = "RunConfig.InsertLumiSection") insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 1 }, transaction = False) insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 2 }, transaction = False) insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 3 }, transaction = False) insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 4 }, transaction = False) insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 5 }, transaction = False) insertStreamDAO = daoFactory(classname = "RunConfig.InsertStream") insertStreamDAO.execute(binds = { 'STREAM' : "A" }, transaction = False) insertCMSSVersionDAO = daoFactory(classname = "RunConfig.InsertCMSSWVersion") insertCMSSVersionDAO.execute(binds = { 'VERSION' : "CMSSW_4_2_7" }, transaction = False) insertStreamCMSSWVersionDAO = daoFactory(classname = "RunConfig.InsertStreamCMSSWVersion") insertStreamCMSSWVersionDAO.execute(binds = { 'RUN' : 1, 'STREAM' : 'A', 'VERSION' : "CMSSW_4_2_7" }, transaction = False) insertStreamerDAO = daoFactory(classname = "RunConfig.InsertStreamer") insertStreamerDAO.execute(streamerPNN = "SomePNN", binds = { 'RUN' : 1, 'P5_ID' : 1, 'LUMI' : 4, 'STREAM' : "A", 'LFN' : "/testLFN/A", 'FILESIZE' : 100, 'EVENTS' : 100, 'TIME' : int(time.time()) }, transaction = False) insertStreamFilesetDAO = daoFactory(classname = "RunConfig.InsertStreamFileset") insertStreamFilesetDAO.execute(1, "A", "TestFileset1") self.fileset1 = Fileset(name = "TestFileset1") self.fileset2 = Fileset(name = "TestFileset2") self.fileset1.load() self.fileset2.create() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow2 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow2", task="Test") workflow1.create() workflow2.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Repack", type = "Repack") self.subscription2 = Subscription(fileset = self.fileset2, workflow = workflow2, split_algo = "RepackMerge", type = "RepackMerge") self.subscription1.create() self.subscription2.create() myThread.dbi.processData("""INSERT INTO wmbs_workflow_output (WORKFLOW_ID, OUTPUT_IDENTIFIER, OUTPUT_FILESET) VALUES (%d, 'SOMEOUTPUT', %d) """ % (workflow1.id, self.fileset2.id), transaction = False) # keep for later self.insertSplitLumisDAO = daoFactory(classname = "JobSplitting.InsertSplitLumis") self.insertClosedLumiDAO = daoFactory(classname = "RunLumiCloseout.InsertClosedLumi") self.feedStreamersDAO = daoFactory(classname = "Tier0Feeder.FeedStreamers") self.acquireFilesDAO = wmbsDaoFactory(classname = "Subscriptions.AcquireFiles") self.completeFilesDAO = wmbsDaoFactory(classname = "Subscriptions.CompleteFiles") self.currentTime = int(time.time()) # default split parameters self.splitArgs = {} self.splitArgs['minInputSize'] = 2.1 * 1024 * 1024 * 1024 self.splitArgs['maxInputSize'] = 4.0 * 1024 * 1024 * 1024 self.splitArgs['maxInputEvents'] = 100000000 self.splitArgs['maxInputFiles'] = 1000 self.splitArgs['maxEdmSize'] = 20 * 1024 * 1024 * 1024 self.splitArgs['maxOverSize'] = 10 * 1024 * 1024 * 1024 self.SplitArgs['maxLatency'] = 50000 return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() return def deleteSplitLumis(self): """ _deleteSplitLumis_ """ myThread = threading.currentThread() myThread.dbi.processData("""DELETE FROM lumi_section_split_active """, transaction = False) return def test00(self): """ _test00_ Test that the job name prefix feature works Test max edm size threshold for single lumi small lumi, followed by over-large lumi expect 1 job for small lumi and 4 jobs for over-large """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(2 * lumi): newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxEdmSize'] = 13000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 3, "ERROR: JobFactory didn't create three jobs") job = jobGroups[0].jobs[0] self.assertTrue(job['name'].startswith("RepackMerge-"), "ERROR: Job has wrong name") self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 3, "ERROR: Job does not process 3 files") job = jobGroups[0].jobs[2] self.assertEqual(len(job.getFiles()), 1, "ERROR: Job does not process 1 file") return def test01(self): """ _test01_ Test max size threshold for single lumi small lumi, followed by large lumi expect 1 job for small lumi and 1 job for large """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(2): newFile = File(makeUUID(), size = 1000 * lumi, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputSize'] = 3000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test02(self): """ _test02_ Test max event threshold for single lumi small lumi, followed by large lumi expect 1 job for small lumi and 1 job for large """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(2): newFile = File(makeUUID(), size = 1000, events = 100 * lumi) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputEvents'] = 300 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test03(self): """ _test03_ Test max input files threshold for single lumi small lumi, followed by large lumi expect 1 job for small lumi and 1 job for large """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(lumi * 2): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputFiles'] = 3 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") return def test04(self): """ _test04_ Test max size threshold for multi lumi 3 same size lumis """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 3]: for i in range(2): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 3000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputSize'] = 5000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset2.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test05(self): """ _test05_ Test max event threshold for multi lumi 3 same size lumis """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 3]: for i in range(2): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 3000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputEvents'] = 500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset2.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test06(self): """ _test06_ Test max input files threshold for multi lumi 3 same size lumis """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 3]: for i in range(2): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 3000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputFiles'] = 5 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset2.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test07(self): """ _test07_ Test over merge one small lumi, one large lumi (small below min size, large below max size, but both together above max size) """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(2): newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 3000 mySplitArgs['maxInputSize'] = 9000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") return def test08(self): """ _test08_ Test under merge (over merge size threshold) one small lumi, one large lumi (small below min size, large below max size, but both together above max size) """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(2): newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 3000 mySplitArgs['maxInputSize'] = 9000 mySplitArgs['maxOverSize'] = 9500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") self.fileset2.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test09(self): """ _test09_ Test under merge (over merge event threshold) one small lumi, one large lumi (small below min size, large below max size, but both together above max size) It was changed due to maxinputevents not being used anymore. """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2]: for i in range(2): newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 1500 mySplitArgs['maxInputSize'] = 9000 mySplitArgs['maxOverSize'] = 9500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") self.fileset2.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") return def test10(self): """ _test10_ Test merging of multiple lumis with holes in the lumi sequence Hole is due to no streamer files for the lumi Multi lumi input It only works with a single hole, as it creates a merged file even with it being of a smaller size than the mininputsize. It was changed due to the maxinputevents not being used anymore """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 4]: for i in range(2): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomePNN", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['minInputSize'] = 100000 mySplitArgs['maxInputSize'] = 200000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.insertClosedLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 3, 'STREAM' : "A", 'FILECOUNT' : 0, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime }, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") return
class SiblingProcessingBasedTest(unittest.TestCase): """ _SiblingProcessingBasedTest_ Test SiblingProcessing job splitting. """ def setUp(self): """ _setUp_ Setup the database connections and schema. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="Locations.New") locationAction.execute("T2_CH_CERN", pnn="T2_CH_CERN") locationAction.execute("T1_US_FNAL", pnn="T1_US_FNAL_Disk") self.testFilesetA = Fileset(name="FilesetA") self.testFilesetA.create() self.testFilesetB = Fileset(name="FilesetB") self.testFilesetB.create() self.testFileA = File("testFileA", size=1000, events=100, locations=set(["T2_CH_CERN"])) self.testFileA.create() self.testFileB = File("testFileB", size=1000, events=100, locations=set(["T2_CH_CERN"])) self.testFileB.create() self.testFileC = File("testFileC", size=1000, events=100, locations=set(["T2_CH_CERN"])) self.testFileC.create() self.testFilesetA.addFile(self.testFileA) self.testFilesetA.addFile(self.testFileB) self.testFilesetA.addFile(self.testFileC) self.testFilesetA.commit() self.testFileD = File("testFileD", size=1000, events=100, locations=set(["T2_CH_CERN"])) self.testFileD.create() self.testFileE = File("testFileE", size=1000, events=100, locations=set(["T2_CH_CERN"])) self.testFileE.create() self.testFileF = File("testFileF", size=1000, events=100, locations=set(["T2_CH_CERN"])) self.testFileF.create() self.testFilesetB.addFile(self.testFileD) self.testFilesetB.addFile(self.testFileE) self.testFilesetB.addFile(self.testFileF) self.testFilesetB.commit() testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test") testWorkflowA.create() testWorkflowB = Workflow(spec="specB.xml", owner="Steve", name="wfB", task="Test") testWorkflowB.create() testWorkflowC = Workflow(spec="specC.xml", owner="Steve", name="wfC", task="Test") testWorkflowC.create() testWorkflowD = Workflow(spec="specD.xml", owner="Steve", name="wfD", task="Test") testWorkflowD.create() self.testSubscriptionA = Subscription(fileset=self.testFilesetA, workflow=testWorkflowA, split_algo="FileBased", type="Processing") self.testSubscriptionA.create() self.testSubscriptionB = Subscription(fileset=self.testFilesetB, workflow=testWorkflowB, split_algo="FileBased", type="Processing") self.testSubscriptionB.create() self.testSubscriptionC = Subscription(fileset=self.testFilesetB, workflow=testWorkflowC, split_algo="FileBased", type="Processing") self.testSubscriptionC.create() self.testSubscriptionD = Subscription(fileset=self.testFilesetB, workflow=testWorkflowD, split_algo="FileBased", type="Processing") self.testSubscriptionD.create() deleteWorkflow = Workflow(spec="specE.xml", owner="Steve", name="wfE", task="Test") deleteWorkflow.create() self.deleteSubscriptionA = Subscription( fileset=self.testFilesetA, workflow=deleteWorkflow, split_algo="SiblingProcessingBased", type="Cleanup") self.deleteSubscriptionA.create() self.deleteSubscriptionB = Subscription( fileset=self.testFilesetB, workflow=deleteWorkflow, split_algo="SiblingProcessingBased", type="Cleanup") self.deleteSubscriptionB.create() return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def testSiblingProcessing(self): """ _testSiblingProcessing_ Verify that the sibling processing split works correctly dealing with failed files and acquiring files correctly. """ splitter = SplitterFactory() deleteFactoryA = splitter(package="WMCore.WMBS", subscription=self.deleteSubscriptionA) deleteFactoryB = splitter(package="WMCore.WMBS", subscription=self.deleteSubscriptionB) result = deleteFactoryA() assert len(result) == 0, \ "Error: No jobs should be returned." result = deleteFactoryB() assert len(result) == 0, \ "Error: No jobs should be returned." self.testSubscriptionA.completeFiles(self.testFileA) result = deleteFactoryA(files_per_job=1) assert len(result) == 1, \ "Error: Only one jobgroup should be returned." assert len(result[0].jobs) == 1, \ "Error: There should only be one job in the jobgroup." assert result[0].jobs[0]["possiblePSN"] == set(["T2_CH_CERN"]), \ "Error: possiblePSN is wrong." assert len(result[0].jobs[0]["input_files"]) == 1, \ "Error: Job should only have one input file." assert result[0].jobs[0]["input_files"][0]["lfn"] == "testFileA", \ "Error: Input file for job is wrong." result = deleteFactoryB(files_per_job=1) assert len(result) == 0, \ "Error: Second subscription should have no jobs." result = deleteFactoryA(files_per_job=1) assert len(result) == 0, \ "Error: No jobs should have been created." self.testSubscriptionB.completeFiles(self.testFileD) self.testSubscriptionC.failFiles(self.testFileD) result = deleteFactoryA(files_per_job=1) assert len(result) == 0, \ "Error: No jobs should have been created." result = deleteFactoryB(files_per_job=1) assert len(result) == 0, \ "Error: No jobs should have been created." self.testSubscriptionD.failFiles(self.testFileD) result = deleteFactoryA(files_per_job=1) assert len(result) == 0, \ "Error: No jobs should have been created." result = deleteFactoryB(files_per_job=1) assert len(result) == 0, \ "Error: No job groups should have been created." self.testSubscriptionB.completeFiles([self.testFileE, self.testFileF]) self.testSubscriptionC.completeFiles([self.testFileE, self.testFileF]) self.testSubscriptionD.completeFiles([self.testFileE, self.testFileF]) result = deleteFactoryB(files_per_job=10) assert len(result) == 0, \ "Error: No jobs should have been created." self.testFilesetB.markOpen(False) result = deleteFactoryB(files_per_job=10) assert len(result) == 1, \ "Error: One jobgroup should have been returned." assert len(result[0].jobs) == 1, \ "Error: There should only be one job in the jobgroup." assert len(result[0].jobs[0]["input_files"]) == 2, \ "Error: Job should only have one input file." lfns = [ result[0].jobs[0]["input_files"][0]["lfn"], result[0].jobs[0]["input_files"][1]["lfn"] ] assert "testFileE" in lfns, \ "Error: TestFileE missing from job input." assert "testFileF" in lfns, \ "Error: TestFileF missing from job input." self.assertEqual(len(self.deleteSubscriptionB.availableFiles()), 0, "Error: There should be no available files.") completeFiles = self.deleteSubscriptionB.filesOfStatus("Completed") self.assertEqual(len(completeFiles), 1, "Error: There should only be one complete file.") self.assertEqual( list(completeFiles)[0]["lfn"], "testFileD", "Error: Test file D should be complete.") return def testMultipleLocations(self): """ _testMultipleLocations_ Verify that the sibling processing based algorithm doesn't create jobs that run over files at multiple sites. """ testFile1 = File("testFile1", size=1000, events=100, locations=set(["T1_US_FNAL_Disk"])) testFile1.create() testFile2 = File("testFile2", size=1000, events=100, locations=set(["T1_US_FNAL_Disk"])) testFile2.create() testFile3 = File("testFile3", size=1000, events=100, locations=set(["T1_US_FNAL_Disk"])) testFile3.create() self.testFilesetA.addFile(testFile1) self.testFilesetA.addFile(testFile2) self.testFilesetA.addFile(testFile3) self.testFilesetA.commit() self.testFilesetA.markOpen(False) self.testSubscriptionA.completeFiles([testFile1, testFile2, testFile3]) self.testSubscriptionA.completeFiles( [self.testFileA, self.testFileB, self.testFileC]) splitter = SplitterFactory() deleteFactoryA = splitter(package="WMCore.WMBS", subscription=self.deleteSubscriptionA) result = deleteFactoryA(files_per_job=50) assert len(result) == 2, \ "Error: Wrong number of jobgroups returned." goldenFilesA = ["testFileA", "testFileB", "testFileC"] goldenFilesB = ["testFile1", "testFile2", "testFile3"] for jobGroup in result: assert len(jobGroup.jobs) == 1, \ "Error: Wrong number of jobs in jobgroup." assert len(jobGroup.jobs[0]["input_files"]) == 3, \ "Error: Wrong number of input files in job." jobSite = jobGroup.jobs[0]["possiblePSN"] assert (jobSite == set(["T2_CH_CERN"]) or jobSite == set(["T1_US_FNAL"])), \ "Error: Wrong site for job." if jobSite == set(["T2_CH_CERN"]): goldenFiles = goldenFilesA else: goldenFiles = goldenFilesB for jobFile in jobGroup.jobs[0]["input_files"]: goldenFiles.remove(jobFile["lfn"]) assert len(goldenFiles) == 0, \ "Error: Files are missing." return def testLargeNumberOfFiles(self): """ _testLargeNumberOfFiles_ Setup a subscription with 500 files and verify that the splitting algo works correctly. """ testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test") testWorkflowA.create() testWorkflowB = Workflow(spec="specB.xml", owner="Steve", name="wfB", task="Test") testWorkflowB.create() testFileset = Fileset(name="TestFileset") testFileset.create() allFiles = [] for i in range(500): testFile = File(str(i), size=1000, events=100, locations=set(["T2_CH_CERN"])) testFile.create() allFiles.append(testFile) testFileset.addFile(testFile) testFileset.commit() testSubscriptionA = Subscription(fileset=testFileset, workflow=testWorkflowA, split_algo="FileBased", type="Processing") testSubscriptionA.create() testSubscriptionB = Subscription(fileset=testFileset, workflow=testWorkflowB, split_algo="SiblingProcessingBased", type="Processing") testSubscriptionB.create() testSubscriptionA.completeFiles(allFiles) splitter = SplitterFactory() deleteFactoryA = splitter(package="WMCore.WMBS", subscription=testSubscriptionB) result = deleteFactoryA(files_per_job=50) self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.") self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.") return def testFilesWithoutOtherSubscriptions(self): """ _testFilesWithoutOtherSubscriptions_ Test the case where files only in the delete subscription can happen if cleanup of the other subscriptions is fast """ testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test") testWorkflowA.create() testFileset = Fileset(name="TestFileset") testFileset.create() allFiles = [] for i in range(500): testFile = File(str(i), size=1000, events=100, locations=set(["T2_CH_CERN"])) testFile.create() allFiles.append(testFile) testFileset.addFile(testFile) testFileset.commit() testSubscriptionA = Subscription(fileset=testFileset, workflow=testWorkflowA, split_algo="SiblingProcessingBased", type="Processing") testSubscriptionA.create() splitter = SplitterFactory() deleteFactoryA = splitter(package="WMCore.WMBS", subscription=testSubscriptionA) result = deleteFactoryA(files_per_job=50) self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.") self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.") return
def __call__(self, filesetToProcess): """ The algorithm itself """ global LOCK # Get configuration initObj = WMInit() initObj.setLogging() initObj.setDatabaseConnection(os.getenv("DATABASE"), \ os.getenv('DIALECT'), os.getenv("DBSOCK")) myThread = threading.currentThread() daofactory = DAOFactory(package = "WMCore.WMBS" , \ logger = myThread.logger, \ dbinterface = myThread.dbi) locationNew = daofactory(classname="Locations.New") getFileLoc = daofactory(classname="Files.GetLocation") logging.debug("the T0Feeder is processing %s" % \ filesetToProcess.name) logging.debug("the fileset name %s" % \ (filesetToProcess.name).split(":")[0]) startRun = (filesetToProcess.name).split(":")[3] fileType = (filesetToProcess.name).split(":")[2] # url builder primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1] processedDataset = (( filesetToProcess.name).split(":")[0]).split('/')[2] dataTier = (((filesetToProcess.name\ ).split(":")[0]).split('/')[3]).split('-')[0] # Fisrt call to T0 db for this fileset # Here add test for the closed fileset LASTIME = filesetToProcess.lastUpdate url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \ (fileType, LASTIME, primaryDataset,processedDataset, dataTier) tries = 1 while True: try: myRequester = JSONRequests(url="vocms52.cern.ch:8889") requestResult = myRequester.get(\ url+"/"+"?return_type=text/json%2Bdas") newFilesList = requestResult[0]["results"] except: logging.debug("T0Reader call error...") if tries == self.maxRetries: return else: tries += 1 continue logging.debug("T0ASTRun queries done ...") now = time.time() filesetToProcess.last_update = now LASTIME = int(newFilesList['end_time']) + 1 break # process all files if len(newFilesList['files']): LOCK.acquire() try: locationNew.execute(siteName="caf.cern.ch", seName="caf.cern.ch") except Exception as e: logging.debug("Error when adding new location...") logging.debug(e) logging.debug(format_exc()) for files in newFilesList['files']: # Assume parents aren't asked newfile = File(str(files['lfn']), \ size = files['file_size'], events = files['events']) try: if newfile.exists() == False: newfile.create() else: newfile.loadData() #Add run test if already exist for run in files['runs']: if startRun != 'None' and int(startRun) <= int(run): # ToDo: Distinguish between # filestA-RunX and filesetA-Run[0-9]* filesetRun = Fileset( name = (((\ filesetToProcess.name).split(':')[0]).split('/')[0]\ )+'/'+(((filesetToProcess.name).split(':')[0]).split\ ('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\ ).split('/')[2])+'/'+((((filesetToProcess.name).split\ (':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\ (run)+":"+":".join((filesetToProcess.name).split(':')[1:] \ ) ) if filesetRun.exists() == False: filesetRun.create() else: filesetRun.loadData() # Add test runs already there # (for growing dataset) - # to support file with different runs and lumi if not newfile['runs']: runSet = set() runSet.add(Run(run, *files['runs'][run])) newfile.addRunSet(runSet) fileLoc = getFileLoc.execute(file=files['lfn']) if 'caf.cern.ch' not in fileLoc: newfile.setLocation("caf.cern.ch") filesetRun.addFile(newfile) logging.debug( "new file created/loaded added by T0ASTRun...") filesetRun.commit() except Exception as e: logging.debug("Error when adding new files in T0ASTRun...") logging.debug(e) logging.debug(format_exc()) filesetToProcess.setLastUpdate\ (int(newFilesList['end_time']) + 1) filesetToProcess.commit() LOCK.release() else: logging.debug("nothing to do...") # For re-opned fileset or empty, try until the purge time if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime: filesetToProcess.setLastUpdate(time.time()) filesetToProcess.commit() if LASTIME: myRequester = JSONRequests(url="vocms52.cern.ch:8889") requestResult = myRequester.get("/tier0/runs") for listRun in requestResult[0]: if int(startRun) <= int(listRun['run']): if listRun['status'] =='CloseOutExport' or \ listRun['status'] =='Complete' or listRun['status'] ==\ 'CloseOutT1Skimming': closeFileset = Fileset( name = (((\ filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\ (((filesetToProcess.name).split(':')[0]).split('/')[1]\ )+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\ [2])+'/'+((((filesetToProcess.name).split(':')[0]).split\ ('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\ +":"+":".join((filesetToProcess.name).split(':')[1:] ) ) if closeFileset.exists() != False: closeFileset = Fileset(id=closeFileset.exists()) closeFileset.loadData() if closeFileset.open == True: closeFileset.markOpen(False) # Commit the fileset filesetToProcess.commit() # Commit the fileset logging.debug("Test purge in T0ASTRun ...") filesetToProcess.load() LASTIME = filesetToProcess.lastUpdate if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime: filesetToProcess.markOpen(False) logging.debug("Purge Done...") filesetToProcess.commit()
class WMBSHelper(WMConnectionBase): """ _WMBSHelper_ Interface between the WorkQueue and WMBS. """ def __init__(self, wmSpec, taskName, blockName=None, mask=None, cachepath='.'): """ _init_ Initialize DAOs and other things needed. """ self.block = blockName self.mask = mask self.wmSpec = wmSpec self.topLevelTask = wmSpec.getTask(taskName) self.cachepath = cachepath self.isDBS = True self.topLevelFileset = None self.topLevelSubscription = None self.topLevelTaskDBSBufferId = None self.mergeOutputMapping = {} # Initiate the pieces you need to run your own DAOs WMConnectionBase.__init__(self, "WMCore.WMBS") myThread = threading.currentThread() self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) # DAOs from WMBS for file commit self.setParentage = self.daofactory(classname="Files.SetParentage") self.setFileRunLumi = self.daofactory(classname="Files.AddRunLumi") self.setFileLocation = self.daofactory(classname="Files.SetLocationForWorkQueue") self.setFileAddChecksum = self.daofactory(classname="Files.AddChecksumByLFN") self.addFileAction = self.daofactory(classname="Files.Add") self.addToFileset = self.daofactory(classname="Files.AddDupsToFileset") self.getLocations = self.daofactory(classname="Locations.ListSites") self.getLocationInfo = self.daofactory(classname="Locations.GetSiteInfo") # DAOs from DBSBuffer self.dbsCreateFiles = self.dbsDaoFactory(classname="DBSBufferFiles.Add") self.dbsSetLocation = self.dbsDaoFactory(classname="DBSBufferFiles.SetLocationByLFN") self.dbsInsertLocation = self.dbsDaoFactory(classname="DBSBufferFiles.AddLocation") self.dbsSetChecksum = self.dbsDaoFactory(classname="DBSBufferFiles.AddChecksumByLFN") self.dbsInsertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow") # Added for file creation bookkeeping self.dbsFilesToCreate = [] self.addedLocations = [] self.wmbsFilesToCreate = [] self.insertedBogusDataset = -1 return def createSandbox(self): """Create the runtime sandbox""" sandboxCreator = SandboxCreator() sandboxCreator.makeSandbox(self.cachepath, self.wmSpec) def createTopLevelFileset(self, topLevelFilesetName=None): """ _createTopLevelFileset_ Create the top level fileset for the workflow. If the name of the top level fileset is not given create one. """ if topLevelFilesetName is None: filesetName = ("%s-%s" % (self.wmSpec.name(), self.wmSpec.getTopLevelTask()[0].name())) if self.block: filesetName += "-%s" % self.block if self.mask: from hashlib import md5 mask_string = ",".join(["%s=%s" % (x, self.mask[x]) for x in sorted(self.mask)]) filesetName += "-%s" % md5(mask_string).hexdigest() else: filesetName = topLevelFilesetName self.topLevelFileset = Fileset(filesetName) self.topLevelFileset.create() return def outputFilesetName(self, task, outputModuleName): """ _outputFilesetName_ Generate an output fileset name for the given task and output module. """ if task.taskType() == "Merge": outputFilesetName = "%s/merged-%s" % (task.getPathName(), outputModuleName) else: outputFilesetName = "%s/unmerged-%s" % (task.getPathName(), outputModuleName) return outputFilesetName def createSubscription(self, task, fileset, alternativeFilesetClose=False): """ _createSubscription_ Create subscriptions in the database. This includes workflows in WMBS and DBSBuffer, output maps, datasets and phedex subscriptions, and filesets for each task below and including the given task. """ sub = self._createSubscriptionsInWMBS(task, fileset, alternativeFilesetClose) self._createWorkflowsInDBSBuffer() self._createDatasetSubscriptionsInDBSBuffer() return sub def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() # FIXME: Let workflow put in values if spec is missing them workflow = Workflow(spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"], dn=self.wmSpec.getOwner().get("dn", "unknown"), group=self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"), name=self.wmSpec.name(), task=task.getPathName(), wfType=self.wmSpec.getDashboardActivity(), alternativeFilesetClose=alternativeFilesetClose, priority=self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset=fileset, workflow=workflow, split_algo=task.jobSplittingAlgorithm(), type=task.getPrimarySubType()) if subscription.exists(): subscription.load() msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)" self.logger.info(msg % (subscription['id'], task.getPathName())) else: subscription.create() for site in task.siteWhitelist(): subscription.addWhiteBlackList([{"site_name": site, "valid": True}]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{"site_name": site, "valid": False}]) if self.topLevelSubscription is None: self.topLevelSubscription = subscription logging.info("Top level subscription created: %s", subscription["id"]) else: logging.info("Child subscription created: %s", subscription["id"]) outputModules = task.getOutputModulesForTask() ignoredOutputModules = task.getIgnoredOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): if outputModuleName in ignoredOutputModules: logging.info("IgnoredOutputModule set for %s, skipping fileset creation.", outputModuleName) continue outputFileset = Fileset(self.outputFilesetName(task, outputModuleName)) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: if childTask.taskType() == "Merge": mergedOutputFileset = Fileset(self.outputFilesetName(childTask, "Merged")) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr(getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset != None: self.mergeOutputMapping[mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS(childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset is None: workflow.addOutput(outputModuleName, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName, outputFileset, mergedOutputFileset) return self.topLevelSubscription def addMCFakeFile(self): """Add a fake file for wmbs to run production over""" needed = ['FirstEvent', 'FirstLumi', 'FirstRun', 'LastEvent', 'LastLumi', 'LastRun'] for key in needed: if self.mask and self.mask.get(key) is None: msg = 'Invalid value "%s" for %s' % (self.mask.get(key), key) raise WorkQueueWMBSException(msg) locations = set() for site in self.getLocations.execute(conn=self.getDBConn(), transaction=self.existingTransaction()): try: siteInfo = self.getLocationInfo.execute(site, conn=self.getDBConn(), transaction=self.existingTransaction()) if not siteInfo: self.logger.info('Skipping MonteCarlo injection to site "%s" as unknown to wmbs' % site) continue locations.add(siteInfo[0]['pnn']) except Exception as ex: self.logger.error('Error getting storage element for "%s": %s' % (site, str(ex))) if not locations: msg = 'No locations to inject Monte Carlo work to, unable to proceed' raise WorkQueueWMBSException(msg) mcFakeFileName = ("MCFakeFile-%s" % self.topLevelFileset.name).encode('ascii', 'ignore') wmbsFile = File(lfn=mcFakeFileName, first_event=self.mask['FirstEvent'], last_event=self.mask['LastEvent'], events=self.mask['LastEvent'] - self.mask['FirstEvent'] + 1, # inclusive range locations=locations, merged=False, # merged causes dbs parentage relation ) if self.mask: lumis = range(self.mask['FirstLumi'], self.mask['LastLumi'] + 1) # inclusive range wmbsFile.addRun(Run(self.mask['FirstRun'], *lumis)) # assume run number static else: wmbsFile.addRun(Run(1, 1)) wmbsFile['inFileset'] = True # file is not a parent logging.info("WMBS File: %s on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations']) self.wmbsFilesToCreate.append(wmbsFile) totalFiles = self.topLevelFileset.addFilesToWMBSInBulk(self.wmbsFilesToCreate, self.wmSpec.name(), isDBS=self.isDBS) self.topLevelFileset.markOpen(False) return totalFiles def createSubscriptionAndAddFiles(self, block): """ _createSubscriptionAndAddFiles_ Create the subscription and add files at one time to put everything in one transaction. """ self.beginTransaction() self.createTopLevelFileset() try: sub = self.createSubscription(self.topLevelTask, self.topLevelFileset) except Exception as ex: myThread = threading.currentThread() myThread.transaction.rollback() msg = traceback.format_exc() logging.error("Failed to create subscription %s", msg) raise ex if block != None: logging.info('"%s" Injecting block %s (%d files) into wmbs', self.wmSpec.name(), self.block, len(block['Files'])) addedFiles = self.addFiles(block) # For MC case else: logging.info( '"%s" Injecting production %s:%s:%s - %s:%s:%s (run:lumi:event) into wmbs', self.wmSpec.name(), self.mask['FirstRun'], self.mask['FirstLumi'], self.mask['FirstEvent'], self.mask['LastRun'], self.mask['LastLumi'], self.mask['LastEvent']) addedFiles = self.addMCFakeFile() self.commitTransaction(existingTransaction=False) return sub, addedFiles def addFiles(self, block): """ _addFiles_ create wmbs files from given dbs block. as well as run lumi update """ if self.topLevelTask.getInputACDC(): self.isDBS = False for acdcFile in self.validFiles(block['Files']): self._addACDCFileToWMBSFile(acdcFile) else: self.isDBS = True for dbsFile in self.validFiles(block['Files']): self._addDBSFileToWMBSFile(dbsFile, block['PhEDExNodeNames']) # Add files to WMBS totalFiles = self.topLevelFileset.addFilesToWMBSInBulk(self.wmbsFilesToCreate, self.wmSpec.name(), isDBS=self.isDBS) # Add files to DBSBuffer self._createFilesInDBSBuffer() self.topLevelFileset.markOpen(block.get('IsOpen', False)) return totalFiles def getMergeOutputMapping(self): """ _getMergeOutputMapping_ retrieves the relationship between primary dataset and merge output fileset ids for all merge tasks created """ return self.mergeOutputMapping def _createWorkflowsInDBSBuffer(self): """ _createWorkflowsInDBSBuffer_ Register workflow information and settings in dbsbuffer for all tasks that will potentially produce any output in this spec. """ for task in self.wmSpec.listOutputProducingTasks(): workflow_id = self.dbsInsertWorkflow.execute(self.wmSpec.name(), task, self.wmSpec.getBlockCloseMaxWaitTime(), self.wmSpec.getBlockCloseMaxFiles(), self.wmSpec.getBlockCloseMaxEvents(), self.wmSpec.getBlockCloseMaxSize(), conn=self.getDBConn(), transaction=self.existingTransaction()) if task == self.topLevelTask.getPathName(): self.topLevelTaskDBSBufferId = workflow_id def _createDatasetSubscriptionsInDBSBuffer(self): """ _createDatasetSubscriptionsInDBSBuffer_ Insert the subscriptions defined in the workload for the output datasets with the different options. """ subInfo = self.wmSpec.getSubscriptionInformation() for dataset in subInfo: dbsDataset = DBSBufferDataset(path=dataset) dbsDataset.create() dbsDataset.addSubscription(subInfo[dataset]) return def _createFilesInDBSBuffer(self): """ _createFilesInDBSBuffer_ It does the actual job of creating things in DBSBuffer """ if len(self.dbsFilesToCreate) == 0: # Whoops, nothing to do! return dbsFileTuples = [] dbsFileLoc = [] dbsCksumBinds = [] locationsToAdd = [] selfChecksums = None # The first thing we need to do is add the datasetAlgo # Assume all files in a pass come from one datasetAlgo? if self.insertedBogusDataset == -1: self.insertedBogusDataset = self.dbsFilesToCreate[0].insertDatasetAlgo() for dbsFile in self.dbsFilesToCreate: # Append a tuple in the format specified by DBSBufferFiles.Add # Also run insertDatasetAlgo lfn = dbsFile['lfn'] selfChecksums = dbsFile['checksums'] newTuple = (lfn, dbsFile['size'], dbsFile['events'], self.insertedBogusDataset, dbsFile['status'], self.topLevelTaskDBSBufferId) if newTuple not in dbsFileTuples: dbsFileTuples.append(newTuple) if len(dbsFile['newlocations']) < 1: msg = '' msg += "File created without any locations!\n" msg += "File lfn: %s\n" % (lfn) msg += "Rejecting this group of files in DBS!\n" logging.error(msg) raise WorkQueueWMBSException(msg) for jobLocation in dbsFile['newlocations']: if jobLocation not in self.addedLocations: # If we don't have it, try and add it locationsToAdd.append(jobLocation) self.addedLocations.append(jobLocation) dbsFileLoc.append({'lfn': lfn, 'pnn': jobLocation}) if selfChecksums: # If we have checksums we have to create a bind # For each different checksum for entry in selfChecksums.keys(): dbsCksumBinds.append({'lfn': lfn, 'cksum': selfChecksums[entry], 'cktype': entry}) for jobLocation in locationsToAdd: self.dbsInsertLocation.execute(siteName=jobLocation, conn=self.getDBConn(), transaction=self.existingTransaction()) self.dbsCreateFiles.execute(files=dbsFileTuples, conn=self.getDBConn(), transaction=self.existingTransaction()) self.dbsSetLocation.execute(binds=dbsFileLoc, conn=self.getDBConn(), transaction=self.existingTransaction()) if len(dbsCksumBinds) > 0: self.dbsSetChecksum.execute(bulkList=dbsCksumBinds, conn=self.getDBConn(), transaction=self.existingTransaction()) # Now that we've created those files, clear the list self.dbsFilesToCreate = [] return def _addToDBSBuffer(self, dbsFile, checksums, locations): """ This step is just for increase the performance for Accountant doesn't neccessary to check the parentage """ dbsBuffer = DBSBufferFile(lfn=dbsFile["LogicalFileName"], size=dbsFile["FileSize"], events=dbsFile["NumberOfEvents"], checksums=checksums, locations=locations, status="GLOBAL") dbsBuffer.setDatasetPath('bogus') dbsBuffer.setAlgorithm(appName="cmsRun", appVer="Unknown", appFam="Unknown", psetHash="Unknown", configContent="Unknown") if not dbsBuffer.exists(): self.dbsFilesToCreate.append(dbsBuffer) # dbsBuffer.create() return def _addDBSFileToWMBSFile(self, dbsFile, storageElements, inFileset=True): """ There are two assumptions made to make this method behave properly, 1. DBS returns only one level of ParentList. If DBS returns multiple level of parentage, it will be still get handled. However that might not be what we wanted. In that case, restrict to one level. 2. Assumes parents files are in the same location as child files. This is not True in general case, but workquue should only select work only where child and parent files are in the same location """ wmbsParents = [] dbsFile.setdefault("ParentList", []) for parent in dbsFile["ParentList"]: wmbsParents.append(self._addDBSFileToWMBSFile(parent, storageElements, inFileset=False)) checksums = {} if dbsFile.get('Checksum'): checksums['cksum'] = dbsFile['Checksum'] if dbsFile.get('Adler32'): checksums['adler32'] = dbsFile['Adler32'] wmbsFile = File(lfn=dbsFile["LogicalFileName"], size=dbsFile["FileSize"], events=dbsFile["NumberOfEvents"], checksums=checksums, # TODO: need to get list of parent lfn parents=wmbsParents, locations=set(storageElements)) for lumi in dbsFile['LumiList']: if isinstance(lumi['LumiSectionNumber'], list): run = Run(lumi['RunNumber'], *lumi['LumiSectionNumber']) else: run = Run(lumi['RunNumber'], lumi['LumiSectionNumber']) wmbsFile.addRun(run) self._addToDBSBuffer(dbsFile, checksums, storageElements) logging.info("WMBS File: %s\n on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations']) wmbsFile['inFileset'] = bool(inFileset) self.wmbsFilesToCreate.append(wmbsFile) return wmbsFile def _convertACDCFileToDBSFile(self, acdcFile): """ convert ACDCFiles to dbs file format """ dbsFile = {} dbsFile["LogicalFileName"] = acdcFile["lfn"] dbsFile["FileSize"] = acdcFile["size"] dbsFile["NumberOfEvents"] = acdcFile["events"] return dbsFile def _addACDCFileToWMBSFile(self, acdcFile, inFileset=True): """ adds the ACDC files into WMBS database """ wmbsParents = [] for parent in acdcFile["parents"]: parent = self._addACDCFileToWMBSFile(DatastructFile(lfn=parent, locations=acdcFile["locations"]), inFileset=False) wmbsParents.append(parent) # pass empty check sum since it won't be updated to dbs anyway checksums = {} wmbsFile = File(lfn=str(acdcFile["lfn"]), size=acdcFile["size"], events=acdcFile["events"], first_event=acdcFile.get('first_event', 0), last_event=acdcFile.get('last_event', 0), checksums=checksums, parents=wmbsParents, locations=acdcFile["locations"], merged=acdcFile.get('merged', True)) ## TODO need to get the lumi lists for run in acdcFile['runs']: wmbsFile.addRun(run) dbsFile = self._convertACDCFileToDBSFile(acdcFile) self._addToDBSBuffer(dbsFile, checksums, acdcFile["locations"]) logging.info("WMBS File: %s\n on Location: %s", wmbsFile['lfn'], wmbsFile['newlocations']) wmbsFile['inFileset'] = bool(inFileset) self.wmbsFilesToCreate.append(wmbsFile) return wmbsFile def validFiles(self, files): """ Apply lumi mask and or run white/black list and return files which have one or more of the requested lumis """ runWhiteList = self.topLevelTask.inputRunWhitelist() runBlackList = self.topLevelTask.inputRunBlacklist() lumiMask = self.topLevelTask.getLumiMask() blackMask = None if lumiMask: # We have a lumiMask, so use it and modify with run white/black list if runWhiteList: lumiMask.selectRuns(runWhiteList) if runBlackList: lumiMask.removeRuns(runBlackList) elif runWhiteList: # We have a run whitelist, subtract off blacklist lumiMask = LumiList(runs=runWhiteList) if runBlackList: # We only have a blacklist, so make a black mask out of it instead lumiMask.removeRuns(runBlackList) else: lumiMask = None if runBlackList: blackMask = LumiList(runs=runBlackList) results = [] for f in files: if isinstance(f, basestring) or "LumiList" not in f: results.append(f) continue # Create a LumiList from the WMBS info runLumis = {} for x in f['LumiList']: if x['RunNumber'] in runLumis: runLumis[x['RunNumber']].extend(x['LumiSectionNumber']) else: runLumis[x['RunNumber']] = x['LumiSectionNumber'] fileLumiList = LumiList(runsAndLumis=runLumis) if lumiMask: if fileLumiList & lumiMask: # At least one lumi from file is in lumiMask results.append(f) elif blackMask: if fileLumiList - blackMask: # At least one lumi from file is not in blackMask results.append(f) else: # There is effectively no mask results.append(f) return results
def __call__(self, filesetToProcess): """ The algorithm itself """ global LOCK # Get configuration initObj = WMInit() initObj.setLogging() initObj.setDatabaseConnection(os.getenv("DATABASE"), \ os.getenv('DIALECT'), os.getenv("DBSOCK")) myThread = threading.currentThread() daofactory = DAOFactory(package = "WMCore.WMBS" , \ logger = myThread.logger, \ dbinterface = myThread.dbi) locationNew = daofactory(classname = "Locations.New") getFileLoc = daofactory(classname = "Files.GetLocation") logging.debug("the T0Feeder is processing %s" % \ filesetToProcess.name) logging.debug("the fileset name %s" % \ (filesetToProcess.name).split(":")[0]) startRun = (filesetToProcess.name).split(":")[3] fileType = (filesetToProcess.name).split(":")[2] # url builder primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1] processedDataset = ((filesetToProcess.name).split(":")[0]).split('/')[2] dataTier = (((filesetToProcess.name\ ).split(":")[0]).split('/')[3]).split('-')[0] # Fisrt call to T0 db for this fileset # Here add test for the closed fileset LASTIME = filesetToProcess.lastUpdate url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \ (fileType, LASTIME, primaryDataset,processedDataset, dataTier) tries = 1 while True: try: myRequester = JSONRequests(url = "vocms52.cern.ch:8889") requestResult = myRequester.get(\ url+"/"+"?return_type=text/json%2Bdas") newFilesList = requestResult[0]["results"] except: logging.debug("T0Reader call error...") if tries == self.maxRetries: return else: tries += 1 continue logging.debug("T0ASTRun queries done ...") now = time.time() filesetToProcess.last_update = now LASTIME = int(newFilesList['end_time']) + 1 break # process all files if len(newFilesList['files']): LOCK.acquire() try: locationNew.execute(siteName = "caf.cern.ch", seName = "caf.cern.ch") except Exception as e: logging.debug("Error when adding new location...") logging.debug(e) logging.debug( format_exc() ) for files in newFilesList['files']: # Assume parents aren't asked newfile = File(str(files['lfn']), \ size = files['file_size'], events = files['events']) try: if newfile.exists() == False : newfile.create() else: newfile.loadData() #Add run test if already exist for run in files['runs']: if startRun != 'None' and int(startRun) <= int(run): # ToDo: Distinguish between # filestA-RunX and filesetA-Run[0-9]* filesetRun = Fileset( name = (((\ filesetToProcess.name).split(':')[0]).split('/')[0]\ )+'/'+(((filesetToProcess.name).split(':')[0]).split\ ('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\ ).split('/')[2])+'/'+((((filesetToProcess.name).split\ (':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\ (run)+":"+":".join((filesetToProcess.name).split(':')[1:] \ ) ) if filesetRun.exists() == False : filesetRun.create() else: filesetRun.loadData() # Add test runs already there # (for growing dataset) - # to support file with different runs and lumi if not newfile['runs']: runSet = set() runSet.add(Run( run, *files['runs'][run])) newfile.addRunSet(runSet) fileLoc = getFileLoc.execute(file = files['lfn']) if 'caf.cern.ch' not in fileLoc: newfile.setLocation("caf.cern.ch") filesetRun.addFile(newfile) logging.debug("new file created/loaded added by T0ASTRun...") filesetRun.commit() except Exception as e: logging.debug("Error when adding new files in T0ASTRun...") logging.debug(e) logging.debug( format_exc() ) filesetToProcess.setLastUpdate\ (int(newFilesList['end_time']) + 1) filesetToProcess.commit() LOCK.release() else: logging.debug("nothing to do...") # For re-opned fileset or empty, try until the purge time if (int(now)/3600 - LASTIME/3600) > self.reopenTime: filesetToProcess.setLastUpdate(time.time()) filesetToProcess.commit() if LASTIME: myRequester = JSONRequests(url = "vocms52.cern.ch:8889") requestResult = myRequester.get("/tier0/runs") for listRun in requestResult[0]: if int(startRun) <= int(listRun['run']): if listRun['status'] =='CloseOutExport' or \ listRun['status'] =='Complete' or listRun['status'] ==\ 'CloseOutT1Skimming': closeFileset = Fileset( name = (((\ filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\ (((filesetToProcess.name).split(':')[0]).split('/')[1]\ )+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\ [2])+'/'+((((filesetToProcess.name).split(':')[0]).split\ ('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\ +":"+":".join((filesetToProcess.name).split(':')[1:] ) ) if closeFileset.exists() != False : closeFileset = Fileset( id = closeFileset.exists()) closeFileset.loadData() if closeFileset.open == True: closeFileset.markOpen(False) # Commit the fileset filesetToProcess.commit() # Commit the fileset logging.debug("Test purge in T0ASTRun ...") filesetToProcess.load() LASTIME = filesetToProcess.lastUpdate if (int(now)/3600 - LASTIME/3600) > self.purgeTime: filesetToProcess.markOpen(False) logging.debug("Purge Done...") filesetToProcess.commit()
closeFileset = Fileset( name = (((\ filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\ (((filesetToProcess.name).split(':')[0]).split('/')[1]\ )+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\ [2])+'/'+((((filesetToProcess.name).split(':')[0]).split\ ('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\ +":"+":".join((filesetToProcess.name).split(':')[1:] ) ) if closeFileset.exists() != False: closeFileset = Fileset(id=closeFileset.exists()) closeFileset.loadData() if closeFileset.open == True: closeFileset.markOpen(False) # Commit the fileset filesetToProcess.commit() # Commit the fileset logging.debug("Test purge in T0ASTRun ...") filesetToProcess.load() LASTIME = filesetToProcess.lastUpdate if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime: filesetToProcess.markOpen(False) logging.debug("Purge Done...") filesetToProcess.commit()
def createSubscription(self, topLevelFilesetName = None, task = None, fileset = None): """ _createSubscription_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ if task == None or fileset == None: self.createTopLevelFileset(topLevelFilesetName) sub = None for topLevelTask in self.wmSpec.getTopLevelTask(): sub = self.createSubscription(topLevelFilesetName, topLevelTask, self.topLevelFileset) return sub # create runtime sandbox for workflow self.createSandbox() workflow = Workflow(spec = self.wmSpec.specUrl(), owner = self.wmSpec.getOwner()["name"], dn = self.wmSpec.getOwner().get("dn", None), group = self.wmSpec.getOwner().get("group", None), owner_vogroup = self.wmSpec.getOwner().get("vogroup", ''), owner_vorole = self.wmSpec.getOwner().get("vorole", ''), name = self.wmSpec.name(), task = task.getPathName(), wfType = self.wmSpec.getDashboardActivity()) workflow.create() subscription = Subscription(fileset = fileset, workflow = workflow, split_algo = task.jobSplittingAlgorithm(), type = task.taskType()) subscription.create() for site in task.siteWhitelist(): subscription.addWhiteBlackList([{"site_name": site, "valid": True}]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{"site_name": site, "valid": False}]) if self.topLevelSubscription == None: self.topLevelSubscription = subscription logging.info("Top level subscription created: %s" % subscription["id"]) else: logging.info("Child subscription created: %s" % subscription["id"]) outputModules = task.getOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): outputFileset = Fileset(self.outputFilesetName(task, outputModuleName)) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: if childTask.taskType() == "Merge": mergedOutputFileset = Fileset(self.outputFilesetName(childTask, "Merged")) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) self.createSubscription(topLevelFilesetName, childTask, outputFileset) if mergedOutputFileset == None: workflow.addOutput(outputModuleName, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName, outputFileset, mergedOutputFileset) return self.topLevelSubscription
class WMBSMergeBySize(unittest.TestCase): def setUp(self): """ _setUp_ Boiler plate DB setup. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def stuffWMBS(self, injected=True): """ _stuffWMBS_ Insert some dummy jobs, jobgroups, filesets, files and subscriptions into WMBS to test job creation. Three completed job groups each containing several files are injected. Another incomplete job group is also injected. Also files are added to the "Mergeable" subscription as well as to the output fileset for their jobgroups. """ locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="T2_CH_CERN", pnn="T2_CH_CERN") locationAction.execute(siteName="T1_US_FNAL", pnn="T2_CH_CERN") changeStateDAO = self.daoFactory(classname="Jobs.ChangeState") self.mergeFileset = Fileset(name="mergeFileset") self.mergeFileset.create() self.bogusFileset = Fileset(name="bogusFileset") self.bogusFileset.create() self.mergeMergedFileset = Fileset(name="mergeMergedFileset") self.mergeMergedFileset.create() self.bogusMergedFileset = Fileset(name="bogusMergedFileset") self.bogusMergedFileset.create() mergeWorkflow = Workflow(name="mergeWorkflow", spec="bunk2", owner="Steve", task="Test") mergeWorkflow.create() markWorkflow = self.daoFactory(classname="Workflow.MarkInjectedWorkflows") markWorkflow.execute(names=[mergeWorkflow.name], injected=injected) self.mergeSubscription = Subscription(fileset=self.mergeFileset, workflow=mergeWorkflow, split_algo="WMBSMergeBySize") self.mergeSubscription.create() self.bogusSubscription = Subscription(fileset=self.bogusFileset, workflow=mergeWorkflow, split_algo="WMBSMergeBySize") inputFileset = Fileset(name="inputFileset") inputFileset.create() inputWorkflow = Workflow(name="inputWorkflow", spec="input", owner="Steve", task="Test") inputWorkflow.create() inputWorkflow.addOutput("output", self.mergeFileset, self.mergeMergedFileset) inputWorkflow.addOutput("output2", self.bogusFileset, self.bogusMergedFileset) bogusInputWorkflow = Workflow(name="bogusInputWorkflow", spec="input", owner="Steve", task="Test") bogusInputWorkflow.create() inputSubscription = Subscription(fileset=inputFileset, workflow=inputWorkflow) inputSubscription.create() bogusInputSubscription = Subscription(fileset=inputFileset, workflow=bogusInputWorkflow) bogusInputSubscription.create() parentFile1 = File(lfn="parentFile1") parentFile1.create() parentFile2 = File(lfn="parentFile2") parentFile2.create() parentFile3 = File(lfn="parentFile3") parentFile3.create() parentFile4 = File(lfn="parentFile4") parentFile4.create() self.parentFileSite2 = File(lfn="parentFileSite2") self.parentFileSite2.create() jobGroup1 = JobGroup(subscription=inputSubscription) jobGroup1.create() jobGroup2 = JobGroup(subscription=inputSubscription) jobGroup2.create() jobGroup3 = JobGroup(subscription=bogusInputSubscription) jobGroup3.create() testJob1 = Job() testJob1.addFile(parentFile1) testJob1.create(jobGroup1) testJob1["state"] = "cleanout" testJob1["oldstate"] = "new" testJob1["couch_record"] = "somejive" testJob1["retry_count"] = 0 testJob1["outcome"] = "success" testJob1.save() changeStateDAO.execute([testJob1]) testJob1A = Job() testJob1A.addFile(parentFile1) testJob1A.create(jobGroup3) testJob1A["state"] = "cleanout" testJob1A["oldstate"] = "new" testJob1A["couch_record"] = "somejive" testJob1A["retry_count"] = 0 testJob1A["outcome"] = "failure" testJob1A.save() changeStateDAO.execute([testJob1A]) testJob2 = Job() testJob2.addFile(parentFile2) testJob2.create(jobGroup1) testJob2["state"] = "cleanout" testJob2["oldstate"] = "new" testJob2["couch_record"] = "somejive" testJob2["retry_count"] = 0 testJob2["outcome"] = "success" testJob2.save() changeStateDAO.execute([testJob2]) testJob3 = Job() testJob3.addFile(parentFile3) testJob3.create(jobGroup2) testJob3["state"] = "cleanout" testJob3["oldstate"] = "new" testJob3["couch_record"] = "somejive" testJob3["retry_count"] = 0 testJob3["outcome"] = "success" testJob3.save() changeStateDAO.execute([testJob3]) testJob4 = Job() testJob4.addFile(parentFile4) testJob4.create(jobGroup2) testJob4["state"] = "cleanout" testJob4["oldstate"] = "new" testJob4["couch_record"] = "somejive" testJob4["retry_count"] = 0 testJob4["outcome"] = "failure" testJob4.save() changeStateDAO.execute([testJob4]) # We'll simulate a failed split by event job that the merger should # ignore. parentFile5 = File(lfn="parentFile5") parentFile5.create() testJob5 = Job() testJob5.addFile(parentFile5) testJob5.create(jobGroup2) testJob5["state"] = "cleanout" testJob5["oldstate"] = "new" testJob5["couch_record"] = "somejive" testJob5["retry_count"] = 0 testJob5["outcome"] = "success" testJob5.save() changeStateDAO.execute([testJob5]) testJob6 = Job() testJob6.addFile(parentFile5) testJob6.create(jobGroup2) testJob6["state"] = "cleanout" testJob6["oldstate"] = "new" testJob6["couch_record"] = "somejive" testJob6["retry_count"] = 0 testJob6["outcome"] = "failure" testJob6.save() changeStateDAO.execute([testJob6]) testJob7 = Job() testJob7.addFile(self.parentFileSite2) testJob7.create(jobGroup2) testJob7["state"] = "cleanout" testJob7["oldstate"] = "new" testJob7["couch_record"] = "somejive" testJob7["retry_count"] = 0 testJob7["outcome"] = "success" testJob7.save() changeStateDAO.execute([testJob7]) badFile1 = File(lfn="badFile1", size=10241024, events=10241024, first_event=0, locations={"T2_CH_CERN"}) badFile1.addRun(Run(1, *[45])) badFile1.create() badFile1.addParent(parentFile5["lfn"]) file1 = File(lfn="file1", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) file1.addRun(Run(1, *[45])) file1.create() file1.addParent(parentFile1["lfn"]) file2 = File(lfn="file2", size=1024, events=1024, first_event=1024, locations={"T2_CH_CERN"}) file2.addRun(Run(1, *[45])) file2.create() file2.addParent(parentFile1["lfn"]) file3 = File(lfn="file3", size=1024, events=1024, first_event=2048, locations={"T2_CH_CERN"}) file3.addRun(Run(1, *[45])) file3.create() file3.addParent(parentFile1["lfn"]) file4 = File(lfn="file4", size=1024, events=1024, first_event=3072, locations={"T2_CH_CERN"}) file4.addRun(Run(1, *[45])) file4.create() file4.addParent(parentFile1["lfn"]) fileA = File(lfn="fileA", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileA.addRun(Run(1, *[46])) fileA.create() fileA.addParent(parentFile2["lfn"]) fileB = File(lfn="fileB", size=1024, events=1024, first_event=1024, locations={"T2_CH_CERN"}) fileB.addRun(Run(1, *[46])) fileB.create() fileB.addParent(parentFile2["lfn"]) fileC = File(lfn="fileC", size=1024, events=1024, first_event=2048, locations={"T2_CH_CERN"}) fileC.addRun(Run(1, *[46])) fileC.create() fileC.addParent(parentFile2["lfn"]) fileI = File(lfn="fileI", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileI.addRun(Run(2, *[46])) fileI.create() fileI.addParent(parentFile3["lfn"]) fileII = File(lfn="fileII", size=1024, events=1024, first_event=1024, locations={"T2_CH_CERN"}) fileII.addRun(Run(2, *[46])) fileII.create() fileII.addParent(parentFile3["lfn"]) fileIII = File(lfn="fileIII", size=1024, events=1024, first_event=2048, locations={"T2_CH_CERN"}) fileIII.addRun(Run(2, *[46])) fileIII.create() fileIII.addParent(parentFile3["lfn"]) fileIV = File(lfn="fileIV", size=1024, events=1024, first_event=3072, locations={"T2_CH_CERN"}) fileIV.addRun(Run(2, *[46])) fileIV.create() fileIV.addParent(parentFile3["lfn"]) fileX = File(lfn="badFileA", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileX.addRun(Run(1, *[47])) fileX.create() fileX.addParent(parentFile4["lfn"]) fileY = File(lfn="badFileB", size=1024, events=1024, first_event=1024, locations={"T2_CH_CERN"}) fileY.addRun(Run(1, *[47])) fileY.create() fileY.addParent(parentFile4["lfn"]) fileZ = File(lfn="badFileC", size=1024, events=1024, first_event=2048, locations={"T2_CH_CERN"}) fileZ.addRun(Run(1, *[47])) fileZ.create() fileZ.addParent(parentFile4["lfn"]) jobGroup1.output.addFile(file1) jobGroup1.output.addFile(file2) jobGroup1.output.addFile(file3) jobGroup1.output.addFile(file4) jobGroup1.output.addFile(fileA) jobGroup1.output.addFile(fileB) jobGroup1.output.addFile(fileC) jobGroup1.output.commit() jobGroup2.output.addFile(fileI) jobGroup2.output.addFile(fileII) jobGroup2.output.addFile(fileIII) jobGroup2.output.addFile(fileIV) jobGroup2.output.addFile(fileX) jobGroup2.output.addFile(fileY) jobGroup2.output.addFile(fileZ) jobGroup2.output.addFile(badFile1) jobGroup2.output.commit() for fileObj in [file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII, fileIII, fileIV, fileX, fileY, fileZ, badFile1]: self.mergeFileset.addFile(fileObj) self.bogusFileset.addFile(fileObj) self.mergeFileset.commit() self.bogusFileset.commit() return def testMinMergeSize1(self): """ _testMinMergeSize1_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance. Verify that no merge jobs will be produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=20000, max_merge_size=2000000000, max_merge_events=200000000) assert len(result) == 0, \ "ERROR: No job groups should be returned." return def testMinMergeSize1a(self): """ _testMinMergeSize1a_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance and mark the fileset as closed. Verify that one job containing all files is pushed out. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=20000, max_merge_size=200000, max_merge_events=20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % len(result) assert len(result[0].jobs) == 2, \ "Error: Two jobs should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"] goldenFilesB = ["fileI", "fileII", "fileIII", "fileIV"] for job in result[0].jobs: self.assertEqual(job["possiblePSN"], {"T1_US_FNAL", "T2_CH_CERN"}) jobFiles = job.getFiles() if len(jobFiles) == len(goldenFilesA): self.assertEqual(job["estimatedDiskUsage"], 7) goldenFiles = goldenFilesA else: self.assertEqual(job["estimatedDiskUsage"], 4) goldenFiles = goldenFilesB currentRun = 0 currentLumi = 0 currentEvent = 0 for fileObj in jobFiles: fileObj.loadData() assert fileObj["lfn"] in goldenFiles, \ "Error: Unknown file: %s" % fileObj["lfn"] goldenFiles.remove(fileObj["lfn"]) fileRun = list(fileObj["runs"])[0].run fileLumi = min(list(fileObj["runs"])[0]) fileEvent = fileObj["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent return def testMinMergeSize2(self): """ _testMinMergeSize2_ Set the minimum merge size to be 7,167 bytes which is one byte less than the sum of all the file sizes in the largest merge group in the WMBS instance. Verify that one merge job containing all the files in the largest merge group is produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=7167, max_merge_size=20000, max_merge_events=20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %d" % len(result) assert len(result[0].jobs) == 1, \ "ERROR: One job should have been returned." self.assertEqual(result[0].jobs[0]["estimatedDiskUsage"], 7) self.assertEqual(result[0].jobs[0]["possiblePSN"], {"T1_US_FNAL", "T2_CH_CERN"}) jobFiles = list(result[0].jobs)[0].getFiles() goldenFiles = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"] assert len(jobFiles) == len(goldenFiles), \ "ERROR: Merge job should contain %d files." % len(goldenFiles) currentRun = 0 currentLumi = 0 currentEvent = 0 for fileObj in jobFiles: assert fileObj["lfn"] in goldenFiles, \ "Error: Unknown file: %s" % fileObj["lfn"] goldenFiles.remove(fileObj["lfn"]) fileRun = list(fileObj["runs"])[0].run fileLumi = min(list(fileObj["runs"])[0]) fileEvent = fileObj["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent return def testMaxMergeSize1(self): """ _testMaxMergeSize1_ Set the maximum merge size to be two bytes. Verify that three merge jobs are created, one for each job group that exists inside the WMBS instance. Verify that each merge job contains the expected files. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=1, max_merge_size=2, max_merge_events=20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned." self.assertEqual(result[0].jobs[0]["possiblePSN"], {"T1_US_FNAL", "T2_CH_CERN"}) goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: self.assertEqual(job["estimatedDiskUsage"], 4) goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: self.assertEqual(job["estimatedDiskUsage"], 3) goldenFiles = goldenFilesB else: self.assertEqual(job["estimatedDiskUsage"], 4) goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for fileObj in jobFiles: assert fileObj["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." goldenFiles.remove(fileObj["lfn"]) fileRun = list(fileObj["runs"])[0].run fileLumi = min(list(fileObj["runs"])[0]) fileEvent = fileObj["first_event"] if currentRun == 0: continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \ len(goldenFilesC) == 0, \ "ERROR: Files missing from merge jobs." return def testMaxMergeSize2(self): """ _testMaxMergeSize2_ Set the minimum merge size to be one byte larger than the largest job group in the WMBS instance and the max merge size to be one byte larger than the total size of two of the groups. Verify that one merge job is produced with two of the job groups in it. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=4097, max_merge_size=7169, max_merge_events=20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 1, \ "ERROR: One job should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] self.assertEqual(result[0].jobs[0]["estimatedDiskUsage"], 7) self.assertEqual(result[0].jobs[0]["possiblePSN"], {"T1_US_FNAL", "T2_CH_CERN"}) jobFiles = list(result[0].jobs)[0].getFiles() currentRun = 0 currentLumi = 0 currentEvent = 0 for fileObj in jobFiles: if fileObj["lfn"] in goldenFilesA: goldenFilesA.remove(fileObj["lfn"]) elif fileObj["lfn"] in goldenFilesB: goldenFilesB.remove(fileObj["lfn"]) elif fileObj["lfn"] in goldenFilesC: goldenFilesC.remove(fileObj["lfn"]) fileRun = list(fileObj["runs"])[0].run fileLumi = min(list(fileObj["runs"])[0]) fileEvent = fileObj["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesB) == 0 and \ (len(goldenFilesA) == 0 or len(goldenFilesC) == 0), \ "ERROR: Files not allocated to jobs correctly." return def testMaxEvents1(self): """ _testMaxEvents1_ Set the maximum number of events per merge job to 1. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=1, max_merge_size=20000, max_merge_events=1) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned: %s" % len(result[0].jobs) goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] for job in result[0].jobs: self.assertEqual(job["possiblePSN"], {"T1_US_FNAL", "T2_CH_CERN"}) jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: self.assertEqual(job["estimatedDiskUsage"], 4) goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: self.assertEqual(job["estimatedDiskUsage"], 3) goldenFiles = goldenFilesB else: self.assertEqual(job["estimatedDiskUsage"], 4) goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for fileObj in jobFiles: assert fileObj["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." goldenFiles.remove(fileObj["lfn"]) fileRun = list(fileObj["runs"])[0].run fileLumi = min(list(fileObj["runs"])[0]) fileEvent = fileObj["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun) if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \ len(goldenFilesC) == 0, \ "ERROR: Files missing from merge jobs." return def testMaxEvents2(self): """ _testMaxEvents2_ Set the minimum merge size to be one byte larger than the largest job group in the WMBS instance and the max events to be one event larger than the total events in two of the groups. Verify that one merge job is produced with two of the job groups in it. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=4097, max_merge_size=20000, max_merge_events=7169) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 1, \ "ERROR: One job should have been returned." self.assertEqual(result[0].jobs[0]["estimatedDiskUsage"], 7) self.assertEqual(result[0].jobs[0]["possiblePSN"], {"T1_US_FNAL", "T2_CH_CERN"}) goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] jobFiles = list(result[0].jobs)[0].getFiles() currentRun = 0 currentLumi = 0 currentEvent = 0 for fileObj in jobFiles: if fileObj["lfn"] in goldenFilesA: goldenFilesA.remove(fileObj["lfn"]) elif fileObj["lfn"] in goldenFilesB: goldenFilesB.remove(fileObj["lfn"]) elif fileObj["lfn"] in goldenFilesC: goldenFilesC.remove(fileObj["lfn"]) fileRun = list(fileObj["runs"])[0].run fileLumi = min(list(fileObj["runs"])[0]) fileEvent = fileObj["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesB) == 0 and \ (len(goldenFilesA) == 0 or len(goldenFilesC) == 0), \ "ERROR: Files not allocated to jobs correctly." return def testParallelProcessing(self): """ _testParallelProcessing_ Verify that merging works correctly when multiple processing subscriptions are run over the same input files. The merging algorithm should ignore processing jobs that feed into different merge subscriptions. """ locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="T2_CH_CERN", pnn="T2_CH_CERN") locationAction.execute(siteName="T1_US_FNAL", pnn="T2_CH_CERN") mergeFilesetA = Fileset(name="mergeFilesetA") mergeFilesetB = Fileset(name="mergeFilesetB") mergeFilesetA.create() mergeFilesetB.create() mergeMergedFilesetA = Fileset(name="mergeMergedFilesetA") mergeMergedFilesetB = Fileset(name="mergeMergedFilesetB") mergeMergedFilesetA.create() mergeMergedFilesetB.create() mergeWorkflow = Workflow(name="mergeWorkflow", spec="bogus", owner="Steve", task="Test") mergeWorkflow.create() mergeSubscriptionA = Subscription(fileset=mergeFilesetA, workflow=mergeWorkflow, split_algo="WMBSMergeBySize") mergeSubscriptionB = Subscription(fileset=mergeFilesetB, workflow=mergeWorkflow, split_algo="WMBSMergeBySize") mergeSubscriptionA.create() mergeSubscriptionB.create() inputFileset = Fileset(name="inputFileset") inputFileset.create() inputFileA = File(lfn="inputLFNA") inputFileB = File(lfn="inputLFNB") inputFileA.create() inputFileB.create() procWorkflowA = Workflow(name="procWorkflowA", spec="bunk2", owner="Steve", task="Test") procWorkflowA.create() procWorkflowA.addOutput("output", mergeFilesetA, mergeMergedFilesetA) procWorkflowB = Workflow(name="procWorkflowB", spec="bunk3", owner="Steve", task="Test2") procWorkflowB.create() procWorkflowB.addOutput("output", mergeFilesetB, mergeMergedFilesetB) procSubscriptionA = Subscription(fileset=inputFileset, workflow=procWorkflowA, split_algo="EventBased") procSubscriptionA.create() procSubscriptionB = Subscription(fileset=inputFileset, workflow=procWorkflowB, split_algo="EventBased") procSubscriptionB.create() jobGroupA = JobGroup(subscription=procSubscriptionA) jobGroupA.create() jobGroupB = JobGroup(subscription=procSubscriptionB) jobGroupB.create() changeStateDAO = self.daoFactory(classname="Jobs.ChangeState") testJobA = Job() testJobA.addFile(inputFileA) testJobA.create(jobGroupA) testJobA["state"] = "cleanout" testJobA["oldstate"] = "new" testJobA["couch_record"] = "somejive" testJobA["retry_count"] = 0 testJobA["outcome"] = "success" testJobA.save() testJobB = Job() testJobB.addFile(inputFileB) testJobB.create(jobGroupA) testJobB["state"] = "cleanout" testJobB["oldstate"] = "new" testJobB["couch_record"] = "somejive" testJobB["retry_count"] = 0 testJobB["outcome"] = "success" testJobB.save() testJobC = Job() testJobC.addFile(inputFileA) testJobC.create(jobGroupB) testJobC["state"] = "cleanout" testJobC["oldstate"] = "new" testJobC["couch_record"] = "somejive" testJobC["retry_count"] = 0 testJobC["outcome"] = "success" testJobC.save() testJobD = Job() testJobD.addFile(inputFileA) testJobD.create(jobGroupB) testJobD["state"] = "cleanout" testJobD["oldstate"] = "new" testJobD["couch_record"] = "somejive" testJobD["retry_count"] = 0 testJobD["outcome"] = "failure" testJobD.save() testJobE = Job() testJobE.addFile(inputFileB) testJobE.create(jobGroupB) testJobE["state"] = "cleanout" testJobE["oldstate"] = "new" testJobE["couch_record"] = "somejive" testJobE["retry_count"] = 0 testJobE["outcome"] = "success" testJobE.save() testJobF = Job() testJobF.addFile(inputFileB) testJobF.create(jobGroupB) testJobF["state"] = "cleanout" testJobF["oldstate"] = "new" testJobF["couch_record"] = "somejive" testJobF["retry_count"] = 0 testJobF["outcome"] = "failure" testJobF.save() changeStateDAO.execute([testJobA, testJobB, testJobC, testJobD, testJobE, testJobF]) fileA = File(lfn="fileA", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileA.addRun(Run(1, *[45])) fileA.create() fileA.addParent(inputFileA["lfn"]) fileB = File(lfn="fileB", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileB.addRun(Run(1, *[45])) fileB.create() fileB.addParent(inputFileB["lfn"]) jobGroupA.output.addFile(fileA) jobGroupA.output.addFile(fileB) jobGroupA.output.commit() mergeFilesetA.addFile(fileA) mergeFilesetA.addFile(fileB) mergeFilesetA.commit() fileC = File(lfn="fileC", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileC.addRun(Run(1, *[45])) fileC.create() fileC.addParent(inputFileA["lfn"]) fileD = File(lfn="fileD", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileD.addRun(Run(1, *[45])) fileD.create() fileD.addParent(inputFileB["lfn"]) jobGroupB.output.addFile(fileC) jobGroupB.output.addFile(fileD) mergeFilesetB.addFile(fileC) mergeFilesetB.addFile(fileD) mergeFilesetB.commit() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=mergeSubscriptionB) result = jobFactory(min_merge_size=1, max_merge_size=20000, max_merge_events=7169) assert len(result) == 0, \ "Error: No merge jobs should have been created." fileE = File(lfn="fileE", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileE.addRun(Run(1, *[45])) fileE.create() fileE.addParent(inputFileA["lfn"]) fileF = File(lfn="fileF", size=1024, events=1024, first_event=0, locations={"T2_CH_CERN"}) fileF.addRun(Run(1, *[45])) fileF.create() fileF.addParent(inputFileB["lfn"]) jobGroupB.output.addFile(fileE) jobGroupB.output.addFile(fileF) mergeFilesetB.addFile(fileE) mergeFilesetB.addFile(fileF) mergeFilesetB.commit() testJobD["outcome"] = "success" testJobD.save() testJobF["outcome"] = "success" testJobF.save() changeStateDAO.execute([testJobD, testJobF]) result = jobFactory(min_merge_size=1, max_merge_size=20000, max_merge_events=7169) assert len(result) == 1, \ "Error: One merge job should have been created: %s" % len(result) return def testLocationMerging(self): """ _testLocationMerging_ Verify that files residing on different SEs are not merged together in the same job. """ self.stuffWMBS() locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="T1_UK_RAL", pnn="T1_UK_RAL_Disk") fileSite2 = File(lfn="fileSite2", size=4098, events=1024, first_event=0, locations={"T1_UK_RAL_Disk"}) fileSite2.addRun(Run(1, *[46])) fileSite2.create() fileSite2.addParent(self.parentFileSite2["lfn"]) self.mergeFileset.addFile(fileSite2) self.mergeFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=4097, max_merge_size=99999999, max_merge_events=999999999) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 2, \ "ERROR: Two jobs should have been returned." ralJobs = 0 fnalcernJobs = 0 for job in result[0].jobs: if job["possiblePSN"] == {"T1_UK_RAL"}: ralJobs += 1 elif job["possiblePSN"] == {"T1_US_FNAL", "T2_CH_CERN"}: fnalcernJobs += 1 self.assertEqual(ralJobs, 1) self.assertEqual(fnalcernJobs, 1) return def testFilesetCloseout(self): """ _testFilesetCloseout_ Verify that the merge algorithm works correctly when it's input fileset is closed. The split algorithm should create merge jobs for all files regardless of size and then mark any orphaned files (files that are the result of a split by lumi / split by event where one of the parent processing jobs has failed while others have succeeded) as failed so that the fileset closing works. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) # Get out all the good merge jobs out of the way. result = jobFactory(min_merge_size=1, max_merge_size=999999999999, max_merge_events=999999999) # Verify that the bad files are the only "available" files availableAction = self.daoFactory(classname="Subscriptions.GetAvailableFilesMeta") availFiles = availableAction.execute(self.mergeSubscription["id"]) assert len(availFiles) == 4, \ "Error: Wrong number of available files." goldenFiles = ["badFile1", "badFileA", "badFileB", "badFileC"] for availFile in availFiles: assert availFile["lfn"] in goldenFiles, \ "Error: Extra file is available." goldenFiles.remove(availFile["lfn"]) self.mergeFileset.markOpen(False) result = jobFactory(min_merge_size=1, max_merge_size=999999999999, max_merge_events=999999999) assert len(result) == 0, \ "Error: Merging should have returned zero jobs." self.mergeFileset.markOpen(False) availFiles2 = availableAction.execute(self.mergeSubscription["id"]) assert len(availFiles2) == 0, \ "Error: There should be no more available files." failedAction = self.daoFactory(classname="Subscriptions.GetFailedFiles") failedFiles = failedAction.execute(self.mergeSubscription["id"]) assert len(failedFiles) == 4, \ "Error: Wrong number of failed files: %s" % failedFiles goldenIDs = [] for availFile in availFiles: goldenIDs.append(availFile["id"]) for failedFile in failedFiles: assert failedFile["file"] in goldenIDs, \ "Error: Extra failed file." return def testFilesetCloseout2(self): """ _testFilesetCloseout2_ Verify that the fail orphan file code does not fail files that have failed for other workflows. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) # Get out all the good merge jobs out of the way. result = jobFactory(min_merge_size=1, max_merge_size=999999999999, max_merge_events=999999999) self.assertEqual(len(result), 1, "Error: Wrong number of job groups.") self.assertEqual(len(result[0].jobs), 2, "Error: Wrong number of jobs.") failedAction = self.daoFactory(classname="Subscriptions.GetFailedFiles") failedFiles = failedAction.execute(self.mergeSubscription["id"]) self.assertEqual(len(failedFiles), 4, "Error: Wrong number of failed files: %s" % failedFiles) return def testForcedMerge(self): """ _testForcedMerge_ Repeat testMinMergeSize1a, but with non-injected files to assert that this causes no jobgroups to be created. """ self.stuffWMBS(injected=False) self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=20000, max_merge_size=200000, max_merge_events=20000) self.assertEqual(len(result), 0) return
class WMBSMergeBySize(unittest.TestCase): def setUp(self): """ _setUp_ Boiler plate DB setup. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def stuffWMBS(self, injected = True): """ _stuffWMBS_ Insert some dummy jobs, jobgroups, filesets, files and subscriptions into WMBS to test job creation. Three completed job groups each containing several files are injected. Another incomplete job group is also injected. Also files are added to the "Mergeable" subscription as well as to the output fileset for their jobgroups. """ locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "s1", seName = "somese.cern.ch") changeStateDAO = self.daoFactory(classname = "Jobs.ChangeState") self.mergeFileset = Fileset(name = "mergeFileset") self.mergeFileset.create() self.bogusFileset = Fileset(name = "bogusFileset") self.bogusFileset.create() self.mergeMergedFileset = Fileset(name = "mergeMergedFileset") self.mergeMergedFileset.create() self.bogusMergedFileset = Fileset(name = "bogusMergedFileset") self.bogusMergedFileset.create() mergeWorkflow = Workflow(name = "mergeWorkflow", spec = "bunk2", owner = "Steve", task="Test") mergeWorkflow.create() markWorkflow = self.daoFactory(classname = "Workflow.MarkInjectedWorkflows") markWorkflow.execute(names = [mergeWorkflow.name], injected = injected) self.mergeSubscription = Subscription(fileset = self.mergeFileset, workflow = mergeWorkflow, split_algo = "WMBSMergeBySize") self.mergeSubscription.create() self.bogusSubscription = Subscription(fileset = self.bogusFileset, workflow = mergeWorkflow, split_algo = "WMBSMergeBySize") inputFileset = Fileset(name = "inputFileset") inputFileset.create() inputWorkflow = Workflow(name = "inputWorkflow", spec = "input", owner = "Steve", task = "Test") inputWorkflow.create() inputWorkflow.addOutput("output", self.mergeFileset, self.mergeMergedFileset) inputWorkflow.addOutput("output2", self.bogusFileset, self.bogusMergedFileset) bogusInputWorkflow = Workflow(name = "bogusInputWorkflow", spec = "input", owner = "Steve", task = "Test") bogusInputWorkflow.create() inputSubscription = Subscription(fileset = inputFileset, workflow = inputWorkflow) inputSubscription.create() bogusInputSubscription = Subscription(fileset = inputFileset, workflow = bogusInputWorkflow) bogusInputSubscription.create() parentFile1 = File(lfn = "parentFile1") parentFile1.create() parentFile2 = File(lfn = "parentFile2") parentFile2.create() parentFile3 = File(lfn = "parentFile3") parentFile3.create() parentFile4 = File(lfn = "parentFile4") parentFile4.create() self.parentFileSite2 = File(lfn = "parentFileSite2") self.parentFileSite2.create() jobGroup1 = JobGroup(subscription = inputSubscription) jobGroup1.create() jobGroup2 = JobGroup(subscription = inputSubscription) jobGroup2.create() jobGroup3 = JobGroup(subscription = bogusInputSubscription) jobGroup3.create() testJob1 = Job() testJob1.addFile(parentFile1) testJob1.create(jobGroup1) testJob1["state"] = "cleanout" testJob1["oldstate"] = "new" testJob1["couch_record"] = "somejive" testJob1["retry_count"] = 0 testJob1["outcome"] = "success" testJob1.save() changeStateDAO.execute([testJob1]) testJob1A = Job() testJob1A.addFile(parentFile1) testJob1A.create(jobGroup3) testJob1A["state"] = "cleanout" testJob1A["oldstate"] = "new" testJob1A["couch_record"] = "somejive" testJob1A["retry_count"] = 0 testJob1A["outcome"] = "failure" testJob1A.save() changeStateDAO.execute([testJob1A]) testJob2 = Job() testJob2.addFile(parentFile2) testJob2.create(jobGroup1) testJob2["state"] = "cleanout" testJob2["oldstate"] = "new" testJob2["couch_record"] = "somejive" testJob2["retry_count"] = 0 testJob2["outcome"] = "success" testJob2.save() changeStateDAO.execute([testJob2]) testJob3 = Job() testJob3.addFile(parentFile3) testJob3.create(jobGroup2) testJob3["state"] = "cleanout" testJob3["oldstate"] = "new" testJob3["couch_record"] = "somejive" testJob3["retry_count"] = 0 testJob3["outcome"] = "success" testJob3.save() changeStateDAO.execute([testJob3]) testJob4 = Job() testJob4.addFile(parentFile4) testJob4.create(jobGroup2) testJob4["state"] = "cleanout" testJob4["oldstate"] = "new" testJob4["couch_record"] = "somejive" testJob4["retry_count"] = 0 testJob4["outcome"] = "failure" testJob4.save() changeStateDAO.execute([testJob4]) # We'll simulate a failed split by event job that the merger should # ignore. parentFile5 = File(lfn = "parentFile5") parentFile5.create() testJob5 = Job() testJob5.addFile(parentFile5) testJob5.create(jobGroup2) testJob5["state"] = "cleanout" testJob5["oldstate"] = "new" testJob5["couch_record"] = "somejive" testJob5["retry_count"] = 0 testJob5["outcome"] = "success" testJob5.save() changeStateDAO.execute([testJob5]) testJob6 = Job() testJob6.addFile(parentFile5) testJob6.create(jobGroup2) testJob6["state"] = "cleanout" testJob6["oldstate"] = "new" testJob6["couch_record"] = "somejive" testJob6["retry_count"] = 0 testJob6["outcome"] = "failure" testJob6.save() changeStateDAO.execute([testJob6]) testJob7 = Job() testJob7.addFile(self.parentFileSite2) testJob7.create(jobGroup2) testJob7["state"] = "cleanout" testJob7["oldstate"] = "new" testJob7["couch_record"] = "somejive" testJob7["retry_count"] = 0 testJob7["outcome"] = "success" testJob7.save() changeStateDAO.execute([testJob7]) badFile1 = File(lfn = "badFile1", size = 10241024, events = 10241024, first_event = 0, locations = set(["somese.cern.ch"])) badFile1.addRun(Run(1, *[45])) badFile1.create() badFile1.addParent(parentFile5["lfn"]) file1 = File(lfn = "file1", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) file1.addRun(Run(1, *[45])) file1.create() file1.addParent(parentFile1["lfn"]) file2 = File(lfn = "file2", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) file2.addRun(Run(1, *[45])) file2.create() file2.addParent(parentFile1["lfn"]) file3 = File(lfn = "file3", size = 1024, events = 1024, first_event = 2048, locations = set(["somese.cern.ch"])) file3.addRun(Run(1, *[45])) file3.create() file3.addParent(parentFile1["lfn"]) file4 = File(lfn = "file4", size = 1024, events = 1024, first_event = 3072, locations = set(["somese.cern.ch"])) file4.addRun(Run(1, *[45])) file4.create() file4.addParent(parentFile1["lfn"]) fileA = File(lfn = "fileA", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileA.addRun(Run(1, *[46])) fileA.create() fileA.addParent(parentFile2["lfn"]) fileB = File(lfn = "fileB", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) fileB.addRun(Run(1, *[46])) fileB.create() fileB.addParent(parentFile2["lfn"]) fileC = File(lfn = "fileC", size = 1024, events = 1024, first_event = 2048, locations = set(["somese.cern.ch"])) fileC.addRun(Run(1, *[46])) fileC.create() fileC.addParent(parentFile2["lfn"]) fileI = File(lfn = "fileI", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileI.addRun(Run(2, *[46])) fileI.create() fileI.addParent(parentFile3["lfn"]) fileII = File(lfn = "fileII", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) fileII.addRun(Run(2, *[46])) fileII.create() fileII.addParent(parentFile3["lfn"]) fileIII = File(lfn = "fileIII", size = 1024, events = 1024, first_event = 2048, locations = set(["somese.cern.ch"])) fileIII.addRun(Run(2, *[46])) fileIII.create() fileIII.addParent(parentFile3["lfn"]) fileIV = File(lfn = "fileIV", size = 1024, events = 1024, first_event = 3072, locations = set(["somese.cern.ch"])) fileIV.addRun(Run(2, *[46])) fileIV.create() fileIV.addParent(parentFile3["lfn"]) fileX = File(lfn = "badFileA", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileX.addRun(Run(1, *[47])) fileX.create() fileX.addParent(parentFile4["lfn"]) fileY = File(lfn = "badFileB", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) fileY.addRun(Run(1, *[47])) fileY.create() fileY.addParent(parentFile4["lfn"]) fileZ = File(lfn = "badFileC", size = 1024, events = 1024, first_event = 2048, locations = set(["somese.cern.ch"])) fileZ.addRun(Run(1, *[47])) fileZ.create() fileZ.addParent(parentFile4["lfn"]) jobGroup1.output.addFile(file1) jobGroup1.output.addFile(file2) jobGroup1.output.addFile(file3) jobGroup1.output.addFile(file4) jobGroup1.output.addFile(fileA) jobGroup1.output.addFile(fileB) jobGroup1.output.addFile(fileC) jobGroup1.output.commit() jobGroup2.output.addFile(fileI) jobGroup2.output.addFile(fileII) jobGroup2.output.addFile(fileIII) jobGroup2.output.addFile(fileIV) jobGroup2.output.addFile(fileX) jobGroup2.output.addFile(fileY) jobGroup2.output.addFile(fileZ) jobGroup2.output.addFile(badFile1) jobGroup2.output.commit() for file in [file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII, fileIII, fileIV, fileX, fileY, fileZ, badFile1]: self.mergeFileset.addFile(file) self.bogusFileset.addFile(file) self.mergeFileset.commit() self.bogusFileset.commit() return def testMinMergeSize1(self): """ _testMinMergeSize1_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance. Verify that no merge jobs will be produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 20000, max_merge_size = 2000000000, max_merge_events = 200000000) assert len(result) == 0, \ "ERROR: No job groups should be returned." return def testMinMergeSize1a(self): """ _testMinMergeSize1a_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance and mark the fileset as closed. Verify that one job containing all files is pushed out. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 20000, max_merge_size = 200000, max_merge_events = 20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % len(result) assert len(result[0].jobs) == 2, \ "Error: Two jobs should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"] goldenFilesB = ["fileI", "fileII", "fileIII", "fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if len(jobFiles) == len(goldenFilesA): goldenFiles = goldenFilesA else: goldenFiles = goldenFilesB currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: file.loadData() assert file["lfn"] in goldenFiles, \ "Error: Unknown file: %s" % file["lfn"] assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent return def testMinMergeSize2(self): """ _testMinMergeSize2_ Set the minimum merge size to be 7,167 bytes which is one byte less than the sum of all the file sizes in the largest merge group in the WMBS instance. Verify that one merge job containing all the files in the largest merge group is produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 7167, max_merge_size = 20000, max_merge_events = 20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %d" % len(result) assert len(result[0].jobs) == 1, \ "ERROR: One job should have been returned." jobFiles = list(result[0].jobs)[0].getFiles() goldenFiles = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"] assert len(jobFiles) == len(goldenFiles), \ "ERROR: Merge job should contain %d files." % len(goldenFiles) currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file: %s" % file["lfn"] assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent return def testMaxMergeSize1(self): """ _testMaxMergeSize1_ Set the maximum merge size to be two bytes. Verify that three merge jobs are created, one for each job group that exists inside the WMBS instance. Verify that each merge job contains the expected files. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 1, max_merge_size = 2, max_merge_events = 20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB else: goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \ len(goldenFilesC) == 0, \ "ERROR: Files missing from merge jobs." return def testMaxMergeSize2(self): """ _testMaxMergeSize2_ Set the minimum merge size to be one byte larger than the largest job group in the WMBS instance and the max merge size to be one byte larger than the total size of two of the groups. Verify that one merge job is produced with two of the job groups in it. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 4097, max_merge_size = 7169, max_merge_events = 20000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 1, \ "ERROR: One job should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] jobFiles = list(result[0].jobs)[0].getFiles() currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." if file["lfn"] in goldenFilesA: goldenFilesA.remove(file["lfn"]) elif file["lfn"] in goldenFilesB: goldenFilesB.remove(file["lfn"]) elif file["lfn"] in goldenFilesC: goldenFilesC.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesB) == 0 and \ (len(goldenFilesA) == 0 or len(goldenFilesC) == 0), \ "ERROR: Files not allocated to jobs correctly." return def testMaxEvents1(self): """ _testMaxEvents1_ Set the maximum number of events per merge job to 1. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 1, max_merge_size = 20000, max_merge_events = 1) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned: %s" % len(result[0].jobs) goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB else: goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location: %s" % file["locations"] goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun) if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \ len(goldenFilesC) == 0, \ "ERROR: Files missing from merge jobs." return def testMaxEvents2(self): """ _testMaxEvents2_ Set the minimum merge size to be one byte larger than the largest job group in the WMBS instance and the max events to be one event larger than the total events in two of the groups. Verify that one merge job is produced with two of the job groups in it. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 4097, max_merge_size = 20000, max_merge_events = 7169) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 1, \ "ERROR: One job should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4"] goldenFilesB = ["fileA", "fileB", "fileC"] goldenFilesC = ["fileI", "fileII", "fileIII", "fileIV"] jobFiles = list(result[0].jobs)[0].getFiles() currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." if file["lfn"] in goldenFilesA: goldenFilesA.remove(file["lfn"]) elif file["lfn"] in goldenFilesB: goldenFilesB.remove(file["lfn"]) elif file["lfn"] in goldenFilesC: goldenFilesC.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesB) == 0 and \ (len(goldenFilesA) == 0 or len(goldenFilesC) == 0), \ "ERROR: Files not allocated to jobs correctly." return def testParallelProcessing(self): """ _testParallelProcessing_ Verify that merging works correctly when multiple processing subscriptions are run over the same input files. The merging algorithm should ignore processing jobs that feed into different merge subscriptions. """ locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "s1", seName = "somese.cern.ch") mergeFilesetA = Fileset(name = "mergeFilesetA") mergeFilesetB = Fileset(name = "mergeFilesetB") mergeFilesetA.create() mergeFilesetB.create() mergeMergedFilesetA = Fileset(name = "mergeMergedFilesetA") mergeMergedFilesetB = Fileset(name = "mergeMergedFilesetB") mergeMergedFilesetA.create() mergeMergedFilesetB.create() mergeWorkflow = Workflow(name = "mergeWorkflow", spec = "bogus", owner = "Steve", task = "Test") mergeWorkflow.create() mergeSubscriptionA = Subscription(fileset = mergeFilesetA, workflow = mergeWorkflow, split_algo = "WMBSMergeBySize") mergeSubscriptionB = Subscription(fileset = mergeFilesetB, workflow = mergeWorkflow, split_algo = "WMBSMergeBySize") mergeSubscriptionA.create() mergeSubscriptionB.create() inputFileset = Fileset(name = "inputFileset") inputFileset.create() inputFileA = File(lfn = "inputLFNA") inputFileB = File(lfn = "inputLFNB") inputFileA.create() inputFileB.create() procWorkflowA = Workflow(name = "procWorkflowA", spec = "bunk2", owner = "Steve", task = "Test") procWorkflowA.create() procWorkflowA.addOutput("output", mergeFilesetA, mergeMergedFilesetA) procWorkflowB = Workflow(name = "procWorkflowB", spec = "bunk3", owner = "Steve", task = "Test2") procWorkflowB.create() procWorkflowB.addOutput("output", mergeFilesetB, mergeMergedFilesetB) procSubscriptionA = Subscription(fileset = inputFileset, workflow = procWorkflowA, split_algo = "EventBased") procSubscriptionA.create() procSubscriptionB = Subscription(fileset = inputFileset, workflow = procWorkflowB, split_algo = "EventBased") procSubscriptionB.create() jobGroupA = JobGroup(subscription = procSubscriptionA) jobGroupA.create() jobGroupB = JobGroup(subscription = procSubscriptionB) jobGroupB.create() changeStateDAO = self.daoFactory(classname = "Jobs.ChangeState") testJobA = Job() testJobA.addFile(inputFileA) testJobA.create(jobGroupA) testJobA["state"] = "cleanout" testJobA["oldstate"] = "new" testJobA["couch_record"] = "somejive" testJobA["retry_count"] = 0 testJobA["outcome"] = "success" testJobA.save() testJobB = Job() testJobB.addFile(inputFileB) testJobB.create(jobGroupA) testJobB["state"] = "cleanout" testJobB["oldstate"] = "new" testJobB["couch_record"] = "somejive" testJobB["retry_count"] = 0 testJobB["outcome"] = "success" testJobB.save() testJobC = Job() testJobC.addFile(inputFileA) testJobC.create(jobGroupB) testJobC["state"] = "cleanout" testJobC["oldstate"] = "new" testJobC["couch_record"] = "somejive" testJobC["retry_count"] = 0 testJobC["outcome"] = "success" testJobC.save() testJobD = Job() testJobD.addFile(inputFileA) testJobD.create(jobGroupB) testJobD["state"] = "cleanout" testJobD["oldstate"] = "new" testJobD["couch_record"] = "somejive" testJobD["retry_count"] = 0 testJobD["outcome"] = "failure" testJobD.save() testJobE = Job() testJobE.addFile(inputFileB) testJobE.create(jobGroupB) testJobE["state"] = "cleanout" testJobE["oldstate"] = "new" testJobE["couch_record"] = "somejive" testJobE["retry_count"] = 0 testJobE["outcome"] = "success" testJobE.save() testJobF = Job() testJobF.addFile(inputFileB) testJobF.create(jobGroupB) testJobF["state"] = "cleanout" testJobF["oldstate"] = "new" testJobF["couch_record"] = "somejive" testJobF["retry_count"] = 0 testJobF["outcome"] = "failure" testJobF.save() changeStateDAO.execute([testJobA, testJobB, testJobC, testJobD, testJobE, testJobF]) fileA = File(lfn = "fileA", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileA.addRun(Run(1, *[45])) fileA.create() fileA.addParent(inputFileA["lfn"]) fileB = File(lfn = "fileB", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileB.addRun(Run(1, *[45])) fileB.create() fileB.addParent(inputFileB["lfn"]) jobGroupA.output.addFile(fileA) jobGroupA.output.addFile(fileB) jobGroupA.output.commit() mergeFilesetA.addFile(fileA) mergeFilesetA.addFile(fileB) mergeFilesetA.commit() fileC = File(lfn = "fileC", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileC.addRun(Run(1, *[45])) fileC.create() fileC.addParent(inputFileA["lfn"]) fileD = File(lfn = "fileD", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileD.addRun(Run(1, *[45])) fileD.create() fileD.addParent(inputFileB["lfn"]) jobGroupB.output.addFile(fileC) jobGroupB.output.addFile(fileD) mergeFilesetB.addFile(fileC) mergeFilesetB.addFile(fileD) mergeFilesetB.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = mergeSubscriptionB) result = jobFactory(min_merge_size = 1, max_merge_size = 20000, max_merge_events = 7169) assert len(result) == 0, \ "Error: No merge jobs should have been created." fileE = File(lfn = "fileE", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileE.addRun(Run(1, *[45])) fileE.create() fileE.addParent(inputFileA["lfn"]) fileF = File(lfn = "fileF", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileF.addRun(Run(1, *[45])) fileF.create() fileF.addParent(inputFileB["lfn"]) jobGroupB.output.addFile(fileE) jobGroupB.output.addFile(fileF) mergeFilesetB.addFile(fileE) mergeFilesetB.addFile(fileF) mergeFilesetB.commit() testJobD["outcome"] = "success" testJobD.save() testJobF["outcome"] = "success" testJobF.save() changeStateDAO.execute([testJobD, testJobF]) result = jobFactory(min_merge_size = 1, max_merge_size = 20000, max_merge_events = 7169) assert len(result) == 1, \ "Error: One merge job should have been created: %s" % len(result) return def testLocationMerging(self): """ _testLocationMerging_ Verify that files residing on different SEs are not merged together in the same job. """ self.stuffWMBS() locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "s2", seName = "somese2.cern.ch") fileSite2 = File(lfn = "fileSite2", size = 4098, events = 1024, first_event = 0, locations = set(["somese2.cern.ch"])) fileSite2.addRun(Run(1, *[46])) fileSite2.create() fileSite2.addParent(self.parentFileSite2["lfn"]) self.mergeFileset.addFile(fileSite2) self.mergeFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999, max_merge_events = 999999999) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 2, \ "ERROR: Two jobs should have been returned." for job in result[0].jobs: firstInputFile = job.getFiles()[0] baseLocation = list(firstInputFile["locations"])[0] for inputFile in job.getFiles(): assert len(inputFile["locations"]) == 1, \ "Error: Wrong number of locations" assert list(inputFile["locations"])[0] == baseLocation, \ "Error: Wrong location." return def testFilesetCloseout(self): """ _testFilesetCloseout_ Verify that the merge algorithm works correctly when it's input fileset is closed. The split algorithm should create merge jobs for all files regardless of size and then mark any orphaned files (files that are the result of a split by lumi / split by event where one of the parent processing jobs has failed while others have succeeded) as failed so that the fileset closing works. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) # Get out all the good merge jobs out of the way. result = jobFactory(min_merge_size = 1, max_merge_size = 999999999999, max_merge_events = 999999999) # Verify that the bad files are the only "available" files availableAction = self.daoFactory(classname = "Subscriptions.GetAvailableFilesMeta") availFiles = availableAction.execute(self.mergeSubscription["id"]) assert len(availFiles) == 4, \ "Error: Wrong number of available files." goldenFiles = ["badFile1", "badFileA", "badFileB", "badFileC"] for availFile in availFiles: assert availFile["lfn"] in goldenFiles, \ "Error: Extra file is available." goldenFiles.remove(availFile["lfn"]) self.mergeFileset.markOpen(False) result = jobFactory(min_merge_size = 1, max_merge_size = 999999999999, max_merge_events = 999999999) assert len(result) == 0, \ "Error: Merging should have returned zero jobs." self.mergeFileset.markOpen(False) availFiles2 = availableAction.execute(self.mergeSubscription["id"]) assert len(availFiles2) == 0, \ "Error: There should be no more available files." failedAction = self.daoFactory(classname = "Subscriptions.GetFailedFiles") failedFiles = failedAction.execute(self.mergeSubscription["id"]) assert len(failedFiles) == 4, \ "Error: Wrong number of failed files: %s" % failedFiles goldenIDs = [] for availFile in availFiles: goldenIDs.append(availFile["id"]) for failedFile in failedFiles: assert failedFile["file"] in goldenIDs, \ "Error: Extra failed file." return def testFilesetCloseout2(self): """ _testFilesetCloseout2_ Verify that the fail orphan file code does not fail files that have failed for other workflows. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) # Get out all the good merge jobs out of the way. result = jobFactory(min_merge_size = 1, max_merge_size = 999999999999, max_merge_events = 999999999) self.assertEqual(len(result), 1, "Error: Wrong number of job groups.") self.assertEqual(len(result[0].jobs), 2, "Error: Wrong number of jobs.") failedAction = self.daoFactory(classname = "Subscriptions.GetFailedFiles") failedFiles = failedAction.execute(self.mergeSubscription["id"]) self.assertEqual(len(failedFiles), 4, "Error: Wrong number of failed files: %s" % failedFiles) return def testForcedMerge(self): """ _testForcedMerge_ Repeat testMinMergeSize1a, but with non-injected files to assert that this causes no jobgroups to be created. """ self.stuffWMBS(injected = False) self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 20000, max_merge_size = 200000, max_merge_events = 20000) self.assertEqual(len(result), 0) return
def createTestJobGroup( self, config, name="TestWorkthrough", specLocation="spec.xml", error=False, task="/TestWorkload/ReReco" ): """ Creates a group of several jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=specLocation, owner="Simon", name=name, task=task) testWorkflow.create() testWMBSFileset = Fileset(name=name) testWMBSFileset.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation("malpaquet") testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(10, *[12312])) testFileB.setLocation("malpaquet") testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() for i in range(0, self.nJobs): testJob = Job(name=makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob["retry_count"] = 1 testJob["retry_max"] = 10 testJob["mask"].addRunAndLumis(run=10, lumis=[12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() changer = ChangeState(config) report1 = Report() report2 = Report() if error: path1 = os.path.join( WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl" ) path2 = path1 else: path1 = os.path.join( WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/TaskArchiver_t/fwjrs", "mergeReport1.pkl" ) path2 = os.path.join( WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/TaskArchiver_t/fwjrs", "mergeReport2.pkl" ) report1.load(filename=path1) report2.load(filename=path2) changer.propagate(testJobGroup.jobs, "created", "new") changer.propagate(testJobGroup.jobs, "executing", "created") changer.propagate(testJobGroup.jobs, "complete", "executing") for i in range(self.nJobs): if i < self.nJobs / 2: testJobGroup.jobs[i]["fwjr"] = report1 else: testJobGroup.jobs[i]["fwjr"] = report2 changer.propagate(testJobGroup.jobs, "jobfailed", "complete") changer.propagate(testJobGroup.jobs, "exhausted", "jobfailed") changer.propagate(testJobGroup.jobs, "cleanout", "exhausted") testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup
def createTestJobGroup(self, config, name="TestWorkthrough", filesetName="TestFileset", specLocation="spec.xml", error=False, task="/TestWorkload/ReReco", multicore=False): """ Creates a group of several jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=specLocation, owner=self.OWNERDN, name=name, task=task, owner_vogroup="", owner_vorole="") testWorkflow.create() self.inject.execute(names=[name], injected=True) testWMBSFileset = Fileset(name=filesetName) testWMBSFileset.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) outputWMBSFileset = Fileset(name='%sOutput' % filesetName) outputWMBSFileset.create() testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10) testFileC.addRun(Run(10, *[12312])) testFileC.setLocation('malpaquet') testFileC.create() outputWMBSFileset.addFile(testFileC) outputWMBSFileset.commit() outputWMBSFileset.markOpen(0) testWorkflow.addOutput('output', outputWMBSFileset) testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() for i in range(0, self.nJobs): testJob = Job(name=makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() changer = ChangeState(config) report1 = Report() report2 = Report() if error: path1 = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') elif multicore: path1 = os.path.join( WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl") path2 = path1 else: path1 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'mergeReport1.pkl') path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') report1.load(filename=path1) report2.load(filename=path2) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') for i in range(self.nJobs): if i < self.nJobs / 2: testJobGroup.jobs[i]['fwjr'] = report1 else: testJobGroup.jobs[i]['fwjr'] = report2 changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed') changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup
def testMultipleRunHarvesting(self): """ _testMultipleRunHarvesting_ Add some files with multiple runs in each, make sure the jobs are created by location and run. Verify each job mask afterwards. Note that in this test run are splitted between sites, in real life that MUST NOT happen we still don't support that. """ multipleFilesFileset = Fileset(name = "TestFileset") newFile = File("/some/file/test1", size = 1000, events = 100) newFile.addRun(Run(1,*[1,3,4,5,6,7])) newFile.addRun(Run(2,*[1,2,4,5,6,7])) newFile.setLocation('SomeSE') multipleFilesFileset.addFile(newFile) newFile = File("/some/file/test2", size = 1000, events = 100) newFile.addRun(Run(1,*[2,8])) newFile.addRun(Run(2,*[3,8])) newFile.setLocation('SomeSE3') multipleFilesFileset.addFile(newFile) multipleFilesFileset.create() harvestingWorkflow = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset = multipleFilesFileset, workflow = harvestingWorkflow, split_algo = "Harvest", type = "Harvesting") harvestSub.create() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSub) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask") self.finishJobs(jobGroups, harvestSub) newFile = File("/some/file/test3", size = 1000, events = 100) newFile.addRun(Run(1,*range(9,15))) newFile.setLocation('SomeSE3') multipleFilesFileset.addFile(newFile) multipleFilesFileset.commit() time.sleep(2) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask") harvestingWorkflowSib = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflowSib", task="TestSib") harvestingWorkflowSib.create() harvestSubSib = Subscription(fileset = multipleFilesFileset, workflow = harvestingWorkflowSib, split_algo = "Harvest", type = "Harvesting") harvestSubSib.create() jobFactorySib = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSubSib) multipleFilesFileset.markOpen(False) jobGroups = jobFactorySib(periodic_harvest_sibling = True) self.assertEqual(len(jobGroups), 0, "A single job group was created") self.finishJobs(jobGroups, harvestSub) jobGroups = jobFactorySib(periodic_harvest_sibling = True) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
class ParentlessMergeBySizeTest(unittest.TestCase): def setUp(self): """ _setUp_ Boiler plate DB setup. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def stuffWMBS(self): """ _stuffWMBS_ Insert some dummy jobs, jobgroups, filesets, files and subscriptions into WMBS to test job creation. Three completed job groups each containing several files are injected. Another incomplete job group is also injected. Also files are added to the "Mergeable" subscription as well as to the output fileset for their jobgroups. """ locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="s1", seName="somese.cern.ch") locationAction.execute(siteName="s1", seName="somese2.cern.ch") changeStateDAO = self.daoFactory(classname="Jobs.ChangeState") self.mergeFileset = Fileset(name="mergeFileset") self.mergeFileset.create() self.bogusFileset = Fileset(name="bogusFileset") self.bogusFileset.create() mergeWorkflow = Workflow(name="mergeWorkflow", spec="bunk2", owner="Steve", task="Test") mergeWorkflow.create() markWorkflow = self.daoFactory( classname="Workflow.MarkInjectedWorkflows") markWorkflow.execute(names=[mergeWorkflow.name], injected=True) self.mergeSubscription = Subscription( fileset=self.mergeFileset, workflow=mergeWorkflow, split_algo="ParentlessMergeBySize") self.mergeSubscription.create() self.bogusSubscription = Subscription( fileset=self.bogusFileset, workflow=mergeWorkflow, split_algo="ParentlessMergeBySize") file1 = File(lfn="file1", size=1024, events=1024, first_event=0, locations=set(["somese.cern.ch"])) file1.addRun(Run(1, *[45])) file1.create() file2 = File(lfn="file2", size=1024, events=1024, first_event=1024, locations=set(["somese.cern.ch"])) file2.addRun(Run(1, *[45])) file2.create() file3 = File(lfn="file3", size=1024, events=1024, first_event=2048, locations=set(["somese.cern.ch"])) file3.addRun(Run(1, *[45])) file3.create() file4 = File(lfn="file4", size=1024, events=1024, first_event=3072, locations=set(["somese.cern.ch"])) file4.addRun(Run(1, *[45])) file4.create() fileA = File(lfn="fileA", size=1024, events=1024, first_event=0, locations=set(["somese.cern.ch"])) fileA.addRun(Run(1, *[46])) fileA.create() fileB = File(lfn="fileB", size=1024, events=1024, first_event=1024, locations=set(["somese.cern.ch"])) fileB.addRun(Run(1, *[46])) fileB.create() fileC = File(lfn="fileC", size=1024, events=1024, first_event=2048, locations=set(["somese.cern.ch"])) fileC.addRun(Run(1, *[46])) fileC.create() fileI = File(lfn="fileI", size=1024, events=1024, first_event=0, locations=set(["somese.cern.ch"])) fileI.addRun(Run(2, *[46])) fileI.create() fileII = File(lfn="fileII", size=1024, events=1024, first_event=1024, locations=set(["somese.cern.ch"])) fileII.addRun(Run(2, *[46])) fileII.create() fileIII = File(lfn="fileIII", size=1024, events=102400, first_event=2048, locations=set(["somese.cern.ch"])) fileIII.addRun(Run(2, *[46])) fileIII.create() fileIV = File(lfn="fileIV", size=102400, events=1024, first_event=3072, locations=set(["somese.cern.ch"])) fileIV.addRun(Run(2, *[46])) fileIV.create() for file in [ file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII, fileIII, fileIV ]: self.mergeFileset.addFile(file) self.bogusFileset.addFile(file) self.mergeFileset.commit() self.bogusFileset.commit() return def testMinMergeSize1(self): """ _testMinMergeSize1_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance. Verify that no merge jobs will be produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=200000, max_merge_size=2000000000, max_merge_events=200000000) assert len(result) == 0, \ "ERROR: No job groups should be returned." return def testMinMergeSize1(self): """ _testMinMergeSize1_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance. Verify that no merge jobs will be produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=200000, max_merge_size=2000000000, max_merge_events=200000000) assert len(result) == 0, \ "ERROR: No job groups should be returned." return def testMinMergeSize1a(self): """ _testMinMergeSize1a_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance and mark the fileset as closed. Verify that one job containing all files is pushed out. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=200000, max_merge_size=2000000, max_merge_events=2000000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % len(result) assert len(result[0].jobs) == 1, \ "Error: One job should have been returned: %s" % len(result[0].jobs) goldenFiles = [ "file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", "fileI", "fileII", "fileIII", "fileIV" ] jobFiles = result[0].jobs[0].getFiles() currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: file.loadData() assert file["lfn"] in goldenFiles, \ "Error: Unknown file: %s" % file["lfn"] self.assertTrue( file["locations"] == set(["somese.cern.ch", "somese2.cern.ch"]), "Error: File is missing a location.") goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent return def testMaxMergeSize(self): """ _testMaxMergeSize_ Set the maximum merge size to be 100000 bytes. Verify that two merge jobs are created, one for the one large file and another for the rest of the files. Verify that each merge job contains the expected files and that we merge across runs. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=1, max_merge_size=100000, max_merge_events=200000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 2, \ "ERROR: Two jobs should have been returned." goldenFilesA = [ "file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", "fileI", "fileII", "fileIII" ] goldenFilesB = ["fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0, \ "ERROR: Files missing from merge jobs." return def testMaxEvents(self): """ _testMaxEvents_ Verify the the max_merge_events parameter works and that we correctly merge across runs. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=1, max_merge_size=20000000, max_merge_events=100000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 2, \ "ERROR: Two jobs should have been returned: %s" % len(result[0].jobs) goldenFilesA = [ "file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", "fileI", "fileII", "fileIV" ] goldenFilesB = ["fileIII"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location: %s" % file["locations"] goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun) if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \ "ERROR: Files missing from merge jobs." return def testMinMergeSize1aNoRunMerge(self): """ _testMinMergeSize1aNoRunMerge_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance and mark the fileset as closed. Verify that two jobs are pushed out and that we don't merge accross run boundaries. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=200000, max_merge_size=2000000, max_merge_events=2000000, merge_across_runs=False) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % len(result) assert len(result[0].jobs) == 2, \ "Error: Two jobs should have been returned: %s" % len(result[0].jobs) goldenFilesA = [ "file1", "file2", "file3", "file4", "fileA", "fileB", "fileC" ] goldenFilesB = ["fileI", "fileII", "fileIII", "fileIV"] goldenFilesA.sort() goldenFilesB.sort() for job in result[0].jobs: currentRun = 0 currentLumi = 0 currentEvent = 0 jobLFNs = [] for file in job.getFiles(): file.loadData() jobLFNs.append(file["lfn"]) self.assertTrue( file["locations"] == set( ["somese.cern.ch", "somese2.cern.ch"]), "Error: File is missing a location.") fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent jobLFNs.sort() if jobLFNs == goldenFilesA: goldenFilesA = [] else: self.assertEqual(jobLFNs, goldenFilesB, "Error: LFNs do not match.") goldenFilesB = [] return def testMaxMergeSizeNoRunMerge(self): """ _testMaxMergeSizeNoRunMerge_ Set the maximum merge size to be 100000 bytes. Verify that two merge jobs are created, one for the one large file and another for the rest of the files. Verify that each merge job contains the expected files and that we don't merge across run boundaries. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=1, max_merge_size=100000, max_merge_events=200000, merge_across_runs=False) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned." goldenFilesA = [ "file1", "file2", "file3", "file4", "fileA", "fileB", "fileC" ] goldenFilesB = ["fileI", "fileII", "fileIII"] goldenFilesC = ["fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB else: goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: self.assertTrue(file["lfn"] in goldenFiles, "Error: Unknown file in merge jobs.") self.assertTrue(file["locations"] == set(["somese.cern.ch"]), "Error: File is missing a location.") goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue self.assertTrue(fileRun >= currentRun, "ERROR: Files not sorted by run.") if fileRun == currentRun: self.assertTrue(fileLumi >= currentLumi, "ERROR: Files not ordered by lumi") if fileLumi == currentLumi: self.assertTrue(fileEvent >= currentEvent, "ERROR: Files not ordered by first event") currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent self.assertTrue( len(goldenFilesA) == 0 and len(goldenFilesB) == 0, "ERROR: Files missing from merge jobs.") return def testMaxEventsNoRunMerge(self): """ _testMaxEventsNoRunMerge_ Verify that the max events merge parameter works correctly and that we don't merge accross run boundaries. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=1, max_merge_size=20000000, max_merge_events=100000, merge_across_runs=False) self.assertTrue( len(result) == 1, "ERROR: More than one JobGroup returned: %s" % result) self.assertTrue( len(result[0].jobs) == 3, "ERROR: Three jobs should have been returned: %s" % len(result[0].jobs)) goldenFilesA = [ "file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", ] goldenFilesB = ["fileI", "fileII", "fileIV"] goldenFilesC = ["fileIII"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB else: goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: self.assertTrue(file["lfn"] in goldenFiles, "Error: Unknown file in merge jobs.") self.assertTrue( file["locations"] == set(["somese.cern.ch"]), "Error: File is missing a location: %s" % file["locations"]) goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue self.assertTrue( fileRun >= currentRun, "ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun)) if fileRun == currentRun: self.assertTrue(fileLumi >= currentLumi, "ERROR: Files not ordered by lumi") if fileLumi == currentLumi: self.assertTrue( fileEvent >= currentEvent, "ERROR: Files not ordered by first event") currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent self.assertTrue( len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and len(goldenFilesC) == 0, "ERROR: Files missing from merge jobs.") return def testLocationMerging(self): """ _testLocationMerging_ Verify that files residing on different SEs are not merged together in the same job. """ self.stuffWMBS() locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="s2", seName="somese2.cern.ch") fileSite2 = File(lfn="fileSite2", size=4098, events=1024, first_event=0, locations=set(["somese2.cern.ch"])) fileSite2.addRun(Run(1, *[46])) fileSite2.create() self.mergeFileset.addFile(fileSite2) self.mergeFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=4097, max_merge_size=99999999, max_merge_events=999999999, merge_across_runs=False) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned." for job in result[0].jobs: firstInputFile = job.getFiles()[0] baseLocation = list(firstInputFile["locations"])[0] for inputFile in job.getFiles(): assert len(inputFile["locations"]) == 1, \ "Error: Wrong number of locations" assert list(inputFile["locations"])[0] == baseLocation, \ "Error: Wrong location." return def testMaxWaitTime(self): """ _testMaxWaitTime_ Set the max wait times to be negative - this should force all files to merge out immediately Using the first setup as the first merge test which should normally produce no jobGroups """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=200000, max_merge_size=2000000000, max_merge_events=200000000, max_wait_time=-10) # Everything should be in one, small jobGroup self.assertEqual(len(result), 1) self.assertEqual(len(result[0].jobs), 1) job = result[0].jobs[0] # All files should be in one job self.assertEqual(len(job.getFiles()), 11) return def testDifferentSubscritionIDs(self): """ _testDifferentSubscriptionIDs_ Make sure that the merge splitting still runs if the subscription ID is not equal to the workflow ID. """ myThread = threading.currentThread() myThread.transaction.begin() dummyWorkflow = Workflow(name="dummyWorkflow", spec="bunk49", owner="Steve", task="Test2") dummyWorkflow.create() dummyFileset = Fileset(name="dummyFileset") dummyFileset.create() dummySubscription1 = Subscription(fileset=dummyFileset, workflow=dummyWorkflow, split_algo="ParentlessMergeBySize") dummySubscription2 = Subscription(fileset=dummyFileset, workflow=dummyWorkflow, split_algo="ParentlessMergeBySize") dummySubscription1.create() dummySubscription2.create() myThread.transaction.commit() self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=self.mergeSubscription) result = jobFactory(min_merge_size=4097, max_merge_size=99999999, max_merge_events=999999999, merge_across_runs=False) self.assertEqual(len(result), 1) jobGroup = result[0] self.assertEqual(len(jobGroup.jobs), 2) return
class ParentlessMergeBySizeTest(unittest.TestCase): def setUp(self): """ _setUp_ Boiler plate DB setup. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def stuffWMBS(self): """ _stuffWMBS_ Insert some dummy jobs, jobgroups, filesets, files and subscriptions into WMBS to test job creation. Three completed job groups each containing several files are injected. Another incomplete job group is also injected. Also files are added to the "Mergeable" subscription as well as to the output fileset for their jobgroups. """ locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "s1", seName = "somese.cern.ch") changeStateDAO = self.daoFactory(classname = "Jobs.ChangeState") self.mergeFileset = Fileset(name = "mergeFileset") self.mergeFileset.create() self.bogusFileset = Fileset(name = "bogusFileset") self.bogusFileset.create() mergeWorkflow = Workflow(name = "mergeWorkflow", spec = "bunk2", owner = "Steve", task="Test") mergeWorkflow.create() markWorkflow = self.daoFactory(classname = "Workflow.MarkInjectedWorkflows") markWorkflow.execute(names = [mergeWorkflow.name], injected = True) self.mergeSubscription = Subscription(fileset = self.mergeFileset, workflow = mergeWorkflow, split_algo = "ParentlessMergeBySize") self.mergeSubscription.create() self.bogusSubscription = Subscription(fileset = self.bogusFileset, workflow = mergeWorkflow, split_algo = "ParentlessMergeBySize") file1 = File(lfn = "file1", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) file1.addRun(Run(1, *[45])) file1.create() file2 = File(lfn = "file2", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) file2.addRun(Run(1, *[45])) file2.create() file3 = File(lfn = "file3", size = 1024, events = 1024, first_event = 2048, locations = set(["somese.cern.ch"])) file3.addRun(Run(1, *[45])) file3.create() file4 = File(lfn = "file4", size = 1024, events = 1024, first_event = 3072, locations = set(["somese.cern.ch"])) file4.addRun(Run(1, *[45])) file4.create() fileA = File(lfn = "fileA", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileA.addRun(Run(1, *[46])) fileA.create() fileB = File(lfn = "fileB", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) fileB.addRun(Run(1, *[46])) fileB.create() fileC = File(lfn = "fileC", size = 1024, events = 1024, first_event = 2048, locations = set(["somese.cern.ch"])) fileC.addRun(Run(1, *[46])) fileC.create() fileI = File(lfn = "fileI", size = 1024, events = 1024, first_event = 0, locations = set(["somese.cern.ch"])) fileI.addRun(Run(2, *[46])) fileI.create() fileII = File(lfn = "fileII", size = 1024, events = 1024, first_event = 1024, locations = set(["somese.cern.ch"])) fileII.addRun(Run(2, *[46])) fileII.create() fileIII = File(lfn = "fileIII", size = 1024, events = 102400, first_event = 2048, locations = set(["somese.cern.ch"])) fileIII.addRun(Run(2, *[46])) fileIII.create() fileIV = File(lfn = "fileIV", size = 102400, events = 1024, first_event = 3072, locations = set(["somese.cern.ch"])) fileIV.addRun(Run(2, *[46])) fileIV.create() for file in [file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII, fileIII, fileIV]: self.mergeFileset.addFile(file) self.bogusFileset.addFile(file) self.mergeFileset.commit() self.bogusFileset.commit() return def testMinMergeSize1(self): """ _testMinMergeSize1_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance. Verify that no merge jobs will be produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 200000, max_merge_size = 2000000000, max_merge_events = 200000000) assert len(result) == 0, \ "ERROR: No job groups should be returned." return def testMinMergeSize1(self): """ _testMinMergeSize1_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance. Verify that no merge jobs will be produced. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 200000, max_merge_size = 2000000000, max_merge_events = 200000000) assert len(result) == 0, \ "ERROR: No job groups should be returned." return def testMinMergeSize1a(self): """ _testMinMergeSize1a_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance and mark the fileset as closed. Verify that one job containing all files is pushed out. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 200000, max_merge_size = 2000000, max_merge_events = 2000000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % len(result) assert len(result[0].jobs) == 1, \ "Error: One job should have been returned: %s" % len(result[0].jobs) goldenFiles = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", "fileI", "fileII", "fileIII", "fileIV"] jobFiles = result[0].jobs[0].getFiles() currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: file.loadData() assert file["lfn"] in goldenFiles, \ "Error: Unknown file: %s" % file["lfn"] assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent return def testMaxMergeSize(self): """ _testMaxMergeSize_ Set the maximum merge size to be 100000 bytes. Verify that two merge jobs are created, one for the one large file and another for the rest of the files. Verify that each merge job contains the expected files and that we merge across runs. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 1, max_merge_size = 100000, max_merge_events = 200000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 2, \ "ERROR: Two jobs should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", "fileI", "fileII", "fileIII"] goldenFilesB = ["fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0, \ "ERROR: Files missing from merge jobs." return def testMaxEvents(self): """ _testMaxEvents_ Verify the the max_merge_events parameter works and that we correctly merge across runs. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 1, max_merge_size = 20000000, max_merge_events = 100000) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 2, \ "ERROR: Two jobs should have been returned: %s" % len(result[0].jobs) goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC", "fileI", "fileII", "fileIV"] goldenFilesB = ["fileIII"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: assert file["lfn"] in goldenFiles, \ "Error: Unknown file in merge jobs." assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location: %s" % file["locations"] goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun) if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \ "ERROR: Files missing from merge jobs." return def testMinMergeSize1aNoRunMerge(self): """ _testMinMergeSize1aNoRunMerge_ Set the minimum merge size to be 20,000 bytes which is more than the sum of all file sizes in the WMBS instance and mark the fileset as closed. Verify that two jobs are pushed out and that we don't merge accross run boundaries. """ self.stuffWMBS() self.mergeFileset.markOpen(False) splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 200000, max_merge_size = 2000000, max_merge_events = 2000000, merge_across_runs = False) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % len(result) assert len(result[0].jobs) == 2, \ "Error: Two jobs should have been returned: %s" % len(result[0].jobs) goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"] goldenFilesB = ["fileI", "fileII", "fileIII", "fileIV"] goldenFilesA.sort() goldenFilesB.sort() for job in result[0].jobs: currentRun = 0 currentLumi = 0 currentEvent = 0 jobLFNs = [] for file in job.getFiles(): file.loadData() jobLFNs.append(file["lfn"]) assert file["locations"] == set(["somese.cern.ch"]), \ "Error: File is missing a location." fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue assert fileRun >= currentRun, \ "ERROR: Files not sorted by run." if fileRun == currentRun: assert fileLumi >= currentLumi, \ "ERROR: Files not ordered by lumi" if fileLumi == currentLumi: assert fileEvent >= currentEvent, \ "ERROR: Files not ordered by first event" currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent jobLFNs.sort() if jobLFNs == goldenFilesA: goldenFilesA = [] else: self.assertEqual(jobLFNs, goldenFilesB, "Error: LFNs do not match.") goldenFilesB = [] return def testMaxMergeSizeNoRunMerge(self): """ _testMaxMergeSizeNoRunMerge_ Set the maximum merge size to be 100000 bytes. Verify that two merge jobs are created, one for the one large file and another for the rest of the files. Verify that each merge job contains the expected files and that we don't merge across run boundaries. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 1, max_merge_size = 100000, max_merge_events = 200000, merge_across_runs = False) assert len(result) == 1, \ "ERROR: More than one JobGroup returned: %s" % result assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned." goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"] goldenFilesB = ["fileI", "fileII", "fileIII"] goldenFilesC = ["fileIV"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB else: goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: self.assertTrue(file["lfn"] in goldenFiles, "Error: Unknown file in merge jobs.") self.assertTrue(file["locations"] == set(["somese.cern.ch"]), "Error: File is missing a location.") goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue self.assertTrue(fileRun >= currentRun, "ERROR: Files not sorted by run.") if fileRun == currentRun: self.assertTrue(fileLumi >= currentLumi, "ERROR: Files not ordered by lumi") if fileLumi == currentLumi: self.assertTrue(fileEvent >= currentEvent, "ERROR: Files not ordered by first event") currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent self.assertTrue(len(goldenFilesA) == 0 and len(goldenFilesB) == 0, "ERROR: Files missing from merge jobs.") return def testMaxEventsNoRunMerge(self): """ _testMaxEventsNoRunMerge_ Verify that the max events merge parameter works correctly and that we don't merge accross run boundaries. """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 1, max_merge_size = 20000000, max_merge_events = 100000, merge_across_runs = False) self.assertTrue(len(result) == 1, "ERROR: More than one JobGroup returned: %s" % result) self.assertTrue(len(result[0].jobs) == 3, "ERROR: Three jobs should have been returned: %s" % len(result[0].jobs)) goldenFilesA = ["file1", "file2", "file3", "file4", "fileA", "fileB", "fileC",] goldenFilesB = ["fileI", "fileII", "fileIV"] goldenFilesC = ["fileIII"] for job in result[0].jobs: jobFiles = job.getFiles() if jobFiles[0]["lfn"] in goldenFilesA: goldenFiles = goldenFilesA elif jobFiles[0]["lfn"] in goldenFilesB: goldenFiles = goldenFilesB else: goldenFiles = goldenFilesC currentRun = 0 currentLumi = 0 currentEvent = 0 for file in jobFiles: self.assertTrue(file["lfn"] in goldenFiles, "Error: Unknown file in merge jobs.") self.assertTrue(file["locations"] == set(["somese.cern.ch"]), "Error: File is missing a location: %s" % file["locations"]) goldenFiles.remove(file["lfn"]) fileRun = list(file["runs"])[0].run fileLumi = min(list(file["runs"])[0]) fileEvent = file["first_event"] if currentRun == 0: currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent continue self.assertTrue(fileRun >= currentRun, "ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun)) if fileRun == currentRun: self.assertTrue(fileLumi >= currentLumi, "ERROR: Files not ordered by lumi") if fileLumi == currentLumi: self.assertTrue(fileEvent >= currentEvent, "ERROR: Files not ordered by first event") currentRun = fileRun currentLumi = fileLumi currentEvent = fileEvent self.assertTrue(len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and len(goldenFilesC) == 0, "ERROR: Files missing from merge jobs.") return def testLocationMerging(self): """ _testLocationMerging_ Verify that files residing on different SEs are not merged together in the same job. """ self.stuffWMBS() locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "s2", seName = "somese2.cern.ch") fileSite2 = File(lfn = "fileSite2", size = 4098, events = 1024, first_event = 0, locations = set(["somese2.cern.ch"])) fileSite2.addRun(Run(1, *[46])) fileSite2.create() self.mergeFileset.addFile(fileSite2) self.mergeFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999, max_merge_events = 999999999, merge_across_runs = False) assert len(result) == 1, \ "ERROR: More than one JobGroup returned." assert len(result[0].jobs) == 3, \ "ERROR: Three jobs should have been returned." for job in result[0].jobs: firstInputFile = job.getFiles()[0] baseLocation = list(firstInputFile["locations"])[0] for inputFile in job.getFiles(): assert len(inputFile["locations"]) == 1, \ "Error: Wrong number of locations" assert list(inputFile["locations"])[0] == baseLocation, \ "Error: Wrong location." return def testMaxWaitTime(self): """ _testMaxWaitTime_ Set the max wait times to be negative - this should force all files to merge out immediately Using the first setup as the first merge test which should normally produce no jobGroups """ self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 200000, max_merge_size = 2000000000, max_merge_events = 200000000, max_wait_time = -10) # Everything should be in one, small jobGroup self.assertEqual(len(result), 1) self.assertEqual(len(result[0].jobs), 1) job = result[0].jobs[0] # All files should be in one job self.assertEqual(len(job.getFiles()), 11) return def testDifferentSubscritionIDs(self): """ _testDifferentSubscriptionIDs_ Make sure that the merge splitting still runs if the subscription ID is not equal to the workflow ID. """ myThread = threading.currentThread() myThread.transaction.begin() dummyWorkflow = Workflow(name = "dummyWorkflow", spec = "bunk49", owner = "Steve", task="Test2") dummyWorkflow.create() dummyFileset = Fileset(name = "dummyFileset") dummyFileset.create() dummySubscription1 = Subscription(fileset = dummyFileset, workflow = dummyWorkflow, split_algo = "ParentlessMergeBySize") dummySubscription2 = Subscription(fileset = dummyFileset, workflow = dummyWorkflow, split_algo = "ParentlessMergeBySize") dummySubscription1.create() dummySubscription2.create() myThread.transaction.commit() self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999, max_merge_events = 999999999, merge_across_runs = False) self.assertEqual(len(result), 1) jobGroup = result[0] self.assertEqual(len(jobGroup.jobs), 2) return
def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() # FIXME: Let workflow put in values if spec is missing them workflow = Workflow( spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"], dn=self.wmSpec.getOwner().get("dn", "unknown"), group=self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"), name=self.wmSpec.name(), task=task.getPathName(), wfType=self.wmSpec.getDashboardActivity(), alternativeFilesetClose=alternativeFilesetClose, priority=self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset=fileset, workflow=workflow, split_algo=task.jobSplittingAlgorithm(), type=task.getPrimarySubType()) if subscription.exists(): subscription.load() msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)" self.logger.info(msg % (subscription['id'], task.getPathName())) else: subscription.create() for site in task.siteWhitelist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": True }]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": False }]) if self.topLevelSubscription is None: self.topLevelSubscription = subscription logging.info("Top level subscription created: %s", subscription["id"]) else: logging.info("Child subscription created: %s", subscription["id"]) outputModules = task.getOutputModulesForTask() ignoredOutputModules = task.getIgnoredOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): if outputModuleName in ignoredOutputModules: logging.info( "IgnoredOutputModule set for %s, skipping fileset creation.", outputModuleName) continue outputFileset = Fileset( self.outputFilesetName(task, outputModuleName)) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: if childTask.taskType() == "Merge": mergedOutputFileset = Fileset( self.outputFilesetName(childTask, "Merged")) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr( getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset != None: self.mergeOutputMapping[ mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS( childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset is None: workflow.addOutput(outputModuleName, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName, outputFileset, mergedOutputFileset) return self.topLevelSubscription
class PeriodicTest(unittest.TestCase): def setUp(self): """ _setUp_ Create a single subscription with one file. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "site1", seName = "somese.cern.ch") locationAction.execute(siteName = "site2", seName = "otherse.cern.ch") self.testFileset = Fileset(name = "TestFileset1") self.testFileset.create() testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task="Test" ) testWorkflow.create() self.testSubscription = Subscription(fileset = self.testFileset, workflow = testWorkflow, split_algo = "Periodic", type = "Processing") self.testSubscription.create() return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def injectFile(self): """ _injectFile_ Inject a file into the periodic splitting input fileset. """ testFile = File(lfn = "/this/is/a/lfn%s" % time.time(), size = 1000, events = 100, locations = set(["somese.cern.ch"])) testFile.create() self.testFileset.addFile(testFile) self.testFileset.commit() return def verifyFiles(self, wmbsJob): """ _verifyFiles_ Verify that the input files for the job are the same as the files in the input fileset. """ inputFiles = wmbsJob.getFiles() filesetFiles = self.testFileset.getFiles() for inputFile in inputFiles: assert inputFile in filesetFiles, \ "ERROR: Unknown file: %s" % inputFile filesetFiles.remove(inputFile) assert len(filesetFiles) == 0, \ "ERROR: Not all files included in job." return def testPeriodicSplitting(self): """ _testPeriodiciSplitting_ Manipulate the splitting algorithm to test the corner cases. """ splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.testSubscription) # First pass: no jobs exist. The algorithm should create a job # containing all available files. self.injectFile() jobGroups = jobFactory(job_period = 99999999999) assert len(jobGroups) == 1, \ "ERROR: Wrong number of job groups returned: %s" % len(jobGroups) assert len(jobGroups[0].jobs) == 1, \ "ERROR: Jobgroup has wrong number of jobs: %s" % len(jobGroups[0].jobs) wmbsJob = jobGroups[0].jobs.pop() self.verifyFiles(wmbsJob) # Verify that no jobs are generated as the previously issued job has not # completed yet. time.sleep(5) self.injectFile() moreJobGroups = jobFactory(job_period = 1) assert len(moreJobGroups) == 0, \ "ERROR: No jobgroups should be returned." # Complete the job so that the splitting algorithm will generate # another job. wmbsJob["state"] = "cleanout" wmbsJob["oldstate"] = "new" wmbsJob["couch_record"] = "somejive" wmbsJob["retry_count"] = 0 changeStateDAO = self.daoFactory(classname = "Jobs.ChangeState") changeStateDAO.execute([wmbsJob]) # Verify that no jobs will be generated if the period has not yet # expried. self.injectFile() moreJobGroups = jobFactory(job_period = 999999999999) assert len(moreJobGroups) == 0, \ "ERROR: No jobgroups should be returned." # Verify that a job will be generated if the period has expired. time.sleep(5) self.injectFile() jobGroups = jobFactory(job_period = 1) assert len(jobGroups) == 1, \ "ERROR: Wrong number of job groups returned: %s" % len(jobGroups) assert len(jobGroups[0].jobs) == 1, \ "ERROR: Jobgroup has wrong number of jobs: %s" % len(jobGroups[0].jobs) self.verifyFiles(jobGroups[0].jobs.pop()) # Verify that no jobs will be generated in the case that a periodic job # is still running and the fileset has been closed. self.testFileset.markOpen(False) time.sleep(5) self.injectFile() jobGroups = jobFactory(job_period = 1) assert len(jobGroups) == 0, \ "ERROR: Wrong number of job groups returned: %s" % len(jobGroups) # Complete the outstanding job. wmbsJob["state"] = "cleanout" wmbsJob["oldstate"] = "new" wmbsJob["couch_record"] = "somejive" wmbsJob["retry_count"] = 0 changeStateDAO.execute([wmbsJob]) # Verify that when the input fileset is closed and all periodic jobs # are complete a job will not be generated. self.injectFile() jobGroups = jobFactory(job_period = 99999999999) assert len(jobGroups) == 0, \ "ERROR: Wrong number of job groups returned: %s" % len(jobGroups) # Verify that after the final job is complete no more jobs are generated. wmbsJob["state"] = "cleanout" wmbsJob["oldstate"] = "new" wmbsJob["couch_record"] = "somejive" wmbsJob["retry_count"] = 0 changeStateDAO.execute([wmbsJob]) time.sleep(5) self.injectFile() moreJobGroups = jobFactory(job_period = 1) assert len(moreJobGroups) == 0, \ "ERROR: No jobgroups should be returned." return
class HarvestTest(unittest.TestCase): """ _HarvestTest_ Test for EndOfRun job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE") myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE2", "SomeCE") myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE3", "SomeCE2") self.fileset1 = Fileset(name = "TestFileset1") for file in range(11): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Harvest", type = "Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = 'wmagent_jobdump' return config def finishJobs(self, jobGroups, subscription = None): """ _finishJobs_ """ if not subscription: subscription = self.subscription1 for f in subscription.acquiredFiles(): subscription.completeFiles(f) for jobGroup in jobGroups: self.changer.propagate(jobGroup.jobs, 'executing', 'created') self.changer.propagate(jobGroup.jobs, 'complete', 'executing') self.changer.propagate(jobGroup.jobs, 'success', 'complete') self.changer.propagate(jobGroup.jobs, 'cleanout', 'success') return def testHarvestEndOfRunTrigger(self): """ _testDQMHarvestEndOfRunTrigger_ Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset """ self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't") jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 0 , "We got 1 or more jobGroups with an open fileset and no periodic configuration") self.fileset1.markOpen(False) self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?") # We should also check if there are aqcuired files, if there are, there are jobs, # we don't want to fire another jobs while previous are running (output is integrating whatever input) # TODO : The above one we can do when all is done. Not priority jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1 , "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed") return def testPeriodicTrigger(self): """ _testPeriodicTrigger_ """ self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here") # Test timeout (5s for this first test) # there should be no acquired files, if there are, shouldn't be a job #self.subscription1.acquireFiles(self.subscription1.availableFiles().pop()) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 3) self.assertEqual(len(jobGroups), 1 , "Didn't created the first periodic job when there were acquired files") # For the whole thing to work, faking the first job finishing, and putting the files as complete self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for file in range(12,24): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.commit() # Testing that it doesn't create a job unless the delay is past jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 0 , "Created one or more job, when there were non-acquired file and the period is not passed by") time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, and there weren't and the period is passed by") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for file in range(26,36): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.commit() # Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 0 , "Created one or more job, there are new files, but the delay is not past") time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, there are new files and the delay is past") # Last check is whether the job gets all the files or not numFilesJob = jobGroups[0].jobs[0].getFiles() numFilesFileset = self.fileset1.getFiles() self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding files for the first location for file in range(38,48): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.commit() # Then another location for file in range(50,56): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE3') self.fileset1.addFile(newFile) self.fileset1.commit() # We should have jobs in both locations time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups[0].getJobs()), 2 , "We didn't get 2 jobs for 2 locations") firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0] secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0] self.assertEqual(firstJobLocation, 'SomeSite', "First job location is not SomeSite") self.assertEqual(secondJobLocation, 'SomeSite2', "Second job location is not SomeSite2") self.finishJobs(jobGroups) for file in range(60,65): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(2,*[2])) newFile.setLocation('SomeSE3') self.fileset1.addFile(newFile) self.fileset1.commit() for file in range(70,75): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(3,*[3])) newFile.setLocation('SomeSE3') self.fileset1.addFile(newFile) self.fileset1.commit() time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) # This is one of the most "complicated" tests so worth to comment, 4 jobs should be created # 1 - all previous files from SomeSE and run = 1 (a lot, like ~45) # 2 - Few files from SomeSE3, Run = 1 # 3 - Few files from SomeSE3, Run = 2 # 4 - Few files from SomeSE3, Run = 3 self.assertEqual(len(jobGroups[0].getJobs()), 4 , "We didn't get 4 jobs for adding 2 different runs to SomeSE3") return def testMultipleRunHarvesting(self): """ _testMultipleRunHarvesting_ Add some files with multiple runs in each, make sure the jobs are created by location and run. Verify each job mask afterwards. Note that in this test run are splitted between sites, in real life that MUST NOT happen we still don't support that. """ multipleFilesFileset = Fileset(name = "TestFileset") newFile = File("/some/file/test1", size = 1000, events = 100) newFile.addRun(Run(1,*[1,3,4,5,6,7])) newFile.addRun(Run(2,*[1,2,4,5,6,7])) newFile.setLocation('SomeSE') multipleFilesFileset.addFile(newFile) newFile = File("/some/file/test2", size = 1000, events = 100) newFile.addRun(Run(1,*[2,8])) newFile.addRun(Run(2,*[3,8])) newFile.setLocation('SomeSE3') multipleFilesFileset.addFile(newFile) multipleFilesFileset.create() harvestingWorkflow = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset = multipleFilesFileset, workflow = harvestingWorkflow, split_algo = "Harvest", type = "Harvesting") harvestSub.create() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSub) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask") self.finishJobs(jobGroups, harvestSub) newFile = File("/some/file/test3", size = 1000, events = 100) newFile.addRun(Run(1,*range(9,15))) newFile.setLocation('SomeSE3') multipleFilesFileset.addFile(newFile) multipleFilesFileset.commit() time.sleep(2) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask") harvestingWorkflowSib = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflowSib", task="TestSib") harvestingWorkflowSib.create() harvestSubSib = Subscription(fileset = multipleFilesFileset, workflow = harvestingWorkflowSib, split_algo = "Harvest", type = "Harvesting") harvestSubSib.create() jobFactorySib = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSubSib) multipleFilesFileset.markOpen(False) jobGroups = jobFactorySib(periodic_harvest_sibling = True) self.assertEqual(len(jobGroups), 0, "A single job group was created") self.finishJobs(jobGroups, harvestSub) jobGroups = jobFactorySib(periodic_harvest_sibling = True) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
def testMultipleRunHarvesting(self): """ _testMultipleRunHarvesting_ Add some files with multiple runs in each, make sure the jobs are created by location and run. Verify each job mask afterwards. Note that in this test run are splitted between sites, in real life that MUST NOT happen we still don't support that. """ multipleFilesFileset = Fileset(name="TestFileset") newFile = File("/some/file/test1", size=1000, events=100) newFile.addRun(Run(1, *[1, 3, 4, 5, 6, 7])) newFile.addRun(Run(2, *[1, 2, 4, 5, 6, 7])) newFile.setLocation('T1_US_FNAL_Disk') multipleFilesFileset.addFile(newFile) newFile = File("/some/file/test2", size=1000, events=100) newFile.addRun(Run(1, *[2, 8])) newFile.addRun(Run(2, *[3, 8])) newFile.setLocation('T2_CH_CERN') multipleFilesFileset.addFile(newFile) multipleFilesFileset.create() harvestingWorkflow = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((str(run), lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) self.finishJobs(jobGroups, harvestSub) newFile = File("/some/file/test3", size=1000, events=100) newFile.addRun(Run(1, *range(9, 15))) newFile.setLocation('T2_CH_CERN') multipleFilesFileset.addFile(newFile) multipleFilesFileset.commit() time.sleep(2) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) harvestingWorkflowSib = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflowSib", task="TestSib") harvestingWorkflowSib.create() harvestSubSib = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflowSib, split_algo="Harvest", type="Harvesting") harvestSubSib.create() jobFactorySib = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSubSib) multipleFilesFileset.markOpen(False) jobGroups = jobFactorySib(periodic_harvest_sibling=True) self.assertEqual(len(jobGroups), 0, "A single job group was created") self.finishJobs(jobGroups, harvestSub) jobGroups = jobFactorySib(periodic_harvest_sibling=True) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
def setupExpressWorkflow(self): """ _setupExpressWorkflow_ Populate WMBS with a express-like workflow, every subscription must be unfinished at first """ workflowName = 'Express_Run481516_StreamZFast' secondLevelTasks = ['ExpressMergewrite_StreamZFast_DQM', 'ExpressMergewrite_ExpressPhysics_FEVT', 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO', 'ExpressCleanupUnmergedwrite_StreamZFast_DQM', 'ExpressCleanupUnmergedwrite_ExpressPhysics_FEVT', 'ExpressCleanupUnmergedwrite_StreamZFast_ALCARECO'] alcaHarvestTask = 'ExpressAlcaSkimwrite_StreamZFast_ALCARECOAlcaHarvestALCARECOStreamPromptCalibProd' dqmHarvestTask = 'ExpressMergewrite_StreamZFast_DQMEndOfRunDQMHarvestMerged' self.stateMap = {'Merge': [], 'Harvesting': [], 'Processing Done': []} self.orderedStates = ['Merge', 'Harvesting', 'Processing Done'] # Populate WMStats self.requestDBWriter.insertGenericRequest({'RequestName': workflowName}) self.requestDBWriter.updateRequestStatus(workflowName, 'Closed') # Create a wmspec in disk workload = newWorkload(workflowName) expressTask = workload.newTask('Express') for task in secondLevelTasks: secondLevelTask = expressTask.addTask(task) if task == 'ExpressAlcaSkimwrite_StreamZFast_ALCARECO': secondLevelTask.addTask(alcaHarvestTask) elif task == 'ExpressMergewrite_StreamZFast_DQM': secondLevelTask.addTask(dqmHarvestTask) specPath = os.path.join(self.testDir, 'Express.pkl') workload.save(specPath) # Populate WMBS sharedFileset = Fileset(name='TestFileset') sharedFileset.create() sharedFileset.markOpen(False) options = {'spec': specPath, 'owner': 'ItsAMeMario', 'name': workflowName, 'wfType': 'tier0'} topLevelWorkflow = Workflow(task='/%s/Express' % workflowName, **options) topLevelWorkflow.create() topLevelSub = Subscription(sharedFileset, topLevelWorkflow) topLevelSub.create() self.stateMap['Merge'].append(topLevelSub) for task in [x for x in secondLevelTasks if not x.count('CleanupUnmerged')]: secondLevelWorkflow = Workflow(task='/%s/Express/%s' % (workflowName, task), **options) secondLevelWorkflow.create() mergeSub = Subscription(sharedFileset, secondLevelWorkflow) mergeSub.create() self.stateMap['Harvesting'].append(mergeSub) for (parent, child) in [('ExpressAlcaSkimwrite_StreamZFast_ALCARECO', alcaHarvestTask), ('ExpressMergewrite_StreamZFast_DQM', dqmHarvestTask)]: harvestingWorkflow = Workflow(task='/%s/Express/%s/%s' % (workflowName, parent, child), **options) harvestingWorkflow.create() harvestingSub = Subscription(sharedFileset, harvestingWorkflow) harvestingSub.create() self.stateMap['Processing Done'].append(harvestingSub) return
class HarvestTest(unittest.TestCase): """ _HarvestTest_ Test for EndOfRun job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN") self.fileset1 = Fileset(name="TestFileset1") for fileNum in range(11): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset=self.fileset1, workflow=workflow1, split_algo="Harvest", type="Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = 'wmagent_jobdump' return config def finishJobs(self, jobGroups, subscription=None): """ _finishJobs_ """ if not subscription: subscription = self.subscription1 for f in subscription.acquiredFiles(): subscription.completeFiles(f) for jobGroup in jobGroups: self.changer.propagate(jobGroup.jobs, 'executing', 'created') self.changer.propagate(jobGroup.jobs, 'complete', 'executing') self.changer.propagate(jobGroup.jobs, 'success', 'complete') self.changer.propagate(jobGroup.jobs, 'cleanout', 'success') return def testHarvestEndOfRunTrigger(self): """ _testDQMHarvestEndOfRunTrigger_ Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset """ self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 0, "We got 1 or more jobGroups with an open fileset and no periodic configuration") self.fileset1.markOpen(False) self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?") # We should also check if there are aqcuired files, if there are, there are jobs, # we don't want to fire another jobs while previous are running (output is integrating whatever input) # TODO : The above one we can do when all is done. Not priority jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1, "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed") return def testPeriodicTrigger(self): """ _testPeriodicTrigger_ """ self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here") # Test timeout (5s for this first test) # there should be no acquired files, if there are, shouldn't be a job # self.subscription1.acquireFiles(self.subscription1.availableFiles().pop()) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=3) self.assertEqual(len(jobGroups), 1, "Didn't created the first periodic job when there were acquired files") # For the whole thing to work, faking the first job finishing, and putting the files as complete self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for fileNum in range(12, 24): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.commit() # Testing that it doesn't create a job unless the delay is past jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 0, "Created one or more job, when there were non-acquired file and the period is not passed by") time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, and there weren't and the period is passed by") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for fileNum in range(26, 36): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.commit() # Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 0, "Created one or more job, there are new files, but the delay is not past") time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, there are new files and the delay is past") # Last check is whether the job gets all the files or not numFilesJob = jobGroups[0].jobs[0].getFiles() numFilesFileset = self.fileset1.getFiles() self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding files for the first location for fileNum in range(38, 48): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.commit() # Then another location for fileNum in range(50, 56): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T2_CH_CERN') self.fileset1.addFile(newFile) self.fileset1.commit() # We should have jobs in both locations time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups[0].getJobs()), 2, "We didn't get 2 jobs for 2 locations") firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0] secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0] self.assertEqual(firstJobLocation, 'T2_CH_CERN') self.assertEqual(secondJobLocation, 'T1_US_FNAL') self.finishJobs(jobGroups) for fileNum in range(60, 65): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(2, *[2])) newFile.setLocation('T2_CH_CERN') self.fileset1.addFile(newFile) self.fileset1.commit() for fileNum in range(70, 75): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(3, *[3])) newFile.setLocation('T2_CH_CERN') self.fileset1.addFile(newFile) self.fileset1.commit() time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) # This is one of the most "complicated" tests so worth to comment, 4 jobs should be created # 1 - all previous files from SomeSE and run = 1 (a lot, like ~45) # 2 - Few files from SomeSE3, Run = 1 # 3 - Few files from SomeSE3, Run = 2 # 4 - Few files from SomeSE3, Run = 3 self.assertEqual(len(jobGroups[0].getJobs()), 4, "We didn't get 4 jobs for adding 2 different runs to SomeSE3") return def testMultipleRunHarvesting(self): """ _testMultipleRunHarvesting_ Add some files with multiple runs in each, make sure the jobs are created by location and run. Verify each job mask afterwards. Note that in this test run are splitted between sites, in real life that MUST NOT happen we still don't support that. """ multipleFilesFileset = Fileset(name="TestFileset") newFile = File("/some/file/test1", size=1000, events=100) newFile.addRun(Run(1, *[1, 3, 4, 5, 6, 7])) newFile.addRun(Run(2, *[1, 2, 4, 5, 6, 7])) newFile.setLocation('T1_US_FNAL_Disk') multipleFilesFileset.addFile(newFile) newFile = File("/some/file/test2", size=1000, events=100) newFile.addRun(Run(1, *[2, 8])) newFile.addRun(Run(2, *[3, 8])) newFile.setLocation('T2_CH_CERN') multipleFilesFileset.addFile(newFile) multipleFilesFileset.create() harvestingWorkflow = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((str(run), lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) self.finishJobs(jobGroups, harvestSub) newFile = File("/some/file/test3", size=1000, events=100) newFile.addRun(Run(1, *range(9, 15))) newFile.setLocation('T2_CH_CERN') multipleFilesFileset.addFile(newFile) multipleFilesFileset.commit() time.sleep(2) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) harvestingWorkflowSib = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflowSib", task="TestSib") harvestingWorkflowSib.create() harvestSubSib = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflowSib, split_algo="Harvest", type="Harvesting") harvestSubSib.create() jobFactorySib = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSubSib) multipleFilesFileset.markOpen(False) jobGroups = jobFactorySib(periodic_harvest_sibling=True) self.assertEqual(len(jobGroups), 0, "A single job group was created") self.finishJobs(jobGroups, harvestSub) jobGroups = jobFactorySib(periodic_harvest_sibling=True) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) def testMultiRunHarvesting(self): """ _testMultiRunHarvesting_ Provided a fileset with a couple of files and different runs, create a single job for all the runs at a specific location, which also adds a baggage to the job (True) which is later on looked up by SetupCMSSWPSet. """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(dqmHarvestUnit="multiRun") self.assertEqual(len(jobGroups), 1) for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 1) for job in jobGroup.jobs: baggage = job.getBaggage() self.assertTrue(getattr(baggage, "multiRun", False), "It's supposed to be a multiRun job") self.assertEqual(getattr(baggage, "runLimits", ""), "-1-6") def testByRunHarvesting(self): """ _testByRunHarvesting_ Provided a fileset with a couple of files and 4 different runs, create one single job per run and location. The multiRun baggage should be false in this case. """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!") harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1, "Should have created 1 job group") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs") for job in jobGroup.jobs: baggage = job.getBaggage() self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job") def testByRunAndRunWhitelist(self): """ _testByRunAndRunWhitelist_ Create harvesting jobs by run for the runs provided in the RunWhitelist """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(runWhitelist=[1, 3]) self.assertEqual(len(jobGroups), 1, "One jobgroup per location") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 2) def testByRunAndRunBlacklist(self): """ _testByRunAndRunWhitelist_ Create harvesting jobs by run for the runs provided in the RunWhitelist """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3]) self.assertEqual(len(jobGroups), 1, "One jobgroup per location") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 3)
def createTestJobGroup(self, config, name = "TestWorkthrough", filesetName = "TestFileset", specLocation = "spec.xml", error = False, task = "/TestWorkload/ReReco", type = "Processing"): """ Creates a group of several jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec = specLocation, owner = self.OWNERDN, name = name, task = task, owner_vogroup="", owner_vorole="") testWorkflow.create() self.inject.execute(names = [name], injected = True) testWMBSFileset = Fileset(name = filesetName) testWMBSFileset.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) outputWMBSFileset = Fileset(name = '%sOutput' % filesetName) outputWMBSFileset.create() testFileC = File(lfn = "/this/is/a/lfnC" , size = 1024, events = 10) testFileC.addRun(Run(10, *[12312])) testFileC.setLocation('malpaquet') testFileC.create() outputWMBSFileset.addFile(testFileC) outputWMBSFileset.commit() outputWMBSFileset.markOpen(0) testWorkflow.addOutput('output', outputWMBSFileset) testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow, type = type) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() for i in range(0,self.nJobs): testJob = Job(name = makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() changer = ChangeState(config) report1 = Report() report2 = Report() if error: path1 = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') else: path1 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'mergeReport1.pkl') path2 = os.path.join(WMCore.WMBase.getTestBase(), 'WMComponent_t/TaskArchiver_t/fwjrs', 'logCollectReport2.pkl') report1.load(filename = path1) report2.load(filename = path2) changer.propagate(testJobGroup.jobs, 'created', 'new') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') for i in range(self.nJobs): if i < self.nJobs/2: testJobGroup.jobs[i]['fwjr'] = report1 else: testJobGroup.jobs[i]['fwjr'] = report2 changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed') changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff') changer.propagate(testJobGroup.jobs, 'executing', 'created') changer.propagate(testJobGroup.jobs, 'complete', 'executing') changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete') changer.propagate(testJobGroup.jobs, 'retrydone', 'jobfailed') changer.propagate(testJobGroup.jobs, 'exhausted', 'retrydone') changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup
def __call__(self, parameters): """ Perform the work required with the given parameters """ DefaultSlave.__call__(self, parameters) # Handle the message message = self.messageArgs # Lock on the running feeders list myThread = threading.currentThread() myThread.runningFeedersLock.acquire() # Create empty fileset if fileset.name doesn't exist filesetName = message["dataset"] feederType = message["FeederType"] fileType = message["FileType"] startRun = message["StartRun"] logging.debug("Dataset " + filesetName + " arrived") fileset = Fileset(name = filesetName+':'\ +feederType+':'+fileType+':'+startRun) # Check if the fileset is already there if fileset.exists() == False: # Empty fileset creation fileset.create() fileset.setLastUpdate(0) logging.info("Fileset %s whith id %s is added" \ %(fileset.name, str(fileset.id))) # Get feeder type feederType = message["FeederType"] # Check if there is a running feeder if feederType in myThread.runningFeeders: logging.info("HAVE FEEDER " + feederType + " RUNNING") logging.info(myThread.runningFeeders[feederType]) else: logging.info("NO FEEDER " + feederType + " RUNNING") # Check if we have a feeder in DB if self.queries.checkFeeder(feederType): # Have feeder, get info logging.info("Getting Feeder from DB") feederId = self.queries.getFeederId(feederType) logging.info(feederId) myThread.runningFeeders[feederType] = feederId else: # Create feeder logging.info("Adding Feeder to DB") self.queries.addFeeder(feederType, "StatePath") feederId = self.queries.getFeederId(feederType) logging.info(feederId) myThread.runningFeeders[feederType] = feederId # Fileset/Feeder association self.queries.addFilesetToManage(fileset.id, \ myThread.runningFeeders[feederType]) logging.info("Fileset %s is added to feeder %s" %(fileset.id, \ myThread.runningFeeders[feederType])) else: # If fileset already exist a new subscription # will be created for its workflow logging.info("Fileset exists: Subscription will be created for it") # Open it if close fileset.load() if fileset.open == False: fileset.markOpen(True) logging.info("Getting Feeder from DB") feederId = self.queries.getFeederId(feederType) logging.info(feederId) myThread.runningFeeders[feederType] = feederId self.queries.addFilesetToManage(fileset.id, \ myThread.runningFeeders[feederType]) logging.info("Fileset %s is added to feeder %s" %(fileset.id, \ myThread.runningFeeders[feederType])) myThread.runningFeedersLock.release() myThread.msgService.finish()
def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose = False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() #FIXME: Let workflow put in values if spec is missing them workflow = Workflow(spec = self.wmSpec.specUrl(), owner = self.wmSpec.getOwner()["name"], dn = self.wmSpec.getOwner().get("dn", "unknown"), group = self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup = self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole = self.wmSpec.getOwner().get("vorole", "DEFAULT"), name = self.wmSpec.name(), task = task.getPathName(), wfType = self.wmSpec.getDashboardActivity(), alternativeFilesetClose = alternativeFilesetClose, priority = self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset = fileset, workflow = workflow, split_algo = task.jobSplittingAlgorithm(), type = task.getPrimarySubType()) if subscription.exists(): subscription.load() msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)" self.logger.info(msg % (subscription['id'], task.getPathName())) else: subscription.create() for site in task.siteWhitelist(): subscription.addWhiteBlackList([{"site_name": site, "valid": True}]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{"site_name": site, "valid": False}]) if self.topLevelSubscription == None: self.topLevelSubscription = subscription logging.info("Top level subscription created: %s" % subscription["id"]) else: logging.info("Child subscription created: %s" % subscription["id"]) outputModules = task.getOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): outputFileset = Fileset(self.outputFilesetName(task, outputModuleName)) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: if childTask.taskType() == "Merge": mergedOutputFileset = Fileset(self.outputFilesetName(childTask, "Merged")) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr(getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset != None: self.mergeOutputMapping[mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS(childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset == None: workflow.addOutput(outputModuleName, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName, outputFileset, mergedOutputFileset) return self.topLevelSubscription
closeFileset = Fileset( name = (((\ filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\ (((filesetToProcess.name).split(':')[0]).split('/')[1]\ )+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\ [2])+'/'+((((filesetToProcess.name).split(':')[0]).split\ ('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\ +":"+":".join((filesetToProcess.name).split(':')[1:] ) ) if closeFileset.exists() != False : closeFileset = Fileset( id = closeFileset.exists()) closeFileset.loadData() if closeFileset.open == True: closeFileset.markOpen(False) # Commit the fileset filesetToProcess.commit() # Commit the fileset logging.debug("Test purge in T0ASTRun ...") filesetToProcess.load() LASTIME = filesetToProcess.lastUpdate if (int(now)/3600 - LASTIME/3600) > self.purgeTime: filesetToProcess.markOpen(False) logging.debug("Purge Done...")
def createTestJobGroup(self, name = "TestWorkthrough", specLocation = "spec.xml", error = False, task = "/TestWorkload/ReReco", nJobs = 10): """ _createTestJobGroup_ Generate a test WMBS JobGroup with real FWJRs """ myThread = threading.currentThread() testWorkflow = Workflow(spec = specLocation, owner = "Simon", name = name, task = task) testWorkflow.create() testWMBSFileset = Fileset(name = name) testWMBSFileset.create() testFileA = File(lfn = makeUUID(), size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = makeUUID(), size = 1024, events = 10) testFileB.addRun(Run(10, *[12312])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() for i in range(0, nJobs): testJob = Job(name = makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() report = Report() if error: path = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") else: path = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "PerformanceReport2.pkl") report.load(filename = path) self.changeState.propagate(testJobGroup.jobs, 'created', 'new') self.changeState.propagate(testJobGroup.jobs, 'executing', 'created') self.changeState.propagate(testJobGroup.jobs, 'complete', 'executing') for job in testJobGroup.jobs: job['fwjr'] = report self.changeState.propagate(testJobGroup.jobs, 'jobfailed', 'complete') self.changeState.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed') self.changeState.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup
def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() # FIXME: Let workflow put in values if spec is missing them workflow = Workflow(spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"], dn=self.wmSpec.getOwner().get("dn", "unknown"), group=self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"), name=self.wmSpec.name(), task=task.getPathName(), wfType=self.wmSpec.getDashboardActivity(), alternativeFilesetClose=alternativeFilesetClose, priority=self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset=fileset, workflow=workflow, split_algo=task.jobSplittingAlgorithm(), type=task.getPrimarySubType()) subscription.create() ### FIXME: I'm pretty sure we can improve how we handle this site white/black list for site in task.siteWhitelist(): subscription.addWhiteBlackList([{"site_name": site, "valid": True}]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{"site_name": site, "valid": False}]) if self.topLevelSubscription is None: self.topLevelSubscription = subscription logging.info("Top level subscription %s created for %s", subscription["id"], self.wmSpec.name()) else: logging.info("Child subscription %s created for %s", subscription["id"], self.wmSpec.name()) outputModules = task.getOutputModulesForTask() ignoredOutputModules = task.getIgnoredOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): if outputModuleName in ignoredOutputModules: msg = "%s has %s as IgnoredOutputModule, skipping fileset creation." logging.info(msg, task.getPathName(), outputModuleName) continue dataTier = getattr(getattr(outputModule, outputModuleName), "dataTier", '') filesetName = self.outputFilesetName(task, outputModuleName, dataTier) outputFileset = Fileset(filesetName) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: childDatatier = getattr(childTask.data.input, 'dataTier', '') if childTask.taskType() in ["Cleanup", "Merge"] and childDatatier != dataTier: continue elif childTask.taskType() == "Merge" and childDatatier == dataTier: filesetName = self.outputFilesetName(childTask, "Merged", dataTier) mergedOutputFileset = Fileset(filesetName) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr(getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset is not None: self.mergeOutputMapping[mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS(childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset is None: workflow.addOutput(outputModuleName + dataTier, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName + dataTier, outputFileset, mergedOutputFileset) return
class SiblingProcessingBasedTest(unittest.TestCase): """ _SiblingProcessingBasedTest_ Test SiblingProcessing job splitting. """ def setUp(self): """ _setUp_ Setup the database connections and schema. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = daofactory(classname = "Locations.New") locationAction.execute("T2_CH_CERN", pnn = "T2_CH_CERN") locationAction.execute("T1_US_FNAL", pnn = "T1_US_FNAL_Disk") self.testFilesetA = Fileset(name = "FilesetA") self.testFilesetA.create() self.testFilesetB = Fileset(name = "FilesetB") self.testFilesetB.create() self.testFileA = File("testFileA", size = 1000, events = 100, locations = set(["T2_CH_CERN"])) self.testFileA.create() self.testFileB = File("testFileB", size = 1000, events = 100, locations = set(["T2_CH_CERN"])) self.testFileB.create() self.testFileC = File("testFileC", size = 1000, events = 100, locations = set(["T2_CH_CERN"])) self.testFileC.create() self.testFilesetA.addFile(self.testFileA) self.testFilesetA.addFile(self.testFileB) self.testFilesetA.addFile(self.testFileC) self.testFilesetA.commit() self.testFileD = File("testFileD", size = 1000, events = 100, locations = set(["T2_CH_CERN"])) self.testFileD.create() self.testFileE = File("testFileE", size = 1000, events = 100, locations = set(["T2_CH_CERN"])) self.testFileE.create() self.testFileF = File("testFileF", size = 1000, events = 100, locations = set(["T2_CH_CERN"])) self.testFileF.create() self.testFilesetB.addFile(self.testFileD) self.testFilesetB.addFile(self.testFileE) self.testFilesetB.addFile(self.testFileF) self.testFilesetB.commit() testWorkflowA = Workflow(spec = "specA.xml", owner = "Steve", name = "wfA", task = "Test") testWorkflowA.create() testWorkflowB = Workflow(spec = "specB.xml", owner = "Steve", name = "wfB", task = "Test") testWorkflowB.create() testWorkflowC = Workflow(spec = "specC.xml", owner = "Steve", name = "wfC", task = "Test") testWorkflowC.create() testWorkflowD = Workflow(spec = "specD.xml", owner = "Steve", name = "wfD", task = "Test") testWorkflowD.create() self.testSubscriptionA = Subscription(fileset = self.testFilesetA, workflow = testWorkflowA, split_algo = "FileBased", type = "Processing") self.testSubscriptionA.create() self.testSubscriptionB = Subscription(fileset = self.testFilesetB, workflow = testWorkflowB, split_algo = "FileBased", type = "Processing") self.testSubscriptionB.create() self.testSubscriptionC = Subscription(fileset = self.testFilesetB, workflow = testWorkflowC, split_algo = "FileBased", type = "Processing") self.testSubscriptionC.create() self.testSubscriptionD = Subscription(fileset = self.testFilesetB, workflow = testWorkflowD, split_algo = "FileBased", type = "Processing") self.testSubscriptionD.create() deleteWorkflow = Workflow(spec = "specE.xml", owner = "Steve", name = "wfE", task = "Test") deleteWorkflow.create() self.deleteSubscriptionA = Subscription(fileset = self.testFilesetA, workflow = deleteWorkflow, split_algo = "SiblingProcessingBased", type = "Cleanup") self.deleteSubscriptionA.create() self.deleteSubscriptionB = Subscription(fileset = self.testFilesetB, workflow = deleteWorkflow, split_algo = "SiblingProcessingBased", type = "Cleanup") self.deleteSubscriptionB.create() return def tearDown(self): """ _tearDown_ Clear out WMBS. """ self.testInit.clearDatabase() return def testSiblingProcessing(self): """ _testSiblingProcessing_ Verify that the sibling processing split works correctly dealing with failed files and acquiring files correctly. """ splitter = SplitterFactory() deleteFactoryA = splitter(package = "WMCore.WMBS", subscription = self.deleteSubscriptionA) deleteFactoryB = splitter(package = "WMCore.WMBS", subscription = self.deleteSubscriptionB) result = deleteFactoryA() assert len(result) == 0, \ "Error: No jobs should be returned." result = deleteFactoryB() assert len(result) == 0, \ "Error: No jobs should be returned." self.testSubscriptionA.completeFiles(self.testFileA) result = deleteFactoryA(files_per_job = 1) assert len(result) == 1, \ "Error: Only one jobgroup should be returned." assert len(result[0].jobs) == 1, \ "Error: There should only be one job in the jobgroup." assert result[0].jobs[0]["possiblePSN"] == set(["T2_CH_CERN"]), \ "Error: possiblePSN is wrong." assert len(result[0].jobs[0]["input_files"]) == 1, \ "Error: Job should only have one input file." assert result[0].jobs[0]["input_files"][0]["lfn"] == "testFileA", \ "Error: Input file for job is wrong." result = deleteFactoryB(files_per_job = 1) assert len(result) == 0, \ "Error: Second subscription should have no jobs." result = deleteFactoryA(files_per_job = 1) assert len(result) == 0, \ "Error: No jobs should have been created." self.testSubscriptionB.completeFiles(self.testFileD) self.testSubscriptionC.failFiles(self.testFileD) result = deleteFactoryA(files_per_job = 1) assert len(result) == 0, \ "Error: No jobs should have been created." result = deleteFactoryB(files_per_job = 1) assert len(result) == 0, \ "Error: No jobs should have been created." self.testSubscriptionD.failFiles(self.testFileD) result = deleteFactoryA(files_per_job = 1) assert len(result) == 0, \ "Error: No jobs should have been created." result = deleteFactoryB(files_per_job = 1) assert len(result) == 0, \ "Error: No job groups should have been created." self.testSubscriptionB.completeFiles([self.testFileE, self.testFileF]) self.testSubscriptionC.completeFiles([self.testFileE, self.testFileF]) self.testSubscriptionD.completeFiles([self.testFileE, self.testFileF]) result = deleteFactoryB(files_per_job = 10) assert len(result) == 0, \ "Error: No jobs should have been created." self.testFilesetB.markOpen(False) result = deleteFactoryB(files_per_job = 10) assert len(result) == 1, \ "Error: One jobgroup should have been returned." assert len(result[0].jobs) == 1, \ "Error: There should only be one job in the jobgroup." assert len(result[0].jobs[0]["input_files"]) == 2, \ "Error: Job should only have one input file." lfns = [result[0].jobs[0]["input_files"][0]["lfn"], result[0].jobs[0]["input_files"][1]["lfn"]] assert "testFileE" in lfns, \ "Error: TestFileE missing from job input." assert "testFileF" in lfns, \ "Error: TestFileF missing from job input." self.assertEqual(len(self.deleteSubscriptionB.availableFiles()), 0, "Error: There should be no available files.") completeFiles = self.deleteSubscriptionB.filesOfStatus("Completed") self.assertEqual(len(completeFiles), 1, "Error: There should only be one complete file.") self.assertEqual(list(completeFiles)[0]["lfn"], "testFileD", "Error: Test file D should be complete.") return def testMultipleLocations(self): """ _testMultipleLocations_ Verify that the sibling processing based algorithm doesn't create jobs that run over files at multiple sites. """ testFile1 = File("testFile1", size = 1000, events = 100, locations = set(["T1_US_FNAL_Disk"])) testFile1.create() testFile2 = File("testFile2", size = 1000, events = 100, locations = set(["T1_US_FNAL_Disk"])) testFile2.create() testFile3 = File("testFile3", size = 1000, events = 100, locations = set(["T1_US_FNAL_Disk"])) testFile3.create() self.testFilesetA.addFile(testFile1) self.testFilesetA.addFile(testFile2) self.testFilesetA.addFile(testFile3) self.testFilesetA.commit() self.testFilesetA.markOpen(False) self.testSubscriptionA.completeFiles([testFile1, testFile2, testFile3]) self.testSubscriptionA.completeFiles([self.testFileA, self.testFileB, self.testFileC]) splitter = SplitterFactory() deleteFactoryA = splitter(package = "WMCore.WMBS", subscription = self.deleteSubscriptionA) result = deleteFactoryA(files_per_job = 50) assert len(result) == 2, \ "Error: Wrong number of jobgroups returned." goldenFilesA = ["testFileA", "testFileB", "testFileC"] goldenFilesB = ["testFile1", "testFile2", "testFile3"] for jobGroup in result: assert len(jobGroup.jobs) == 1, \ "Error: Wrong number of jobs in jobgroup." assert len(jobGroup.jobs[0]["input_files"]) == 3, \ "Error: Wrong number of input files in job." jobSite = jobGroup.jobs[0]["possiblePSN"] assert (jobSite == set(["T2_CH_CERN"]) or jobSite == set(["T1_US_FNAL"])), \ "Error: Wrong site for job." if jobSite == set(["T2_CH_CERN"]): goldenFiles = goldenFilesA else: goldenFiles = goldenFilesB for jobFile in jobGroup.jobs[0]["input_files"]: goldenFiles.remove(jobFile["lfn"]) assert len(goldenFiles) == 0, \ "Error: Files are missing." return def testLargeNumberOfFiles(self): """ _testLargeNumberOfFiles_ Setup a subscription with 500 files and verify that the splitting algo works correctly. """ testWorkflowA = Workflow(spec = "specA.xml", owner = "Steve", name = "wfA", task = "Test") testWorkflowA.create() testWorkflowB = Workflow(spec = "specB.xml", owner = "Steve", name = "wfB", task = "Test") testWorkflowB.create() testFileset = Fileset(name = "TestFileset") testFileset.create() allFiles = [] for i in range(500): testFile = File(str(i), size = 1000, events = 100, locations = set(["T2_CH_CERN"])) testFile.create() allFiles.append(testFile) testFileset.addFile(testFile) testFileset.commit() testSubscriptionA = Subscription(fileset = testFileset, workflow = testWorkflowA, split_algo = "FileBased", type = "Processing") testSubscriptionA.create() testSubscriptionB = Subscription(fileset = testFileset, workflow = testWorkflowB, split_algo = "SiblingProcessingBased", type = "Processing") testSubscriptionB.create() testSubscriptionA.completeFiles(allFiles) splitter = SplitterFactory() deleteFactoryA = splitter(package = "WMCore.WMBS", subscription = testSubscriptionB) result = deleteFactoryA(files_per_job = 50) self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.") self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.") return def testFilesWithoutOtherSubscriptions(self): """ _testFilesWithoutOtherSubscriptions_ Test the case where files only in the delete subscription can happen if cleanup of the other subscriptions is fast """ testWorkflowA = Workflow(spec = "specA.xml", owner = "Steve", name = "wfA", task = "Test") testWorkflowA.create() testFileset = Fileset(name = "TestFileset") testFileset.create() allFiles = [] for i in range(500): testFile = File(str(i), size = 1000, events = 100, locations = set(["T2_CH_CERN"])) testFile.create() allFiles.append(testFile) testFileset.addFile(testFile) testFileset.commit() testSubscriptionA = Subscription(fileset = testFileset, workflow = testWorkflowA, split_algo = "SiblingProcessingBased", type = "Processing") testSubscriptionA.create() splitter = SplitterFactory() deleteFactoryA = splitter(package = "WMCore.WMBS", subscription = testSubscriptionA) result = deleteFactoryA(files_per_job = 50) self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.") self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.") return
class RepackTest(unittest.TestCase): """ _RepackTest_ Test for Repack job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["T0.WMBS"]) self.splitterFactory = SplitterFactory(package = "T0.JobSplitting") myThread = threading.currentThread() daoFactory = DAOFactory(package = "T0.WMBS", logger = logging, dbinterface = myThread.dbi) myThread.dbi.processData("""INSERT INTO wmbs_location (id, site_name, state) VALUES (1, 'SomeSite', 1) """, transaction = False) myThread.dbi.processData("""INSERT INTO wmbs_location_senames (location, se_name) VALUES (1, 'SomeSE') """, transaction = False) myThread.dbi.processData("""INSERT INTO wmbs_location_senames (location, se_name) VALUES (1, 'SomeSE2') """, transaction = False) insertRunDAO = daoFactory(classname = "RunConfig.InsertRun") insertRunDAO.execute(binds = { 'RUN' : 1, 'TIME' : int(time.time()), 'HLTKEY' : "someHLTKey" }, transaction = False) insertLumiDAO = daoFactory(classname = "RunConfig.InsertLumiSection") for lumi in [1, 2, 3, 4]: insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : lumi }, transaction = False) insertStreamDAO = daoFactory(classname = "RunConfig.InsertStream") insertStreamDAO.execute(binds = { 'STREAM' : "A" }, transaction = False) insertStreamFilesetDAO = daoFactory(classname = "RunConfig.InsertStreamFileset") insertStreamFilesetDAO.execute(1, "A", "TestFileset1") self.fileset1 = Fileset(name = "TestFileset1") self.fileset1.load() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Repack", type = "Repack") self.subscription1.create() # keep for later self.insertClosedLumiDAO = daoFactory(classname = "RunLumiCloseout.InsertClosedLumi") self.currentTime = int(time.time()) # default split parameters self.splitArgs = {} self.splitArgs['maxSizeSingleLumi'] = 20*1024*1024*1024 self.splitArgs['maxSizeMultiLumi'] = 10*1024*1024*1024 self.splitArgs['maxInputEvents'] = 500000 self.splitArgs['maxInputFiles'] = 1000 return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() return def getNumActiveSplitLumis(self): """ _getNumActiveSplitLumis_ helper function that counts the number of active split lumis """ myThread = threading.currentThread() results = myThread.dbi.processData("""SELECT COUNT(*) FROM lumi_section_split_active """, transaction = False)[0].fetchall() return results[0][0] def test00(self): """ _test00_ Test that the job name prefix feature works Test multi lumi size threshold Multi lumi input """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 3, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) mySplitArgs['maxSizeMultiLumi'] = self.splitArgs['maxSizeMultiLumi'] jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxSizeMultiLumi'] = 5000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertTrue(job['name'].startswith("Repack-"), "ERROR: Job has wrong name") self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertTrue(job['name'].startswith("Repack-"), "ERROR: Job has wrong name") self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return def test01(self): """ _test01_ Test multi lumi event threshold Multi lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 3, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append( { 'RUN' : 1, 'LUMI' : lumi, 'STREAM' : "A", 'FILECOUNT' : filecount, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime } ) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.insertClosedLumiDAO.execute(binds = insertClosedLumiBinds, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputEvents'] = 500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return def test02(self): """ _test02_ Test single lumi size threshold Single lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1]: filecount = 8 for i in range(filecount): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append( { 'RUN' : 1, 'LUMI' : lumi, 'STREAM' : "A", 'FILECOUNT' : filecount, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime } ) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.insertClosedLumiDAO.execute(binds = insertClosedLumiBinds, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxSizeSingleLumi'] = 6500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 6, "ERROR: Job does not process 6 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") self.assertEqual(self.getNumActiveSplitLumis(), 1, "ERROR: Split lumis were not created") return def test03(self): """ _test03_ Test single lumi event threshold Single lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1]: filecount = 8 for i in range(filecount): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append( { 'RUN' : 1, 'LUMI' : lumi, 'STREAM' : "A", 'FILECOUNT' : filecount, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime } ) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.insertClosedLumiDAO.execute(binds = insertClosedLumiBinds, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputEvents'] = 650 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 6, "ERROR: Job does not process 6 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") self.assertEqual(self.getNumActiveSplitLumis(), 1, "ERROR: Split lumis were not created") return def test04(self): """ _test04_ Test streamer count threshold (only multi lumi) Multi lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 3, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append( { 'RUN' : 1, 'LUMI' : lumi, 'STREAM' : "A", 'FILECOUNT' : filecount, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime } ) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.insertClosedLumiDAO.execute(binds = insertClosedLumiBinds, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputFiles'] = 5 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return def test05(self): """ _test05_ Test repacking of multiple lumis with holes in the lumi sequence Multi lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append( { 'RUN' : 1, 'LUMI' : lumi, 'STREAM' : "A", 'FILECOUNT' : filecount, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime } ) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.insertClosedLumiDAO.execute(binds = insertClosedLumiBinds, transaction = False) mySplitArgs['maxInputFiles'] = 5 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.insertClosedLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 3, 'STREAM' : "A", 'FILECOUNT' : 0, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime }, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") self.assertEqual(len(jobGroups[0].jobs[0].getFiles()), 4, "ERROR: first job does not process 4 files") return def test06(self): """ _test06_ Test repacking of 3 lumis 2 small lumis (single job), followed by a big one (multiple jobs) files for lumi 1 and 2 are below multi-lumi thresholds files for lumi 3 are above single-lumi threshold """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 3]: filecount = 2 for i in range(filecount): if lumi == 3: nevents = 500 else: nevents = 100 newFile = File(makeUUID(), size = 1000, events = nevents) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append( { 'RUN' : 1, 'LUMI' : lumi, 'STREAM' : "A", 'FILECOUNT' : filecount, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime } ) self.fileset1.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) self.insertClosedLumiDAO.execute(binds = insertClosedLumiBinds, transaction = False) mySplitArgs['maxInputEvents'] = 900 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 3, "ERROR: JobFactory didn't create three jobs") self.assertEqual(len(jobGroups[0].jobs[0].getFiles()), 4, "ERROR: first job does not process 4 files") self.assertEqual(len(jobGroups[0].jobs[1].getFiles()), 1, "ERROR: second job does not process 1 file") self.assertEqual(len(jobGroups[0].jobs[2].getFiles()), 1, "ERROR: third job does not process 1 file") return
class RepackTest(unittest.TestCase): """ _RepackTest_ Test for Repack job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["T0.WMBS"]) self.splitterFactory = SplitterFactory(package="T0.JobSplitting") myThread = threading.currentThread() daoFactory = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=myThread.dbi) myThread.dbi.processData("""INSERT INTO wmbs_location (id, site_name, state) VALUES (1, 'SomeSite', 1) """, transaction=False) myThread.dbi.processData("""INSERT INTO wmbs_location_senames (location, se_name) VALUES (1, 'SomeSE') """, transaction=False) myThread.dbi.processData("""INSERT INTO wmbs_location_senames (location, se_name) VALUES (1, 'SomeSE2') """, transaction=False) insertRunDAO = daoFactory(classname="RunConfig.InsertRun") insertRunDAO.execute(binds={ 'RUN': 1, 'TIME': int(time.time()), 'HLTKEY': "someHLTKey" }, transaction=False) insertLumiDAO = daoFactory(classname="RunConfig.InsertLumiSection") for lumi in [1, 2, 3, 4]: insertLumiDAO.execute(binds={ 'RUN': 1, 'LUMI': lumi }, transaction=False) insertStreamDAO = daoFactory(classname="RunConfig.InsertStream") insertStreamDAO.execute(binds={'STREAM': "A"}, transaction=False) insertStreamFilesetDAO = daoFactory( classname="RunConfig.InsertStreamFileset") insertStreamFilesetDAO.execute(1, "A", "TestFileset1") self.fileset1 = Fileset(name="TestFileset1") self.fileset1.load() workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset=self.fileset1, workflow=workflow1, split_algo="Repack", type="Repack") self.subscription1.create() # keep for later self.insertClosedLumiDAO = daoFactory( classname="RunLumiCloseout.InsertClosedLumi") self.currentTime = int(time.time()) # default split parameters self.splitArgs = {} self.splitArgs['maxSizeSingleLumi'] = 20 * 1024 * 1024 * 1024 self.splitArgs['maxSizeMultiLumi'] = 10 * 1024 * 1024 * 1024 self.splitArgs['maxInputEvents'] = 500000 self.splitArgs['maxInputFiles'] = 1000 return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() return def getNumActiveSplitLumis(self): """ _getNumActiveSplitLumis_ helper function that counts the number of active split lumis """ myThread = threading.currentThread() results = myThread.dbi.processData("""SELECT COUNT(*) FROM lumi_section_split_active """, transaction=False)[0].fetchall() return results[0][0] def test00(self): """ _test00_ Test that the job name prefix feature works Test multi lumi size threshold Multi lumi input """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 3, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) mySplitArgs['maxSizeMultiLumi'] = self.splitArgs['maxSizeMultiLumi'] jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxSizeMultiLumi'] = 5000 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertTrue(job['name'].startswith("Repack-"), "ERROR: Job has wrong name") self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertTrue(job['name'].startswith("Repack-"), "ERROR: Job has wrong name") self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return def test01(self): """ _test01_ Test multi lumi event threshold Multi lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 3, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "A", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputEvents'] = 500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return def test02(self): """ _test02_ Test single lumi size threshold Single lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1]: filecount = 8 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "A", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxSizeSingleLumi'] = 6500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 6, "ERROR: Job does not process 6 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") self.assertEqual(self.getNumActiveSplitLumis(), 1, "ERROR: Split lumis were not created") return def test03(self): """ _test03_ Test single lumi event threshold Single lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1]: filecount = 8 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "A", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputEvents'] = 650 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 2, "ERROR: JobFactory didn't create two jobs") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 6, "ERROR: Job does not process 6 files") job = jobGroups[0].jobs[1] self.assertEqual(len(job.getFiles()), 2, "ERROR: Job does not process 2 files") self.assertEqual(self.getNumActiveSplitLumis(), 1, "ERROR: Split lumis were not created") return def test04(self): """ _test04_ Test streamer count threshold (only multi lumi) Multi lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 3, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "A", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") mySplitArgs['maxInputFiles'] = 5 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.fileset1.markOpen(False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return def test05(self): """ _test05_ Test repacking of multiple lumis with holes in the lumi sequence Multi lumi input """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 4]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "A", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) mySplitArgs['maxInputFiles'] = 5 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.insertClosedLumiDAO.execute(binds={ 'RUN': 1, 'LUMI': 3, 'STREAM': "A", 'FILECOUNT': 0, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }, transaction=False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") self.assertEqual(len(jobGroups[0].jobs[0].getFiles()), 4, "ERROR: first job does not process 4 files") return def test06(self): """ _test06_ Test repacking of 3 lumis 2 small lumis (single job), followed by a big one (multiple jobs) files for lumi 1 and 2 are below multi-lumi thresholds files for lumi 3 are above single-lumi threshold """ mySplitArgs = self.splitArgs.copy() insertClosedLumiBinds = [] for lumi in [1, 2, 3]: filecount = 2 for i in range(filecount): if lumi == 3: nevents = 500 else: nevents = 100 newFile = File(makeUUID(), size=1000, events=nevents) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "A", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': self.currentTime }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) mySplitArgs['maxInputEvents'] = 900 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 3, "ERROR: JobFactory didn't create three jobs") self.assertEqual(len(jobGroups[0].jobs[0].getFiles()), 4, "ERROR: first job does not process 4 files") self.assertEqual(len(jobGroups[0].jobs[1].getFiles()), 1, "ERROR: second job does not process 1 file") self.assertEqual(len(jobGroups[0].jobs[2].getFiles()), 1, "ERROR: third job does not process 1 file") return
def createTestJobGroup(self, name="TestWorkthrough", specLocation="spec.xml", error=False, task="/TestWorkload/ReReco", nJobs=10): """ _createTestJobGroup_ Generate a test WMBS JobGroup with real FWJRs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=specLocation, owner="Simon", name=name, task=task) testWorkflow.create() testWMBSFileset = Fileset(name=name) testWMBSFileset.create() testFileA = File(lfn=makeUUID(), size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn=makeUUID(), size=1024, events=10) testFileB.addRun(Run(10, *[12312])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFileset.addFile(testFileB) testWMBSFileset.commit() testWMBSFileset.markOpen(0) testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() for i in range(0, nJobs): testJob = Job(name=makeUUID()) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313]) testJobGroup.add(testJob) testJobGroup.commit() report = Report() if error: path = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl") else: path = os.path.join(WMCore.WMBase.getTestBase(), "WMComponent_t/JobAccountant_t/fwjrs", "PerformanceReport2.pkl") report.load(filename=path) self.changeState.propagate(testJobGroup.jobs, 'created', 'new') self.changeState.propagate(testJobGroup.jobs, 'executing', 'created') self.changeState.propagate(testJobGroup.jobs, 'complete', 'executing') for job in testJobGroup.jobs: job['fwjr'] = report self.changeState.propagate(testJobGroup.jobs, 'jobfailed', 'complete') self.changeState.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed') self.changeState.propagate(testJobGroup.jobs, 'cleanout', 'exhausted') testSubscription.completeFiles([testFileA, testFileB]) return testJobGroup