def testB_NoFileSplitNoHardLimit(self): """ _testB_NoFileSplitNoHardLimit_ In this case we don't split on file boundaries, check different combination of files make sure we make the most of the splitting, e.g. include many zero event files in a single job. """ splitter = SplitterFactory() # Create 100 files with 7 lumi per file and 0 events per lumi on average. testSubscription = self.createSubscription(nFiles=100, lumisPerFile=7, twoSites=False, nEventsPerFile=0) jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) # First test, the optimal settings are 360 events per job # As we have files with 0 events per lumi, this will configure the splitting to # a single job containing all files jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=360, performance=self.performanceParams) self.assertEqual(len(jobGroups), 0, "There aren't enough events, so it should have 0 job groups") # we close this fileset to get it moving fileset = testSubscription.getFileset() fileset.markOpen(False) jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=360, performance=self.performanceParams) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 1, "There should be 1 job") self.assertEqual(len(jobs[0]['input_files']), 100, "All 100 files must be in the job") self.assertEqual(jobs[0]['estimatedMemoryUsage'], 2300) self.assertEqual(jobs[0]['estimatedDiskUsage'], 0) self.assertEqual(jobs[0]['estimatedJobTime'], 0) # Create 7 files, each one with different lumi/event distributions testFileset = Fileset(name="FilesetA") testFileset.create() testFileA = self.createFile("/this/is/file1", 250, 0, 5, "T2_CH_CERN") testFileB = self.createFile("/this/is/file2", 600, 1, 1, "T2_CH_CERN") testFileC = self.createFile("/this/is/file3", 1200, 2, 2, "T2_CH_CERN") testFileD = self.createFile("/this/is/file4", 100, 3, 1, "T2_CH_CERN") testFileE = self.createFile("/this/is/file5", 30, 4, 1, "T2_CH_CERN") testFileF = self.createFile("/this/is/file6", 10, 5, 1, "T2_CH_CERN") testFileG = self.createFile("/this/is/file7", 151, 6, 3, "T2_CH_CERN") testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.addFile(testFileC) testFileset.addFile(testFileD) testFileset.addFile(testFileE) testFileset.addFile(testFileF) testFileset.addFile(testFileG) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiBased", type="Processing") testSubscription.create() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) # Optimal settings are: jobs with 150 events per job # This means, the first file must be splitted in 3 lumis per job which would leave room # for another lumi in the second job, but the second file has a lumi too big for that # The 3rd job only contains the second file, the fourth and fifth job split the third file jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=150, performance=self.performanceParams) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 7, "7 jobs must be in the jobgroup") self.assertEqual(jobs[0]["mask"].getRunAndLumis(), {0: [[0, 2]]}, "Wrong mask for the first job") self.assertEqual(jobs[0]["estimatedJobTime"], 150 * 12) self.assertEqual(jobs[0]["estimatedDiskUsage"], 150 * 400) self.assertEqual(jobs[1]["mask"].getRunAndLumis(), {0: [[3, 4]]}, "Wrong mask for the second job") self.assertEqual(jobs[1]["estimatedJobTime"], 100 * 12) self.assertEqual(jobs[1]["estimatedDiskUsage"], 100 * 400) self.assertEqual(jobs[2]["mask"].getRunAndLumis(), {1: [[1, 1]]}, "Wrong mask for the third job") self.assertEqual(jobs[2]["estimatedJobTime"], 600 * 12) self.assertEqual(jobs[2]["estimatedDiskUsage"], 600 * 400) self.assertEqual(jobs[3]["mask"].getRunAndLumis(), {2: [[4, 4]]}, "Wrong mask for the fourth job") self.assertEqual(jobs[3]["estimatedJobTime"], 600 * 12) self.assertEqual(jobs[3]["estimatedDiskUsage"], 600 * 400) self.assertEqual(jobs[4]["mask"].getRunAndLumis(), {2: [[5, 5]]}, "Wrong mask for the fifth job") self.assertEqual(jobs[4]["estimatedJobTime"], 600 * 12) self.assertEqual(jobs[4]["estimatedDiskUsage"], 600 * 400) self.assertEqual(jobs[5]["mask"].getRunAndLumis(), {3: [[3, 3]], 4: [[4, 4]], 5: [[5, 5]]}, "Wrong mask for the sixth job") self.assertEqual(jobs[5]["estimatedJobTime"], 140 * 12) self.assertEqual(jobs[5]["estimatedDiskUsage"], 140 * 400) self.assertEqual(jobs[6]["mask"].getRunAndLumis(), {6: [[18, 20]]}, "Wrong mask for the seventh job") self.assertEqual(jobs[6]["estimatedJobTime"], 150 * 12) self.assertEqual(jobs[6]["estimatedDiskUsage"], 150 * 400) for job in jobs: self.assertEqual(job["estimatedMemoryUsage"], 2300) # Test interactions of this algorithm with splitOnRun = True # Make 2 files, one with 3 runs and a second one with the last run of the first fileA = File(lfn="/this/is/file1a", size=1000, events=2400) lumiListA = [] lumiListB = [] lumiListC = [] for lumi in range(8): lumiListA.append(1 + lumi) lumiListB.append(1 + lumi) lumiListC.append(1 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.addRun(Run(2, *lumiListA)) fileA.addRun(Run(3, *lumiListA)) fileA.setLocation("T1_US_FNAL_Disk") fileB = self.createFile('/this/is/file2a', 200, 3, 5, "T1_US_FNAL_Disk") testFileset = Fileset(name='FilesetB') testFileset.create() testFileset.addFile(fileA) testFileset.addFile(fileB) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiBased", type="Processing") testSubscription.create() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) # The settings for this splitting are 700 events per job jobGroups = jobFactory(splitOnRun=True, halt_job_on_file_boundaries=False, events_per_job=700, performance=self.performanceParams) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 6, "Six jobs must be in the jobgroup") self.assertEqual(jobs[0]["estimatedJobTime"], 700 * 12) self.assertEqual(jobs[0]["estimatedDiskUsage"], 700 * 400) self.assertEqual(jobs[1]["estimatedJobTime"], 100 * 12) self.assertEqual(jobs[1]["estimatedDiskUsage"], 100 * 400) self.assertEqual(jobs[2]["estimatedJobTime"], 700 * 12) self.assertEqual(jobs[2]["estimatedDiskUsage"], 700 * 400) self.assertEqual(jobs[3]["estimatedJobTime"], 100 * 12) self.assertEqual(jobs[3]["estimatedDiskUsage"], 100 * 400) self.assertEqual(jobs[4]["estimatedJobTime"], 700 * 12) self.assertEqual(jobs[4]["estimatedDiskUsage"], 700 * 400) self.assertEqual(jobs[5]["estimatedJobTime"], 300 * 12) self.assertEqual(jobs[5]["estimatedDiskUsage"], 300 * 400)
def stuffWMBS(self, workflowURL, name): """ _stuffWMBS_ Insert some dummy jobs, jobgroups, filesets, files and subscriptions into WMBS to test job creation. Three completed job groups each containing several files are injected. Another incomplete job group is also injected. Also files are added to the "Mergeable" subscription as well as to the output fileset for their jobgroups. """ locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="s1", pnn="somese.cern.ch") mergeFileset = Fileset(name="mergeFileset") mergeFileset.create() bogusFileset = Fileset(name="bogusFileset") bogusFileset.create() mergeWorkflow = Workflow(spec=workflowURL, owner="mnorman", name=name, task="/TestWorkload/ReReco") mergeWorkflow.create() mergeSubscription = Subscription(fileset=mergeFileset, workflow=mergeWorkflow, split_algo="ParentlessMergeBySize") mergeSubscription.create() dummySubscription = Subscription(fileset=bogusFileset, workflow=mergeWorkflow, split_algo="ParentlessMergeBySize") file1 = File(lfn="file1", size=1024, events=1024, first_event=0, locations={"somese.cern.ch"}) file1.addRun(Run(1, *[45])) file1.create() file2 = File(lfn="file2", size=1024, events=1024, first_event=1024, locations={"somese.cern.ch"}) file2.addRun(Run(1, *[45])) file2.create() file3 = File(lfn="file3", size=1024, events=1024, first_event=2048, locations={"somese.cern.ch"}) file3.addRun(Run(1, *[45])) file3.create() file4 = File(lfn="file4", size=1024, events=1024, first_event=3072, locations={"somese.cern.ch"}) file4.addRun(Run(1, *[45])) file4.create() fileA = File(lfn="fileA", size=1024, events=1024, first_event=0, locations={"somese.cern.ch"}) fileA.addRun(Run(1, *[46])) fileA.create() fileB = File(lfn="fileB", size=1024, events=1024, first_event=1024, locations={"somese.cern.ch"}) fileB.addRun(Run(1, *[46])) fileB.create() fileC = File(lfn="fileC", size=1024, events=1024, first_event=2048, locations={"somese.cern.ch"}) fileC.addRun(Run(1, *[46])) fileC.create() fileI = File(lfn="fileI", size=1024, events=1024, first_event=0, locations={"somese.cern.ch"}) fileI.addRun(Run(2, *[46])) fileI.create() fileII = File(lfn="fileII", size=1024, events=1024, first_event=1024, locations={"somese.cern.ch"}) fileII.addRun(Run(2, *[46])) fileII.create() fileIII = File(lfn="fileIII", size=1024, events=1024, first_event=2048, locations={"somese.cern.ch"}) fileIII.addRun(Run(2, *[46])) fileIII.create() fileIV = File(lfn="fileIV", size=1024 * 1000000, events=1024, first_event=3072, locations={"somese.cern.ch"}) fileIV.addRun(Run(2, *[46])) fileIV.create() for fileObj in [ file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII, fileIII, fileIV ]: mergeFileset.addFile(fileObj) bogusFileset.addFile(fileObj) mergeFileset.commit() bogusFileset.commit() return
def testGetParentLFNs(self): """ _testGetParentLFNs_ Create three files and set them to be parents of a fourth file. Check to make sure that getParentLFNs() on the child file returns the correct LFNs. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentC = DBSBufferFile(lfn="/this/is/a/parent/lfnC", size=1024, events=20) testFileParentC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentC.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileParentC.create() testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.addRun(Run(1, *[45])) testFile.create() testFile.addParents([ testFileParentA["lfn"], testFileParentB["lfn"], testFileParentC["lfn"] ]) parentLFNs = testFile.getParentLFNs() assert len(parentLFNs) == 3, \ "ERROR: Child does not have the right amount of parents" goldenLFNs = [ "/this/is/a/parent/lfnA", "/this/is/a/parent/lfnB", "/this/is/a/parent/lfnC" ] for parentLFN in parentLFNs: assert parentLFN in goldenLFNs, \ "ERROR: Unknown parent lfn" goldenLFNs.remove(parentLFN) testFile.delete() testFileParentA.delete() testFileParentB.delete() testFileParentC.delete() return
def execute(self, jobID, fileSelection=None, conn=None, transaction=False): """ _execute_ Execute the SQL for the given job ID and then format and return the result. """ if type(jobID) == list: if len(jobID) < 1: # Nothing to do return [] binds = jobID else: binds = {"jobid": jobID} result = self.dbi.processData(self.sql, binds, conn=conn, transaction=transaction) jobList = self.formatJobs(result) filesResult = self.dbi.processData(self.fileSQL, binds, conn=conn, transaction=transaction) fileList = self.formatDict(filesResult) fileBinds = [] if fileSelection: fileList = [ x for x in fileList if x['lfn'] in fileSelection[x['jobid']] ] for x in fileList: # Add new runs x['newRuns'] = [] # Assemble unique list of binds if not {'fileid': x['id']} in fileBinds: fileBinds.append({'fileid': x['id']}) parentList = [] if len(fileBinds) > 0: parentResult = self.dbi.processData(self.parentSQL, fileBinds, conn=conn, transaction=transaction) parentList = self.formatDict(parentResult) lumiResult = self.dbi.processData(self.runLumiSQL, fileBinds, conn=conn, transaction=transaction) lumiList = self.formatDict(lumiResult) lumiDict = {} for l in lumiList: if not l['fileid'] in lumiDict.keys(): lumiDict[l['fileid']] = [] lumiDict[l['fileid']].append(l) for f in fileList: fileRuns = {} if f['id'] in lumiDict.keys(): for l in lumiDict[f['id']]: run = l['run'] lumi = l['lumi'] numEvents = l['num_events'] fileRuns.setdefault(run, []) fileRuns[run].append((lumi, numEvents)) for r in fileRuns.keys(): newRun = Run(runNumber=r) newRun.lumis = fileRuns[r] f['newRuns'].append(newRun) filesForJobs = {} for f in fileList: jobid = f['jobid'] if not jobid in filesForJobs.keys(): filesForJobs[jobid] = {} if f['id'] not in filesForJobs[jobid].keys(): wmbsFile = File(id=f['id']) wmbsFile.update(f) wmbsFile['locations'].add(f['pnn']) for r in wmbsFile['newRuns']: wmbsFile.addRun(r) for entry in parentList: if entry['id'] == f['id']: wmbsFile['parents'].add(entry['lfn']) filesForJobs[jobid][f['id']] = wmbsFile else: # If the file is there, just add the location filesForJobs[jobid][f['id']]['locations'].add(f['pnn']) for j in jobList: if j['id'] in filesForJobs.keys(): j['input_files'] = filesForJobs[j['id']].values() return jobList
def createFilesWithChildren(self, moreParentFiles, acqEra): """ _createFilesWithChildren_ Create several parentless files and then create child files. """ parentFiles = [] childFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % acqEra for i in range(10): testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % acqEra) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentFiles.append(testFile) baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % acqEra for i in range(5): testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % acqEra) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(20): lumis.append((i * 20) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") testFile.addParents([parentFiles[i * 2]["lfn"], parentFiles[i * 2 + 1]["lfn"]]) testFile.addParents([moreParentFiles[i * 2]["lfn"], moreParentFiles[i * 2 + 1]["lfn"]]) childFiles.append(testFile) return parentFiles, childFiles
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="Locations.New") locationAction.execute(siteName="site1", seName="somese.cern.ch") self.multipleFileFileset = Fileset(name="TestFileset1") self.multipleFileFileset.create() for i in range(10): newFile = File(makeUUID(), size=1000, events=100, locations=set(["somese.cern.ch"])) newFile.addRun(Run(i, *[45 + i])) newFile.create() self.multipleFileFileset.addFile(newFile) self.multipleFileFileset.commit() self.singleFileFileset = Fileset(name="TestFileset2") self.singleFileFileset.create() newFile = File("/some/file/name", size=1000, events=100, locations=set(["somese.cern.ch"])) newFile.addRun(Run(1, *[45])) newFile.create() self.singleFileFileset.addFile(newFile) self.singleFileFileset.commit() self.multipleFileLumiset = Fileset(name="TestFileset3") self.multipleFileLumiset.create() for i in range(10): newFile = File(makeUUID(), size=1000, events=100, locations=set(["somese.cern.ch"])) newFile.addRun(Run(1, *[45 + i / 3])) newFile.create() self.multipleFileLumiset.addFile(newFile) self.multipleFileLumiset.commit() self.singleLumiFileset = Fileset(name="TestFileset4") self.singleLumiFileset.create() for i in range(10): newFile = File(makeUUID(), size=1000, events=100, locations=set(["somese.cern.ch"])) newFile.addRun(Run(1, *[45])) newFile.create() self.singleLumiFileset.addFile(newFile) self.singleLumiFileset.commit() testWorkflow = Workflow(spec="spec.xml", owner="mnorman", name="wf001", task="Test") testWorkflow.create() self.multipleFileSubscription = Subscription( fileset=self.multipleFileFileset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.singleFileSubscription = Subscription( fileset=self.singleFileFileset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.multipleLumiSubscription = Subscription( fileset=self.multipleFileLumiset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.singleLumiSubscription = Subscription( fileset=self.singleLumiFileset, workflow=testWorkflow, split_algo="FixedDelay", type="Processing") self.multipleFileSubscription.create() self.singleFileSubscription.create() self.multipleLumiSubscription.create() self.singleLumiSubscription.create() return
def testLumiMask(self): """ _testLumiMask_ Test that we can use a lumi-mask to filter good runs/lumis. """ splitter = SplitterFactory() # Create 3 files with 100 events per lumi: # - file1 with 1 run of 8 lumis # - file2 with 2 runs of 2 lumis each # - file3 with 1 run of 5 lumis fileA = File(lfn="/this/is/file1", size=1000, events=800) fileB = File(lfn="/this/is/file2", size=1000, events=400) fileC = File(lfn="/this/is/file3", size=1000, events=500) lumiListA = [] for lumi in range(8): lumiListA.append(10 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.setLocation("somese.cern.ch") lumiListB1 = [] lumiListB2 = [] for lumi in range(2): lumiListB1.append(20 + lumi) lumiListB2.append(30 + lumi) fileB.addRun(Run(2, *lumiListB1)) fileB.addRun(Run(3, *lumiListB2)) fileB.setLocation("somese.cern.ch") lumiListC = [] for lumi in range(5): lumiListC.append(40 + lumi) fileC.addRun(Run(4, *lumiListC)) fileC.setLocation("somese.cern.ch") testFileset = Fileset(name='Fileset') testFileset.addFile(fileA) testFileset.addFile(fileB) testFileset.addFile(fileC) testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # Use a lumi-mask = {1: [[10,14]], 2: [[20,21]], 4: [[40,41]]} jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=850, runs=['1', '2', '4'], lumis=['10,14', '20,21', '40,41'], performance=self.performanceParams) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 2, "Two jobs must be in the jobgroup") processedLumis = LumiList() for job in jobs: processedLumis += LumiList(compactList=job['mask'].getRunAndLumis()) correctLumis = LumiList(compactList={1: [[10, 14]], 2: [[20, 21]], 4: [[40, 41]]}) self.assertEqual(processedLumis.getCMSSWString(), correctLumis.getCMSSWString())
def createTestJobGroup(self, nJobs=10, retry_count=1, workloadPath='test', fwjrPath=None, workloadName=makeUUID()): """ Creates a group of several jobs """ myThread = threading.currentThread() myThread.transaction.begin() testWorkflow = Workflow(spec=workloadPath, owner="cmsdataops", group="cmsdataops", name=workloadName, task="/TestWorkload/ReReco") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFile0 = File(lfn="/this/is/a/parent", size=1024, events=10) testFile0.addRun(Run(10, *[12312])) testFile0.setLocation('malpaquet') testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10, first_event=88) testFileA.addRun(Run(10, *[12312, 12313])) testFileA.setLocation('malpaquet') testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10, first_event=88) testFileB.addRun(Run(10, *[12314, 12315, 12316])) testFileB.setLocation('malpaquet') testFile0.create() testFileA.create() testFileB.create() testFileA.addParent(lfn="/this/is/a/parent") testFileB.addParent(lfn="/this/is/a/parent") for i in range(0, nJobs): testJob = Job(name=makeUUID()) testJob['retry_count'] = retry_count testJob['retry_max'] = 10 testJob['mask'].addRunAndLumis(run=10, lumis=[12312]) testJob['mask'].addRunAndLumis(run=10, lumis=[12314, 12316]) testJob['mask']['FirstEvent'] = 100 testJob['cache_dir'] = os.path.join(self.testDir, testJob['name']) testJob['fwjr_path'] = fwjrPath os.mkdir(testJob['cache_dir']) testJobGroup.add(testJob) testJob.create(group=testJobGroup) testJob.addFile(testFileA) testJob.addFile(testFileB) testJob.save() testJobGroup.commit() testSubscription.acquireFiles(files=[testFileA, testFileB]) testSubscription.save() myThread.transaction.commit() return testJobGroup
def stuffACDCDatabase(self, numFiles=50, lumisPerFile=20, lumisPerACDCRecord=2): """ _stuffACDCDatabase_ Fill the ACDC database with ACDC records, both for processing and merge """ filesetName = '/%s/DataProcessing' % self.workflowName owner = 'unknown' group = 'unknown' for i in range(numFiles): for j in range(1, lumisPerFile + 1, lumisPerACDCRecord): lfn = '/store/data/a/%d' % i acdcFile = File(lfn=lfn, size=100, events=250, locations=self.validLocations, merged=1) run = Run( i + 1, *range(j, min(j + lumisPerACDCRecord, lumisPerFile + 1))) acdcFile.addRun(run) acdcDoc = { 'collection_name': self.workflowName, 'collection_type': 'ACDC.CollectionTypes.DataCollection', 'files': { lfn: acdcFile }, 'fileset_name': filesetName, 'owner': { 'user': owner, 'group': group } } self.acdcDB.queue(acdcDoc) filesetName = '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName for i in range(numFiles): for j in range(1, lumisPerFile + 1, lumisPerACDCRecord): lfn = '/store/unmerged/b/%d' % i acdcFile = File(lfn=lfn, size=100, events=250, locations=set([choice(self.validLocations)]), merged=0) run = Run( i + 1, *range(j, min(j + lumisPerACDCRecord, lumisPerFile + 1))) acdcFile.addRun(run) acdcDoc = { 'collection_name': self.workflowName, 'collection_type': 'ACDC.CollectionTypes.DataCollection', 'files': { lfn: acdcFile }, 'fileset_name': filesetName, 'owner': { 'user': owner, 'group': group } } self.acdcDB.queue(acdcDoc) self.acdcDB.commit() return
def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the datbase. We'll inject files with the location set as an SE name as well as a PhEDEx node name as well. """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) insertWorkflow = buffer3Factory(classname="InsertWorkflow") insertWorkflow.execute("BogusRequest", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) datasetAction = uploadFactory(classname="NewDataset") createAction = uploadFactory(classname="CreateBlocks") datasetAction.execute(datasetPath=self.testDatasetA) datasetAction.execute(datasetPath=self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBufferBlock(name=self.blockAName, location="srm-cms.cern.ch", datasetpath=None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBufferBlock(name=self.blockBName, location="srm-cms.cern.ch", datasetpath=None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks=[newBlockA, newBlockB]) bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "LOCAL") fileStatus.execute(testFileB["lfn"], "LOCAL") fileStatus.execute(testFileC["lfn"], "LOCAL") fileStatus.execute(testFileD["lfn"], "LOCAL") fileStatus.execute(testFileE["lfn"], "LOCAL") associateWorkflow = buffer3Factory( classname="DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask") return
class DataDiscovery(TaskAction): """ I am the abstract class for the data discovery. Taking care of generalizing different data discovery possibilities. Implementing only a common method to return a properly formatted output. """ def formatOutput(self, task, requestname, datasetfiles, locations): """ Receives as input the result of the data location discovery operations and fill up the WMCore objects. """ self.logger.debug(" Formatting data discovery output ") # TEMPORARY pnn_psn_map = {} sbj = SiteDBJSON({ "key": self.config.TaskWorker.cmskey, "cert": self.config.TaskWorker.cmscert }) wmfiles = [] event_counter = 0 lumi_counter = 0 file_counter = 0 uniquelumis = set() ## Loop over the sorted list of files. for lfn, infos in datasetfiles.iteritems(): ## Skip the file if the block has not been found or has no locations. if not infos['BlockName'] in locations or not locations[ infos['BlockName']]: self.logger.warning( "Skipping %s because its block (%s) has no locations" % (lfn, infos['BlockName'])) continue ## Skip the file if it is not in VALID state. if not infos.get('ValidFile', True): self.logger.warning("Skipping invalid file %s" % lfn) continue if task['tm_use_parent'] == 1 and len(infos['Parents']) == 0: raise TaskWorkerException( "The CRAB3 server backend refuses to submit jobs to the Grid scheduler\n" + "because you specified useParents=True but some your files have no" + "parents.\nExample: " + lfn) ## Createa a WMCore File object. wmfile = File(lfn=lfn, events=infos['NumberOfEvents'], size=infos['Size'], checksums=infos['Checksums'], parents=infos['Parents']) wmfile['block'] = infos['BlockName'] wmfile['locations'] = [] for pnn in locations[infos['BlockName']]: if pnn and pnn not in pnn_psn_map: self.logger.debug("Translating PNN %s" % pnn) try: pnn_psn_map[pnn] = sbj.PNNtoPSN(pnn) except KeyError, ke: self.logger.error( "Impossible translating %s to a CMS name through SiteDB" % pnn) pnn_psn_map[pnn] = '' except httplib.HTTPException, ex: self.logger.error("Couldn't map SE to site: %s" % pnn) print "Couldn't map SE to site: %s" % pnn print "got problem: %s" % ex print "got another problem: %s" % ex.__dict__ if pnn and pnn in pnn_psn_map: if type(pnn_psn_map[pnn]) == list: wmfile['locations'].extend(pnn_psn_map[pnn]) else: wmfile['locations'].append(pnn_psn_map[pnn]) wmfile['workflow'] = requestname event_counter += infos['NumberOfEvents'] for run, lumis in infos['Lumis'].iteritems(): wmfile.addRun(Run(run, *lumis)) for lumi in lumis: uniquelumis.add((run, lumi)) lumi_counter += len(lumis) wmfiles.append(wmfile) file_counter += 1
def test10(self): """ _test10_ Test merging of multiple lumis with holes in the lumi sequence Hole is due to no streamer files for the lumi Multi lumi input """ mySplitArgs = self.splitArgs.copy() for lumi in [1, 2, 5]: for i in range(2): newFile = File(makeUUID(), size = 1000, events = 100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave = False) newFile.create() self.fileset2.addFile(newFile) self.fileset2.commit() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription2) mySplitArgs['maxInputEvents'] = 500 jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.insertClosedLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 3, 'STREAM' : "A", 'FILECOUNT' : 0, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime }, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.insertClosedLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 4, 'STREAM' : "A", 'FILECOUNT' : 1, 'INSERT_TIME' : self.currentTime, 'CLOSE_TIME' : self.currentTime }, transaction = False) self.feedStreamersDAO.execute(transaction = False) self.fileset1.loadData() jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") for fileid in self.fileset1.getFiles(type = 'id'): self.acquireFilesDAO.execute(self.subscription1['id'], fileid, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") for fileid in self.fileset1.getFiles(type = 'id'): self.completeFilesDAO.execute(self.subscription1['id'], fileid, transaction = False) jobGroups = jobFactory(**mySplitArgs) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create one job") job = jobGroups[0].jobs[0] self.assertEqual(len(job.getFiles()), 4, "ERROR: Job does not process 4 files") return
def testChunking(self): """ _testChunking_ Insert a workload and files that have several distinct sets of locations. Verify that the chunks are created correctly and that they only groups files that have the same set of locations. Also verify that the chunks are pulled out of ACDC correctly. """ dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc") testFileA = File(lfn=makeUUID(), size=1024, events=1024) testFileA.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileA.addRun(Run(1, 1, 2)) testFileB = File(lfn=makeUUID(), size=1024, events=1024) testFileB.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileB.addRun(Run(1, 3, 4)) testFileC = File(lfn=makeUUID(), size=1024, events=1024) testFileC.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileC.addRun(Run(1, 5, 6)) testJobA = self.getMinimalJob() testJobA.addFile(testFileA) testJobA.addFile(testFileB) testJobA.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024) testFileD.setLocation(["cmssrm.fnal.gov"]) testFileD.addRun(Run(2, 1, 2)) testFileE = File(lfn=makeUUID(), size=1024, events=1024) testFileE.setLocation(["cmssrm.fnal.gov"]) testFileE.addRun(Run(2, 3, 4)) testJobB = self.getMinimalJob() testJobB.addFile(testFileD) testJobB.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024, parents={"/some/parent/F"}) testFileF.setLocation(["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"]) testFileF.addRun(Run(3, 1, 2)) testFileG = File(lfn=makeUUID(), size=1024, events=1024, parents={"/some/parent/G"}) testFileG.setLocation(["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"]) testFileG.addRun(Run(3, 3, 4)) testFileH = File(lfn=makeUUID(), size=1024, events=1024, parents={"/some/parent/H"}) testFileH.setLocation(["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"]) testFileH.addRun(Run(3, 5, 6)) testJobC = self.getMinimalJob() testJobC.addFile(testFileF) testJobC.addFile(testFileG) testJobC.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024, merged=True) testFileI.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileI.addRun(Run(4, 1, 2)) testFileJ = File(lfn=makeUUID(), size=1024, events=1024, merged=True) testFileJ.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileJ.addRun(Run(4, 3, 4)) testFileK = File(lfn=makeUUID(), size=1024, events=1024, merged=True) testFileK.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"]) testFileK.addRun(Run(4, 5, 6)) testJobD = self.getMinimalJob() testJobD.addFile(testFileI) testJobD.addFile(testFileJ) testJobD.addFile(testFileK) dcs.failedJobs([testJobA, testJobB, testJobC, testJobD]) chunks = dcs.chunkFileset("ACDCTest", "/ACDCTest/reco", chunkSize=5) self.assertEqual(len(chunks), 4, "Error: There should be four chunks: %s" % len(chunks)) goldenMetaData = {1: {"lumis": 2, "locations": ["castor.cern.ch", "cmssrm.fnal.gov"], "events": 1024}, 2: {"lumis": 4, "locations": ["cmssrm.fnal.gov"], "events": 2048}, 3: {"lumis": 6, "locations": ["castor.cern.ch", "cmssrm.fnal.gov", "srm.ral.uk"], "events": 3072}, 5: {"lumis": 10, "locations": ["castor.cern.ch", "cmssrm.fnal.gov"], "events": 5120}} testFiles = [testFileA, testFileB, testFileC, testFileI, testFileJ, testFileK] lastFile = testFileA for testFile in testFiles: if lastFile["lfn"] < testFile["lfn"]: lastFile = testFile testFiles.remove(lastFile) goldenFiles = {1: [lastFile], 2: [testFileD, testFileE], 3: [testFileF, testFileG, testFileH], 5: testFiles} for chunk in chunks: chunkMetaData = dcs.getChunkInfo("ACDCTest", "/ACDCTest/reco", chunk["offset"], chunk["files"]) self.assertEqual(chunkMetaData["files"], chunk["files"]) self.assertEqual(chunkMetaData["lumis"], chunk["lumis"]) self.assertEqual(chunkMetaData["events"], chunk["events"]) self.assertEqual(chunkMetaData["locations"], chunk["locations"]) self.assertTrue(chunk["files"] in goldenMetaData, "Error: Extra chunk found.") self.assertEqual(chunk["lumis"], goldenMetaData[chunk["files"]]["lumis"], "Error: Lumis in chunk is wrong.") self.assertEqual(chunk["locations"], goldenMetaData[chunk["files"]]["locations"], "Error: Locations in chunk is wrong.") self.assertEqual(chunk["events"], goldenMetaData[chunk["files"]]["events"], "Error: Events in chunk is wrong.") del goldenMetaData[chunk["files"]] chunkFiles = dcs.getChunkFiles("ACDCTest", "/ACDCTest/reco", chunk["offset"], chunk["files"]) self.assertTrue(chunk["files"] in goldenFiles, "Error: Extra chunk found.") goldenChunkFiles = goldenFiles[chunk["files"]] self.assertEqual(len(chunkFiles), len(goldenChunkFiles)) for chunkFile in chunkFiles: foundFile = None for goldenChunkFile in goldenChunkFiles: if chunkFile["lfn"] == goldenChunkFile["lfn"]: foundFile = goldenChunkFile break self.assertIsNotNone(foundFile, "Error: Missing chunk file: %s, %s" % (chunkFiles, goldenChunkFiles)) self.assertEqual(set(foundFile["parents"]), chunkFile["parents"], "Error: File parents should match.") self.assertEqual(foundFile["merged"], chunkFile["merged"], "Error: File merged status should match.") self.assertEqual(foundFile["locations"], chunkFile["locations"], "Error: File locations should match.") self.assertEqual(foundFile["events"], chunkFile["events"]) self.assertEqual(foundFile["size"], chunkFile["size"]) self.assertEqual(len(foundFile["runs"]), len(chunkFile["runs"]), "Error: Wrong number of runs.") for run in foundFile["runs"]: runMatch = False for chunkRun in chunkFile["runs"]: if chunkRun.run == run.run and chunkRun.lumis == run.lumis: runMatch = True break self.assertTrue(runMatch, "Error: Run information is wrong.") del goldenFiles[chunk["files"]] singleChunk = dcs.singleChunkFileset("ACDCTest", "/ACDCTest/reco") self.assertEqual(singleChunk, {"offset": 0, "files": 11, "events": 11264, "lumis": 22, "locations": {"castor.cern.ch", "cmssrm.fnal.gov", "srm.ral.uk"}}, "Error: Single chunk metadata is wrong") return
def testGetLumiWhitelist(self): """ _testGetLumiWhitelist_ Verify that the ACDC whitelist generation code works correctly. We'll add jobs with the following lumi info: # Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12] # Run 2, lumis [5, 6, 7], [10, 11, 12], [15] # Run 3, lumis [20] And should get out a whitelist that looks like this: {"1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]]} """ dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc") testFileA = File(lfn=makeUUID(), size=1024, events=1024) testFileA.addRun(Run(1, 1, 2)) testFileB = File(lfn=makeUUID(), size=1024, events=1024) testFileB.addRun(Run(1, 3)) testJobA = self.getMinimalJob() testJobA.addFile(testFileA) testJobA.addFile(testFileB) testFileC = File(lfn=makeUUID(), size=1024, events=1024) testFileC.addRun(Run(1, 4, 6)) testJobB = self.getMinimalJob() testJobB.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024) testFileD.addRun(Run(1, 7)) testJobC = self.getMinimalJob() testJobC.addFile(testFileD) testFileE = File(lfn=makeUUID(), size=1024, events=1024) testFileE.addRun(Run(1, 11, 12)) testJobD = self.getMinimalJob() testJobD.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024) testFileF.addRun(Run(2, 5, 6, 7)) testJobE = self.getMinimalJob() testJobE.addFile(testFileF) testFileG = File(lfn=makeUUID(), size=1024, events=1024) testFileG.addRun(Run(2, 10, 11, 12)) testJobF = self.getMinimalJob() testJobF.addFile(testFileG) testFileH = File(lfn=makeUUID(), size=1024, events=1024) testFileH.addRun(Run(2, 15)) testJobG = self.getMinimalJob() testJobG.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024) testFileI.addRun(Run(3, 20)) testJobH = self.getMinimalJob() testJobH.addFile(testFileI) testFileJ = File(lfn=makeUUID(), size=1024, events=1024) testFileJ.addRun(Run(1, 9)) testJobI = self.getMinimalJob() testJobI.addFile(testFileJ) dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE, testJobF, testJobG, testJobH, testJobI]) whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(len(whiteList), 3, "Error: There should be 3 runs.") self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]], "Error: Whitelist for run 1 is wrong.") self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]], "Error: Whitelist for run 2 is wrong.") self.assertEqual(whiteList["3"], [[20, 20]], "Error: Whitelist for run 3 is wrong.") correctLumiList = LumiList(compactList={"1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]]}) testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(correctLumiList.getCMSSWString(), testLumiList.getCMSSWString()) return
def testC_ACDCTest(self): """ _ACDCTest_ Test whether we can get a goodRunList out of ACDC and process it correctly. """ workload = self.createTestWorkload() dcs = DataCollectionService(url=self.testInit.couchUrl, database=self.testInit.couchDbName) testFileA = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileA.addRun(Run(1, 1, 2)) testFileA.create() testFileB = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileB.addRun(Run(1, 3)) testFileB.create() testJobA = getJob(workload) testJobA.addFile(testFileA) testJobA.addFile(testFileB) testFileC = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileC.addRun(Run(1, 4, 6)) testFileC.create() testJobB = getJob(workload) testJobB.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileD.addRun(Run(1, 7)) testFileD.create() testJobC = getJob(workload) testJobC.addFile(testFileD) testFileE = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileE.addRun(Run(1, 11, 12)) testFileE.create() testJobD = getJob(workload) testJobD.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileF.addRun(Run(2, 5, 6, 7)) testFileF.create() testJobE = getJob(workload) testJobE.addFile(testFileF) testFileG = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileG.addRun(Run(2, 10, 11, 12)) testFileG.create() testJobF = getJob(workload) testJobF.addFile(testFileG) testFileH = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileH.addRun(Run(2, 15)) testFileH.create() testJobG = getJob(workload) testJobG.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileI.addRun(Run(3, 20)) testFileI.create() testJobH = getJob(workload) testJobH.addFile(testFileI) testFileJ = File(lfn=makeUUID(), size=1024, events=1024, locations="T1_US_FNAL_Disk") testFileJ.addRun(Run(1, 9)) testFileJ.create() testJobI = getJob(workload) testJobI.addFile(testFileJ) # dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE, # testJobF, testJobG, testJobH, testJobI]) dcs.failedJobs([testJobA, testJobD, testJobH]) baseName = makeUUID() testFileset = Fileset(name=baseName) testFileset.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.addFile(testFileC) testFileset.addFile(testFileD) testFileset.addFile(testFileE) testFileset.addFile(testFileF) testFileset.addFile(testFileG) testFileset.addFile(testFileH) testFileset.addFile(testFileI) testFileset.addFile(testFileJ) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="LumiBased", type="Processing") testSubscription.create() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroups = jobFactory( lumis_per_job=100, halt_job_on_file_boundaries=False, splitOnRun=True, collectionName=workload.name(), filesetName=workload.getTask("reco").getPathName(), owner="evansde77", group="DMWM", couchURL=self.testInit.couchUrl, couchDB=self.testInit.couchDbName, performance=self.performanceParams) self.assertEqual(jobGroups[0].jobs[0]['mask'].getRunAndLumis(), {1: [[1, 2], [3, 3], [11, 12]]}) self.assertEqual(jobGroups[0].jobs[1]['mask'].getRunAndLumis(), {3: [[20, 20]]}) return
def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["T0.WMBS"]) self.splitterFactory = SplitterFactory(package = "T0.JobSplitting") myThread = threading.currentThread() daoFactory = DAOFactory(package = "T0.WMBS", logger = logging, dbinterface = myThread.dbi) wmbsDaoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) myThread.dbi.processData("""INSERT INTO wmbs_location (id, site_name, state) VALUES (1, 'SomeSite', 1) """, transaction = False) myThread.dbi.processData("""INSERT INTO wmbs_location_pnn (location, pnn) VALUES (1, 'SomePNN') """, transaction = False) insertRunDAO = daoFactory(classname = "RunConfig.InsertRun") insertRunDAO.execute(binds = { 'RUN' : 1, 'HLTKEY' : "someHLTKey" }, transaction = False) insertLumiDAO = daoFactory(classname = "RunConfig.InsertLumiSection") insertLumiDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 1 }, transaction = False) insertStreamDAO = daoFactory(classname = "RunConfig.InsertStream") insertStreamDAO.execute(binds = { 'STREAM' : "Express" }, transaction = False) insertStreamFilesetDAO = daoFactory(classname = "RunConfig.InsertStreamFileset") insertStreamFilesetDAO.execute(1, "Express", "TestFileset1") insertStreamerDAO = daoFactory(classname = "RunConfig.InsertStreamer") insertStreamerDAO.execute(binds = { 'RUN' : 1, 'LUMI' : 1, 'STREAM' : "Express", 'TIME' : int(time.time()), 'LFN' : "/streamer", 'FILESIZE' : 0, 'EVENTS' : 0 }, transaction = False) insertPromptCalibrationDAO = daoFactory(classname = "RunConfig.InsertPromptCalibration") insertPromptCalibrationDAO.execute( { 'RUN' : 1, 'STREAM' : "Express" }, transaction = False) self.fileset1 = Fileset(name = "TestFileset1") self.fileset1.create() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Condition", type = "Condition") self.subscription1.create() # set parentage chain and sqlite fileset alcaRecoFile = File("/alcareco", size = 0, events = 0) alcaRecoFile.addRun(Run(1, *[1])) alcaRecoFile.setLocation("SomePNN", immediateSave = False) alcaRecoFile.create() alcaPromptFile = File("/alcaprompt", size = 0, events = 0) alcaPromptFile.addRun(Run(1, *[1])) alcaPromptFile.setLocation("SomePNN", immediateSave = False) alcaPromptFile.create() sqliteFile = File("/sqlite", size = 0, events = 0) sqliteFile.create() self.fileset1.addFile(sqliteFile) self.fileset1.commit() results = myThread.dbi.processData("""SELECT lfn FROM wmbs_file_details """, transaction = False)[0].fetchall() setParentageDAO = wmbsDaoFactory(classname = "Files.SetParentage") setParentageDAO.execute(binds = [ { 'parent' : "/streamer", 'child' : "/alcareco" }, { 'parent' : "/alcareco", 'child' : "/alcaprompt" }, { 'parent' : "/alcaprompt", 'child' : "/sqlite" } ], transaction = False) # default split parameters self.splitArgs = {} self.splitArgs['runNumber'] = 1 self.splitArgs['streamName'] = "Express" return
def test_NotEnoughEvents(self): """ _test_NotEnoughEvents_ Checks whether jobs are not created when there are not enough files (actually, events) according to the events_per_job requested to the splitter algorithm """ splitter = SplitterFactory() # Very small fileset (single file) without enough events testSubscription = self.createSubscription(nFiles=1, lumisPerFile=2) jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroups = jobFactory(lumis_per_job=5, performance=self.performanceParams) self.assertEqual(len(jobGroups), 0) # Still a small fileset (two files) without enough events testSubscription = self.createSubscription(nFiles=2, lumisPerFile=2) jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroups = jobFactory(lumis_per_job=5, performance=self.performanceParams, splitOnRun=False) self.assertEqual(len(jobGroups), 0) # Finally an acceptable fileset size (three files) with enough events testSubscription = self.createSubscription(nFiles=3, lumisPerFile=2) jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroups = jobFactory(lumis_per_job=5, performance=self.performanceParams, splitOnRun=False) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 2) self.assertEqual(len(jobs[0]['input_files']), 3) self.assertEqual(len(jobs[1]['input_files']), 1) self.assertEqual(jobs[0]['mask'].getRunAndLumis(), { 0: [[0, 1]], 1: [[100, 101]], 2: [[200, 200]] }) self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {2: [[201, 201]]}) # Test fileset with a single run and splitOnRun=True testFileset = Fileset(name="FilesetA") for i in range(3): testFile = File(lfn="/this/is/file%s" % i, size=1024, events=200, locations="T1_US_FNAL_Disk") lumis = [i * 2 + val for val in range(1, 3)] testFile.addRun(Run(1, lumis)) testFile.create() testFileset.addFile(testFile) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="LumiBased", type="Processing") testSubscription.create() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroups = jobFactory(lumis_per_job=5, performance=self.performanceParams) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 2) self.assertEqual(len(jobs[0]['input_files']), 3) self.assertEqual(len(jobs[1]['input_files']), 1) self.assertEqual(jobs[0]['mask'].getRunAndLumis(), {1: [[1, 2], [3, 4], [5, 5]]}) self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {1: [[6, 6]]}) return
def formatOutput(self, task, requestname, datasetfiles, locations, tempDir): """ Receives as input the result of the data location discovery operations and fill up the WMCore objects. """ self.logger.debug(" Formatting data discovery output ") wmfiles = [] event_counter = 0 lumi_counter = 0 uniquelumis = set() datasetLumis = {} ## Loop over the sorted list of files. # can't affort one message from CRIC per file, unless critical ! previousLogLevel = self.logger.getEffectiveLevel() resourceCatalog = CRIC(logger=self.logger) self.logger.setLevel(logging.ERROR) for lfn, infos in datasetfiles.iteritems(): ## Skip the file if the block has not been found or has no locations. if not infos['BlockName'] in locations or not locations[ infos['BlockName']]: self.logger.warning( "Skipping %s because its block (%s) has no locations" % (lfn, infos['BlockName'])) continue ## Skip the file if it is not in VALID state. if not infos.get('ValidFile', True): self.logger.warning("Skipping invalid file %s" % lfn) continue if task['tm_use_parent'] == 1 and len(infos['Parents']) == 0: raise TaskWorkerException( "The CRAB3 server backend refuses to submit jobs to the Grid scheduler\n" + "because you specified useParents=True but some your files have no" + "parents.\nExample: " + lfn) ## Create a WMCore File object. try: size = infos['FileSize'] checksums = { 'Checksum': infos['Checksum'], 'Adler32': infos['Adler32'], 'Md5': infos['Md5'] } except: #This is so that the task worker does not crash if an old version of WMCore is used (the interface of an API suddenly changed). # We may want to remove the try/except and the following two lines eventually, but keeping them for the moment so other devels won't be affected #See this WMCore commit: https://github.com/dmwm/WMCore/commit/2afc01ae571390f5fa009dd258be757adac89c28#diff-374b7a6640288184175057234e393e1cL204 size = infos['Size'] checksums = infos['Checksums'] wmfile = File(lfn=lfn, events=infos['NumberOfEvents'], size=size, checksums=checksums, parents=infos['Parents']) wmfile['block'] = infos['BlockName'] try: wmfile['locations'] = resourceCatalog.PNNstoPSNs( locations[wmfile['block']]) except Exception as ex: self.logger.error( "Impossible translating %s to a CMS name through CMS Resource Catalog" % locations[wmfile['block']]) self.logger.error("got this exception:\n %s" % ex) raise wmfile['workflow'] = requestname event_counter += infos['NumberOfEvents'] for run, lumis in infos['Lumis'].iteritems(): datasetLumis.setdefault(run, []).extend(lumis) wmfile.addRun(Run(run, *lumis)) for lumi in lumis: uniquelumis.add((run, lumi)) lumi_counter += len(lumis) wmfiles.append(wmfile) self.logger.setLevel(previousLogLevel) uniquelumis = len(uniquelumis) self.logger.debug('Tot events found: %d' % event_counter) self.logger.debug('Tot lumis found: %d' % uniquelumis) self.logger.debug('Duplicate lumis found: %d' % (lumi_counter - uniquelumis)) self.logger.debug('Tot files found: %d' % len(wmfiles)) self.logger.debug( "Starting to create compact lumilists for input dataset") datasetLumiList = LumiList(runsAndLumis=datasetLumis) datasetLumis = datasetLumiList.getCompactList() datasetDuplicateLumis = datasetLumiList.getDuplicates().getCompactList( ) self.logger.debug( "Finished to create compact lumilists for input dataset") with open(os.path.join(tempDir, "input_dataset_lumis.json"), "w") as fd: json.dump(datasetLumis, fd) with open(os.path.join(tempDir, "input_dataset_duplicate_lumis.json"), "w") as fd: json.dump(datasetDuplicateLumis, fd) return Result(task=task, result=Fileset(name='FilesToSplit', files=set(wmfiles)))
def testNoFileSplitNoHardLimit(self): """ _testNoFileSplitNoHardLimit_ In this case we don't split on file boundaries, check different combination of files make sure we make the most of the splitting, e.g. include many zero event files in a single job. """ splitter = SplitterFactory() # Create 100 files with 7 lumi per file and 0 events per lumi on average. testSubscription = self.createSubscription(nFiles=100, lumisPerFile=7, twoSites=False, nEventsPerFile=0) jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # First test, the optimal settings are 360 events per job. As we have files with 0 events per lumi, this will # configure the splitting to a single job containing all files jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=360, performance=self.performanceParams) # One job in one job group with 100 files self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 1) self.assertEqual(len(jobs[0]['input_files']), 100) # Create 7 files, each one with different lumi/event distributions testFileset = Fileset(name="FilesetA") testFileA = self.createFile("/this/is/file1", 250, 0, 5, "blenheim") testFileB = self.createFile("/this/is/file2", 600, 1, 1, "blenheim") testFileC = self.createFile("/this/is/file3", 1200, 2, 2, "blenheim") testFileD = self.createFile("/this/is/file4", 100, 3, 1, "blenheim") testFileE = self.createFile("/this/is/file5", 30, 4, 1, "blenheim") testFileF = self.createFile("/this/is/file6", 10, 5, 1, "blenheim") testFileG = self.createFile("/this/is/file7", 153, 6, 3, "blenheim") testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.addFile(testFileC) testFileset.addFile(testFileD) testFileset.addFile(testFileE) testFileset.addFile(testFileF) testFileset.addFile(testFileG) testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # Split the work targeting 150 events per job jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=150, performance=self.performanceParams) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 7) # Test interactions of this algorithm with splitOnRun = True # Make 2 files, one with 3 runs and a second one with the last run of the first fileA = File(lfn="/this/is/file1", size=1000, events=2400) lumiListA = [] lumiListB = [] lumiListC = [] for lumi in range(8): lumiListA.append(1 + lumi) lumiListB.append(1 + lumi) lumiListC.append(1 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.addRun(Run(2, *lumiListA)) fileA.addRun(Run(3, *lumiListA)) fileA.setLocation("malpaquet") fileB = self.createFile('/this/is/file2', 200, 3, 5, "malpaquet") testFileset = Fileset(name='FilesetB') testFileset.addFile(fileA) testFileset.addFile(fileB) testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # The settings for this splitting are 700 events per job jobGroups = jobFactory(splitOnRun=True, halt_job_on_file_boundaries=False, events_per_job=700, performance=self.performanceParams) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 6) # Make sure each job has one run for job in jobs: self.assertEqual(len(job['mask'].getRunAndLumis()), 1)
def test00(self): """ _test00_ Test that the job name prefix feature works Test event threshold (single job creation) Test that only closed lumis are used Test check on express release """ insertClosedLumiBinds = [] for lumi in [1]: filecount = 2 for i in range(filecount): newFile = File(makeUUID(), size=1000, events=100) newFile.addRun(Run(1, *[lumi])) newFile.setLocation("SomeSE", immediateSave=False) newFile.create() self.fileset1.addFile(newFile) insertClosedLumiBinds.append({ 'RUN': 1, 'LUMI': lumi, 'STREAM': "Express", 'FILECOUNT': filecount, 'INSERT_TIME': self.currentTime, 'CLOSE_TIME': 0 }) self.fileset1.commit() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(maxInputEvents=200) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds, transaction=False) jobGroups = jobFactory(maxInputEvents=200) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.finalCloseLumis() jobGroups = jobFactory(maxInputEvents=200) self.assertEqual(len(jobGroups), 0, "ERROR: JobFactory should have returned no JobGroup") self.releaseExpressDAO.execute(binds={'RUN': 1}, transaction=False) jobGroups = jobFactory(maxInputEvents=200) self.assertEqual(len(jobGroups), 1, "ERROR: JobFactory didn't return one JobGroup") self.assertEqual(len(jobGroups[0].jobs), 1, "ERROR: JobFactory didn't create a single job") job = jobGroups[0].jobs[0] self.assertTrue(job['name'].startswith("Express-"), "ERROR: Job has wrong name") self.assertEqual(self.getNumActiveSplitLumis(), 0, "ERROR: Split lumis were created") return
def testRunWhiteList(self): """ _testRunWhiteList_ Test that we can use a run white list to filter good runs/lumis. """ splitter = SplitterFactory() # Create 3 files with 100 events per lumi: # - file1 with 1 run of 8 lumis # - file2 with 2 runs of 2 lumis each # - file3 with 1 run of 5 lumis fileA = File(lfn="/this/is/file1", size=1000, events=800) fileB = File(lfn="/this/is/file2", size=1000, events=400) fileC = File(lfn="/this/is/file3", size=1000, events=500) lumiListA = [] for lumi in range(8): lumiListA.append(10 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.setLocation("somese.cern.ch") lumiListB1 = [] lumiListB2 = [] for lumi in range(2): lumiListB1.append(20 + lumi) lumiListB2.append(30 + lumi) fileB.addRun(Run(2, *lumiListB1)) fileB.addRun(Run(3, *lumiListB2)) fileB.setLocation("somese.cern.ch") lumiListC = [] for lumi in range(5): lumiListC.append(40 + lumi) fileC.addRun(Run(4, *lumiListC)) fileC.setLocation("somese.cern.ch") testFileset = Fileset(name='Fileset') testFileset.addFile(fileA) testFileset.addFile(fileB) testFileset.addFile(fileC) testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # Split with no breaks jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=725, runWhitelist=[1, 4], performance=self.performanceParams) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 2) for job in jobs: for run in job['mask'].getRunAndLumis(): self.assertIn(run, [1, 4]) # Re-split with a break on runs jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=True, events_per_job=595, runWhitelist=[1, 3, 4], performance=self.performanceParams) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 4) self.enforceLimits(jobs=jobs, runsPerJob=1) for job in jobs: for run in job['mask'].getRunAndLumis(): self.assertIn(run, [1, 3, 4]) # Re-split with a break on files jobGroups = jobFactory(halt_job_on_file_boundaries=True, splitOnRun=False, events_per_job=595, runWhitelist=[1, 2, 3], performance=self.performanceParams) self.assertEqual(len(jobGroups), 1) jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 3) self.enforceLimits(jobs=jobs, filesPerJob=1) for job in jobs: for run in job['mask'].getRunAndLumis(): self.assertIn(run, [1, 2, 3])
def testB_NoFileSplitNoHardLimit(self): """ _testB_NoFileSplitNoHardLimit_ In this case we don't split on file boundaries, check different combination of files make sure we make the most of the splitting, e.g. include many zero event files in a single job. """ splitter = SplitterFactory() #Create 100 files with 7 lumi per file and 0 events per lumi on average. testSubscription = self.createSubscription(nFiles=100, lumisPerFile=7, twoSites=False, nEventsPerFile=0) jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) #First test, the optimal settings are 360 events per job #As we have files with 0 events per lumi, this will configure the splitting to #a single job containing all files jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=360) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 1, "There should be 1 job") self.assertEqual(len(jobs[0]['input_files']), 100, "All 100 files must be in the job") #Create 7 files, each one with different lumi/event distributions testFileset = Fileset(name="FilesetA") testFileset.create() testFileA = self.createFile("/this/is/file1", 250, 0, 5, "otherse.cern.ch") testFileB = self.createFile("/this/is/file2", 600, 1, 1, "otherse.cern.ch") testFileC = self.createFile("/this/is/file3", 1200, 2, 2, "otherse.cern.ch") testFileD = self.createFile("/this/is/file4", 100, 3, 1, "otherse.cern.ch") testFileE = self.createFile("/this/is/file5", 30, 4, 1, "otherse.cern.ch") testFileF = self.createFile("/this/is/file6", 10, 5, 1, "otherse.cern.ch") testFileG = self.createFile("/this/is/file7", 151, 6, 3, "otherse.cern.ch") testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.addFile(testFileC) testFileset.addFile(testFileD) testFileset.addFile(testFileE) testFileset.addFile(testFileF) testFileset.addFile(testFileG) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiBased", type="Processing") testSubscription.create() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) #Optimal settings are: jobs with 150 events per job #This means, the first file must be splitted in 3 lumis per job which would leave room #for another lumi in the second job, but the second file has a lumi too big for that #The 3rd job only contains the second file, the fourth and fifth job split the third file jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=150) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 8, "Eight jobs must be in the jobgroup") self.assertEqual(jobs[0]["mask"].getRunAndLumis(), {0L: [[0L, 2L]]}, "Wrong mask for the first job") self.assertEqual(jobs[1]["mask"].getRunAndLumis(), {0L: [[3L, 4L]]}, "Wrong mask for the second job") self.assertEqual(jobs[2]["mask"].getRunAndLumis(), {1L: [[1L, 1L]]}, "Wrong mask for the third job") self.assertEqual(jobs[3]["mask"].getRunAndLumis(), {2L: [[4L, 4L]]}, "Wrong mask for the fourth job") self.assertEqual(jobs[4]["mask"].getRunAndLumis(), {2L: [[5L, 5L]]}, "Wrong mask for the fifth job") self.assertEqual(jobs[5]["mask"].getRunAndLumis(), { 3L: [[3L, 3L]], 4L: [[4L, 4L]], 5L: [[5L, 5L]] }, "Wrong mask for the sixth job") self.assertEqual(jobs[6]["mask"].getRunAndLumis(), {6L: [[18L, 19L]]}, "Wrong mask for the seventh job") self.assertEqual(jobs[7]["mask"].getRunAndLumis(), {6L: [[20L, 20L]]}, "Wrong mask for the seventh job") #Test interactions of this algorithm with splitOnRun = True #Make 2 files, one with 3 runs and a second one with the last run of the first fileA = File(lfn="/this/is/file1a", size=1000, events=2400) lumiListA = [] lumiListB = [] lumiListC = [] for lumi in range(8): lumiListA.append(1 + lumi) lumiListB.append(1 + lumi) lumiListC.append(1 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.addRun(Run(2, *lumiListA)) fileA.addRun(Run(3, *lumiListA)) fileA.setLocation("somese.cern.ch") fileB = self.createFile('/this/is/file2a', 200, 3, 5, "somese.cern.ch") testFileset = Fileset(name='FilesetB') testFileset.create() testFileset.addFile(fileA) testFileset.addFile(fileB) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiBased", type="Processing") testSubscription.create() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) #The settings for this splitting are 700 events per job jobGroups = jobFactory(splitOnRun=True, halt_job_on_file_boundaries=False, events_per_job=700) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 6, "Six jobs must be in the jobgroup")
class WMBSHelper(WMConnectionBase): """ _WMBSHelper_ Interface between the WorkQueue and WMBS. """ def __init__(self, wmSpec, taskName, blockName=None, mask=None, cachepath='.'): """ _init_ Initialize DAOs and other things needed. """ self.block = blockName self.mask = mask self.wmSpec = wmSpec self.topLevelTask = wmSpec.getTask(taskName) self.cachepath = cachepath self.isDBS = True self.topLevelFileset = None self.topLevelSubscription = None self.topLevelTaskDBSBufferId = None self.mergeOutputMapping = {} # Initiate the pieces you need to run your own DAOs WMConnectionBase.__init__(self, "WMCore.WMBS") myThread = threading.currentThread() self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) # DAOs from WMBS for file commit self.setParentage = self.daofactory(classname="Files.SetParentage") self.setFileRunLumi = self.daofactory(classname="Files.AddRunLumi") self.setFileLocation = self.daofactory( classname="Files.SetLocationForWorkQueue") self.setFileAddChecksum = self.daofactory( classname="Files.AddChecksumByLFN") self.addFileAction = self.daofactory(classname="Files.Add") self.addToFileset = self.daofactory(classname="Files.AddDupsToFileset") self.getLocations = self.daofactory(classname="Locations.ListSites") self.getLocationInfo = self.daofactory( classname="Locations.GetSiteInfo") # DAOs from DBSBuffer self.dbsCreateFiles = self.dbsDaoFactory( classname="DBSBufferFiles.Add") self.dbsSetLocation = self.dbsDaoFactory( classname="DBSBufferFiles.SetLocationByLFN") self.dbsInsertLocation = self.dbsDaoFactory( classname="DBSBufferFiles.AddLocation") self.dbsSetChecksum = self.dbsDaoFactory( classname="DBSBufferFiles.AddChecksumByLFN") self.dbsInsertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow") # Added for file creation bookkeeping self.dbsFilesToCreate = [] self.addedLocations = [] self.wmbsFilesToCreate = [] self.insertedBogusDataset = -1 return def createSandbox(self): """Create the runtime sandbox""" sandboxCreator = SandboxCreator() sandboxCreator.makeSandbox(self.cachepath, self.wmSpec) def createTopLevelFileset(self, topLevelFilesetName=None): """ _createTopLevelFileset_ Create the top level fileset for the workflow. If the name of the top level fileset is not given create one. """ if topLevelFilesetName == None: filesetName = ( "%s-%s" % (self.wmSpec.name(), self.wmSpec.getTopLevelTask()[0].name())) if self.block: filesetName += "-%s" % self.block if self.mask: from hashlib import md5 mask_string = ",".join( ["%s=%s" % (x, self.mask[x]) for x in sorted(self.mask)]) filesetName += "-%s" % md5(mask_string).hexdigest() else: filesetName = topLevelFilesetName self.topLevelFileset = Fileset(filesetName) self.topLevelFileset.create() return def outputFilesetName(self, task, outputModuleName): """ _outputFilesetName_ Generate an output fileset name for the given task and output module. """ if task.taskType() == "Merge": outputFilesetName = "%s/merged-%s" % (task.getPathName(), outputModuleName) else: outputFilesetName = "%s/unmerged-%s" % (task.getPathName(), outputModuleName) return outputFilesetName def createSubscription(self, task, fileset, alternativeFilesetClose=False): """ _createSubscription_ Create subscriptions in the database. This includes workflows in WMBS and DBSBuffer, output maps, datasets and phedex subscriptions, and filesets for each task below and including the given task. """ sub = self._createSubscriptionsInWMBS(task, fileset, alternativeFilesetClose) self._createWorkflowsInDBSBuffer() self._createDatasetSubscriptionsInDBSBuffer() return sub def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False): """ __createSubscriptionsInWMBS_ Create subscriptions in WMBS for all the tasks in the spec. This includes filesets, workflows and the output map for each task. """ # create runtime sandbox for workflow self.createSandbox() #FIXME: Let workflow put in values if spec is missing them workflow = Workflow( spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"], dn=self.wmSpec.getOwner().get("dn", "unknown"), group=self.wmSpec.getOwner().get("group", "unknown"), owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"), owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"), name=self.wmSpec.name(), task=task.getPathName(), wfType=self.wmSpec.getDashboardActivity(), alternativeFilesetClose=alternativeFilesetClose, priority=self.wmSpec.priority()) workflow.create() subscription = Subscription(fileset=fileset, workflow=workflow, split_algo=task.jobSplittingAlgorithm(), type=task.getPrimarySubType()) if subscription.exists(): subscription.load() msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)" self.logger.info(msg % (subscription['id'], task.getPathName())) else: subscription.create() for site in task.siteWhitelist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": True }]) for site in task.siteBlacklist(): subscription.addWhiteBlackList([{ "site_name": site, "valid": False }]) if self.topLevelSubscription == None: self.topLevelSubscription = subscription logging.info("Top level subscription created: %s" % subscription["id"]) else: logging.info("Child subscription created: %s" % subscription["id"]) outputModules = task.getOutputModulesForTask() for outputModule in outputModules: for outputModuleName in outputModule.listSections_(): outputFileset = Fileset( self.outputFilesetName(task, outputModuleName)) outputFileset.create() outputFileset.markOpen(True) mergedOutputFileset = None for childTask in task.childTaskIterator(): if childTask.data.input.outputModule == outputModuleName: if childTask.taskType() == "Merge": mergedOutputFileset = Fileset( self.outputFilesetName(childTask, "Merged")) mergedOutputFileset.create() mergedOutputFileset.markOpen(True) primaryDataset = getattr( getattr(outputModule, outputModuleName), "primaryDataset", None) if primaryDataset != None: self.mergeOutputMapping[ mergedOutputFileset.id] = primaryDataset self._createSubscriptionsInWMBS( childTask, outputFileset, alternativeFilesetClose) if mergedOutputFileset == None: workflow.addOutput(outputModuleName, outputFileset, outputFileset) else: workflow.addOutput(outputModuleName, outputFileset, mergedOutputFileset) return self.topLevelSubscription def addMCFakeFile(self): """Add a fake file for wmbs to run production over""" needed = [ 'FirstEvent', 'FirstLumi', 'FirstRun', 'LastEvent', 'LastLumi', 'LastRun' ] for key in needed: if self.mask and self.mask.get(key) is None: raise RuntimeError, 'Invalid value "%s" for %s' % ( self.mask.get(key), key) locations = set() for site in self.getLocations.execute( conn=self.getDBConn(), transaction=self.existingTransaction()): try: siteInfo = self.getLocationInfo.execute( site, conn=self.getDBConn(), transaction=self.existingTransaction()) if not siteInfo: self.logger.info( 'Skipping MonteCarlo injection to site "%s" as unknown to wmbs' % site) continue locations.add(siteInfo[0]['se_name']) except StandardError, ex: self.logger.error( 'Error getting storage element for "%s": %s' % (site, str(ex))) if not locations: raise RuntimeError, "No locations to inject Monte Carlo work to, unable to proceed" mcFakeFileName = "MCFakeFile-%s" % self.topLevelFileset.name wmbsFile = File( lfn=mcFakeFileName, first_event=self.mask['FirstEvent'], last_event=self.mask['LastEvent'], events=self.mask['LastEvent'] - self.mask['FirstEvent'] + 1, # inclusive range locations=locations, merged=False, # merged causes dbs parentage relation ) if self.mask: lumis = range(self.mask['FirstLumi'], self.mask['LastLumi'] + 1) # inclusive range wmbsFile.addRun(Run(self.mask['FirstRun'], *lumis)) # assume run number static else: wmbsFile.addRun(Run(1, 1)) wmbsFile['inFileset'] = True # file is not a parent logging.info("WMBS File: %s on Location: %s" % (wmbsFile['lfn'], wmbsFile['newlocations'])) self.wmbsFilesToCreate.append(wmbsFile) totalFiles = self.topLevelFileset.addFilesToWMBSInBulk( self.wmbsFilesToCreate, self.wmSpec.name(), isDBS=self.isDBS) self.topLevelFileset.markOpen(False) return totalFiles
def testMask(self): """ _testMask_ Test the new mask setup """ testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") testWorkflow.create() testFileset = Fileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testFileA = File(lfn=makeUUID(), locations="test.site.ch") testFileB = File(lfn=makeUUID(), locations="test.site.ch") testFileA.create() testFileB.create() testFileset.addFile([testFileA, testFileB]) testFileset.commit() testSubscription.acquireFiles([testFileA, testFileB]) testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testJob = Job() testJob['mask'].addRunAndLumis(run=100, lumis=[101, 102]) testJob['mask'].addRunAndLumis(run=200, lumis=[201, 202]) testJob.create(group=testJobGroup) loadJob = Job(id=testJob.exists()) loadJob.loadData() runs = loadJob['mask'].getRunAndLumis() self.assertEqual(len(runs), 2) self.assertEqual(runs[100], [[101, 102]]) self.assertEqual(runs[200], [[201, 202]]) bigRun = Run(100, *[101, 102, 103, 104]) badRun = Run(300, *[1001, 1002]) result = loadJob['mask'].filterRunLumisByMask([bigRun, badRun]) self.assertEqual(len(result), 1) alteredRun = result.pop() self.assertEqual(alteredRun.run, 100) self.assertEqual(alteredRun.lumis, [101, 102]) run0 = Run(300, *[1001, 1002]) run1 = Run(300, *[1001, 1002]) loadJob['mask'].filterRunLumisByMask([run0, run1]) return
def algorithm(self, *args, **kwargs): """ _algorithm_ An event base splitting algorithm. All available files are split into a set number of events per job. """ eventsPerJob = int(kwargs.get("events_per_job", 100)) eventsPerLumi = int(kwargs.get("events_per_lumi", eventsPerJob)) getParents = kwargs.get("include_parents", False) lheInput = kwargs.get("lheInputFiles", False) collectionName = kwargs.get('collectionName', None) timePerEvent, sizePerEvent, memoryRequirement = \ self.getPerformanceParameters(kwargs.get('performance', {})) acdcFileList = [] deterministicPileup = kwargs.get('deterministicPileup', False) if eventsPerJob <= 0 or eventsPerLumi <= 0: msg = "events_per_job and events_per_lumi must be positive. Their values are: " msg += "events_per_job: %d, events_per_lumi: %d" % (eventsPerJob, eventsPerLumi) raise RuntimeError(msg) if deterministicPileup and self.package == 'WMCore.WMBS': getJobNumber = self.daoFactory(classname="Jobs.GetNumberOfJobsPerWorkflow") self.nJobs = getJobNumber.execute(workflow=self.subscription.getWorkflow().id) logging.info('Creating jobs in DeterministicPileup mode for %s', self.subscription.workflowName()) # If we have runLumi info, we need to load it from couch if collectionName: try: from WMCore.ACDC.DataCollectionService import DataCollectionService couchURL = kwargs.get('couchURL') couchDB = kwargs.get('couchDB') filesetName = kwargs.get('filesetName') collectionName = kwargs.get('collectionName') logging.info('Creating jobs for ACDC fileset %s', filesetName) dcs = DataCollectionService(couchURL, couchDB) acdcFileList = dcs.getProductionACDCInfo(collectionName, filesetName) except Exception as ex: msg = "Exception while trying to load goodRunList\n" msg += "Refusing to create any jobs.\n" msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) return totalJobs = 0 locationDict = self.sortByLocation() for location in locationDict: self.newGroup() fileList = locationDict[location] getRunLumiInformation = False for f in fileList: if f['lfn'].startswith("MCFakeFile"): # We have one MCFakeFile, then it needs run information getRunLumiInformation = True break if getRunLumiInformation: if self.package == 'WMCore.WMBS': loadRunLumi = self.daoFactory(classname="Files.GetBulkRunLumi") fileLumis = loadRunLumi.execute(files=fileList) for f in fileList: lumiDict = fileLumis.get(f['id'], {}) for run in lumiDict.keys(): f.addRun(run=Run(run, *lumiDict[run])) for f in fileList: currentEvent = f['first_event'] eventsInFile = f['events'] runs = list(f['runs']) # We got the runs, clean the file. f['runs'] = set() if getParents: parentLFNs = self.findParent(lfn=f['lfn']) for lfn in parentLFNs: parent = File(lfn=lfn) f['parents'].add(parent) if acdcFileList: if f['lfn'] in [x['lfn'] for x in acdcFileList]: totalJobs = self.createACDCJobs(f, acdcFileList, timePerEvent, sizePerEvent, memoryRequirement, lheInput, eventsPerJob, eventsPerLumi, deterministicPileup, totalJobs) continue if not f['lfn'].startswith("MCFakeFile"): # Very very uncommon, but it has real input dataset if eventsInFile >= eventsPerJob: while currentEvent < eventsInFile: self.newJob(name=self.getJobName(length=totalJobs)) self.currentJob.addFile(f) if eventsPerJob + currentEvent < eventsInFile: jobTime = eventsPerJob * timePerEvent diskRequired = eventsPerJob * sizePerEvent self.currentJob["mask"].setMaxAndSkipEvents(eventsPerJob, currentEvent) else: jobTime = (eventsInFile - currentEvent) * timePerEvent diskRequired = (eventsInFile - currentEvent) * sizePerEvent self.currentJob["mask"].setMaxAndSkipEvents(None, currentEvent) self.currentJob.addResourceEstimates(jobTime=jobTime, memory=memoryRequirement, disk=diskRequired) if deterministicPileup: self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob) logging.debug("Job created for real input with %s", self.currentJob) currentEvent += eventsPerJob totalJobs += 1 else: self.newJob(name=self.getJobName(length=totalJobs)) self.currentJob.addFile(f) jobTime = eventsInFile * timePerEvent diskRequired = eventsInFile * sizePerEvent self.currentJob.addResourceEstimates(jobTime=jobTime, memory=memoryRequirement, disk=diskRequired) if deterministicPileup: self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob) logging.debug("Last job created for real input with %s", self.currentJob) totalJobs += 1 else: # This assumes there's only one run which is the case for MC lumis = runs[0].lumis (firstLumi, lastLumi) = (min(lumis), max(lumis)) currentLumi = firstLumi totalEvents = 0 if eventsInFile >= eventsPerJob: while totalEvents < eventsInFile: self.newJob(name=self.getJobName(length=totalJobs)) self.currentJob.addFile(f) self.currentJob.addBaggageParameter("lheInputFiles", lheInput) lumisPerJob = int(ceil(float(eventsPerJob) / eventsPerLumi)) # Limit the number of events to a unsigned 32bit int eventsRemaining = eventsInFile - totalEvents if (currentEvent + eventsPerJob - 1) > (2 ** 32 - 1) and ( currentEvent + eventsRemaining - 1) > (2 ** 32 - 1): currentEvent = 1 if eventsRemaining > eventsPerJob: self.currentJob["mask"].setMaxAndSkipEvents(eventsPerJob, currentEvent) self.currentJob["mask"].setMaxAndSkipLumis(lumisPerJob, currentLumi) jobTime = eventsPerJob * timePerEvent diskRequired = eventsPerJob * sizePerEvent else: jobTime = eventsRemaining * timePerEvent diskRequired = eventsRemaining * sizePerEvent lumisPerJob = int(ceil(float(eventsRemaining) / eventsPerLumi)) self.currentJob["mask"].setMaxAndSkipEvents(eventsRemaining, currentEvent) self.currentJob["mask"].setMaxAndSkipLumis(lumisPerJob, currentLumi) if deterministicPileup: self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob) currentLumi += lumisPerJob currentEvent += eventsPerJob totalEvents += eventsPerJob totalJobs += 1 self.currentJob.addResourceEstimates(jobTime=jobTime, memory=memoryRequirement, disk=diskRequired) else: self.newJob(name=self.getJobName(length=totalJobs)) self.currentJob.addFile(f) # For MC we use firstEvent instead of skipEvents so set it to 1 # We must check for events going over 2**32 - 1 here too if (eventsInFile + currentEvent - 1) > (2 ** 32 - 1): currentEvent = 1 self.currentJob["mask"].setMaxAndSkipEvents(eventsInFile, currentEvent) self.currentJob["mask"].setMaxAndSkipLumis(lastLumi - currentLumi + 1, currentLumi) jobTime = eventsInFile * timePerEvent diskRequired = eventsInFile * sizePerEvent self.currentJob.addResourceEstimates(jobTime=jobTime, memory=memoryRequirement, disk=diskRequired) if deterministicPileup: self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob) totalJobs += 1
def testB_AlgoMigration(self): """ _AlgoMigration_ Test our ability to migrate multiple algos to global Do this by creating, mid-poll, two separate batches of files One with the same dataset but a different algo One with the same algo, but a different dataset See that they both get to global """ #raise nose.SkipTest myThread = threading.currentThread() config = self.createConfig() self.injectWorkflow(MaxWaitTime=20) name = "ThisIsATest_%s" % (makeUUID()) tier = "RECO" nFiles = 12 files = self.getFiles(name=name, tier=tier, nFiles=nFiles) datasetPath = '/%s/%s/%s' % (name, name, tier) # Load components that are necessary to check status factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") dbinterface = factory.loadObject("UploadToDBS") dbsInterface = DBSInterface(config=config) localAPI = dbsInterface.getAPIRef() globeAPI = dbsInterface.getAPIRef(globalRef=True) testDBSUpload = DBSUploadPoller(config=config) testDBSUpload.algorithm() # There should now be one block result = listBlocks(apiRef=globeAPI, datasetPath=datasetPath) self.assertEqual(len(result), 1) # Okay, by now, the first migration should have gone through. # Now create a second batch of files with the same dataset # but a different algo. for i in range(0, nFiles): testFile = DBSBufferFile(lfn='%s-batch2-%i' % (name, i), size=1024, events=20, checksums={'cksum': 1}, locations="malpaquet") testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam=tier, psetHash="GIBBERISH_PART2", configContent=self.configURL) testFile.setDatasetPath(datasetPath) testFile.addRun(Run(1, *[46])) testFile.create() # Have to do things twice to get parents testDBSUpload.algorithm() testDBSUpload.algorithm() # There should now be two blocks result = listBlocks(apiRef=globeAPI, datasetPath=datasetPath) self.assertEqual(len(result), 2) # Now create another batch of files with the original algo # But in a different dataset for i in range(0, nFiles): testFile = DBSBufferFile(lfn='%s-batch3-%i' % (name, i), size=1024, events=20, checksums={'cksum': 1}, locations="malpaquet") testFile.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam=tier, psetHash="GIBBERISH", configContent=self.configURL) testFile.setDatasetPath('/%s/%s_3/%s' % (name, name, tier)) testFile.addRun(Run(1, *[46])) testFile.create() # Do it twice for parentage. testDBSUpload.algorithm() testDBSUpload.algorithm() # There should now be one block result = listBlocks(apiRef=globeAPI, datasetPath='/%s/%s_3/%s' % (name, name, tier)) self.assertEqual(len(result), 1) # Well, all the blocks got there, so we're done return
def __call__(self, filesetToProcess): """ The algorithm itself """ global LOCK # Get configuration initObj = WMInit() initObj.setLogging() initObj.setDatabaseConnection(os.getenv("DATABASE"), \ os.getenv('DIALECT'), os.getenv("DBSOCK")) myThread = threading.currentThread() daofactory = DAOFactory(package = "WMCore.WMBS" , \ logger = myThread.logger, \ dbinterface = myThread.dbi) locationNew = daofactory(classname="Locations.New") getFileLoc = daofactory(classname="Files.GetLocation") fileInFileset = daofactory(classname="Files.InFileset") logging.debug("the T0Feeder is processing %s" % \ filesetToProcess.name) logging.debug("the fileset name %s" % \ (filesetToProcess.name).split(":")[0]) # Get the start Run if asked startRun = (filesetToProcess.name).split(":")[3] fileType = (filesetToProcess.name).split(":")[2] LASTIME = filesetToProcess.lastUpdate # url builder primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1] processedDataset = (( filesetToProcess.name).split(":")[0]).split('/')[2] dataTier = ((filesetToProcess.name\ ).split(":")[0]).split('/')[3] url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \ (fileType, LASTIME, primaryDataset,processedDataset, dataTier) tries = 1 while True: try: myRequester = JSONRequests(url="vocms52.cern.ch:8889") requestResult = myRequester.get(\ url+"/"+"?return_type=text/json%2Bdas") newFilesList = requestResult[0]["results"] except: logging.debug("T0Reader call error...") if tries == self.maxRetries: return else: tries += 1 continue logging.debug("T0 queries done ...") now = time.time() LASTIME = int(newFilesList['end_time']) + 1 break # process all files if len(newFilesList['files']): try: locationNew.execute(siteName="caf.cern.ch", seName="caf.cern.ch") except Exception as e: logging.debug("Error when adding new location...") logging.debug(e) logging.debug(format_exc()) for files in newFilesList['files']: # Assume parents aren't asked newfile = File(str(files['lfn']), \ size = files['file_size'], events = files['events']) try: LOCK.acquire() if newfile.exists() == False: newfile.create() for run in files['runs']: runSet = set() runSet.add(Run(run, *files['runs'][run])) newfile.addRunSet(runSet) else: newfile.loadData() fileLoc = getFileLoc.execute(file=files['lfn']) if 'caf.cern.ch' not in fileLoc: newfile.setLocation("caf.cern.ch") # else: # logging.debug("File already associated to %s" %fileLoc) LOCK.release() if len(newfile["runs"]): val = 0 for run in newfile['runs']: if run.run < int(startRun): val = 1 break if not val: listFile = fileInFileset.execute\ (filesetToProcess.id) if {'fileid': newfile['id']} not in listFile: filesetToProcess.addFile(newfile) filesetToProcess.setLastUpdate\ (int(newFilesList['end_time']) + 1) filesetToProcess.commit() logging.debug( "new file created/loaded added by T0AST..." ) except Exception as e: logging.debug("Error when adding new files in T0AST...") logging.debug(e) logging.debug(format_exc()) LOCK.release() filesetToProcess.commit() else: logging.debug("nothing to do in T0AST...") # For reopned fileset or empty # try until the purge time is reached if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime: filesetToProcess.setLastUpdate(time.time()) filesetToProcess.commit() # Commit the fileset logging.debug("Test purge in T0AST ...") filesetToProcess.load() LASTIME = filesetToProcess.lastUpdate if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime: filesetToProcess.markOpen(False) logging.debug("Purge Done...") filesetToProcess.commit()
def testCreateDeleteExists(self): """ Create and then delete a job and workflow. Use the workunit class's exists() method to determine if the workunit has been written to the database before the job is created, after the job has been created, and after the workflow has been deleted. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(1, *[45])) testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(1, *[46])) testFileA.create() testFileB.create() testJob = Job(name="TestJob", files=[testFileA, testFileB]) testWU1 = WorkUnit(taskID=testWorkflow.id, fileid=testFileA['id'], runLumi=Run(1, *[45])) testWU2 = WorkUnit(taskID=testWorkflow.id, fileid=testFileB['id'], runLumi=Run(1, *[46])) self.assertFalse(testWU1.exists(), "WorkUnit exists before job was created") self.assertFalse(testWU2.exists(), "WorkUnit exists before job was created") testJob.create(group=testJobGroup) self.assertTrue(testWU1.exists(), "WorkUnit does not exist after job was created") self.assertTrue(testWU2.exists(), "WorkUnit does not exist after job was created") testJob.delete() self.assertTrue(testWU1.exists(), "WorkUnit does not exist after job is deleted") self.assertTrue(testWU2.exists(), "WorkUnit does not exist after job is deleted") testWorkflow.delete() self.assertFalse(testWU1.exists(), "WorkUnit exists after workflow is deleted") self.assertFalse(testWU2.exists(), "WorkUnit exists after workflow is deleted") return
def testAddChildTransaction(self): """ _testAddChildTransaction_ Add a child to some parent files and make sure that all the parentage information is loaded/stored correctly from the database. Rollback the addition of one of the childs and then verify that it does in fact only have one parent. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileParentA.addChildren("/this/is/a/lfn") myThread = threading.currentThread() myThread.transaction.begin() testFileParentB.addChildren("/this/is/a/lfn") testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load(parentage=1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, \ "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, \ "ERROR: Some parents are missing" myThread.transaction.rollback() testFileB.load(parentage=1) goldenFiles = [testFileParentA] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, \ "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, \ "ERROR: Some parents are missing" return
def createJobsLocationWise(self, fileset, endOfRun, dqmHarvestUnit, lumiMask, goodRunList): myThread = threading.currentThread() fileset.loadData(parentage=0) allFiles = fileset.getFiles() # sort by location and run locationDict = {} runDict = {} for fileInfo in allFiles: locSet = frozenset(fileInfo['locations']) runSet = fileInfo.getRuns() if len(locSet) == 0: logging.error("File %s has no locations!", fileInfo['lfn']) if len(runSet) == 0: logging.error("File %s has no run information!", fileInfo['lfn']) # Populate a dictionary with [location][run] so we can split jobs according to those different combinations if locSet not in locationDict.keys(): locationDict[locSet] = {} fileInfo['runs'] = set() # Handle jobs with run whitelist/blacklist if goodRunList: runDict[fileInfo['lfn']] = set() for run in runSet: if run.run in goodRunList: runDict[fileInfo['lfn']].add(run) if run.run in locationDict[locSet].keys(): locationDict[locSet][run.run].append(fileInfo) else: locationDict[locSet][run.run] = [fileInfo] elif lumiMask: # it has lumiMask, thus we consider only good run/lumis newRunSet = [] for run in runSet: if not isGoodRun(lumiMask, run.run): continue # then loop over lumis maskedLumis = [] for lumi in run.lumis: if not isGoodLumi(lumiMask, run.run, lumi): continue maskedLumis.append(lumi) if not maskedLumis: continue maskedRun = Run(run.run, *maskedLumis) newRunSet.append(maskedRun) if run.run in locationDict[locSet].keys(): locationDict[locSet][run.run].append(fileInfo) else: locationDict[locSet][run.run] = [fileInfo] if newRunSet: runDict[fileInfo['lfn']] = newRunSet else: # no LumiList and no run white or black list runDict[fileInfo['lfn']] = runSet for run in runSet: if run.run in locationDict[locSet].keys(): locationDict[locSet][run.run].append(fileInfo) else: locationDict[locSet][run.run] = [fileInfo] # create separate jobs for different locations self.newGroup() self.jobCount = 0 baseName = makeUUID() self.newGroup() if endOfRun: harvestType = "EndOfRun" else: harvestType = "Periodic" for location in locationDict.keys(): if dqmHarvestUnit == "byRun": self.createJobByRun(locationDict, location, baseName, harvestType, runDict, endOfRun) else: self.createMultiRunJob(locationDict, location, baseName, harvestType, runDict, endOfRun) return