def testXSetBlock(self): """ _testSetBlock_ Verify that the [Set|Get]Block DAOs work correctly. """ myThread = threading.currentThread() uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createAction = uploadFactory(classname = "SetBlockStatus") createAction.execute(block = "someblockname", locations = ["se1.cern.ch"]) setBlockAction = self.daoFactory(classname = "DBSBufferFiles.SetBlock") getBlockAction = self.daoFactory(classname = "DBSBufferFiles.GetBlock") testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10, locations = "se1.fnal.gov") testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() setBlockAction.execute(lfn = testFile["lfn"], blockName = "someblockname") blockName = getBlockAction.execute(lfn = testFile["lfn"]) assert blockName[0][0] == "someblockname", \ "Error: Incorrect block returned: %s" % blockName[0][0] return
def testCreateDeleteExists(self): """ _testCreateDeleteExists_ Test the create(), delete() and exists() methods of the file class by creating and deleting a file. The exists() method will be called before and after creation and after deletion. """ testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") assert testFile.exists() == False, \ "ERROR: File exists before it was created" testFile.addRun(Run(1, *[45])) testFile.create() assert testFile.exists() > 0, \ "ERROR: File does not exist after it was created" testFile.delete() assert testFile.exists() == False, \ "ERROR: File exists after it has been deleted" return
def getFiles(self, name, tier='RECO', nFiles=12, site="malpaquet", nLumis=1): """ Create some quick dummy test files """ files = [] for f in range(nFiles): testFile = DBSBufferFile( lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size=1024, events=20, checksums={'cksum': 1}) testFile.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) lumis = [] for i in range(nLumis): lumis.append((f * 100000) + i) testFile.addRun(Run(1, *lumis)) testFile.setAcquisitionEra(name.split('-')[0]) testFile.setProcessingVer("0") testFile.setGlobalTag("Weird") testFile.create() testFile.setLocation(site) files.append(testFile) count = 0 for f in files: count += 1 testFileChild = DBSBufferFile( lfn= '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' % (name, site, count), size=1024, events=10, checksums={'cksum': 1}) testFileChild.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([f['lfn']]) return files
def testAddParents(self): """ _testAddParents_ Verify that the addParents() method works correctly even if the parents do not already exist in the database. """ myThread = threading.currentThread() testFile = DBSBufferFile(lfn = "/this/is/a/lfnA", size = 1024, events = 10, locations = "se1.fnal.gov") testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testParent = DBSBufferFile(lfn = "/this/is/a/lfnB", size = 1024, events = 10, locations = "se1.fnal.gov") testParent.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testParent.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RAW") testParent.create() goldenLFNs = ["lfn1", "lfn2", "lfn3", "/this/is/a/lfnB"] testFile.addParents(goldenLFNs) verifyFile = DBSBufferFile(id = testFile["id"]) verifyFile.load(parentage = 1) parentLFNs = verifyFile.getParentLFNs() for parentLFN in parentLFNs: self.assertTrue(parentLFN in goldenLFNs, "Error: unknown lfn %s" % parentLFN) goldenLFNs.remove(parentLFN) self.assertEqual(len(goldenLFNs), 0, "Error: missing LFNs...") # Check that the bogus dataset is listed as inDBS sqlCommand = """SELECT in_dbs FROM dbsbuffer_algo_dataset_assoc das INNER JOIN dbsbuffer_dataset ds ON das.dataset_id = ds.id WHERE ds.path = 'bogus'""" status = myThread.dbi.processData(sqlCommand)[0].fetchall()[0][0] self.assertEqual(status, 1) # Now make sure the dummy files are listed as being in DBS sqlCommand = """SELECT status FROM dbsbuffer_file df INNER JOIN dbsbuffer_algo_dataset_assoc das ON das.id = df.dataset_algo INNER JOIN dbsbuffer_dataset ds ON das.dataset_id = ds.id WHERE ds.path = '/bogus/dataset/path' """ status = myThread.dbi.processData(sqlCommand)[0].fetchall() for entry in status: self.assertEqual(entry, ('AlreadyInDBS',)) return
def testGetParentStatusDAO(self): """ _testGetParentStatusDAO_ Verify that the GetParentStatus DAO correctly returns the status of a file's children. """ testFileChild = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024, events = 20) testFileChild.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChild.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChild.create() testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChild.addParents([testFile["lfn"]]) getStatusAction = self.daoFactory(classname = "DBSBufferFiles.GetParentStatus") parentStatus = getStatusAction.execute(testFileChild["lfn"]) assert len(parentStatus) == 1, \ "ERROR: Wrong number of statuses returned." assert parentStatus[0] == "NOTUPLOADED", \ "ERROR: Wrong status returned." return
def testFilesize(self): """ _testFilesize_ Test storing and loading the file information from dbsbuffer_file. Make sure filesize can be bigger than 32 bits """ checksums = {"adler32": "adler32", "cksum": "cksum"} testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=3221225472, events=1500000, checksums=checksums) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_7_6_0", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn=testFileA["lfn"]) testFileB.load() self.assertEqual(testFileB["size"], 3221225472, "Error: the filesize should be 3GB") self.assertEqual(testFileB["events"], 1500000, "Error: the number of events should be 1.5M")
def testProperties(self): """ _testProperties_ Test added tags that use DBSBuffer to transfer from workload to DBS """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.setValidStatus(validStatus="VALID") testFileA.setProcessingVer(ver="ProcVer") testFileA.setAcquisitionEra(era="AcqEra") testFileA.setGlobalTag(globalTag="GlobalTag") testFileA.setDatasetParent(datasetParent="Parent") testFileA.create() # There are no accessors for these things because load is never called action = self.daoFactory2(classname="LoadInfoFromDAS") das = action.execute(ids=[1])[0] self.assertEqual(das['Parent'], 'Parent') self.assertEqual(das['GlobalTag'], 'GlobalTag') self.assertEqual(das['ValidStatus'], 'VALID') return
def testSetLocation(self): """ _testSetLocation_ Create a file and add a couple locations. Load the file from the database to make sure that the locations were set correctly. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave = False) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testSetLocation(self): """ _testSetLocation_ Create a file and add a couple locations. Load the file from the database to make sure that the locations were set correctly. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave=False) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testSetLocationByLFN(self): """ _testSetLocationByLFN_ """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() setLocationAction = self.daoFactory( classname="DBSBufferFiles.SetLocationByLFN") setLocationAction.execute(binds={ 'lfn': "/this/is/a/lfn", 'pnn': 'se1.cern.ch' }) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() self.assertEqual(testFileB['locations'], set(['se1.cern.ch'])) return
def testAddCKSumByLFN(self): """ _testAddCKSumByLFN_ """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() checksums = {"adler32": "adler32", "cksum": "cksum", "md5": "md5"} setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN") binds = [{'lfn': "/this/is/a/lfn", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/lfn", 'cktype': 'cksum', 'cksum': 101}] setCksumAction.execute(bulkList = binds) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() self.assertEqual(testFileB['checksums'], {'adler32': '201', 'cksum': '101'}) return
def testCreateTransaction(self): """ _testCreateTransaction_ Begin a transaction and then create a file in the database. Afterwards, rollback the transaction. Use the File class's exists() method to to verify that the file doesn't exist before it was created, exists after it was created and doesn't exist after the transaction was rolled back. """ myThread = threading.currentThread() myThread.transaction.begin() testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") assert testFile.exists() == False, \ "ERROR: File exists before it was created" testFile.addRun(Run(1, *[45])) testFile.create() assert testFile.exists() > 0, \ "ERROR: File does not exist after it was created" myThread.transaction.rollback() assert testFile.exists() == False, \ "ERROR: File exists after transaction was rolled back." return
def testCreateTransaction(self): """ _testCreateTransaction_ Begin a transaction and then create a file in the database. Afterwards, rollback the transaction. Use the File class's exists() method to to verify that the file doesn't exist before it was created, exists after it was created and doesn't exist after the transaction was rolled back. """ myThread = threading.currentThread() myThread.transaction.begin() testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") assert testFile.exists() == False, \ "ERROR: File exists before it was created" testFile.addRun(Run(1, *[45])) testFile.create() assert testFile.exists() > 0, \ "ERROR: File does not exist after it was created" myThread.transaction.rollback() assert testFile.exists() == False, \ "ERROR: File exists after transaction was rolled back." return
def testCreateDeleteExists(self): """ _testCreateDeleteExists_ Test the create(), delete() and exists() methods of the file class by creating and deleting a file. The exists() method will be called before and after creation and after deletion. """ testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") assert testFile.exists() == False, \ "ERROR: File exists before it was created" testFile.addRun(Run(1, *[45])) testFile.create() assert testFile.exists() > 0, \ "ERROR: File does not exist after it was created" testFile.delete() assert testFile.exists() == False, \ "ERROR: File exists after it has been deleted" return
def testSetLocationByLFN(self): """ _testSetLocationByLFN_ """ myThread = threading.currentThread() testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() setLocationAction = self.daoFactory(classname = "DBSBufferFiles.SetLocationByLFN") setLocationAction.execute(binds = {'lfn': "/this/is/a/lfn", 'sename': 'se1.cern.ch'}) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() self.assertEqual(testFileB['locations'], set(['se1.cern.ch'])) return
def testProperties(self): """ _testProperties_ Test added tags that use DBSBuffer to transfer from workload to DBS """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.setValidStatus(validStatus = "VALID") testFileA.setProcessingVer(ver = "ProcVer") testFileA.setAcquisitionEra(era = "AcqEra") testFileA.setGlobalTag(globalTag = "GlobalTag") testFileA.setDatasetParent(datasetParent = "Parent") testFileA.create() # There are no accessors for these things because load is never called action = self.daoFactory2(classname = "LoadInfoFromDAS") das = action.execute(ids = [1])[0] self.assertEqual(das['Parent'], 'Parent') self.assertEqual(das['GlobalTag'], 'GlobalTag') self.assertEqual(das['ValidStatus'], 'VALID')
def getFiles(self, name, tier, nFiles=12, site="malpaquet"): """ Create some quick dummy test files """ files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn="%s-%s-%i" % (name, site, f), size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm( appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL ) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run(1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) testFileChild = DBSBufferFile(lfn="%s-%s-child" % (name, site), size=1024, events=10, checksums={"cksum": 1}) testFileChild.setAlgorithm( appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL ) testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x["lfn"] for x in files]) return files
def createTestFiles(self): """ _createTestFiles_ Create some dbsbuffer test files with different statuses :return: """ phedexStatus = self.dbsbufferFactory(classname="DBSBufferFiles.SetPhEDExStatus") for i in range(0, 4): lfn = "/path/to/some/lfn" + str(i) # Two files should be InDBS, two files should be NOTUPLOADED if i in [0,2]: status = 'InDBS' else: status = 'NOTUPLOADED' testDBSFile = DBSBufferFile(lfn=lfn, size=600000, events=60000, status=status, workflowId=1) testDBSFile.setAlgorithm(appName="cmsRun", appVer="UNKNOWN", appFam="RECO", psetHash="SOMEHASH" + str(i), configContent="SOMECONTENT") testDBSFile.setDatasetPath("/path/to/some/dataset") testDBSFile.create() # Create all four combinations of status(InDBS,NOTUPLOADED) and in_phedex(0,1) if i in [0,1]: phedexStatus.execute(lfn, 1)
def testAddRunSet(self): """ _testAddRunSet_ Test the ability to add run and lumi information to a file. """ testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10, locations="se1.fnal.gov") testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() runSet = set() runSet.add(Run(1, *[45])) runSet.add(Run(2, *[67, 68])) testFile.addRunSet(runSet) assert (runSet - testFile["runs"]) == set(), \ "Error: addRunSet is not updating set correctly"
def prepareDBSFiles(self): """ _prepareDBSFiles_ Retrieve the information from the JSON input data and create DBSFile objects that can be registered in the database. """ timestamp = time.strftime('%m%d%y_%H%M%S') for fileEntry in self.inputData: # Get all the info out of a standard named dataset datasetInfo = str(fileEntry["dataset"]) tokens = datasetInfo.split('/') primDs = tokens[1] procDs = tokens[2] dataTier = tokens[3] procDsTokens = procDs.split('-') acqEra = procDsTokens[0] procVer = procDsTokens[-1][1:] ckSumInfo = fileEntry["checksums"] for entry in ckSumInfo: ckSumInfo[entry] = str(ckSumInfo[entry]) # Build the basic dbsBuffer file dbsFile = DBSBufferFile(lfn = str(fileEntry["lfn"]), size = int(fileEntry.get("size", 0)), events = int(fileEntry.get("events", 0)), checksums = ckSumInfo, status = "NOTUPLOADED") dbsFile.setAlgorithm(appName = "cmsRun", appVer = str(fileEntry.get("cmssw", "LEGACY")), appFam = "Legacy", psetHash = "GIBBERISH", configContent = "None;;None;;None") dbsFile.setDatasetPath("/%s/%s/%s" % (primDs, procDs, dataTier)) dbsFile.setValidStatus(validStatus = "PRODUCTION") dbsFile.setProcessingVer(ver = procVer) dbsFile.setAcquisitionEra(era = acqEra) dbsFile.setGlobalTag(globalTag = str(fileEntry.get('globalTag', "LEGACY"))) # Build a representative task name dbsFile['task'] = '/LegacyInsertionTask_%s/Insertion' % timestamp # Get the runs and lumis runsAndLumis = fileEntry.get("runsAndLumis", {}) for run in runsAndLumis: newRun = Run(runNumber = int(run)) newRun.extend([int(x) for x in runsAndLumis[run]]) dbsFile.addRun(newRun) # Complete the file information with the location and queue it dbsFile.setLocation(se = str(fileEntry["location"]), immediateSave = False) self.dbsFilesToCreate.append(dbsFile) self.inputData = None return
def testLocationsConstructor(self): """ _testLocationsConstructor_ Test to make sure that locations passed into the File() constructor are loaded from and save to the database correctly. Also test to make sure that the class behaves well when the location is passed in as a single string instead of a set. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10, locations=set(["se1.fnal.gov"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn="/this/is/a/lfn2", size=1024, events=10, locations="se1.fnal.gov") testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileB.addRun(Run(1, *[45])) testFileB.create() testFileC = DBSBufferFile(id=testFileA["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" testFileC = DBSBufferFile(id=testFileB["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def getFiles(self, name, tier, nFiles=12, site="malpaquet", workflowName=None, taskPath=None, noChild=False): """ Create some quick dummy test files """ if workflowName is not None and taskPath is not None: workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath) else: workflowId = self.injectWorkflow() files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn='%s-%s-%i' % (name, site, f), size=1024, events=20, checksums={'cksum': 1}, workflowId=workflowId) testFile.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run(1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) if not noChild: testFileChild = DBSBufferFile(lfn='%s-%s-child' % (name, site), size=1024, events=10, checksums={'cksum': 1}, workflowId=workflowId) testFileChild.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL) testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def createFilesWithChildren(self, moreParentFiles, acqEra): """ _createFilesWithChildren_ Create several parentless files and then create child files. """ parentFiles = [] childFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(10): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RAW", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentFiles.append(testFile) baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % (acqEra) for i in range(5): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % (acqEra)) lumis = [] for j in range(20): lumis.append((i * 20) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") testFile.addParents([parentFiles[i * 2]["lfn"], parentFiles[i * 2 + 1]["lfn"]]) testFile.addParents([moreParentFiles[i * 2]["lfn"], moreParentFiles[i * 2 + 1]["lfn"]]) childFiles.append(testFile) return (parentFiles, childFiles)
def testSetLocationTransaction(self): """ _testSetLocationTransaction_ Create a file at specific locations and commit everything to the database. Reload the file from the database and verify that the locations are correct. Rollback the database transaction and once again reload the file. Verify that the original locations are back. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov"]) myThread = threading.currentThread() myThread.transaction.begin() testFileA.setLocation(["se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave=False) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" myThread.transaction.rollback() testFileB.load() goldenLocations = ["se1.fnal.gov"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def getFiles(self, name, tier, nFiles=12, site="malpaquet", nLumis=1): """ _getFiles_ Create some dummy test files. """ files = [] (acqEra, procVer) = name.split("-") baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra, procVer) for f in range(nFiles): testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer)) lumis = [] for i in range(nLumis): lumis.append((f * 1000000) + i) testFile.addRun(Run(1, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("0") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation(site) files.append(testFile) baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra, procVer) testFileChild = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=10, checksums={'cksum': 1}) testFileChild.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW-RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" % (acqEra, procVer)) testFileChild.addRun(Run(1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def testSetLocationTransaction(self): """ _testSetLocationTransaction_ Create a file at specific locations and commit everything to the database. Reload the file from the database and verify that the locations are correct. Rollback the database transaction and once again reload the file. Verify that the original locations are back. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov"]) myThread = threading.currentThread() myThread.transaction.begin() testFileA.setLocation(["se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave = False) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" myThread.transaction.rollback() testFileB.load() goldenLocations = ["se1.fnal.gov"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testLocationsConstructor(self): """ _testLocationsConstructor_ Test to make sure that locations passed into the File() constructor are loaded from and save to the database correctly. Also test to make sure that the class behaves well when the location is passed in as a single string instead of a set. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10, locations = set(["se1.fnal.gov"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = "/this/is/a/lfn2", size = 1024, events = 10, locations = "se1.fnal.gov") testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileB.addRun(Run( 1, *[45])) testFileB.create() testFileC = DBSBufferFile(id = testFileA["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" testFileC = DBSBufferFile(id = testFileB["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def addFileToDBS(self, jobReportFile, task, errorDataset=False): """ _addFileToDBS_ Add a file that was output from a job to the DBS buffer. """ datasetInfo = jobReportFile["dataset"] dbsFile = DBSBufferFile(lfn=jobReportFile["lfn"], size=jobReportFile["size"], events=jobReportFile["events"], checksums=jobReportFile["checksums"], status="NOTUPLOADED") dbsFile.setAlgorithm(appName=datasetInfo["applicationName"], appVer=datasetInfo["applicationVersion"], appFam=jobReportFile["module_label"], psetHash="GIBBERISH", configContent=jobReportFile.get('configURL')) if errorDataset: dbsFile.setDatasetPath( "/%s/%s/%s" % (datasetInfo["primaryDataset"] + "-Error", datasetInfo["processedDataset"], datasetInfo["dataTier"])) else: dbsFile.setDatasetPath( "/%s/%s/%s" % (datasetInfo["primaryDataset"], datasetInfo["processedDataset"], datasetInfo["dataTier"])) dbsFile.setValidStatus( validStatus=jobReportFile.get("validStatus", None)) dbsFile.setProcessingVer(ver=jobReportFile.get('processingVer', None)) dbsFile.setAcquisitionEra( era=jobReportFile.get('acquisitionEra', None)) dbsFile.setGlobalTag(globalTag=jobReportFile.get('globalTag', None)) #TODO need to find where to get the prep id dbsFile.setPrepID(prep_id=jobReportFile.get('prep_id', None)) dbsFile['task'] = task for run in jobReportFile["runs"]: newRun = Run(runNumber=run.run) newRun.extend(run.lumis) dbsFile.addRun(newRun) dbsFile.setLocation(pnn=list(jobReportFile["locations"])[0], immediateSave=False) self.dbsFilesToCreate.append(dbsFile) return
def injectFilesFromDBS(inputFileset, datasetPath): """ _injectFilesFromDBS_ """ print("injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name)) args = {} args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray( path=datasetPath, retriveList=["retrive_lumi", "retrive_run"]) # Limiter on number of files dbsResults = dbsResults[0:20] print(" found %d files, inserting into wmbs..." % (len(dbsResults))) for dbsResult in dbsResults: myFile = File(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"], events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]}, locations="cmssrm.fnal.gov", merged=True) myRun = Run(runNumber=dbsResult["LumiList"][0]["RunNumber"]) for lumi in dbsResult["LumiList"]: myRun.appendLumi(lumi["LumiSectionNumber"]) myFile.addRun(myRun) myFile.create() inputFileset.addFile(myFile) dbsFile = DBSBufferFile(lfn=dbsResult["LogicalFileName"], size=dbsResult["FileSize"], events=dbsResult["NumberOfEvents"], checksums={"cksum": dbsResult["Checksum"]}, locations="cmssrm.fnal.gov", status="NOTUPLOADED") dbsFile.setDatasetPath(datasetPath) dbsFile.setAlgorithm(appName="cmsRun", appVer="Unknown", appFam="Unknown", psetHash="Unknown", configContent="Unknown") dbsFile.create() inputFileset.commit() inputFileset.markOpen(False) return
def createParentFiles(self, acqEra, nFiles=10, workflowName='TestWorkload', taskPath='/TestWorkload/DataTest'): """ _createParentFiles_ Create several parentless files in DBSBuffer. This simulates raw files in the T0. """ workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath) parentlessFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(nFiles): testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}, workflowId=workflowId) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentlessFiles.append(testFile) return parentlessFiles
def testLoad(self): """ _testLoad_ Test the loading of file meta data using the ID of a file and the LFN of a file. """ checksums = {"adler32": "adler32", "cksum": "cksum", "md5": "md5"} testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10, checksums=checksums) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn=testFileA["lfn"]) testFileB.load() testFileC = DBSBufferFile(id=testFileA["id"]) testFileC.load() assert testFileA == testFileB, \ "ERROR: File load by LFN didn't work" assert testFileA == testFileC, \ "ERROR: File load by ID didn't work" assert isinstance(testFileB["id"], int) or isinstance(testFileB["id"], long), \ "ERROR: File id is not an integer type." assert isinstance(testFileB["size"], int) or isinstance(testFileB["size"], long), \ "ERROR: File size is not an integer type." assert isinstance(testFileB["events"], int) or isinstance(testFileB["events"], long), \ "ERROR: File events is not an integer type." assert isinstance(testFileC["id"], int) or isinstance(testFileC["id"], long), \ "ERROR: File id is not an integer type." assert isinstance(testFileC["size"], int) or isinstance(testFileC["size"], long), \ "ERROR: File size is not an integer type." assert isinstance(testFileC["events"], int) or isinstance(testFileC["events"], long), \ "ERROR: File events is not an integer type." testFileA.delete() return
def testGetChildrenDAO(self): """ _testGetChildrenDAO_ Verify that the GetChildren DAO correctly returns the LFNs of a file's children. """ testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024, events = 20) testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024, events = 20) testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024, events = 20) testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) getChildrenAction = self.daoFactory(classname = "DBSBufferFiles.GetChildren") childLFNs = getChildrenAction.execute(testFile["lfn"]) assert len(childLFNs) == 3, \ "ERROR: Parent does not have the right amount of children." goldenLFNs = ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC"] for childLFN in childLFNs: assert childLFN in goldenLFNs, \ "ERROR: Unknown child lfn" goldenLFNs.remove(childLFN) return
def testGetParentLFNs(self): """ _testGetParentLFNs_ Create three files and set them to be parents of a fourth file. Check to make sure that getParentLFNs() on the child file returns the correct LFNs. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentC = DBSBufferFile(lfn="/this/is/a/parent/lfnC", size=1024, events=20) testFileParentC.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileParentC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentC.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileParentC.create() testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.addRun(Run(1, *[45])) testFile.create() testFile.addParents([testFileParentA["lfn"], testFileParentB["lfn"], testFileParentC["lfn"]]) parentLFNs = testFile.getParentLFNs() assert len(parentLFNs) == 3, "ERROR: Child does not have the right amount of parents" goldenLFNs = ["/this/is/a/parent/lfnA", "/this/is/a/parent/lfnB", "/this/is/a/parent/lfnC"] for parentLFN in parentLFNs: assert parentLFN in goldenLFNs, "ERROR: Unknown parent lfn" goldenLFNs.remove(parentLFN) testFile.delete() testFileParentA.delete() testFileParentB.delete() testFileParentC.delete() return
def addFileToDBS(self, jobReportFile, task, errorDataset = False): """ _addFileToDBS_ Add a file that was output from a job to the DBS buffer. """ datasetInfo = jobReportFile["dataset"] dbsFile = DBSBufferFile(lfn = jobReportFile["lfn"], size = jobReportFile["size"], events = jobReportFile["events"], checksums = jobReportFile["checksums"], status = "NOTUPLOADED") dbsFile.setAlgorithm(appName = datasetInfo["applicationName"], appVer = datasetInfo["applicationVersion"], appFam = jobReportFile["module_label"], psetHash = "GIBBERISH", configContent = jobReportFile.get('configURL')) if errorDataset: dbsFile.setDatasetPath("/%s/%s/%s" % (datasetInfo["primaryDataset"] + "-Error", datasetInfo["processedDataset"], datasetInfo["dataTier"])) else: dbsFile.setDatasetPath("/%s/%s/%s" % (datasetInfo["primaryDataset"], datasetInfo["processedDataset"], datasetInfo["dataTier"])) dbsFile.setValidStatus(validStatus = jobReportFile.get("validStatus", None)) dbsFile.setProcessingVer(ver = jobReportFile.get('processingVer', None)) dbsFile.setAcquisitionEra(era = jobReportFile.get('acquisitionEra', None)) dbsFile.setGlobalTag(globalTag = jobReportFile.get('globalTag', None)) #TODO need to find where to get the prep id dbsFile.setPrepID(prep_id = jobReportFile.get('prep_id', None)) dbsFile['task'] = task for run in jobReportFile["runs"]: newRun = Run(runNumber = run.run) newRun.extend(run.lumis) dbsFile.addRun(newRun) dbsFile.setLocation(pnn = list(jobReportFile["locations"])[0], immediateSave = False) self.dbsFilesToCreate.append(dbsFile) return
def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", workflowName = None, taskPath = None, noChild = False): """ Create some quick dummy test files """ if workflowName is not None and taskPath is not None: workflowId = self.injectWorkflow(workflowName = workflowName, taskPath = taskPath) else: workflowId = self.injectWorkflow() files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn = '%s-%s-%i' % (name, site, f), size = 1024, events = 20, checksums = {'cksum': 1}, workflowId = workflowId) testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = self.configURL) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run( 1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) if not noChild: testFileChild = DBSBufferFile(lfn = '%s-%s-child' %(name, site), size = 1024, events = 10, checksums = {'cksum': 1}, workflowId = workflowId) testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = self.configURL) testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name)) testFileChild.addRun(Run( 1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1): """ Create some quick dummy test files """ files = [] for f in range(nFiles): testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024, events = 20, checksums = {'cksum': 1}) testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) lumis = [] for i in range(nLumis): lumis.append((f * 100000) + i) testFile.addRun(Run( 1, *lumis)) testFile.setAcquisitionEra(name.split('-')[0]) testFile.setProcessingVer("0") testFile.setGlobalTag("Weird") testFile.create() testFile.setLocation(site) files.append(testFile) count = 0 for f in files: count += 1 testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024, events = 10, checksums = {'cksum': 1}) testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name)) testFileChild.addRun(Run( 1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([f['lfn']]) return files
def testAddRunSet(self): """ _testAddRunSet_ Test the ability to add run and lumi information to a file. """ testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10, locations="se1.fnal.gov") testFile.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() runSet = set() runSet.add(Run(1, *[45])) runSet.add(Run(2, *[67, 68])) testFile.addRunSet(runSet) assert (runSet - testFile["runs"]) == set(), "Error: addRunSet is not updating set correctly"
def _addToDBSBuffer(self, dbsFile, checksums, locations): """ This step is just for increase the performance for Accountant doesn't neccessary to check the parentage """ dbsBuffer = DBSBufferFile(lfn=dbsFile["LogicalFileName"], size=dbsFile["FileSize"], events=dbsFile["NumberOfEvents"], checksums=checksums, locations=locations, status="GLOBAL", inPhedex=1) dbsBuffer.setDatasetPath('bogus') dbsBuffer.setAlgorithm(appName="cmsRun", appVer="Unknown", appFam="Unknown", psetHash="Unknown", configContent="Unknown") self.dbsFilesToCreate.add(dbsBuffer) return
def testLoad(self): """ _testLoad_ Test the loading of file meta data using the ID of a file and the LFN of a file. """ checksums = {"adler32": "adler32", "cksum": "cksum", "md5": "md5"} testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10, checksums = checksums) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn = testFileA["lfn"]) testFileB.load() testFileC = DBSBufferFile(id = testFileA["id"]) testFileC.load() assert testFileA == testFileB, \ "ERROR: File load by LFN didn't work" assert testFileA == testFileC, \ "ERROR: File load by ID didn't work" assert type(testFileB["id"]) == int or type(testFileB["id"]) == long, \ "ERROR: File id is not an integer type." assert type(testFileB["size"]) == int or type(testFileB["size"]) == long, \ "ERROR: File size is not an integer type." assert type(testFileB["events"]) == int or type(testFileB["events"]) == long, \ "ERROR: File events is not an integer type." assert type(testFileC["id"]) == int or type(testFileC["id"]) == long, \ "ERROR: File id is not an integer type." assert type(testFileC["size"]) == int or type(testFileC["size"]) == long, \ "ERROR: File size is not an integer type." assert type(testFileC["events"]) == int or type(testFileC["events"]) == long, \ "ERROR: File events is not an integer type." testFileA.delete() return
def createParentFiles(self, acqEra, nFiles=10, workflowName="TestWorkload", taskPath="/TestWorkload/DataTest"): """ _createParentFiles_ Create several parentless files in DBSBuffer. This simulates raw files in the T0. """ workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath) parentlessFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(nFiles): testFile = DBSBufferFile( lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}, workflowId=workflowId ) testFile.setAlgorithm( appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW", psetHash="GIBBERISH", configContent="MOREGIBBERISH", ) testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) testFile["block_close_max_wait_time"] = 1000000 testFile["block_close_max_events"] = 1000000 testFile["block_close_max_size"] = 1000000 testFile["block_close_max_files"] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentlessFiles.append(testFile) return parentlessFiles
def testProperties(self): """ _testProperties_ Test added tags that use DBSBuffer to transfer from workload to DBS """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.setValidStatus(validStatus="VALID") testFileA.setProcessingVer(ver="ProcVer") testFileA.setAcquisitionEra(era="AcqEra") testFileA.setGlobalTag(globalTag="GlobalTag") testFileA.setDatasetParent(datasetParent="Parent") testFileA.create() return
def testFilesize(self): """ _testFilesize_ Test storing and loading the file information from dbsbuffer_file. Make sure filesize can be bigger than 32 bits """ checksums = {"adler32": "adler32", "cksum": "cksum"} testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 3221225472, events = 1500000, checksums = checksums) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_7_6_0", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn = testFileA["lfn"]) testFileB.load() self.assertEqual(testFileB["size"], 3221225472, "Error: the filesize should be 3GB") self.assertEqual(testFileB["events"], 1500000, "Error: the number of events should be 1.5M")
def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", nLumis = 1): """ _getFiles_ Create some dummy test files. """ files = [] (acqEra, procVer) = name.split("-") baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra, procVer) for f in range(nFiles): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer)) lumis = [] for i in range(nLumis): lumis.append((f * 1000000) + i) testFile.addRun(Run( 1, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("0") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation(site) files.append(testFile) baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra, procVer) testFileChild = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 10, checksums = {'cksum': 1}) testFileChild.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RAW-RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" %(acqEra, procVer)) testFileChild.addRun(Run( 1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def testProperties(self): """ _testProperties_ Test added tags that use DBSBuffer to transfer from workload to DBS """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.setValidStatus(validStatus = "VALID") testFileA.setProcessingVer(ver = "ProcVer") testFileA.setAcquisitionEra(era = "AcqEra") testFileA.setGlobalTag(globalTag = "GlobalTag") testFileA.setDatasetParent(datasetParent = "Parent") testFileA.create() return
def testCountFilesDAO(self): """ _testCountFilesDAO_ Verify that the CountFiles DAO object functions correctly. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfnA", size=1024, events=10, locations="se1.fnal.gov") testFileA.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn="/this/is/a/lfnB", size=1024, events=10, locations="se1.fnal.gov") testFileB.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileB.create() testFileC = DBSBufferFile(lfn="/this/is/a/lfnC", size=1024, events=10, locations="se1.fnal.gov") testFileC.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileC.create() countAction = self.daoFactory(classname="CountFiles") assert countAction.execute() == 3, "Error: Wrong number of files counted in DBS Buffer." return
def injectFilesFromDBS(inputFileset, datasetPath): """ _injectFilesFromDBS_ """ print "injecting files from %s into %s, please wait..." % (datasetPath, inputFileset.name) args={} args["url"] = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" args["version"] = "DBS_2_0_9" args["mode"] = "GET" dbsApi = DbsApi(args) dbsResults = dbsApi.listFileArray(path = datasetPath, retriveList = ["retrive_lumi", "retrive_run"]) dbsResults = dbsResults[0:10] print " found %d files, inserting into wmbs..." % (len(dbsResults)) for dbsResult in dbsResults: myFile = File(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"], events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]}, locations = "cmssrm.fnal.gov", merged = True) myRun = Run(runNumber = dbsResult["LumiList"][0]["RunNumber"]) for lumi in dbsResult["LumiList"]: myRun.lumis.append(lumi["LumiSectionNumber"]) myFile.addRun(myRun) myFile.create() inputFileset.addFile(myFile) dbsFile = DBSBufferFile(lfn = dbsResult["LogicalFileName"], size = dbsResult["FileSize"], events = dbsResult["NumberOfEvents"], checksums = {"cksum": dbsResult["Checksum"]}, locations = "cmssrm.fnal.gov", status = "LOCAL") dbsFile.setDatasetPath(datasetPath) dbsFile.setAlgorithm(appName = "cmsRun", appVer = "Unknown", appFam = "Unknown", psetHash = "Unknown", configContent = "Unknown") dbsFile.create() inputFileset.commit() inputFileset.markOpen(False) return
def createTestFiles(self): """ _createTestFiles_ Create some dbsbuffer test files with different statuses :return: """ phedexStatus = self.dbsbufferFactory( classname="DBSBufferFiles.SetPhEDExStatus") for i in range(0, 4): lfn = "/path/to/some/lfn" + str(i) # Two files should be InDBS, two files should be NOTUPLOADED if i in [0, 2]: status = 'InDBS' else: status = 'NOTUPLOADED' testDBSFile = DBSBufferFile(lfn=lfn, size=600000, events=60000, status=status, workflowId=1) testDBSFile.setAlgorithm(appName="cmsRun", appVer="UNKNOWN", appFam="RECO", psetHash="SOMEHASH" + str(i), configContent="SOMECONTENT") testDBSFile.setDatasetPath("/path/to/some/dataset") testDBSFile.create() # Create all four combinations of status(InDBS,NOTUPLOADED) and in_phedex(0,1) if i in [0, 1]: phedexStatus.execute(lfn, 1)
def testAddChild(self): """ _testAddChild_ Add a child to some parent files and make sure that all the parentage information is loaded/stored correctly from the database. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileParentA.addChildren("/this/is/a/lfn") testFileParentB.addChildren("/this/is/a/lfn") testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load(parentage=1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, \ "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, \ "ERROR: Some parents are missing" return
def testCountFilesDAO(self): """ _testCountFilesDAO_ Verify that the CountFiles DAO object functions correctly. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfnA", size=1024, events=10, locations="se1.fnal.gov") testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn="/this/is/a/lfnB", size=1024, events=10, locations="se1.fnal.gov") testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileB.create() testFileC = DBSBufferFile(lfn="/this/is/a/lfnC", size=1024, events=10, locations="se1.fnal.gov") testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileC.create() countAction = self.daoFactory(classname="CountFiles") assert countAction.execute() == 3, \ "Error: Wrong number of files counted in DBS Buffer." return
def testAddChild(self): """ _testAddChild_ Add a child to some parent files and make sure that all the parentage information is loaded/stored correctly from the database. """ testFileParentA = DBSBufferFile(lfn = "/this/is/a/parent/lfnA", size = 1024, events = 20) testFileParentA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run( 1, *[45])) testFileParentB = DBSBufferFile(lfn = "/this/is/a/parent/lfnB", size = 1024, events = 20) testFileParentB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run( 1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileParentA.addChildren("/this/is/a/lfn") testFileParentB.addChildren("/this/is/a/lfn") testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load(parentage = 1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, \ "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, \ "ERROR: Some parents are missing" return
def createFilesWithChildren(self, moreParentFiles, acqEra): """ _createFilesWithChildren_ Create several parentless files and then create child files. """ parentFiles = [] childFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(10): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RAW", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentFiles.append(testFile) baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % (acqEra) for i in range(5): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % (acqEra)) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(20): lumis.append((i * 20) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") testFile.addParents([parentFiles[i * 2]["lfn"], parentFiles[i * 2 + 1]["lfn"]]) testFile.addParents([moreParentFiles[i * 2]["lfn"], moreParentFiles[i * 2 + 1]["lfn"]]) childFiles.append(testFile) return (parentFiles, childFiles)
def testGetParentLFNs(self): """ _testGetParentLFNs_ Create three files and set them to be parents of a fourth file. Check to make sure that getParentLFNs() on the child file returns the correct LFNs. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentC = DBSBufferFile(lfn="/this/is/a/parent/lfnC", size=1024, events=20) testFileParentC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentC.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileParentC.create() testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.addRun(Run(1, *[45])) testFile.create() testFile.addParents([ testFileParentA["lfn"], testFileParentB["lfn"], testFileParentC["lfn"] ]) parentLFNs = testFile.getParentLFNs() assert len(parentLFNs) == 3, \ "ERROR: Child does not have the right amount of parents" goldenLFNs = [ "/this/is/a/parent/lfnA", "/this/is/a/parent/lfnB", "/this/is/a/parent/lfnC" ] for parentLFN in parentLFNs: assert parentLFN in goldenLFNs, \ "ERROR: Unknown parent lfn" goldenLFNs.remove(parentLFN) testFile.delete() testFileParentA.delete() testFileParentB.delete() testFileParentC.delete() return
def testBulkLoad(self): """ _testBulkLoad_ Can we load in bulk? """ addToBuffer = DBSBufferUtil() testFileChildA = DBSBufferFile(lfn="/this/is/a/child/lfnA", size=1024, events=20) testFileChildA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildB = DBSBufferFile(lfn="/this/is/a/child/lfnB", size=1024, events=20) testFileChildB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildC = DBSBufferFile(lfn="/this/is/a/child/lfnC", size=1024, events=20) testFileChildC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"]) runSet = set() runSet.add(Run(1, *[45])) runSet.add(Run(2, *[67, 68])) testFileChildA.addRunSet(runSet) testFileChildB.addRunSet(runSet) testFileChildC.addRunSet(runSet) testFileChildA.save() testFileChildB.save() testFileChildC.save() setCksumAction = self.daoFactory( classname="DBSBufferFiles.AddChecksumByLFN") binds = [{ 'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201 }, { 'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101 }, { 'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201 }, { 'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101 }, { 'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201 }, { 'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101 }] setCksumAction.execute(bulkList=binds) testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) binds = [{ 'id': testFileChildA.exists() }, { 'id': testFileChildB.exists() }, { 'id': testFileChildC.exists() }] listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs=binds) # print listOfFiles compareList = [ 'locations', 'psetHash', 'configContent', 'appName', 'appVer', 'appFam', 'events', 'datasetPath', 'runs' ] for f in listOfFiles: self.assertTrue( f['lfn'] in [ "/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC" ], "Unknown file in loaded results") self.assertEqual(f['checksums'], { 'adler32': '201', 'cksum': '101' }) for parent in f['parents']: self.assertEqual(parent['lfn'], testFile['lfn']) for key in compareList: self.assertEqual(f[key], testFileChildA[key])
def testReportHandling(self): """ _testReportHandling_ Verify that we're able to parse a CMSSW report, convert it to a Report() style report, pickle it and then have the accountant process it. """ self.procPath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(self.procPath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl") cmsRunStep = myReport.retrieveStep("cmsRun1") cmsRunStep.status = 0 myReport.setTaskName('/TestWF/None') myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath) pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000) pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") pFile.setDatasetPath("/bogus/dataset/path") #pFile.addRun(Run(1, *[45])) pFile.create() config = self.createConfig(workerThreads = 1) accountant = JobAccountantPoller(config) accountant.setup() accountant.algorithm() self.verifyJobSuccess(self.testJob["id"]) self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1")) inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root") inputFile.load() self.testMergeJob = Job(name = "testMergeJob", files = [inputFile]) self.testMergeJob.create(group = self.mergeJobGroup) self.testMergeJob["state"] = "complete" self.stateChangeAction.execute(jobs = [self.testMergeJob]) self.mergePath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml") myReport = Report("mergeReco") myReport.parse(self.mergePath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("mergeReco"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1", "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1", "dataTier": "RECO"} fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl") myReport.setTaskName('/MergeWF/None') cmsRunStep = myReport.retrieveStep("mergeReco") cmsRunStep.status = 0 myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath) accountant.algorithm() self.verifyJobSuccess(self.testMergeJob["id"]) self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco")) return