def testSetLocation(self): """ _testSetLocation_ Create a file and add a couple locations. Load the file from the database to make sure that the locations were set correctly. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave = False) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testSetLocation(self): """ _testSetLocation_ Create a file and add a couple locations. Load the file from the database to make sure that the locations were set correctly. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave=False) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testAddParents(self): """ _testAddParents_ Verify that the addParents() method works correctly even if the parents do not already exist in the database. """ myThread = threading.currentThread() testFile = DBSBufferFile(lfn = "/this/is/a/lfnA", size = 1024, events = 10, locations = "se1.fnal.gov") testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testParent = DBSBufferFile(lfn = "/this/is/a/lfnB", size = 1024, events = 10, locations = "se1.fnal.gov") testParent.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testParent.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RAW") testParent.create() goldenLFNs = ["lfn1", "lfn2", "lfn3", "/this/is/a/lfnB"] testFile.addParents(goldenLFNs) verifyFile = DBSBufferFile(id = testFile["id"]) verifyFile.load(parentage = 1) parentLFNs = verifyFile.getParentLFNs() for parentLFN in parentLFNs: self.assertTrue(parentLFN in goldenLFNs, "Error: unknown lfn %s" % parentLFN) goldenLFNs.remove(parentLFN) self.assertEqual(len(goldenLFNs), 0, "Error: missing LFNs...") # Check that the bogus dataset is listed as inDBS sqlCommand = """SELECT in_dbs FROM dbsbuffer_algo_dataset_assoc das INNER JOIN dbsbuffer_dataset ds ON das.dataset_id = ds.id WHERE ds.path = 'bogus'""" status = myThread.dbi.processData(sqlCommand)[0].fetchall()[0][0] self.assertEqual(status, 1) # Now make sure the dummy files are listed as being in DBS sqlCommand = """SELECT status FROM dbsbuffer_file df INNER JOIN dbsbuffer_algo_dataset_assoc das ON das.id = df.dataset_algo INNER JOIN dbsbuffer_dataset ds ON das.dataset_id = ds.id WHERE ds.path = '/bogus/dataset/path' """ status = myThread.dbi.processData(sqlCommand)[0].fetchall() for entry in status: self.assertEqual(entry, ('AlreadyInDBS',)) return
def testFilesize(self): """ _testFilesize_ Test storing and loading the file information from dbsbuffer_file. Make sure filesize can be bigger than 32 bits """ checksums = {"adler32": "adler32", "cksum": "cksum"} testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=3221225472, events=1500000, checksums=checksums) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_7_6_0", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn=testFileA["lfn"]) testFileB.load() self.assertEqual(testFileB["size"], 3221225472, "Error: the filesize should be 3GB") self.assertEqual(testFileB["events"], 1500000, "Error: the number of events should be 1.5M")
def testAddCKSumByLFN(self): """ _testAddCKSumByLFN_ """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() checksums = {"adler32": "adler32", "cksum": "cksum", "md5": "md5"} setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN") binds = [{'lfn': "/this/is/a/lfn", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/lfn", 'cktype': 'cksum', 'cksum': 101}] setCksumAction.execute(bulkList = binds) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() self.assertEqual(testFileB['checksums'], {'adler32': '201', 'cksum': '101'}) return
def testSetLocationByLFN(self): """ _testSetLocationByLFN_ """ myThread = threading.currentThread() testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() setLocationAction = self.daoFactory(classname = "DBSBufferFiles.SetLocationByLFN") setLocationAction.execute(binds = {'lfn': "/this/is/a/lfn", 'sename': 'se1.cern.ch'}) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() self.assertEqual(testFileB['locations'], set(['se1.cern.ch'])) return
def testSetLocationByLFN(self): """ _testSetLocationByLFN_ """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() setLocationAction = self.daoFactory( classname="DBSBufferFiles.SetLocationByLFN") setLocationAction.execute(binds={ 'lfn': "/this/is/a/lfn", 'pnn': 'se1.cern.ch' }) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() self.assertEqual(testFileB['locations'], set(['se1.cern.ch'])) return
def testLocationsConstructor(self): """ _testLocationsConstructor_ Test to make sure that locations passed into the File() constructor are loaded from and save to the database correctly. Also test to make sure that the class behaves well when the location is passed in as a single string instead of a set. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10, locations=set(["se1.fnal.gov"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn="/this/is/a/lfn2", size=1024, events=10, locations="se1.fnal.gov") testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileB.addRun(Run(1, *[45])) testFileB.create() testFileC = DBSBufferFile(id=testFileA["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" testFileC = DBSBufferFile(id=testFileB["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testAddChild(self): """ _testAddChild_ Add a child to some parent files and make sure that all the parentage information is loaded/stored correctly from the database. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileParentA.addChildren("/this/is/a/lfn") testFileParentB.addChildren("/this/is/a/lfn") testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load(parentage=1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, \ "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, \ "ERROR: Some parents are missing" return
def testSetLocationTransaction(self): """ _testSetLocationTransaction_ Create a file at specific locations and commit everything to the database. Reload the file from the database and verify that the locations are correct. Rollback the database transaction and once again reload the file. Verify that the original locations are back. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov"]) myThread = threading.currentThread() myThread.transaction.begin() testFileA.setLocation(["se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave=False) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" myThread.transaction.rollback() testFileB.load() goldenLocations = ["se1.fnal.gov"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testSetLocationTransaction(self): """ _testSetLocationTransaction_ Create a file at specific locations and commit everything to the database. Reload the file from the database and verify that the locations are correct. Rollback the database transaction and once again reload the file. Verify that the original locations are back. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov"]) myThread = threading.currentThread() myThread.transaction.begin() testFileA.setLocation(["se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave = False) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" myThread.transaction.rollback() testFileB.load() goldenLocations = ["se1.fnal.gov"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testLocationsConstructor(self): """ _testLocationsConstructor_ Test to make sure that locations passed into the File() constructor are loaded from and save to the database correctly. Also test to make sure that the class behaves well when the location is passed in as a single string instead of a set. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10, locations = set(["se1.fnal.gov"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = "/this/is/a/lfn2", size = 1024, events = 10, locations = "se1.fnal.gov") testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileB.addRun(Run( 1, *[45])) testFileB.create() testFileC = DBSBufferFile(id = testFileA["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" testFileC = DBSBufferFile(id = testFileB["id"]) testFileC.load() goldenLocations = ["se1.fnal.gov"] for location in testFileC["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testAddChild(self): """ _testAddChild_ Add a child to some parent files and make sure that all the parentage information is loaded/stored correctly from the database. """ testFileParentA = DBSBufferFile(lfn = "/this/is/a/parent/lfnA", size = 1024, events = 20) testFileParentA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run( 1, *[45])) testFileParentB = DBSBufferFile(lfn = "/this/is/a/parent/lfnB", size = 1024, events = 20) testFileParentB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run( 1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileParentA.addChildren("/this/is/a/lfn") testFileParentB.addChildren("/this/is/a/lfn") testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load(parentage = 1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, \ "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, \ "ERROR: Some parents are missing" return
def testLoad(self): """ _testLoad_ Test the loading of file meta data using the ID of a file and the LFN of a file. """ checksums = {"adler32": "adler32", "cksum": "cksum", "md5": "md5"} testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10, checksums=checksums) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn=testFileA["lfn"]) testFileB.load() testFileC = DBSBufferFile(id=testFileA["id"]) testFileC.load() assert testFileA == testFileB, \ "ERROR: File load by LFN didn't work" assert testFileA == testFileC, \ "ERROR: File load by ID didn't work" assert isinstance(testFileB["id"], int) or isinstance(testFileB["id"], long), \ "ERROR: File id is not an integer type." assert isinstance(testFileB["size"], int) or isinstance(testFileB["size"], long), \ "ERROR: File size is not an integer type." assert isinstance(testFileB["events"], int) or isinstance(testFileB["events"], long), \ "ERROR: File events is not an integer type." assert isinstance(testFileC["id"], int) or isinstance(testFileC["id"], long), \ "ERROR: File id is not an integer type." assert isinstance(testFileC["size"], int) or isinstance(testFileC["size"], long), \ "ERROR: File size is not an integer type." assert isinstance(testFileC["events"], int) or isinstance(testFileC["events"], long), \ "ERROR: File events is not an integer type." testFileA.delete() return
def testLoad(self): """ _testLoad_ Test the loading of file meta data using the ID of a file and the LFN of a file. """ checksums = {"adler32": "adler32", "cksum": "cksum", "md5": "md5"} testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10, checksums = checksums) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn = testFileA["lfn"]) testFileB.load() testFileC = DBSBufferFile(id = testFileA["id"]) testFileC.load() assert testFileA == testFileB, \ "ERROR: File load by LFN didn't work" assert testFileA == testFileC, \ "ERROR: File load by ID didn't work" assert type(testFileB["id"]) == int or type(testFileB["id"]) == long, \ "ERROR: File id is not an integer type." assert type(testFileB["size"]) == int or type(testFileB["size"]) == long, \ "ERROR: File size is not an integer type." assert type(testFileB["events"]) == int or type(testFileB["events"]) == long, \ "ERROR: File events is not an integer type." assert type(testFileC["id"]) == int or type(testFileC["id"]) == long, \ "ERROR: File id is not an integer type." assert type(testFileC["size"]) == int or type(testFileC["size"]) == long, \ "ERROR: File size is not an integer type." assert type(testFileC["events"]) == int or type(testFileC["events"]) == long, \ "ERROR: File events is not an integer type." testFileA.delete() return
def addFile(self, file, dataset=0): """ Add the file to the buffer """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() bufferFile = DBSBufferFile(lfn=file['LFN'], size=file['Size'], events=file['TotalEvents'], cksum=file['Checksum'], dataset=dataset) runLumiList = file.getLumiSections() runList = [x['RunNumber'] for x in runLumiList] for runNumber in runList: lumis = [ int(y['LumiSectionNumber']) for y in runLumiList if y['RunNumber'] == runNumber ] run = Run(runNumber, *lumis) bufferFile.addRun(run) if bufferFile.exists() == False: bufferFile.create() bufferFile.setLocation(se=file['SEName'], immediateSave=True) else: bufferFile.load() # Lets add the file to DBS Buffer as well #UPDATE File Count self.updateDSFileCount(dataset=dataset) #Parent files bufferFile.addParents(file.inputFiles) self.commitTransaction(existingTransaction) return
def testFilesize(self): """ _testFilesize_ Test storing and loading the file information from dbsbuffer_file. Make sure filesize can be bigger than 32 bits """ checksums = {"adler32": "adler32", "cksum": "cksum"} testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 3221225472, events = 1500000, checksums = checksums) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_7_6_0", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.create() testFileB = DBSBufferFile(lfn = testFileA["lfn"]) testFileB.load() self.assertEqual(testFileB["size"], 3221225472, "Error: the filesize should be 3GB") self.assertEqual(testFileB["events"], 1500000, "Error: the number of events should be 1.5M")
def addFile(self, file, dataset=0): """ Add the file to the buffer """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() bufferFile = DBSBufferFile( lfn=file["LFN"], size=file["Size"], events=file["TotalEvents"], cksum=file["Checksum"], dataset=dataset ) runLumiList = file.getLumiSections() runList = [x["RunNumber"] for x in runLumiList] for runNumber in runList: lumis = [int(y["LumiSectionNumber"]) for y in runLumiList if y["RunNumber"] == runNumber] run = Run(runNumber, *lumis) bufferFile.addRun(run) if bufferFile.exists() == False: bufferFile.create() bufferFile.setLocation(se=file["SEName"], immediateSave=True) else: bufferFile.load() # Lets add the file to DBS Buffer as well # UPDATE File Count self.updateDSFileCount(dataset=dataset) # Parent files bufferFile.addParents(file.inputFiles) self.commitTransaction(existingTransaction) return
def testAddChildTransaction(self): """ _testAddChildTransaction_ Add a child to some parent files and make sure that all the parentage information is loaded/stored correctly from the database. Rollback the addition of one of the childs and then verify that it does in fact only have one parent. """ testFileParentA = DBSBufferFile(lfn="/this/is/a/parent/lfnA", size=1024, events=20) testFileParentA.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileParentA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentA.addRun(Run(1, *[45])) testFileParentB = DBSBufferFile(lfn="/this/is/a/parent/lfnB", size=1024, events=20) testFileParentB.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileParentB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileParentB.addRun(Run(1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileParentA.addChildren("/this/is/a/lfn") myThread = threading.currentThread() myThread.transaction.begin() testFileParentB.addChildren("/this/is/a/lfn") testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load(parentage=1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, "ERROR: Some parents are missing" myThread.transaction.rollback() testFileB.load(parentage=1) goldenFiles = [testFileParentA] for parentFile in testFileB["parents"]: assert parentFile in goldenFiles, "ERROR: Unknown parent file" goldenFiles.remove(parentFile) assert len(goldenFiles) == 0, "ERROR: Some parents are missing" return