def testSetLocation(self): """ _testSetLocation_ Create a file and add a couple locations. Load the file from the database to make sure that the locations were set correctly. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave=False) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def getFiles(self, name, tier, nFiles=12, site="malpaquet"): """ Create some quick dummy test files """ files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn="%s-%s-%i" % (name, site, f), size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm( appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL ) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run(1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) testFileChild = DBSBufferFile(lfn="%s-%s-child" % (name, site), size=1024, events=10, checksums={"cksum": 1}) testFileChild.setAlgorithm( appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL ) testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x["lfn"] for x in files]) return files
def testSetLocation(self): """ _testSetLocation_ Create a file and add a couple locations. Load the file from the database to make sure that the locations were set correctly. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave = False) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def getFiles(self, name, tier='RECO', nFiles=12, site="malpaquet", nLumis=1): """ Create some quick dummy test files """ files = [] for f in range(nFiles): testFile = DBSBufferFile( lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size=1024, events=20, checksums={'cksum': 1}) testFile.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) lumis = [] for i in range(nLumis): lumis.append((f * 100000) + i) testFile.addRun(Run(1, *lumis)) testFile.setAcquisitionEra(name.split('-')[0]) testFile.setProcessingVer("0") testFile.setGlobalTag("Weird") testFile.create() testFile.setLocation(site) files.append(testFile) count = 0 for f in files: count += 1 testFileChild = DBSBufferFile( lfn= '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' % (name, site, count), size=1024, events=10, checksums={'cksum': 1}) testFileChild.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([f['lfn']]) return files
def prepareDBSFiles(self): """ _prepareDBSFiles_ Retrieve the information from the JSON input data and create DBSFile objects that can be registered in the database. """ timestamp = time.strftime('%m%d%y_%H%M%S') for fileEntry in self.inputData: # Get all the info out of a standard named dataset datasetInfo = str(fileEntry["dataset"]) tokens = datasetInfo.split('/') primDs = tokens[1] procDs = tokens[2] dataTier = tokens[3] procDsTokens = procDs.split('-') acqEra = procDsTokens[0] procVer = procDsTokens[-1][1:] ckSumInfo = fileEntry["checksums"] for entry in ckSumInfo: ckSumInfo[entry] = str(ckSumInfo[entry]) # Build the basic dbsBuffer file dbsFile = DBSBufferFile(lfn = str(fileEntry["lfn"]), size = int(fileEntry.get("size", 0)), events = int(fileEntry.get("events", 0)), checksums = ckSumInfo, status = "NOTUPLOADED") dbsFile.setAlgorithm(appName = "cmsRun", appVer = str(fileEntry.get("cmssw", "LEGACY")), appFam = "Legacy", psetHash = "GIBBERISH", configContent = "None;;None;;None") dbsFile.setDatasetPath("/%s/%s/%s" % (primDs, procDs, dataTier)) dbsFile.setValidStatus(validStatus = "PRODUCTION") dbsFile.setProcessingVer(ver = procVer) dbsFile.setAcquisitionEra(era = acqEra) dbsFile.setGlobalTag(globalTag = str(fileEntry.get('globalTag', "LEGACY"))) # Build a representative task name dbsFile['task'] = '/LegacyInsertionTask_%s/Insertion' % timestamp # Get the runs and lumis runsAndLumis = fileEntry.get("runsAndLumis", {}) for run in runsAndLumis: newRun = Run(runNumber = int(run)) newRun.extend([int(x) for x in runsAndLumis[run]]) dbsFile.addRun(newRun) # Complete the file information with the location and queue it dbsFile.setLocation(se = str(fileEntry["location"]), immediateSave = False) self.dbsFilesToCreate.append(dbsFile) self.inputData = None return
def getFiles(self, name, tier, nFiles=12, site="malpaquet", workflowName=None, taskPath=None, noChild=False): """ Create some quick dummy test files """ if workflowName is not None and taskPath is not None: workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath) else: workflowId = self.injectWorkflow() files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn='%s-%s-%i' % (name, site, f), size=1024, events=20, checksums={'cksum': 1}, workflowId=workflowId) testFile.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run(1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) if not noChild: testFileChild = DBSBufferFile(lfn='%s-%s-child' % (name, site), size=1024, events=10, checksums={'cksum': 1}, workflowId=workflowId) testFileChild.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent=self.configURL) testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def createFilesWithChildren(self, moreParentFiles, acqEra): """ _createFilesWithChildren_ Create several parentless files and then create child files. """ parentFiles = [] childFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(10): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RAW", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentFiles.append(testFile) baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % (acqEra) for i in range(5): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % (acqEra)) lumis = [] for j in range(20): lumis.append((i * 20) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") testFile.addParents([parentFiles[i * 2]["lfn"], parentFiles[i * 2 + 1]["lfn"]]) testFile.addParents([moreParentFiles[i * 2]["lfn"], moreParentFiles[i * 2 + 1]["lfn"]]) childFiles.append(testFile) return (parentFiles, childFiles)
def getFiles(self, name, tier, nFiles=12, site="malpaquet", nLumis=1): """ _getFiles_ Create some dummy test files. """ files = [] (acqEra, procVer) = name.split("-") baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra, procVer) for f in range(nFiles): testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer)) lumis = [] for i in range(nLumis): lumis.append((f * 1000000) + i) testFile.addRun(Run(1, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("0") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation(site) files.append(testFile) baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra, procVer) testFileChild = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=10, checksums={'cksum': 1}) testFileChild.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW-RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" % (acqEra, procVer)) testFileChild.addRun(Run(1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def addFileToDBS(self, jobReportFile, task, errorDataset=False): """ _addFileToDBS_ Add a file that was output from a job to the DBS buffer. """ datasetInfo = jobReportFile["dataset"] dbsFile = DBSBufferFile(lfn=jobReportFile["lfn"], size=jobReportFile["size"], events=jobReportFile["events"], checksums=jobReportFile["checksums"], status="NOTUPLOADED") dbsFile.setAlgorithm(appName=datasetInfo["applicationName"], appVer=datasetInfo["applicationVersion"], appFam=jobReportFile["module_label"], psetHash="GIBBERISH", configContent=jobReportFile.get('configURL')) if errorDataset: dbsFile.setDatasetPath( "/%s/%s/%s" % (datasetInfo["primaryDataset"] + "-Error", datasetInfo["processedDataset"], datasetInfo["dataTier"])) else: dbsFile.setDatasetPath( "/%s/%s/%s" % (datasetInfo["primaryDataset"], datasetInfo["processedDataset"], datasetInfo["dataTier"])) dbsFile.setValidStatus( validStatus=jobReportFile.get("validStatus", None)) dbsFile.setProcessingVer(ver=jobReportFile.get('processingVer', None)) dbsFile.setAcquisitionEra( era=jobReportFile.get('acquisitionEra', None)) dbsFile.setGlobalTag(globalTag=jobReportFile.get('globalTag', None)) #TODO need to find where to get the prep id dbsFile.setPrepID(prep_id=jobReportFile.get('prep_id', None)) dbsFile['task'] = task for run in jobReportFile["runs"]: newRun = Run(runNumber=run.run) newRun.extend(run.lumis) dbsFile.addRun(newRun) dbsFile.setLocation(pnn=list(jobReportFile["locations"])[0], immediateSave=False) self.dbsFilesToCreate.append(dbsFile) return
def createParentFiles(self, acqEra, nFiles=10, workflowName='TestWorkload', taskPath='/TestWorkload/DataTest'): """ _createParentFiles_ Create several parentless files in DBSBuffer. This simulates raw files in the T0. """ workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath) parentlessFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(nFiles): testFile = DBSBufferFile(lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}, workflowId=workflowId) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentlessFiles.append(testFile) return parentlessFiles
def addFileToDBS(self, jobReportFile, task, errorDataset = False): """ _addFileToDBS_ Add a file that was output from a job to the DBS buffer. """ datasetInfo = jobReportFile["dataset"] dbsFile = DBSBufferFile(lfn = jobReportFile["lfn"], size = jobReportFile["size"], events = jobReportFile["events"], checksums = jobReportFile["checksums"], status = "NOTUPLOADED") dbsFile.setAlgorithm(appName = datasetInfo["applicationName"], appVer = datasetInfo["applicationVersion"], appFam = jobReportFile["module_label"], psetHash = "GIBBERISH", configContent = jobReportFile.get('configURL')) if errorDataset: dbsFile.setDatasetPath("/%s/%s/%s" % (datasetInfo["primaryDataset"] + "-Error", datasetInfo["processedDataset"], datasetInfo["dataTier"])) else: dbsFile.setDatasetPath("/%s/%s/%s" % (datasetInfo["primaryDataset"], datasetInfo["processedDataset"], datasetInfo["dataTier"])) dbsFile.setValidStatus(validStatus = jobReportFile.get("validStatus", None)) dbsFile.setProcessingVer(ver = jobReportFile.get('processingVer', None)) dbsFile.setAcquisitionEra(era = jobReportFile.get('acquisitionEra', None)) dbsFile.setGlobalTag(globalTag = jobReportFile.get('globalTag', None)) #TODO need to find where to get the prep id dbsFile.setPrepID(prep_id = jobReportFile.get('prep_id', None)) dbsFile['task'] = task for run in jobReportFile["runs"]: newRun = Run(runNumber = run.run) newRun.extend(run.lumis) dbsFile.addRun(newRun) dbsFile.setLocation(pnn = list(jobReportFile["locations"])[0], immediateSave = False) self.dbsFilesToCreate.append(dbsFile) return
def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", workflowName = None, taskPath = None, noChild = False): """ Create some quick dummy test files """ if workflowName is not None and taskPath is not None: workflowId = self.injectWorkflow(workflowName = workflowName, taskPath = taskPath) else: workflowId = self.injectWorkflow() files = [] for f in range(0, nFiles): testFile = DBSBufferFile(lfn = '%s-%s-%i' % (name, site, f), size = 1024, events = 20, checksums = {'cksum': 1}, workflowId = workflowId) testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = self.configURL) testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) testFile.addRun(Run( 1, *[f])) testFile.setGlobalTag("aGlobalTag") testFile.create() testFile.setLocation(site) files.append(testFile) if not noChild: testFileChild = DBSBufferFile(lfn = '%s-%s-child' %(name, site), size = 1024, events = 10, checksums = {'cksum': 1}, workflowId = workflowId) testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = self.configURL) testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name)) testFileChild.addRun(Run( 1, *[45])) testFileChild.setGlobalTag("aGlobalTag") testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1): """ Create some quick dummy test files """ files = [] for f in range(nFiles): testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024, events = 20, checksums = {'cksum': 1}) testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) lumis = [] for i in range(nLumis): lumis.append((f * 100000) + i) testFile.addRun(Run( 1, *lumis)) testFile.setAcquisitionEra(name.split('-')[0]) testFile.setProcessingVer("0") testFile.setGlobalTag("Weird") testFile.create() testFile.setLocation(site) files.append(testFile) count = 0 for f in files: count += 1 testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024, events = 10, checksums = {'cksum': 1}) testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name)) testFileChild.addRun(Run( 1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([f['lfn']]) return files
def addFile(self, file, dataset=0): """ Add the file to the buffer """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() bufferFile = DBSBufferFile(lfn=file['LFN'], size=file['Size'], events=file['TotalEvents'], cksum=file['Checksum'], dataset=dataset) runLumiList = file.getLumiSections() runList = [x['RunNumber'] for x in runLumiList] for runNumber in runList: lumis = [ int(y['LumiSectionNumber']) for y in runLumiList if y['RunNumber'] == runNumber ] run = Run(runNumber, *lumis) bufferFile.addRun(run) if bufferFile.exists() == False: bufferFile.create() bufferFile.setLocation(se=file['SEName'], immediateSave=True) else: bufferFile.load() # Lets add the file to DBS Buffer as well #UPDATE File Count self.updateDSFileCount(dataset=dataset) #Parent files bufferFile.addParents(file.inputFiles) self.commitTransaction(existingTransaction) return
def createParentFiles(self, acqEra, nFiles=10, workflowName="TestWorkload", taskPath="/TestWorkload/DataTest"): """ _createParentFiles_ Create several parentless files in DBSBuffer. This simulates raw files in the T0. """ workflowId = self.injectWorkflow(workflowName=workflowName, taskPath=taskPath) parentlessFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(nFiles): testFile = DBSBufferFile( lfn=baseLFN + makeUUID() + ".root", size=1024, events=20, checksums={"cksum": 1}, workflowId=workflowId ) testFile.setAlgorithm( appName="cmsRun", appVer="CMSSW_3_1_1", appFam="RAW", psetHash="GIBBERISH", configContent="MOREGIBBERISH", ) testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) testFile["block_close_max_wait_time"] = 1000000 testFile["block_close_max_events"] = 1000000 testFile["block_close_max_size"] = 1000000 testFile["block_close_max_files"] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentlessFiles.append(testFile) return parentlessFiles
def getFiles(self, name, tier, nFiles = 12, site = "malpaquet", nLumis = 1): """ _getFiles_ Create some dummy test files. """ files = [] (acqEra, procVer) = name.split("-") baseLFN = "/store/data/%s/Cosmics/RECO/%s/000/143/316/" % (acqEra, procVer) for f in range(nFiles): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-%s/RECO" % (acqEra, procVer)) lumis = [] for i in range(nLumis): lumis.append((f * 1000000) + i) testFile.addRun(Run( 1, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("0") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation(site) files.append(testFile) baseLFN = "/store/data/%s/Cosmics/RAW-RECO/%s/000/143/316/" % (acqEra, procVer) testFileChild = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 10, checksums = {'cksum': 1}) testFileChild.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RAW-RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChild.setDatasetPath("/Cosmics/%s-%s/RAW-RECO" %(acqEra, procVer)) testFileChild.addRun(Run( 1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([x['lfn'] for x in files]) return files
def addFile(self, file, dataset=0): """ Add the file to the buffer """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() bufferFile = DBSBufferFile( lfn=file["LFN"], size=file["Size"], events=file["TotalEvents"], cksum=file["Checksum"], dataset=dataset ) runLumiList = file.getLumiSections() runList = [x["RunNumber"] for x in runLumiList] for runNumber in runList: lumis = [int(y["LumiSectionNumber"]) for y in runLumiList if y["RunNumber"] == runNumber] run = Run(runNumber, *lumis) bufferFile.addRun(run) if bufferFile.exists() == False: bufferFile.create() bufferFile.setLocation(se=file["SEName"], immediateSave=True) else: bufferFile.load() # Lets add the file to DBS Buffer as well # UPDATE File Count self.updateDSFileCount(dataset=dataset) # Parent files bufferFile.addParents(file.inputFiles) self.commitTransaction(existingTransaction) return
def testSetLocationTransaction(self): """ _testSetLocationTransaction_ Create a file at specific locations and commit everything to the database. Reload the file from the database and verify that the locations are correct. Rollback the database transaction and once again reload the file. Verify that the original locations are back. """ testFileA = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run(1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov"]) myThread = threading.currentThread() myThread.transaction.begin() testFileA.setLocation(["se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave=False) testFileB = DBSBufferFile(id=testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" myThread.transaction.rollback() testFileB.load() goldenLocations = ["se1.fnal.gov"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testSetLocationTransaction(self): """ _testSetLocationTransaction_ Create a file at specific locations and commit everything to the database. Reload the file from the database and verify that the locations are correct. Rollback the database transaction and once again reload the file. Verify that the original locations are back. """ testFileA = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileA.setLocation(["se1.fnal.gov"]) myThread = threading.currentThread() myThread.transaction.begin() testFileA.setLocation(["se1.cern.ch"]) testFileA.setLocation(["bunkse1.fnal.gov", "bunkse1.cern.ch"], immediateSave = False) testFileB = DBSBufferFile(id = testFileA["id"]) testFileB.load() goldenLocations = ["se1.fnal.gov", "se1.cern.ch"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" myThread.transaction.rollback() testFileB.load() goldenLocations = ["se1.fnal.gov"] for location in testFileB["locations"]: assert location in goldenLocations, \ "ERROR: Unknown file location" goldenLocations.remove(location) assert len(goldenLocations) == 0, \ "ERROR: Some locations are missing" return
def testBulkLoad(self): """ _testBulkLoad_ Can we load in bulk? """ addToBuffer = DBSBufferUtil() bulkLoad = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID") testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024, events = 20) testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024, events = 20) testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024, events = 20) testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"]) runSet = set() runSet.add(Run( 1, *[45])) runSet.add(Run( 2, *[67, 68])) testFileChildA.addRunSet(runSet) testFileChildB.addRunSet(runSet) testFileChildC.addRunSet(runSet) testFileChildA.save() testFileChildB.save() testFileChildC.save() setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN") binds = [{'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101}, {'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101}, {'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101}] setCksumAction.execute(bulkList = binds) testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) binds = [{'id': testFileChildA.exists()}, {'id': testFileChildB.exists()}, {'id': testFileChildC.exists()}] listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs = binds) #print listOfFiles compareList = ['locations', 'psetHash', 'configContent', 'appName', 'appVer', 'appFam', 'events', 'datasetPath', 'runs'] for f in listOfFiles: self.assertTrue(f['lfn'] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC"], "Unknown file in loaded results") self.assertEqual(f['checksums'], {'adler32': '201', 'cksum': '101'}) for parent in f['parents']: self.assertEqual(parent['lfn'], testFile['lfn']) for key in compareList: self.assertEqual(f[key], testFileChildA[key])
def testBulkLoad(self): """ _testBulkLoad_ Can we load in bulk? """ addToBuffer = DBSBufferUtil() testFileChildA = DBSBufferFile(lfn="/this/is/a/child/lfnA", size=1024, events=20) testFileChildA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildB = DBSBufferFile(lfn="/this/is/a/child/lfnB", size=1024, events=20) testFileChildB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildC = DBSBufferFile(lfn="/this/is/a/child/lfnC", size=1024, events=20) testFileChildC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"]) runSet = set() runSet.add(Run(1, *[45])) runSet.add(Run(2, *[67, 68])) testFileChildA.addRunSet(runSet) testFileChildB.addRunSet(runSet) testFileChildC.addRunSet(runSet) testFileChildA.save() testFileChildB.save() testFileChildC.save() setCksumAction = self.daoFactory( classname="DBSBufferFiles.AddChecksumByLFN") binds = [{ 'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201 }, { 'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101 }, { 'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201 }, { 'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101 }, { 'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201 }, { 'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101 }] setCksumAction.execute(bulkList=binds) testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) binds = [{ 'id': testFileChildA.exists() }, { 'id': testFileChildB.exists() }, { 'id': testFileChildC.exists() }] listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs=binds) # print listOfFiles compareList = [ 'locations', 'psetHash', 'configContent', 'appName', 'appVer', 'appFam', 'events', 'datasetPath', 'runs' ] for f in listOfFiles: self.assertTrue( f['lfn'] in [ "/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC" ], "Unknown file in loaded results") self.assertEqual(f['checksums'], { 'adler32': '201', 'cksum': '101' }) for parent in f['parents']: self.assertEqual(parent['lfn'], testFile['lfn']) for key in compareList: self.assertEqual(f[key], testFileChildA[key])
def createFilesWithChildren(self, moreParentFiles, acqEra): """ _createFilesWithChildren_ Create several parentless files and then create child files. """ parentFiles = [] childFiles = [] baseLFN = "/store/data/%s/Cosmics/RAW/v1/000/143/316/" % (acqEra) for i in range(10): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RAW", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RAW" % (acqEra)) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(10): lumis.append((i * 10) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") parentFiles.append(testFile) baseLFN = "/store/data/%s/Cosmics/RECO/v1/000/143/316/" % (acqEra) for i in range(5): testFile = DBSBufferFile(lfn = baseLFN + makeUUID() + ".root", size = 1024, events = 20, checksums = {"cksum": 1}) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/%s-v1/RECO" % (acqEra)) testFile['block_close_max_wait_time'] = 1000000 testFile['block_close_max_events'] = 1000000 testFile['block_close_max_size'] = 1000000 testFile['block_close_max_files'] = 1000000 lumis = [] for j in range(20): lumis.append((i * 20) + j) testFile.addRun(Run(143316, *lumis)) testFile.setAcquisitionEra(acqEra) testFile.setProcessingVer("1") testFile.setGlobalTag("START54::All") testFile.create() testFile.setLocation("malpaquet") testFile.addParents([parentFiles[i * 2]["lfn"], parentFiles[i * 2 + 1]["lfn"]]) testFile.addParents([moreParentFiles[i * 2]["lfn"], moreParentFiles[i * 2 + 1]["lfn"]]) childFiles.append(testFile) return (parentFiles, childFiles)
def testBulkLoad(self): """ _testBulkLoad_ Can we load in bulk? """ bufferFactory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database.Interface") addToBuffer = bufferFactory.loadObject("AddToBuffer") bulkLoad = self.daoFactory(classname="DBSBufferFiles.LoadBulkFilesByID") testFileChildA = DBSBufferFile(lfn="/this/is/a/child/lfnA", size=1024, events=20) testFileChildA.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildB = DBSBufferFile(lfn="/this/is/a/child/lfnB", size=1024, events=20) testFileChildB.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildC = DBSBufferFile(lfn="/this/is/a/child/lfnC", size=1024, events=20) testFileChildC.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"]) runSet = set() runSet.add(Run(1, *[45])) runSet.add(Run(2, *[67, 68])) testFileChildA.addRunSet(runSet) testFileChildB.addRunSet(runSet) testFileChildC.addRunSet(runSet) testFileChildA.save() testFileChildB.save() testFileChildC.save() setCksumAction = self.daoFactory(classname="DBSBufferFiles.AddChecksumByLFN") binds = [ {"lfn": "/this/is/a/child/lfnA", "cktype": "adler32", "cksum": 201}, {"lfn": "/this/is/a/child/lfnA", "cktype": "cksum", "cksum": 101}, {"lfn": "/this/is/a/child/lfnB", "cktype": "adler32", "cksum": 201}, {"lfn": "/this/is/a/child/lfnB", "cktype": "cksum", "cksum": 101}, {"lfn": "/this/is/a/child/lfnC", "cktype": "adler32", "cksum": 201}, {"lfn": "/this/is/a/child/lfnC", "cktype": "cksum", "cksum": 101}, ] setCksumAction.execute(bulkList=binds) testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10) testFile.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) binds = [{"id": testFileChildA.exists()}, {"id": testFileChildB.exists()}, {"id": testFileChildC.exists()}] listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs=binds) # print listOfFiles compareList = [ "locations", "psetHash", "configContent", "appName", "appVer", "appFam", "events", "datasetPath", "runs", ] for f in listOfFiles: self.assertTrue( f["lfn"] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC"], "Unknown file in loaded results", ) self.assertEqual(f["checksums"], {"adler32": "201", "cksum": "101"}) for parent in f["parents"]: self.assertEqual(parent["lfn"], testFile["lfn"]) for key in compareList: self.assertEqual(f[key], testFileChildA[key])