def insertFiles(self, fwkJobRep, insertDetectorData=False): """ _insertFiles_ Process the files in the FwkJobReport instance and insert them into the associated datasets A list of affected fileblock names is returned both for merged and unmerged fileblocks. Only merged blocks will have to be managed. #for merged file #blocks to facilitate management of those blocks. #This list is not populated for processing jobs since we dont really #care about the processing job blocks. """ insertLists = {} orderedHashes = [] affectedBlocks = set() if len(fwkJobRep.files) <= 0: msg = "Error in DBSWriter.insertFiles\n" msg += "No files found in FrameWorkJobReport for:\n" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for outFile in fwkJobRep.sortFiles(): # // # // Convert each file into a DBS File object # // seName = None if outFile.has_key("SEName"): if outFile["SEName"]: seName = outFile["SEName"] logging.debug("SEname associated to file is: %s" % seName) ## remove the fallback to site se-name if no SE is associated to File ## because it's likely that there is some stage out problem if there ## is no SEName associated to the file. # if not seName: # if fwkJobRep.siteDetails.has_key("se-name"): # seName = fwkJobRep.siteDetails['se-name'] # seName = str(seName) # logging.debug("site SEname: %s"%seName) if not seName: msg = "Error in DBSWriter.insertFiles\n" msg += "No SEname associated to files in FrameWorkJobReport for " # msg += "No SEname found in FrameWorkJobReport for " msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) try: if insertDetectorData: dbsFiles = DBSWriterObjects.createDBSFiles(outFile, fwkJobRep.jobType, self.dbs) else: dbsFiles = DBSWriterObjects.createDBSFiles(outFile, fwkJobRep.jobType) except DbsException, ex: msg = "Error in DBSWriter.insertFiles:\n" msg += "Error creating DbsFile instances for file:\n" msg += "%s\n" % outFile["LFN"] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if len(dbsFiles) <= 0: msg = "No DbsFile instances created. Not enough info in the FrameWorkJobReport for" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for f in dbsFiles: datasetName = makeDBSDSName(f) hashName = "%s-%s" % (seName, datasetName) if not insertLists.has_key(hashName): insertLists[hashName] = _InsertFileList(seName, datasetName) insertLists[hashName].append(f) if not orderedHashes.count(hashName): orderedHashes.append(hashName)
def insertFiles(self, fwkJobRep, insertDetectorData=False): """ _insertFiles_ Process the files in the FwkJobReport instance and insert them into the associated datasets A list of affected fileblock names is returned both for merged and unmerged fileblocks. Only merged blocks will have to be managed. #for merged file #blocks to facilitate management of those blocks. #This list is not populated for processing jobs since we dont really #care about the processing job blocks. """ insertLists = {} orderedHashes = [] affectedBlocks = set() if len(fwkJobRep.files) <= 0: msg = "Error in DBSWriter.insertFiles\n" msg += "No files found in FrameWorkJobReport for:\n" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for outFile in fwkJobRep.sortFiles(): # // # // Convert each file into a DBS File object #// seName = None if outFile.has_key("SEName"): if outFile['SEName']: seName = outFile['SEName'] logging.debug("SEname associated to file is: %s" % seName) ## remove the fallback to site se-name if no SE is associated to File ## because it's likely that there is some stage out problem if there ## is no SEName associated to the file. # if not seName: # if fwkJobRep.siteDetails.has_key("se-name"): # seName = fwkJobRep.siteDetails['se-name'] # seName = str(seName) # logging.debug("site SEname: %s"%seName) if not seName: msg = "Error in DBSWriter.insertFiles\n" msg += "No SEname associated to files in FrameWorkJobReport for " # msg += "No SEname found in FrameWorkJobReport for " msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) try: if (insertDetectorData): dbsFiles = DBSWriterObjects.createDBSFiles( outFile, fwkJobRep.jobType, self.dbs) else: dbsFiles = DBSWriterObjects.createDBSFiles( outFile, fwkJobRep.jobType) except DbsException, ex: msg = "Error in DBSWriter.insertFiles:\n" msg += "Error creating DbsFile instances for file:\n" msg += "%s\n" % outFile['LFN'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if len(dbsFiles) <= 0: msg = "No DbsFile instances created. Not enough info in the FrameWorkJobReport for" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for f in dbsFiles: datasetName = makeDBSDSName(f) hashName = "%s-%s" % (seName, datasetName) if not insertLists.has_key(hashName): insertLists[hashName] = _InsertFileList( seName, datasetName) insertLists[hashName].append(f) if not orderedHashes.count(hashName): orderedHashes.append(hashName)
def insertFiles(self, fwkJobRep, insertDetectorData=False): """ _insertFiles_ Process the files in the FwkJobReport instance and insert them into the associated datasets A list of affected fileblock names is returned both for merged and unmerged fileblocks. Only merged blocks will have to be managed. #for merged file #blocks to facilitate management of those blocks. #This list is not populated for processing jobs since we dont really #care about the processing job blocks. """ insertLists = {} orderedHashes = [] affectedBlocks = set() if len(fwkJobRep.files) <= 0: msg = "Error in DBSWriter.insertFiles\n" msg += "No files found in FrameWorkJobReport for:\n" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for outFile in fwkJobRep.sortFiles(): # // # // Convert each file into a DBS File object #// seName = None if "SEName" in outFile: if outFile['SEName']: seName = outFile['SEName'] logging.debug("SEname associated to file is: %s" % seName) ## remove the fallback to site se-name if no SE is associated to File ## because it's likely that there is some stage out problem if there ## is no SEName associated to the file. # if not seName: # if fwkJobRep.siteDetails.has_key("se-name"): # seName = fwkJobRep.siteDetails['se-name'] # seName = str(seName) # logging.debug("site SEname: %s"%seName) if not seName: msg = "Error in DBSWriter.insertFiles\n" msg += "No SEname associated to files in FrameWorkJobReport for " # msg += "No SEname found in FrameWorkJobReport for " msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) try: if (insertDetectorData): dbsFiles = DBSWriterObjects.createDBSFiles( outFile, fwkJobRep.jobType, self.dbs) else: dbsFiles = DBSWriterObjects.createDBSFiles( outFile, fwkJobRep.jobType) except DbsException as ex: msg = "Error in DBSWriter.insertFiles:\n" msg += "Error creating DbsFile instances for file:\n" msg += "%s\n" % outFile['LFN'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if len(dbsFiles) <= 0: msg = "No DbsFile instances created. Not enough info in the FrameWorkJobReport for" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for f in dbsFiles: datasetName = makeDBSDSName(f) hashName = "%s-%s" % (seName, datasetName) if hashName not in insertLists: insertLists[hashName] = _InsertFileList( seName, datasetName) insertLists[hashName].append(f) if not orderedHashes.count(hashName): orderedHashes.append(hashName) # //Processing Jobs: # // Insert the lists of sorted files into the appropriate #// fileblocks for hash in orderedHashes: fileList = insertLists[hash] procDataset = fileList[0]['Dataset'] try: fileBlock = DBSWriterObjects.getDBSFileBlock( self.dbs, procDataset, fileList.seName) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot retrieve FileBlock for dataset:\n" msg += " %s\n" % procDataset msg += "In Storage Element:\n %s\n" % fileList.seName msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if fwkJobRep.jobType == "Merge": # // # // Merge files #// for mergedFile in fileList: mergedFile['Block'] = fileBlock affectedBlocks.add(fileBlock['Name']) msg = "calling: self.dbs.insertMergedFile(%s, %s)" % (str( mergedFile['ParentList']), str(mergedFile)) logging.debug(msg) try: self.dbs.insertMergedFile(mergedFile['ParentList'], mergedFile) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert merged file:\n" msg += " %s\n" % mergedFile['LogicalFileName'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) logging.debug( "Inserted merged file: %s to FileBlock: %s" % (mergedFile['LogicalFileName'], fileBlock['Name'])) else: # // # // Processing files #// affectedBlocks.add(fileBlock['Name']) msg = "calling: self.dbs.insertFiles(%s, %s, %s)" % ( str(procDataset), str(list(fileList)), str(fileBlock)) logging.debug(msg) try: self.dbs.insertFiles(procDataset, list(fileList), fileBlock) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" msg += " %s\n" % ([x['LogicalFileName'] for x in fileList], ) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) logging.debug("Inserted files: %s to FileBlock: %s" % (([x['LogicalFileName'] for x in fileList]), fileBlock['Name'])) return list(affectedBlocks)
def insertFiles(self, fwkJobRep, insertDetectorData = False): """ _insertFiles_ Process the files in the FwkJobReport instance and insert them into the associated datasets A list of affected fileblock names is returned both for merged and unmerged fileblocks. Only merged blocks will have to be managed. #for merged file #blocks to facilitate management of those blocks. #This list is not populated for processing jobs since we dont really #care about the processing job blocks. """ insertLists = {} orderedHashes = [] affectedBlocks = set() if len(fwkJobRep.files)<=0: msg = "Error in DBSWriter.insertFiles\n" msg += "No files found in FrameWorkJobReport for:\n" msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId msg += " Workflow: %s"%fwkJobRep.workflowSpecId raise DBSWriterError(msg) for outFile in fwkJobRep.sortFiles(): # // # // Convert each file into a DBS File object #// pnn = None if "PNN" in outFile: if outFile['PNN'] : pnn = outFile['PNN'] logging.debug("PNN associated to file is: %s"%pnn) ## remove the fallback to site se-name if no SE is associated to File ## because it's likely that there is some stage out problem if there ## is no SEName associated to the file. # if not seName: # if fwkJobRep.siteDetails.has_key("se-name"): # seName = fwkJobRep.siteDetails['se-name'] # seName = str(seName) # logging.debug("site SEname: %s"%seName) if not pnn: msg = "Error in DBSWriter.insertFiles\n" msg += "No PNN associated to files in FrameWorkJobReport for " # msg += "No SEname found in FrameWorkJobReport for " msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId msg += " Workflow: %s"%fwkJobRep.workflowSpecId raise DBSWriterError(msg) try: if ( insertDetectorData ): dbsFiles = DBSWriterObjects.createDBSFiles(outFile, fwkJobRep.jobType, self.dbs) else: dbsFiles = DBSWriterObjects.createDBSFiles(outFile, fwkJobRep.jobType) except DbsException as ex: msg = "Error in DBSWriter.insertFiles:\n" msg += "Error creating DbsFile instances for file:\n" msg += "%s\n" % outFile['LFN'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if len(dbsFiles)<=0: msg="No DbsFile instances created. Not enough info in the FrameWorkJobReport for" msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId msg += " Workflow: %s"%fwkJobRep.workflowSpecId raise DBSWriterError(msg) for f in dbsFiles: datasetName = makeDBSDSName(f) hashName = "%s-%s" % (pnn, datasetName) if hashName not in insertLists: insertLists[hashName] = _InsertFileList(pnn, datasetName) insertLists[hashName].append(f) if not orderedHashes.count(hashName): orderedHashes.append(hashName) # //Processing Jobs: # // Insert the lists of sorted files into the appropriate #// fileblocks for hash in orderedHashes: fileList = insertLists[hash] procDataset = fileList[0]['Dataset'] try: fileBlock = DBSWriterObjects.getDBSFileBlock( self.dbs, procDataset, fileList.pnn) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot retrieve FileBlock for dataset:\n" msg += " %s\n" % procDataset # msg += "In Storage Element:\n %s\n" % fileList.seName msg += "In PNN:\n %s\n" % fileList.pnn msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if fwkJobRep.jobType == "Merge": # // # // Merge files #// for mergedFile in fileList: mergedFile['Block'] = fileBlock affectedBlocks.add(fileBlock['Name']) msg="calling: self.dbs.insertMergedFile(%s, %s)" % (str(mergedFile['ParentList']),str(mergedFile)) logging.debug(msg) try: self.dbs.insertMergedFile(mergedFile['ParentList'], mergedFile) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert merged file:\n" msg += " %s\n" % mergedFile['LogicalFileName'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) logging.debug("Inserted merged file: %s to FileBlock: %s"%(mergedFile['LogicalFileName'],fileBlock['Name'])) else: # // # // Processing files #// affectedBlocks.add(fileBlock['Name']) msg="calling: self.dbs.insertFiles(%s, %s, %s)" % (str(procDataset),str(list(fileList)),str(fileBlock)) logging.debug(msg) try: self.dbs.insertFiles(procDataset, list(fileList), fileBlock) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" msg += " %s\n" % ( [ x['LogicalFileName'] for x in fileList ], ) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) logging.debug("Inserted files: %s to FileBlock: %s"%( ([ x['LogicalFileName'] for x in fileList ]),fileBlock['Name'])) return list(affectedBlocks)