def insertFiles(self, fwkJobRep, insertDetectorData = False): """ _insertFiles_ Process the files in the FwkJobReport instance and insert them into the associated datasets A list of affected fileblock names is returned both for merged and unmerged fileblocks. Only merged blocks will have to be managed. #for merged file #blocks to facilitate management of those blocks. #This list is not populated for processing jobs since we dont really #care about the processing job blocks. """ insertLists = {} orderedHashes = [] affectedBlocks = set() if len(fwkJobRep.files)<=0: msg = "Error in DBSWriter.insertFiles\n" msg += "No files found in FrameWorkJobReport for:\n" msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId msg += " Workflow: %s"%fwkJobRep.workflowSpecId raise DBSWriterError(msg) for outFile in fwkJobRep.sortFiles(): # // # // Convert each file into a DBS File object #// seName = None if outFile.has_key("SEName"): if outFile['SEName'] : seName = outFile['SEName'] logging.debug("SEname associated to file is: %s"%seName) ## remove the fallback to site se-name if no SE is associated to File ## because it's likely that there is some stage out problem if there ## is no SEName associated to the file. # if not seName: # if fwkJobRep.siteDetails.has_key("se-name"): # seName = fwkJobRep.siteDetails['se-name'] # seName = str(seName) # logging.debug("site SEname: %s"%seName) if not seName: msg = "Error in DBSWriter.insertFiles\n" msg += "No SEname associated to files in FrameWorkJobReport for " # msg += "No SEname found in FrameWorkJobReport for " msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId msg += " Workflow: %s"%fwkJobRep.workflowSpecId raise DBSWriterError(msg) try: if ( insertDetectorData ): dbsFiles = DBSWriterObjects.createDBSFiles(outFile, fwkJobRep.jobType, self.dbs) else: dbsFiles = DBSWriterObjects.createDBSFiles(outFile, fwkJobRep.jobType) except DbsException, ex: msg = "Error in DBSWriter.insertFiles:\n" msg += "Error creating DbsFile instances for file:\n" msg += "%s\n" % outFile['LFN'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if len(dbsFiles)<=0: msg="No DbsFile instances created. Not enough info in the FrameWorkJobReport for" msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId msg += " Workflow: %s"%fwkJobRep.workflowSpecId raise DBSWriterError(msg) for f in dbsFiles: datasetName = makeDBSDSName(f) hashName = "%s-%s" % (seName, datasetName) if not insertLists.has_key(hashName): insertLists[hashName] = _InsertFileList(seName, datasetName) insertLists[hashName].append(f) if not orderedHashes.count(hashName): orderedHashes.append(hashName)
def insertFiles(self, fwkJobRep, insertDetectorData=False): """ _insertFiles_ Process the files in the FwkJobReport instance and insert them into the associated datasets A list of affected fileblock names is returned both for merged and unmerged fileblocks. Only merged blocks will have to be managed. #for merged file #blocks to facilitate management of those blocks. #This list is not populated for processing jobs since we dont really #care about the processing job blocks. """ insertLists = {} orderedHashes = [] affectedBlocks = set() if len(fwkJobRep.files) <= 0: msg = "Error in DBSWriter.insertFiles\n" msg += "No files found in FrameWorkJobReport for:\n" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for outFile in fwkJobRep.sortFiles(): # // # // Convert each file into a DBS File object #// seName = None if outFile.has_key("SEName"): if outFile['SEName']: seName = outFile['SEName'] logging.debug("SEname associated to file is: %s" % seName) ## remove the fallback to site se-name if no SE is associated to File ## because it's likely that there is some stage out problem if there ## is no SEName associated to the file. # if not seName: # if fwkJobRep.siteDetails.has_key("se-name"): # seName = fwkJobRep.siteDetails['se-name'] # seName = str(seName) # logging.debug("site SEname: %s"%seName) if not seName: msg = "Error in DBSWriter.insertFiles\n" msg += "No SEname associated to files in FrameWorkJobReport for " # msg += "No SEname found in FrameWorkJobReport for " msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) try: if (insertDetectorData): dbsFiles = DBSWriterObjects.createDBSFiles( outFile, fwkJobRep.jobType, self.dbs) else: dbsFiles = DBSWriterObjects.createDBSFiles( outFile, fwkJobRep.jobType) except DbsException, ex: msg = "Error in DBSWriter.insertFiles:\n" msg += "Error creating DbsFile instances for file:\n" msg += "%s\n" % outFile['LFN'] msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) if len(dbsFiles) <= 0: msg = "No DbsFile instances created. Not enough info in the FrameWorkJobReport for" msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId msg += " Workflow: %s" % fwkJobRep.workflowSpecId raise DBSWriterError(msg) for f in dbsFiles: datasetName = makeDBSDSName(f) hashName = "%s-%s" % (seName, datasetName) if not insertLists.has_key(hashName): insertLists[hashName] = _InsertFileList( seName, datasetName) insertLists[hashName].append(f) if not orderedHashes.count(hashName): orderedHashes.append(hashName)