def insertFiles(self, fwkJobRep, insertDetectorData = False):
"""
_insertFiles_
Process the files in the FwkJobReport instance and insert
them into the associated datasets
A list of affected fileblock names is returned both for merged
and unmerged fileblocks. Only merged blocks will have to be managed.
#for merged file
#blocks to facilitate management of those blocks.
#This list is not populated for processing jobs since we dont really
#care about the processing job blocks.
"""
insertLists = {}
orderedHashes = []
affectedBlocks = set()
if len(fwkJobRep.files)<=0:
msg = "Error in DBSWriter.insertFiles\n"
msg += "No files found in FrameWorkJobReport for:\n"
msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId
msg += " Workflow: %s"%fwkJobRep.workflowSpecId
raise DBSWriterError(msg)
for outFile in fwkJobRep.sortFiles():
# //
# // Convert each file into a DBS File object
#//
seName = None
if outFile.has_key("SEName"):
if outFile['SEName'] :
seName = outFile['SEName']
logging.debug("SEname associated to file is: %s"%seName)
## remove the fallback to site se-name if no SE is associated to File
## because it's likely that there is some stage out problem if there
## is no SEName associated to the file.
# if not seName:
# if fwkJobRep.siteDetails.has_key("se-name"):
# seName = fwkJobRep.siteDetails['se-name']
# seName = str(seName)
# logging.debug("site SEname: %s"%seName)
if not seName:
msg = "Error in DBSWriter.insertFiles\n"
msg += "No SEname associated to files in FrameWorkJobReport for "
# msg += "No SEname found in FrameWorkJobReport for "
msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId
msg += " Workflow: %s"%fwkJobRep.workflowSpecId
raise DBSWriterError(msg)
try:
if ( insertDetectorData ):
dbsFiles = DBSWriterObjects.createDBSFiles(outFile,
fwkJobRep.jobType,
self.dbs)
else:
dbsFiles = DBSWriterObjects.createDBSFiles(outFile,
fwkJobRep.jobType)
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles:\n"
msg += "Error creating DbsFile instances for file:\n"
msg += "%s\n" % outFile['LFN']
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
if len(dbsFiles)<=0:
msg="No DbsFile instances created. Not enough info in the FrameWorkJobReport for"
msg += "==> JobSpecId: %s"%fwkJobRep.jobSpecId
msg += " Workflow: %s"%fwkJobRep.workflowSpecId
raise DBSWriterError(msg)
for f in dbsFiles:
datasetName = makeDBSDSName(f)
hashName = "%s-%s" % (seName, datasetName)
if not insertLists.has_key(hashName):
insertLists[hashName] = _InsertFileList(seName,
datasetName)
insertLists[hashName].append(f)
if not orderedHashes.count(hashName):
orderedHashes.append(hashName)
def insertFiles(self, fwkJobRep, insertDetectorData=False):
"""
_insertFiles_
Process the files in the FwkJobReport instance and insert
them into the associated datasets
A list of affected fileblock names is returned both for merged
and unmerged fileblocks. Only merged blocks will have to be managed.
#for merged file
#blocks to facilitate management of those blocks.
#This list is not populated for processing jobs since we dont really
#care about the processing job blocks.
"""
insertLists = {}
orderedHashes = []
affectedBlocks = set()
if len(fwkJobRep.files) <= 0:
msg = "Error in DBSWriter.insertFiles\n"
msg += "No files found in FrameWorkJobReport for:\n"
msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId
msg += " Workflow: %s" % fwkJobRep.workflowSpecId
raise DBSWriterError(msg)
for outFile in fwkJobRep.sortFiles():
# //
# // Convert each file into a DBS File object
#//
seName = None
if outFile.has_key("SEName"):
if outFile['SEName']:
seName = outFile['SEName']
logging.debug("SEname associated to file is: %s" % seName)
## remove the fallback to site se-name if no SE is associated to File
## because it's likely that there is some stage out problem if there
## is no SEName associated to the file.
# if not seName:
# if fwkJobRep.siteDetails.has_key("se-name"):
# seName = fwkJobRep.siteDetails['se-name']
# seName = str(seName)
# logging.debug("site SEname: %s"%seName)
if not seName:
msg = "Error in DBSWriter.insertFiles\n"
msg += "No SEname associated to files in FrameWorkJobReport for "
# msg += "No SEname found in FrameWorkJobReport for "
msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId
msg += " Workflow: %s" % fwkJobRep.workflowSpecId
raise DBSWriterError(msg)
try:
if (insertDetectorData):
dbsFiles = DBSWriterObjects.createDBSFiles(
outFile, fwkJobRep.jobType, self.dbs)
else:
dbsFiles = DBSWriterObjects.createDBSFiles(
outFile, fwkJobRep.jobType)
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles:\n"
msg += "Error creating DbsFile instances for file:\n"
msg += "%s\n" % outFile['LFN']
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
if len(dbsFiles) <= 0:
msg = "No DbsFile instances created. Not enough info in the FrameWorkJobReport for"
msg += "==> JobSpecId: %s" % fwkJobRep.jobSpecId
msg += " Workflow: %s" % fwkJobRep.workflowSpecId
raise DBSWriterError(msg)
for f in dbsFiles:
datasetName = makeDBSDSName(f)
hashName = "%s-%s" % (seName, datasetName)
if not insertLists.has_key(hashName):
insertLists[hashName] = _InsertFileList(
seName, datasetName)
insertLists[hashName].append(f)
if not orderedHashes.count(hashName):
orderedHashes.append(hashName)
