def insertFilesForDBSBuffer(self, files, procDataset, algos, jobType = "NotMerge", insertDetectorData = False):
"""
_insertFiles_
list of files inserted in DBS
"""
#TODO: Whats the purpose of insertDetectorData
if len(files) < 1:
return
affectedBlocks = set()
insertFiles = []
addedRuns=[]
seName = None
#Get the algos in insertable form
ialgos = [DBSWriterObjects.createAlgorithmForInsert(dict(algo)) for algo in algos ]
for outFile in files:
# //
# // Convert each file into a DBS File object
#//
lumiList = []
#Somehing similar should be the real deal when multiple runs/lumi could be returned from wmbs file
for runlumiinfo in outFile.getRuns():
lrun=long(runlumiinfo.run)
run = DbsRun(
RunNumber = lrun,
NumberOfEvents = 0,
NumberOfLumiSections = 0,
TotalLuminosity = 0,
StoreNumber = 0,
StartOfRun = 0,
EndOfRun = 0,
)
#Only added if not added by another file in this loop, why waste a call to DBS
if lrun not in addedRuns:
self.dbs.insertRun(run)
addedRuns.append(lrun) #save it so we do not try to add it again to DBS
logging.debug("run %s added to DBS " % str(lrun))
for alsn in runlumiinfo:
lumi = DbsLumiSection(
LumiSectionNumber = long(alsn),
StartEventNumber = 0,
EndEventNumber = 0,
LumiStartTime = 0,
LumiEndTime = 0,
RunNumber = lrun,
)
lumiList.append(lumi)
logging.debug("lumi list created for the file")
dbsfile = DbsFile(
Checksum = str(outFile['cksum']),
NumberOfEvents = outFile['events'],
LogicalFileName = outFile['lfn'],
FileSize = int(outFile['size']),
Status = "VALID",
ValidationStatus = 'VALID',
FileType = 'EDM',
Dataset = procDataset,
TierList = DBSWriterObjects.makeTierList(procDataset['Path'].split('/')[3]),
AlgoList = ialgos,
LumiList = lumiList,
ParentList = outFile.getParentLFNs(),
#BranchHash = outFile['BranchHash'],
)
#This check comes from ProdAgent, not sure if its required
if len(outFile["locations"]) > 0:
seName = list(outFile["locations"])[0]
logging.debug("SEname associated to file is: %s"%seName)
else:
msg = "Error in DBSWriter.insertFiles\n"
msg += "No SEname associated to file"
#print "FAKING seName for now"
#seName="cmssrm.fnal.gov"
raise DBSWriterError(msg)
insertFiles.append(dbsfile)
# //Processing Jobs:
# // Insert the lists of sorted files into the appropriate
#// fileblocks
try:
fileBlock = DBSWriterObjects.getDBSFileBlock(
self.dbs,
procDataset,
seName)
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot retrieve FileBlock for dataset:\n"
msg += " %s\n" % procDataset['Path']
#msg += "In Storage Element:\n %s\n" % insertFiles.seName
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def insertFilesForDBSBuffer(self,
files,
procDataset,
algos,
jobType="NotMerge",
insertDetectorData=False):
"""
_insertFiles_
list of files inserted in DBS
"""
#TODO: Whats the purpose of insertDetectorData
if len(files) < 1:
return
affectedBlocks = set()
insertFiles = []
addedRuns = []
seName = None
#Get the algos in insertable form
ialgos = [
DBSWriterObjects.createAlgorithmForInsert(dict(algo))
for algo in algos
]
for outFile in files:
# //
# // Convert each file into a DBS File object
#//
lumiList = []
#Somehing similar should be the real deal when multiple runs/lumi could be returned from wmbs file
for runlumiinfo in outFile.getRuns():
lrun = long(runlumiinfo.run)
run = DbsRun(
RunNumber=lrun,
NumberOfEvents=0,
NumberOfLumiSections=0,
TotalLuminosity=0,
StoreNumber=0,
StartOfRun=0,
EndOfRun=0,
)
#Only added if not added by another file in this loop, why waste a call to DBS
if lrun not in addedRuns:
self.dbs.insertRun(run)
addedRuns.append(
lrun) #save it so we do not try to add it again to DBS
logging.debug("run %s added to DBS " % str(lrun))
for alsn in runlumiinfo:
lumi = DbsLumiSection(
LumiSectionNumber=long(alsn),
StartEventNumber=0,
EndEventNumber=0,
LumiStartTime=0,
LumiEndTime=0,
RunNumber=lrun,
)
lumiList.append(lumi)
logging.debug("lumi list created for the file")
dbsfile = DbsFile(
Checksum=str(outFile['cksum']),
NumberOfEvents=outFile['events'],
LogicalFileName=outFile['lfn'],
FileSize=int(outFile['size']),
Status="VALID",
ValidationStatus='VALID',
FileType='EDM',
Dataset=procDataset,
TierList=DBSWriterObjects.makeTierList(
procDataset['Path'].split('/')[3]),
AlgoList=ialgos,
LumiList=lumiList,
ParentList=outFile.getParentLFNs(),
#BranchHash = outFile['BranchHash'],
)
#This check comes from ProdAgent, not sure if its required
if len(outFile["locations"]) > 0:
seName = list(outFile["locations"])[0]
logging.debug("SEname associated to file is: %s" % seName)
else:
msg = "Error in DBSWriter.insertFiles\n"
msg += "No SEname associated to file"
#print "FAKING seName for now"
#seName="cmssrm.fnal.gov"
raise DBSWriterError(msg)
insertFiles.append(dbsfile)
# //Processing Jobs:
# // Insert the lists of sorted files into the appropriate
#// fileblocks
try:
fileBlock = DBSWriterObjects.getDBSFileBlock(
self.dbs, procDataset, seName)
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot retrieve FileBlock for dataset:\n"
msg += " %s\n" % procDataset['Path']
#msg += "In Storage Element:\n %s\n" % insertFiles.seName
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
if not orderedHashes.count(hashName):
orderedHashes.append(hashName)
# //Processing Jobs:
# // Insert the lists of sorted files into the appropriate
#// fileblocks
for hash in orderedHashes:
fileList = insertLists[hash]
procDataset = fileList[0]['Dataset']
try:
fileBlock = DBSWriterObjects.getDBSFileBlock(
self.dbs,
procDataset,
fileList.seName)
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot retrieve FileBlock for dataset:\n"
msg += " %s\n" % procDataset
msg += "In Storage Element:\n %s\n" % fileList.seName
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
if fwkJobRep.jobType == "Merge":
# //
# // Merge files
#//
for mergedFile in fileList:
mergedFile['Block'] = fileBlock
insertLists[hashName].append(f)
if not orderedHashes.count(hashName):
orderedHashes.append(hashName)
# //Processing Jobs:
# // Insert the lists of sorted files into the appropriate
#// fileblocks
for hash in orderedHashes:
fileList = insertLists[hash]
procDataset = fileList[0]['Dataset']
try:
fileBlock = DBSWriterObjects.getDBSFileBlock(
self.dbs, procDataset, fileList.seName)
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot retrieve FileBlock for dataset:\n"
msg += " %s\n" % procDataset
msg += "In Storage Element:\n %s\n" % fileList.seName
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
if fwkJobRep.jobType == "Merge":
# //
# // Merge files
#//
for mergedFile in fileList:
mergedFile['Block'] = fileBlock
affectedBlocks.add(fileBlock['Name'])
