def DBS2Publish(self, good_list):
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list = good_list
print "file_list = ", file_list
####################################################
common.logger.log(
10 - 1,
"fjr with FrameworkJobReport Status='Success', file_list = " +
str(file_list))
common.logger.log(10 - 1, "len(file_list) = " + str(len(file_list)))
if (len(file_list) > 0):
BlocksList = []
common.logger.info("--->>> Start dataset publication")
self.exit_status = self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks = self.publishAJobReport(file, self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10 - 1, "BlocksList = %s" % BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock = dbswriter.manageFileBlock(BlockName,
maxFiles=1)
common.logger.log(10 - 1, "closeBlock %s" % closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s" % ex)
if (len(self.noEventsFiles) > 0):
common.logger.info("--->>> WARNING: " +
str(len(self.noEventsFiles)) +
" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s" % lfn)
if (len(self.noLFN) > 0):
common.logger.info(
"--->>> WARNING: there are " + str(len(self.noLFN)) +
" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s" % pfn)
if (len(self.problemFiles) > 0):
common.logger.info(
"--->>> WARNING: " + str(len(self.problemFiles)) +
" files not published because they had problem with copy to SE"
)
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s" % lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check'] = dataset_to_check
from InspectDBS import InspectDBS
check = InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
def DBS2Publish(self, good_list):
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list=good_list
print "file_list = ", file_list
####################################################
common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list))
common.logger.log(10-1, "len(file_list) = "+str(len(file_list)))
if (len(file_list)>0):
BlocksList=[]
common.logger.info("--->>> Start dataset publication")
self.exit_status=self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks=self.publishAJobReport(file,self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10-1, "BlocksList = %s"%BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1)
common.logger.log(10-1, "closeBlock %s"%closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s"%ex)
if (len(self.noEventsFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s"%lfn)
if (len(self.noLFN)>0):
common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s"%pfn)
if (len(self.problemFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE")
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s"%lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check']=dataset_to_check
from InspectDBS import InspectDBS
check=InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
def run(self):
"""
parse of all xml file on res dir and creation of distionary
"""
task = common._db.getTask()
good_list=[]
for job in task.getJobs():
fjr = self.fjrDirectory + job['outputFiles'][-1]
if (job.runningJob['applicationReturnCode']!=0 or job.runningJob['wrapperReturnCode']!=0): continue
# get FJR filename
fjr = self.fjrDirectory + job['outputFiles'][-1]
reports = readJobReport(fjr)
if len(reports)>0:
if reports[0].status == "Success":
good_list.append(fjr)
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list=good_list
print "file_list = ", file_list
####################################################
common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list))
common.logger.log(10-1, "len(file_list) = "+str(len(file_list)))
if (len(file_list)>0):
BlocksList=[]
common.logger.info("--->>> Start dataset publication")
self.exit_status=self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks=self.publishAJobReport(file,self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10-1, "BlocksList = %s"%BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1)
common.logger.log(10-1, "closeBlock %s"%closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s"%ex)
if (len(self.noEventsFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s"%lfn)
if (len(self.noLFN)>0):
common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s"%pfn)
if (len(self.problemFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE")
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s"%lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check']=dataset_to_check
from InspectDBS import InspectDBS
check=InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
