class BlockManager:
"""
_BlockManager_
File block manager.
Instantiate for a given block and provide API calls to
close the block, migrate it to global DBS and
inject the block to PhEDEx
"""
def __init__(self, blockName, localDbsUrl, globalDbsUrl, datasetPath):
self.block = blockName
self.dataset = datasetPath
self.localDbs = DBSWriter(localDbsUrl)
self.localUrl = localDbsUrl
self.globalDbs = DBSWriter(globalDbsUrl)
self.globalUrl = globalDbsUrl
def closeBlock(self):
"""
_closeBlock_
Close the file block
"""
# //
# // Close block if it has > 0 files in it. IE, force closure of block
#//
self.localDbs.manageFileBlock(self.block, maxFiles=1)
return
def migrateToGlobalDBS(self):
"""
_migrateToGlobalDBS_
Migrate the block to the global DBS Url provided
"""
self.globalDbs.migrateDatasetBlocks(self.localUrl, self.dataset,
[self.block])
return
def injectBlockToPhEDEx(self, phedexConfig, nodes=None):
"""
_injectBlockToPhEDEx_
Inject the file block to PhEDEx
"""
tmdbInjectBlock(
self.globalUrl,
self.dataset,
self.block,
phedexConfig,
"/tmp", # temp dir to create drops
nodes)
return
class BlockManager:
"""
_BlockManager_
File block manager.
Instantiate for a given block and provide API calls to
close the block, migrate it to global DBS and
inject the block to PhEDEx
"""
def __init__(self, blockName, localDbsUrl, globalDbsUrl, datasetPath):
self.block = blockName
self.dataset = datasetPath
self.localDbs = DBSWriter(localDbsUrl)
self.localUrl = localDbsUrl
self.globalDbs = DBSWriter(globalDbsUrl)
self.globalUrl = globalDbsUrl
def closeBlock(self):
"""
_closeBlock_
Close the file block
"""
# //
# // Close block if it has > 0 files in it. IE, force closure of block
#//
self.localDbs.manageFileBlock(self.block, maxFiles=1)
return
def migrateToGlobalDBS(self):
"""
_migrateToGlobalDBS_
Migrate the block to the global DBS Url provided
"""
self.globalDbs.migrateDatasetBlocks(self.localUrl, self.dataset, [self.block])
return
def injectBlockToPhEDEx(self, phedexConfig, nodes=None):
"""
_injectBlockToPhEDEx_
Inject the file block to PhEDEx
"""
tmdbInjectBlock(self.globalUrl, self.dataset, self.block,
phedexConfig,
"/tmp", # temp dir to create drops
nodes)
return
jobReportFile['TotalEvents'] = nEvents
jobReportFile['SEName'] = "srm.cern.ch"
##jobReport.write('FrameworkJobReport.xml')
localDbsUrl = "https://cmst0dbs.cern.ch:8443/cms_dbs_prod_tier0_writer/servlet/DBSServlet"
dbswriter = DBSWriter(localDbsUrl,level='ERROR')
primary = DBSWriterObjects.createPrimaryDataset(datasetStrmr, dbswriter.dbs)
datasetStrmr['ApplicationName'] = appName
datasetStrmr['ApplicationVersion'] = appVersion
datasetStrmr['ApplicationFamily'] = 'DAQ'
datasetStrmr['PSetHash'] = 'NA'
datasetStrmr['PSetContent'] = 'NA'
algo = DBSWriterObjects.createAlgorithm(datasetStrmr, None, dbswriter.dbs)
processed = DBSWriterObjects.createProcessedDataset(primary, algo, datasetStrmr, dbswriter.dbs)
try:
blocks = dbswriter.insertFiles(jobReport, insertDetectorData = True)
except DBSWriterError, ex:
print "%s"%ex
# limit block size to 500
for blockName in blocks:
dbswriter.manageFileBlock(blockName, maxFiles=500)
def DBS2Publish(self, good_list):
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list=good_list
print "file_list = ", file_list
####################################################
common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list))
common.logger.log(10-1, "len(file_list) = "+str(len(file_list)))
if (len(file_list)>0):
BlocksList=[]
common.logger.info("--->>> Start dataset publication")
self.exit_status=self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks=self.publishAJobReport(file,self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10-1, "BlocksList = %s"%BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1)
common.logger.log(10-1, "closeBlock %s"%closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s"%ex)
if (len(self.noEventsFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s"%lfn)
if (len(self.noLFN)>0):
common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s"%pfn)
if (len(self.problemFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE")
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s"%lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check']=dataset_to_check
from InspectDBS import InspectDBS
check=InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
def DBS2Publish(self, good_list):
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list = good_list
print "file_list = ", file_list
####################################################
common.logger.log(
10 - 1,
"fjr with FrameworkJobReport Status='Success', file_list = " +
str(file_list))
common.logger.log(10 - 1, "len(file_list) = " + str(len(file_list)))
if (len(file_list) > 0):
BlocksList = []
common.logger.info("--->>> Start dataset publication")
self.exit_status = self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks = self.publishAJobReport(file, self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10 - 1, "BlocksList = %s" % BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock = dbswriter.manageFileBlock(BlockName,
maxFiles=1)
common.logger.log(10 - 1, "closeBlock %s" % closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s" % ex)
if (len(self.noEventsFiles) > 0):
common.logger.info("--->>> WARNING: " +
str(len(self.noEventsFiles)) +
" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s" % lfn)
if (len(self.noLFN) > 0):
common.logger.info(
"--->>> WARNING: there are " + str(len(self.noLFN)) +
" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s" % pfn)
if (len(self.problemFiles) > 0):
common.logger.info(
"--->>> WARNING: " + str(len(self.problemFiles)) +
" files not published because they had problem with copy to SE"
)
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s" % lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check'] = dataset_to_check
from InspectDBS import InspectDBS
check = InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
def run(self):
"""
parse of all xml file on res dir and creation of distionary
"""
task = common._db.getTask()
good_list=[]
for job in task.getJobs():
fjr = self.fjrDirectory + job['outputFiles'][-1]
if (job.runningJob['applicationReturnCode']!=0 or job.runningJob['wrapperReturnCode']!=0): continue
# get FJR filename
fjr = self.fjrDirectory + job['outputFiles'][-1]
reports = readJobReport(fjr)
if len(reports)>0:
if reports[0].status == "Success":
good_list.append(fjr)
####################################################
if self.no_inp == 1:
file_list = self.remove_input_from_fjr(good_list)
else:
file_list=good_list
print "file_list = ", file_list
####################################################
common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list))
common.logger.log(10-1, "len(file_list) = "+str(len(file_list)))
if (len(file_list)>0):
BlocksList=[]
common.logger.info("--->>> Start dataset publication")
self.exit_status=self.publishDataset(file_list[0])
if (self.exit_status == '1'):
return self.exit_status
common.logger.info("--->>> End dataset publication")
common.logger.info("--->>> Start files publication")
for file in file_list:
Blocks=self.publishAJobReport(file,self.processedData)
if Blocks:
for x in Blocks: # do not allow multiple entries of the same block
if x not in BlocksList:
BlocksList.append(x)
# close the blocks
common.logger.log(10-1, "BlocksList = %s"%BlocksList)
dbswriter = DBSWriter(self.DBSURL)
for BlockName in BlocksList:
try:
closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1)
common.logger.log(10-1, "closeBlock %s"%closeBlock)
except DBSWriterError, ex:
common.logger.info("Close block error %s"%ex)
if (len(self.noEventsFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:")
for lfn in self.noEventsFiles:
common.logger.info("------ LFN: %s"%lfn)
if (len(self.noLFN)>0):
common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN")
for pfn in self.noLFN:
common.logger.info("------ pfn: %s"%pfn)
if (len(self.problemFiles)>0):
common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE")
for lfn in self.problemFiles:
common.logger.info("------ LFN: %s"%lfn)
common.logger.info("--->>> End files publication")
#### FEDE for MULTI ####
for dataset_to_check in self.published_datasets:
self.cfg_params['USER.dataset_to_check']=dataset_to_check
from InspectDBS import InspectDBS
check=InspectDBS(self.cfg_params)
check.checkPublication()
#########################
return self.exit_status
