class BlockManager: """ _BlockManager_ File block manager. Instantiate for a given block and provide API calls to close the block, migrate it to global DBS and inject the block to PhEDEx """ def __init__(self, blockName, localDbsUrl, globalDbsUrl, datasetPath): self.block = blockName self.dataset = datasetPath self.localDbs = DBSWriter(localDbsUrl) self.localUrl = localDbsUrl self.globalDbs = DBSWriter(globalDbsUrl) self.globalUrl = globalDbsUrl def closeBlock(self): """ _closeBlock_ Close the file block """ # // # // Close block if it has > 0 files in it. IE, force closure of block #// self.localDbs.manageFileBlock(self.block, maxFiles=1) return def migrateToGlobalDBS(self): """ _migrateToGlobalDBS_ Migrate the block to the global DBS Url provided """ self.globalDbs.migrateDatasetBlocks(self.localUrl, self.dataset, [self.block]) return def injectBlockToPhEDEx(self, phedexConfig, nodes=None): """ _injectBlockToPhEDEx_ Inject the file block to PhEDEx """ tmdbInjectBlock( self.globalUrl, self.dataset, self.block, phedexConfig, "/tmp", # temp dir to create drops nodes) return
class BlockManager: """ _BlockManager_ File block manager. Instantiate for a given block and provide API calls to close the block, migrate it to global DBS and inject the block to PhEDEx """ def __init__(self, blockName, localDbsUrl, globalDbsUrl, datasetPath): self.block = blockName self.dataset = datasetPath self.localDbs = DBSWriter(localDbsUrl) self.localUrl = localDbsUrl self.globalDbs = DBSWriter(globalDbsUrl) self.globalUrl = globalDbsUrl def closeBlock(self): """ _closeBlock_ Close the file block """ # // # // Close block if it has > 0 files in it. IE, force closure of block #// self.localDbs.manageFileBlock(self.block, maxFiles=1) return def migrateToGlobalDBS(self): """ _migrateToGlobalDBS_ Migrate the block to the global DBS Url provided """ self.globalDbs.migrateDatasetBlocks(self.localUrl, self.dataset, [self.block]) return def injectBlockToPhEDEx(self, phedexConfig, nodes=None): """ _injectBlockToPhEDEx_ Inject the file block to PhEDEx """ tmdbInjectBlock(self.globalUrl, self.dataset, self.block, phedexConfig, "/tmp", # temp dir to create drops nodes) return
jobReportFile['TotalEvents'] = nEvents jobReportFile['SEName'] = "srm.cern.ch" ##jobReport.write('FrameworkJobReport.xml') localDbsUrl = "https://cmst0dbs.cern.ch:8443/cms_dbs_prod_tier0_writer/servlet/DBSServlet" dbswriter = DBSWriter(localDbsUrl,level='ERROR') primary = DBSWriterObjects.createPrimaryDataset(datasetStrmr, dbswriter.dbs) datasetStrmr['ApplicationName'] = appName datasetStrmr['ApplicationVersion'] = appVersion datasetStrmr['ApplicationFamily'] = 'DAQ' datasetStrmr['PSetHash'] = 'NA' datasetStrmr['PSetContent'] = 'NA' algo = DBSWriterObjects.createAlgorithm(datasetStrmr, None, dbswriter.dbs) processed = DBSWriterObjects.createProcessedDataset(primary, algo, datasetStrmr, dbswriter.dbs) try: blocks = dbswriter.insertFiles(jobReport, insertDetectorData = True) except DBSWriterError, ex: print "%s"%ex # limit block size to 500 for blockName in blocks: dbswriter.manageFileBlock(blockName, maxFiles=500)
def DBS2Publish(self, good_list): #################################################### if self.no_inp == 1: file_list = self.remove_input_from_fjr(good_list) else: file_list=good_list print "file_list = ", file_list #################################################### common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list)) common.logger.log(10-1, "len(file_list) = "+str(len(file_list))) if (len(file_list)>0): BlocksList=[] common.logger.info("--->>> Start dataset publication") self.exit_status=self.publishDataset(file_list[0]) if (self.exit_status == '1'): return self.exit_status common.logger.info("--->>> End dataset publication") common.logger.info("--->>> Start files publication") for file in file_list: Blocks=self.publishAJobReport(file,self.processedData) if Blocks: for x in Blocks: # do not allow multiple entries of the same block if x not in BlocksList: BlocksList.append(x) # close the blocks common.logger.log(10-1, "BlocksList = %s"%BlocksList) dbswriter = DBSWriter(self.DBSURL) for BlockName in BlocksList: try: closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1) common.logger.log(10-1, "closeBlock %s"%closeBlock) except DBSWriterError, ex: common.logger.info("Close block error %s"%ex) if (len(self.noEventsFiles)>0): common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:") for lfn in self.noEventsFiles: common.logger.info("------ LFN: %s"%lfn) if (len(self.noLFN)>0): common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN") for pfn in self.noLFN: common.logger.info("------ pfn: %s"%pfn) if (len(self.problemFiles)>0): common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE") for lfn in self.problemFiles: common.logger.info("------ LFN: %s"%lfn) common.logger.info("--->>> End files publication") #### FEDE for MULTI #### for dataset_to_check in self.published_datasets: self.cfg_params['USER.dataset_to_check']=dataset_to_check from InspectDBS import InspectDBS check=InspectDBS(self.cfg_params) check.checkPublication() ######################### return self.exit_status
def DBS2Publish(self, good_list): #################################################### if self.no_inp == 1: file_list = self.remove_input_from_fjr(good_list) else: file_list = good_list print "file_list = ", file_list #################################################### common.logger.log( 10 - 1, "fjr with FrameworkJobReport Status='Success', file_list = " + str(file_list)) common.logger.log(10 - 1, "len(file_list) = " + str(len(file_list))) if (len(file_list) > 0): BlocksList = [] common.logger.info("--->>> Start dataset publication") self.exit_status = self.publishDataset(file_list[0]) if (self.exit_status == '1'): return self.exit_status common.logger.info("--->>> End dataset publication") common.logger.info("--->>> Start files publication") for file in file_list: Blocks = self.publishAJobReport(file, self.processedData) if Blocks: for x in Blocks: # do not allow multiple entries of the same block if x not in BlocksList: BlocksList.append(x) # close the blocks common.logger.log(10 - 1, "BlocksList = %s" % BlocksList) dbswriter = DBSWriter(self.DBSURL) for BlockName in BlocksList: try: closeBlock = dbswriter.manageFileBlock(BlockName, maxFiles=1) common.logger.log(10 - 1, "closeBlock %s" % closeBlock) except DBSWriterError, ex: common.logger.info("Close block error %s" % ex) if (len(self.noEventsFiles) > 0): common.logger.info("--->>> WARNING: " + str(len(self.noEventsFiles)) + " published files contain 0 events are:") for lfn in self.noEventsFiles: common.logger.info("------ LFN: %s" % lfn) if (len(self.noLFN) > 0): common.logger.info( "--->>> WARNING: there are " + str(len(self.noLFN)) + " files not published because they have empty LFN") for pfn in self.noLFN: common.logger.info("------ pfn: %s" % pfn) if (len(self.problemFiles) > 0): common.logger.info( "--->>> WARNING: " + str(len(self.problemFiles)) + " files not published because they had problem with copy to SE" ) for lfn in self.problemFiles: common.logger.info("------ LFN: %s" % lfn) common.logger.info("--->>> End files publication") #### FEDE for MULTI #### for dataset_to_check in self.published_datasets: self.cfg_params['USER.dataset_to_check'] = dataset_to_check from InspectDBS import InspectDBS check = InspectDBS(self.cfg_params) check.checkPublication() ######################### return self.exit_status
def run(self): """ parse of all xml file on res dir and creation of distionary """ task = common._db.getTask() good_list=[] for job in task.getJobs(): fjr = self.fjrDirectory + job['outputFiles'][-1] if (job.runningJob['applicationReturnCode']!=0 or job.runningJob['wrapperReturnCode']!=0): continue # get FJR filename fjr = self.fjrDirectory + job['outputFiles'][-1] reports = readJobReport(fjr) if len(reports)>0: if reports[0].status == "Success": good_list.append(fjr) #################################################### if self.no_inp == 1: file_list = self.remove_input_from_fjr(good_list) else: file_list=good_list print "file_list = ", file_list #################################################### common.logger.log(10-1, "fjr with FrameworkJobReport Status='Success', file_list = "+str(file_list)) common.logger.log(10-1, "len(file_list) = "+str(len(file_list))) if (len(file_list)>0): BlocksList=[] common.logger.info("--->>> Start dataset publication") self.exit_status=self.publishDataset(file_list[0]) if (self.exit_status == '1'): return self.exit_status common.logger.info("--->>> End dataset publication") common.logger.info("--->>> Start files publication") for file in file_list: Blocks=self.publishAJobReport(file,self.processedData) if Blocks: for x in Blocks: # do not allow multiple entries of the same block if x not in BlocksList: BlocksList.append(x) # close the blocks common.logger.log(10-1, "BlocksList = %s"%BlocksList) dbswriter = DBSWriter(self.DBSURL) for BlockName in BlocksList: try: closeBlock=dbswriter.manageFileBlock(BlockName,maxFiles= 1) common.logger.log(10-1, "closeBlock %s"%closeBlock) except DBSWriterError, ex: common.logger.info("Close block error %s"%ex) if (len(self.noEventsFiles)>0): common.logger.info("--->>> WARNING: "+str(len(self.noEventsFiles))+" published files contain 0 events are:") for lfn in self.noEventsFiles: common.logger.info("------ LFN: %s"%lfn) if (len(self.noLFN)>0): common.logger.info("--->>> WARNING: there are "+str(len(self.noLFN))+" files not published because they have empty LFN") for pfn in self.noLFN: common.logger.info("------ pfn: %s"%pfn) if (len(self.problemFiles)>0): common.logger.info("--->>> WARNING: "+str(len(self.problemFiles))+" files not published because they had problem with copy to SE") for lfn in self.problemFiles: common.logger.info("------ LFN: %s"%lfn) common.logger.info("--->>> End files publication") #### FEDE for MULTI #### for dataset_to_check in self.published_datasets: self.cfg_params['USER.dataset_to_check']=dataset_to_check from InspectDBS import InspectDBS check=InspectDBS(self.cfg_params) check.checkPublication() ######################### return self.exit_status