def checkPublication(self): """ check dataset publication in a dbs """ common.logger.info('--->>> Check data publication: dataset '+self.dataset_to_check+' in DBS url '+ self.DBSURL+'\n') # // # // Get API to DBS #// dbsreader = DBSReader(self.DBSURL) # // # // Get list of datasets #// if len(self.dataset_to_check.split('/')) < 4: msg = "the provided dataset name is not correct" raise CrabException(msg) else: primds=self.dataset_to_check.split('/')[1] procds=self.dataset_to_check.split('/')[2] tier=self.dataset_to_check.split('/')[3] datasets=dbsreader.matchProcessedDatasets(primds,tier,procds) if common.debugLevel: print "PrimaryDataset = ", primds print "ProcessedDataset = ", procds print "DataTier = ", tier print "datasets matching your requirements= ", datasets for dataset in datasets: # // # // Get list of blocks for the dataset and their location #// if len(dataset.get('PathList'))==0: print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList')) else: for datasetpath in dataset.get('PathList'): nevttot=0 print "=== dataset %s"%datasetpath ### FEDE ####### if dataset['Description'] != None: print "=== dataset description = ", dataset['Description'] ################ blocks=dbsreader.getFileBlocksInfo(datasetpath) for block in blocks: SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query print "===== File block name: %s" %block['Name'] print " File block located at: ", SEList print " File block status: %s" %block['OpenForWriting'] print " Number of files: %s"%block['NumberOfFiles'] print " Number of Bytes: %s"%block['BlockSize'] print " Number of Events: %s"%block['NumberOfEvents'] if common.debugLevel: print "--------- info about files --------" print " Size \t Events \t LFN \t FileStatus " files=dbsreader.listFilesInBlock(block['Name']) for file in files: print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status']) nevttot = nevttot + block['NumberOfEvents'] print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath) if not common.debugLevel: common.logger.info('You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check='+self.dataset_to_check+' -USER.dbs_url_for_publication='+self.DBSURL+' -debug')
def checkPublication(self): """ check dataset publication in a dbs """ common.logger.info('--->>> Check data publication: dataset ' + self.dataset_to_check + ' in DBS url ' + self.DBSURL + '\n') # // # // Get API to DBS #// dbsreader = DBSReader(self.DBSURL) # // # // Get list of datasets #// if len(self.dataset_to_check.split('/')) < 4: msg = "the provided dataset name is not correct" raise CrabException(msg) else: primds = self.dataset_to_check.split('/')[1] procds = self.dataset_to_check.split('/')[2] tier = self.dataset_to_check.split('/')[3] datasets = dbsreader.matchProcessedDatasets(primds, tier, procds) if common.debugLevel: print "PrimaryDataset = ", primds print "ProcessedDataset = ", procds print "DataTier = ", tier print "datasets matching your requirements= ", datasets for dataset in datasets: # // # // Get list of blocks for the dataset and their location #// if len(dataset.get('PathList')) == 0: print "===== Empty dataset yet /%s/%s with tiers %s" % ( dataset.get('PrimaryDataset')['Name'], dataset.get('Name'), dataset.get('TierList')) else: for datasetpath in dataset.get('PathList'): nevttot = 0 print "=== dataset %s" % datasetpath ### FEDE ####### if dataset['Description'] != None: print "=== dataset description = ", dataset[ 'Description'] ################ blocks = dbsreader.getFileBlocksInfo(datasetpath) for block in blocks: SEList = dbsreader.listFileBlockLocation( block['Name']) # replace that with DLS query print "===== File block name: %s" % block['Name'] print " File block located at: ", SEList print " File block status: %s" % block[ 'OpenForWriting'] print " Number of files: %s" % block[ 'NumberOfFiles'] print " Number of Bytes: %s" % block['BlockSize'] print " Number of Events: %s" % block[ 'NumberOfEvents'] if common.debugLevel: print "--------- info about files --------" print " Size \t Events \t LFN \t FileStatus " files = dbsreader.listFilesInBlock(block['Name']) for file in files: print "%s %s %s %s" % ( file['FileSize'], file['NumberOfEvents'], file['LogicalFileName'], file['Status']) nevttot = nevttot + block['NumberOfEvents'] print "\n total events: %s in dataset: %s\n" % ( nevttot, datasetpath) if not common.debugLevel: common.logger.info( 'You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check=' + self.dataset_to_check + ' -USER.dbs_url_for_publication=' + self.DBSURL + ' -debug')
# // Get list of blocks for the dataset and their location #// for datasetpath in dataset.get('PathList'): nevttot=0 print "===== dataset %s"%datasetpath blocks=dbsreader.getFileBlocksInfo(datasetpath) for block in blocks: SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query print "== File block %s is located at: %s"%(block['Name'],SEList) print "File block name: %s" %block['Name'] print "File block status: %s" %block['OpenForWriting'] print "Number of files: %s"%block['NumberOfFiles'] print "Number of Bytes: %s"%block['BlockSize'] print "Number of Events: %s"%block['NumberOfEvents'] if full: print "--------- info about files --------" print " Size \t Events \t LFN \t FileStatus " files=dbsreader.listFilesInBlock(block['Name']) for file in files: print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status']) nevttot = nevttot + block['NumberOfEvents'] print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath) if len(dataset.get('PathList'))==0: print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList'))