def checkPublication(self):
"""
check dataset publication in a dbs
"""
common.logger.info('--->>> Check data publication: dataset '+self.dataset_to_check+' in DBS url '+ self.DBSURL+'\n')
# //
# // Get API to DBS
#//
dbsreader = DBSReader(self.DBSURL)
# //
# // Get list of datasets
#//
if len(self.dataset_to_check.split('/')) < 4:
msg = "the provided dataset name is not correct"
raise CrabException(msg)
else:
primds=self.dataset_to_check.split('/')[1]
procds=self.dataset_to_check.split('/')[2]
tier=self.dataset_to_check.split('/')[3]
datasets=dbsreader.matchProcessedDatasets(primds,tier,procds)
if common.debugLevel:
print "PrimaryDataset = ", primds
print "ProcessedDataset = ", procds
print "DataTier = ", tier
print "datasets matching your requirements= ", datasets
for dataset in datasets:
# //
# // Get list of blocks for the dataset and their location
#//
if len(dataset.get('PathList'))==0:
print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList'))
else:
for datasetpath in dataset.get('PathList'):
nevttot=0
print "=== dataset %s"%datasetpath
### FEDE #######
if dataset['Description'] != None:
print "=== dataset description = ", dataset['Description']
################
blocks=dbsreader.getFileBlocksInfo(datasetpath)
for block in blocks:
SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query
print "===== File block name: %s" %block['Name']
print " File block located at: ", SEList
print " File block status: %s" %block['OpenForWriting']
print " Number of files: %s"%block['NumberOfFiles']
print " Number of Bytes: %s"%block['BlockSize']
print " Number of Events: %s"%block['NumberOfEvents']
if common.debugLevel:
print "--------- info about files --------"
print " Size \t Events \t LFN \t FileStatus "
files=dbsreader.listFilesInBlock(block['Name'])
for file in files:
print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status'])
nevttot = nevttot + block['NumberOfEvents']
print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath)
if not common.debugLevel:
common.logger.info('You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check='+self.dataset_to_check+' -USER.dbs_url_for_publication='+self.DBSURL+' -debug')
def checkPublication(self):
"""
check dataset publication in a dbs
"""
common.logger.info('--->>> Check data publication: dataset ' +
self.dataset_to_check + ' in DBS url ' +
self.DBSURL + '\n')
# //
# // Get API to DBS
#//
dbsreader = DBSReader(self.DBSURL)
# //
# // Get list of datasets
#//
if len(self.dataset_to_check.split('/')) < 4:
msg = "the provided dataset name is not correct"
raise CrabException(msg)
else:
primds = self.dataset_to_check.split('/')[1]
procds = self.dataset_to_check.split('/')[2]
tier = self.dataset_to_check.split('/')[3]
datasets = dbsreader.matchProcessedDatasets(primds, tier, procds)
if common.debugLevel:
print "PrimaryDataset = ", primds
print "ProcessedDataset = ", procds
print "DataTier = ", tier
print "datasets matching your requirements= ", datasets
for dataset in datasets:
# //
# // Get list of blocks for the dataset and their location
#//
if len(dataset.get('PathList')) == 0:
print "===== Empty dataset yet /%s/%s with tiers %s" % (
dataset.get('PrimaryDataset')['Name'], dataset.get('Name'),
dataset.get('TierList'))
else:
for datasetpath in dataset.get('PathList'):
nevttot = 0
print "=== dataset %s" % datasetpath
### FEDE #######
if dataset['Description'] != None:
print "=== dataset description = ", dataset[
'Description']
################
blocks = dbsreader.getFileBlocksInfo(datasetpath)
for block in blocks:
SEList = dbsreader.listFileBlockLocation(
block['Name']) # replace that with DLS query
print "===== File block name: %s" % block['Name']
print " File block located at: ", SEList
print " File block status: %s" % block[
'OpenForWriting']
print " Number of files: %s" % block[
'NumberOfFiles']
print " Number of Bytes: %s" % block['BlockSize']
print " Number of Events: %s" % block[
'NumberOfEvents']
if common.debugLevel:
print "--------- info about files --------"
print " Size \t Events \t LFN \t FileStatus "
files = dbsreader.listFilesInBlock(block['Name'])
for file in files:
print "%s %s %s %s" % (
file['FileSize'], file['NumberOfEvents'],
file['LogicalFileName'], file['Status'])
nevttot = nevttot + block['NumberOfEvents']
print "\n total events: %s in dataset: %s\n" % (
nevttot, datasetpath)
if not common.debugLevel:
common.logger.info(
'You can obtain more info about files of the dataset using: crab -checkPublication -USER.dataset_to_check='
+ self.dataset_to_check + ' -USER.dbs_url_for_publication=' +
self.DBSURL + ' -debug')
# // Get list of blocks for the dataset and their location
#//
for datasetpath in dataset.get('PathList'):
nevttot=0
print "===== dataset %s"%datasetpath
blocks=dbsreader.getFileBlocksInfo(datasetpath)
for block in blocks:
SEList=dbsreader.listFileBlockLocation(block['Name']) # replace that with DLS query
print "== File block %s is located at: %s"%(block['Name'],SEList)
print "File block name: %s" %block['Name']
print "File block status: %s" %block['OpenForWriting']
print "Number of files: %s"%block['NumberOfFiles']
print "Number of Bytes: %s"%block['BlockSize']
print "Number of Events: %s"%block['NumberOfEvents']
if full:
print "--------- info about files --------"
print " Size \t Events \t LFN \t FileStatus "
files=dbsreader.listFilesInBlock(block['Name'])
for file in files:
print "%s %s %s %s"%(file['FileSize'],file['NumberOfEvents'],file['LogicalFileName'],file['Status'])
nevttot = nevttot + block['NumberOfEvents']
print "\n total events: %s in dataset: %s\n"%(nevttot,datasetpath)
if len(dataset.get('PathList'))==0:
print "===== Empty dataset yet /%s/%s with tiers %s"%(dataset.get('PrimaryDataset')['Name'],dataset.get('Name'),dataset.get('TierList'))
