def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input defaultArguments as well as in the pileupDict
and compare values.
"""
args = {}
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
reader = DBSReader(dbsUrl, **args)
inputArgs = defaultArguments["PileupConfig"]
self.assertEqual(len(inputArgs), len(pileupDict),
"Number of pileup types different.")
for pileupType in inputArgs:
m = ("pileup type '%s' not in PileupFetcher-produced pileup "
"configuration: '%s'" % (pileupType, pileupDict))
self.assertTrue(pileupType in pileupDict, m)
# now query DBS for compare actual results on files lists for each
# pileup type and dataset and location (storage element names)
# pileupDict is saved in the file and now comparing items of this
# configuration with actual DBS results, the structure of pileupDict:
# {"pileupTypeA": {"BlockA": {"FileList": [], "StorageElementNames": []},
# "BlockB": {"FileList": [], "StorageElementName": []}, ....}
for pileupType, datasets in inputArgs.items():
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
for dataset in datasets:
dbsFileBlocks = reader.listFileBlocks(dataset = dataset)
for dbsFileBlockName in dbsFileBlocks:
fileList = [] # list of files in the block (dbsFile["LogicalFileName"])
storageElemNames = set() # list of StorageElementName
# each DBS block has a list under 'StorageElementList', iterate over
storageElements = reader.listFileBlockLocation(dbsFileBlockName)
for storElem in storageElements:
storageElemNames.add(storElem)
# now get list of files in the block
dbsFiles = reader.listFilesInBlock(dbsFileBlockName)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
# now compare the sets:
m = ("StorageElementNames don't agree for pileup type '%s', "
"dataset '%s' in configuration: '%s'" % (pileupType, dataset, pileupDict))
self.assertEqual(set(blockDict[dbsFileBlockName]["StorageElementNames"]), storageElemNames, m)
m = ("FileList don't agree for pileup type '%s', dataset '%s' "
" in configuration: '%s'" % (pileupType, dataset, pileupDict))
print fileList
print blockDict[dbsFileBlockName]["FileList"]
self.assertEqual(sorted(blockDict[dbsFileBlockName]["FileList"]), sorted(fileList))
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
datasets = dbsReader.matchProcessedDatasets(outputDatasetParts[1],
outputDatasetParts[3],
outputDatasetParts[2])
if len(datasets) == 0:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
blockNames = dbsReader.listFileBlocks(datasetName)
for blockName in blockNames:
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
block = dbsReader.listFilesInBlockWithParents(blockName)
blockLocations = set()
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
for blockFile in block:
parentLFNs = []
for fileParent in blockFile["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
def getFiles(datasetName,
runBlacklist,
runWhitelist,
blockBlacklist,
blockWhitelist,
dbsUrl,
fakeLocation=False):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint=dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
class BlockBuster(threading.Thread):
def __init__(self, **args):
threading.Thread.__init__(self)
for k, v in args.items():
setattr(self, k, v)
self.major_failure = False
def run(self):
self.files = {}
logging = self.l
has_parent = self.hp
fakeLocation = self.fl
blockName = self.bn
blockBlacklist = self.bbl
blockWhitelist = self.bwl
if blockBlacklist and blockName in blockBlacklist:
return
if blockWhitelist and blockName not in blockWhitelist:
return
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
dbsReader = DBSReader(endpoint=self.dbs)
replicaInfo = phedexReader.getReplicaInfoForBlocks(block=blockName,
subscribed='y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
if has_parent:
try:
blockFileParents = dbsReader.listFilesInBlockWithParents(
blockName)
except:
print blockName, "does not appear to have a parent, even though it should. Very suspicious"
blockFileParents = dbsReader.listFilesInBlock(blockName)
else:
blockFileParents = dbsReader.listFilesInBlock(blockName)
blockLocations = set()
# load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
PNN = replica["node"]
PSNs = siteDB.PNNtoPSN(PNN)
blockLocations.add(PNN)
#logging.debug("PhEDEx Node Name: %s\tPSNs: %s", PNN, PSNs)
# We cannot upload docs without location, so force it in case it's empty
if not blockLocations:
if fakeLocation:
#logging.info("\t\t %s\tno location", blockName)
blockLocations.update([u'T1_US_FNAL_Disk', u'T2_CH_CERN'])
elif not has_parent: ## this should be the source
logging.info("Blockname: %s\tno location, ABORT",
blockName)
self.major_failure = True
#sys.exit(1)
#logging.info("Blockname: %s\tLocations: %s", blockName, blockLocations)
# for each file on the block
for blockFile in blockFiles:
parentLFNs = []
# populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
# Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection[
"RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection[
"RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(
lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
self.files[blockFile["LogicalFileName"]] = {
"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs
}
return
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset", datasetName, "parts", outputDatasetParts
try:
# retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" %
datasetName)
has_parent = False
try:
parents = dbsReader.listDatasetParents(datasetName)
if parents: has_parent = True
except:
print "Dataset with no parent"
pass
bthreads = []
# traverse each block
for blockName in blockNames:
bthreads.append(
BlockBuster(bn=blockName,
hp=has_parent,
fl=fakeLocation,
bbl=blockBlacklist,
bwl=blockWhitelist,
l=logging,
dbs=dbsUrl))
print len(bthreads), "block query created"
bthreads = ThreadBuster(bthreads, 40, 2., verbose=False)
for t in bthreads:
if t.major_failure:
print "There was a major failure in processing block files"
sys.exit(1)
files.update(t.files)
print len(files)
return files
class DBSReaderTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
#self.endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
self.endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dbs = None
return
@attr("integration")
def testListDatatiers(self):
"""
listDatatiers returns all datatiers available
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listDatatiers()
self.assertTrue('RAW' in results)
self.assertTrue('GEN-SIM-RECO' in results)
self.assertTrue('GEN-SIM' in results)
self.assertFalse('RAW-ALAN' in results)
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listPrimaryDatasets('Jet*')
self.assertTrue('Jet' in results)
self.assertTrue('JetMET' in results)
self.assertTrue('JetMETTau' in results)
self.assertFalse(self.dbs.listPrimaryDatasets('DoesntExist'))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v1')
self.assertEqual(1, len(dataset))
self.assertEqual(['/Jet/Run2011A-v1/RAW'], dataset[0]['PathList'])
self.assertEqual('Run2011A-v1', dataset[0]['Name'])
self.assertFalse(
self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v666'))
def testlistRuns(self):
"""listRuns returns known runs"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRuns(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(block=BLOCK)
self.assertEqual(1, len(runs))
self.assertEqual([173657], runs)
def testlistRunLumis(self):
"""listRunLumis returns known runs and lumicounts (None for DBS3)"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRunLumis(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(173692 in runs)
self.assertEqual(runs[173692], None)
runs = self.dbs.listRunLumis(block=BLOCK)
self.assertEqual(1, len(runs))
self.assertTrue(173657 in runs)
self.assertEqual(runs[173657], None)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
self.dbs = DBSReader(self.endpoint)
datasets = self.dbs.listProcessedDatasets('Jet', 'RAW')
self.assertTrue('Run2011A-v1' in datasets)
self.assertTrue('Run2011B-v1' in datasets)
self.assertFalse(self.dbs.listProcessedDatasets('Jet', 'blah'))
self.assertFalse(self.dbs.listProcessedDatasets('blah', 'RAW'))
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
def testlistDatasetFileDetails(self):
"""testlistDatasetFilesDetails returns lumis, events, and parents of a dataset"""
TESTFILE = '/store/data/Run2011A/HighPileUp/RAW/v1/000/173/658/56484BAB-CBCB-E011-AF00-BCAEC518FF56.root'
self.dbs = DBSReader(self.endpoint)
details = self.dbs.listDatasetFileDetails(DATASET)
self.assertEqual(len(details), 49)
self.assertTrue(TESTFILE in details)
self.assertEqual(details[TESTFILE]['NumberOfEvents'], 545)
self.assertEqual(details[TESTFILE]['file_size'], 286021145)
self.assertEqual(
details[TESTFILE]['BlockName'],
'/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace')
self.assertEqual(details[TESTFILE]['Md5'], 'NOTSET')
self.assertEqual(details[TESTFILE]['md5'], 'NOTSET')
self.assertEqual(details[TESTFILE]['Adler32'], 'a41a1446')
self.assertEqual(details[TESTFILE]['adler32'], 'a41a1446')
self.assertEqual(details[TESTFILE]['Checksum'], '22218315')
self.assertEqual(details[TESTFILE]['check_sum'], '22218315')
self.assertTrue(173658 in details[TESTFILE]['Lumis'])
self.assertEqual(sorted(details[TESTFILE]['Lumis'][173658]), [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
104, 105, 106, 107, 108, 109, 110, 111
])
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset['path'], DATASET)
self.assertEqual(dataset['block'], '')
self.assertEqual(dataset['NumberOfEvents'], 22075)
self.assertEqual(dataset['NumberOfBlocks'], 46)
self.assertEqual(dataset['FileSize'], 4001680824)
self.assertEqual(dataset['file_size'], 4001680824)
self.assertEqual(dataset['NumberOfFiles'], 49)
self.assertEqual(dataset['NumberOfLumis'], 7223)
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block['path'], '')
self.assertEqual(block['block'], BLOCK)
self.assertEqual(block['NumberOfEvents'], 377)
self.assertEqual(block['NumberOfBlocks'], 1)
self.assertEqual(block['FileSize'], 150780132)
self.assertEqual(block['file_size'], 150780132)
self.assertEqual(block['NumberOfFiles'], 2)
self.assertEqual(block['NumberOfLumis'], 94)
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo,
DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET,
BLOCK + 'asas')
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block['Name'] in [x['Name'] for x in blocks])
self.assertEqual(BLOCK, block['Name'])
self.assertEqual(0, block['OpenForWriting'])
self.assertEqual(150780132, block['BlockSize'])
self.assertEqual(2, block['NumberOfFiles'])
# possibly fragile but assume block located at least at cern
sites = [
x['Name'] for x in block['PhEDExNodeList']
if x['Name'].find('CH_CERN') > -1
]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo,
DATASET + 'blah')
self.assertRaises(DBSReaderError,
self.dbs.getFileBlocksInfo,
DATASET,
blockName=BLOCK + 'asas')
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.listFileBlocks(DATASET)
self.assertTrue(BLOCK in blocks)
# block is closed
block = self.dbs.listFileBlocks(DATASET,
blockName=BLOCK,
onlyClosedBlocks=True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.blockExists,
DATASET + '#somethingelse')
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(
FILE in
[x['LogicalFileName'] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock,
DATASET + '#blah')
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listFilesInBlockWithParents(
'/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0'
)
self.assertEqual(4, len(files))
self.assertEqual(
'/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0',
files[0]['block_name'])
self.assertEqual(
'/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0',
files[0]['BlockName'])
self.assertEqual(
'/store/data/Commissioning2015/Cosmics/RAW/v1/000/238/545/00000/1043E89F-2DCF-E411-9CAE-02163E013751.root',
files[0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents,
BLOCK + 'asas')
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(
FILE in
[x['logical_file_name'] for x in self.dbs.lfnsInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + 'asas')
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
WRONG_BLOCK = BLOCK[:-4] + 'abcd'
BLOCK2 = '/HighPileUp/Run2011A-v1/RAW#6021175e-cbfb-11e0-80a9-003048caaace'
DBS_BLOCK = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#fb978442-a61b-413a-b4f4-526e6cdb142e'
DBS_BLOCK2 = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#0b04d417-d734-4ef2-88b0-392c48254dab'
self.dbs = DBSReader(
'https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader/')
# assume one site is cern
sites = [
x for x in self.dbs.listFileBlockLocation(BLOCK)
if x and x.find('CH_CERN') > -1
]
self.assertTrue(sites)
#This block is only found on DBS
self.assertTrue(self.dbs.listFileBlockLocation(DBS_BLOCK))
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(WRONG_BLOCK))
#test bulk call:
## two blocks in phedex
self.assertEqual(2, len(self.dbs.listFileBlockLocation([BLOCK,
BLOCK2])))
## one block in phedex one does not exist
self.assertEqual(
1, len(self.dbs.listFileBlockLocation([BLOCK, WRONG_BLOCK])))
## one in phedex one in dbs
self.assertEqual(
2, len(self.dbs.listFileBlockLocation([BLOCK, DBS_BLOCK])))
## two in dbs
self.assertEqual(
2, len(self.dbs.listFileBlockLocation([DBS_BLOCK, DBS_BLOCK2])))
## one in DBS and one does not exist
self.assertEqual(
1, len(self.dbs.listFileBlockLocation([DBS_BLOCK, WRONG_BLOCK])))
def testGetFileBlock(self):
"""getFileBlock returns block"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block['Files']))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock,
BLOCK + 'asas')
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlockWithParents(
'/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0'
)
self.assertEqual(len(block), 1)
block = block[
'/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0']
self.assertEqual(
'/store/data/Commissioning2015/Cosmics/RAW/v1/000/238/545/00000/1043E89F-2DCF-E411-9CAE-02163E013751.root',
block['Files'][0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents,
BLOCK + 'asas')
def testGetFiles(self):
"""getFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
def testListBlockParents(self):
"""listBlockParents returns block parents"""
self.dbs = DBSReader(self.endpoint)
parents = self.dbs.listBlockParents(
'/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0'
)
self.assertEqual(1, len(parents))
self.assertEqual(
'/Cosmics/Commissioning2015-v1/RAW#942d76fe-cf0e-11e4-afad-001e67ac06a0',
parents[0]['Name'])
sites = [
x for x in parents[0]['PhEDExNodeList'] if x.find("CH_CERN") > -1
]
self.assertTrue(sites)
self.assertFalse(
self.dbs.listBlockParents(
'/Cosmics/Commissioning2015-v1/RAW#942d76fe-cf0e-11e4-afad-001e67ac06a0'
))
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.dbs = DBSReader(self.endpoint)
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertRaises(DBSReaderError, self.dbs.blockToDatasetPath,
BLOCK + 'asas')
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
datasets = dbsReader.matchProcessedDatasets(outputDatasetParts[1],
outputDatasetParts[3],
outputDatasetParts[2])
if len(datasets) == 0:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
blockNames = dbsReader.listFileBlocks(datasetName)
for blockName in blockNames:
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
block = dbsReader.listFilesInBlockWithParents(blockName)
blockLocations = set()
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
for blockFile in block:
parentLFNs = []
for fileParent in blockFile["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
class DBSDataDiscovery(DataDiscovery):
"""Performing the data discovery through CMS DBS service.
"""
# disable pylint warning in next line since they refer to conflict with the main()
# at the bottom of this file which is only used for testing
def __init__(self, config, crabserver='', procnum=-1, rucioClient=None): # pylint: disable=redefined-outer-name
DataDiscovery.__init__(self, config, crabserver, procnum)
self.rucioClient = rucioClient
def checkDatasetStatus(self, dataset, kwargs):
res = self.dbs.dbs.listDatasets(dataset=dataset, detail=1, dataset_access_type='*')
if not res:
raise TaskWorkerException("Cannot find dataset %s in %s DBS instance" % (dataset, self.dbsInstance))
if len(res) > 1:
raise TaskWorkerException("Found more than one dataset while checking in DBS the status of %s" % dataset)
res = res[0]
#import pprint
#self.logger.info("Input dataset details: %s", pprint.pformat(res))
accessType = res['dataset_access_type']
if accessType != 'VALID':
# as per Dima's suggestion https://github.com/dmwm/CRABServer/issues/4739
msgForDeprecDS = "Please contact your physics group if you think the dataset should not be deprecated."
if kwargs['task']['tm_nonvalid_input_dataset'] != 'T':
msg = "CRAB refuses to proceed in getting the details of the dataset %s from DBS, because the dataset is not 'VALID' but '%s'." % (dataset, accessType)
if accessType == 'DEPRECATED':
msg += " (%s)" % (msgForDeprecDS)
msg += " To allow CRAB to consider a dataset that is not 'VALID', set Data.allowNonValidInputDataset = True in the CRAB configuration."
msg += " Notice that this will not force CRAB to run over all files in the dataset;"
msg += " CRAB will still check if there are any valid files in the dataset and run only over those files."
raise TaskWorkerException(msg)
msg = "The input dataset %s is not 'VALID' but '%s'." % (dataset, accessType)
msg += " CRAB will check if there are any valid files in the dataset and run only over those files."
if accessType == 'DEPRECATED':
msg += " %s" % (msgForDeprecDS)
self.uploadWarning(msg, kwargs['task']['user_proxy'], kwargs['task']['tm_taskname'])
return
def keepOnlyDiskRSEs(self, locationsMap):
# get all the RucioStorageElements (RSEs) which are of kind 'Disk'
# locationsMap is a dictionary {block1:[locations], block2:[locations],...}
diskLocationsMap = {}
for block, locations in locationsMap.iteritems():
# as of Sept 2020, tape RSEs ends with _Tape, go for the quick hack
diskRSEs = [rse for rse in locations if not 'Tape' in rse]
if 'T3_CH_CERN_OpenData' in diskRSEs:
diskRSEs.remove('T3_CH_CERN_OpenData') # ignore OpenData until it is accessible by CRAB
if diskRSEs:
# at least some locations are disk
diskLocationsMap[block] = diskRSEs
else:
# no locations are disk, assume that they are tape
# and keep tally of tape-only locations for this dataset
self.tapeLocations = self.tapeLocations.union(set(locations) - set(diskRSEs))
locationsMap.clear() # remove all blocks
locationsMap.update(diskLocationsMap) # add only blocks with disk locations
def checkBlocksSize(self, blocks):
""" Make sure no single blocks has more than 100k lumis. See
https://hypernews.cern.ch/HyperNews/CMS/get/dmDevelopment/2022/1/1/1/1/1/1/2.html
"""
MAX_LUMIS = 100000
for block in blocks:
blockInfo = self.dbs.getDBSSummaryInfo(block=block)
if blockInfo.get('NumberOfLumis', 0) > MAX_LUMIS:
msg = "Block %s contains more than %s lumis.\nThis blows up CRAB server memory" % (block, MAX_LUMIS)
msg += "\nCRAB can only split this by ignoring lumi information. You can do this"
msg += "\nusing FileBased split algorithm and avoiding any additional request"
msg += "\nwich may cause lumi information to be looked up. See CRAB FAQ for more info:"
msg += "\nhttps://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3FAQ"
raise TaskWorkerException(msg)
def requestTapeRecall(self, blockList=[], system='Dynamo', msgHead=''): # pylint: disable=W0102
"""
:param blockList: a list of blocks to recall from Tape to Disk
:param system: a string identifying the DDM system to use 'Dynamo' or 'Rucio' or 'None'
:param msgHead: a string with the initial part of a message to be used for exceptions
:return: nothing: Since data on tape means no submission possible, this function will
always raise a TaskWorkerException to stop the action flow.
The exception message contains details and an attempt is done to upload it to TaskDB
so that crab status can report it
"""
msg = msgHead
if system == 'Rucio':
# need to use crab_tape_recall Rucio account to create containers and create rules
tapeRecallConfig = copy.copy(self.config)
tapeRecallConfig.Services.Rucio_account = 'crab_tape_recall'
rucioClient = getNativeRucioClient(tapeRecallConfig, self.logger) # pylint: disable=redefined-outer-name
# turn input CMS blocks into Rucio dids in cms scope
dids = [{'scope': 'cms', 'name': block} for block in blockList]
# prepare container /TapeRecall/taskname/USER in the service scope
myScope = 'user.crab_tape_recall'
containerName = '/TapeRecall/%s/USER' % self.taskName.replace(':', '.')
containerDid = {'scope':myScope, 'name':containerName}
self.logger.info("Create RUcio container %s", containerName)
try:
rucioClient.add_container(myScope, containerName)
except DataIdentifierAlreadyExists:
self.logger.debug("Container name already exists in Rucio. Keep going")
except Exception as ex:
msg += "Rucio exception creating container: %s" % (str(ex))
raise TaskWorkerException(msg)
try:
rucioClient.attach_dids(myScope, containerName, dids)
except DuplicateContent:
self.logger.debug("Some dids are already in this container. Keep going")
except Exception as ex:
msg += "Rucio exception adding blocks to container: %s" % (str(ex))
raise TaskWorkerException(msg)
self.logger.info("Rucio container %s:%s created with %d blocks", myScope, containerName, len(blockList))
# Compute size of recall request
sizeToRecall = 0
for block in blockList:
replicas = rucioClient.list_dataset_replicas('cms', block)
blockBytes = replicas.next()['bytes'] # pick first replica for each block, they better all have same size
sizeToRecall += blockBytes
TBtoRecall = sizeToRecall // 1e12
if TBtoRecall > 0:
self.logger.info("Total size of data to recall : %d TBytes", TBtoRecall)
else:
self.logger.info("Total size of data to recall : %d GBytes", sizeToRecall/1e9)
if TBtoRecall > 30.:
grouping = 'DATASET' # Rucio DATASET i.e. CMS block !
self.logger.info("Will scatter blocks on multiple sites")
else:
grouping = 'ALL'
self.logger.info("Will place all blocks at a single site")
# create rule
RSE_EXPRESSION = 'ddm_quota>0&(tier=1|tier=2)&rse_type=DISK'
#RSE_EXPRESSION = 'T3_IT_Trieste' # for testing
WEIGHT = 'ddm_quota'
#WEIGHT = None # for testing
LIFETIME = 14 * 24 * 3600 # 14 days
ASK_APPROVAL = False
#ASK_APPROVAL = True # for testing
ACCOUNT = 'crab_tape_recall'
copies = 1
try:
ruleId = rucioClient.add_replication_rule(dids=[containerDid],
copies=copies, rse_expression=RSE_EXPRESSION,
grouping=grouping,
weight=WEIGHT, lifetime=LIFETIME, account=ACCOUNT,
activity='Analysis Input',
comment='Staged from tape for %s' % self.username,
ask_approval=ASK_APPROVAL, asynchronous=True,
)
except DuplicateRule as ex:
# handle "A duplicate rule for this account, did, rse_expression, copies already exists"
# which should only happen when testing, since container name is unique like task name, anyhow...
self.logger.debug("A duplicate rule for this account, did, rse_expression, copies already exists. Use that")
# find the existing rule id
ruleId = rucioClient.list_did_rules(myScope, containerName)
except (InsufficientTargetRSEs, InsufficientAccountLimit, FullStorage) as ex:
msg = "Not enough global quota to issue a tape recall request. Rucio exception:\n%s" % str(ex)
raise TaskWorkerException(msg)
except Exception as ex:
msg += "Rucio exception creating rule: %s" % str(ex)
raise TaskWorkerException(msg)
ruleId = str(ruleId[0]) # from list to singleId and remove unicode
msg += "\nA disk replica has been requested to Rucio (rule ID: %s )" % ruleId
msg += "\nyou can check progress via either of the following two commands:"
msg += "\n rucio rule-info %s" % ruleId
msg += "\n rucio list-rules %s:%s" % (myScope, containerName)
automaticTapeRecallIsImplemented = True
if automaticTapeRecallIsImplemented:
tapeRecallStatus = 'TAPERECALL'
else:
tapeRecallStatus = 'SUBMITFAILED'
configreq = {'workflow': self.taskName,
'taskstatus': tapeRecallStatus,
'ddmreqid': ruleId,
'subresource': 'addddmreqid',
}
try:
tapeRecallStatusSet = self.crabserver.post(api='task', data=urllib.urlencode(configreq))
except HTTPException as hte:
self.logger.exception(hte)
msg = "HTTP Error while contacting the REST Interface %s:\n%s" % (
self.config.TaskWorker.restHost, str(hte))
msg += "\nStoring of %s status and ruleId (%s) failed for task %s" % (
tapeRecallStatus, ruleId, self.taskName)
msg += "\nHTTP Headers are: %s" % hte.headers
raise TaskWorkerException(msg, retry=True)
if tapeRecallStatusSet[2] == "OK":
self.logger.info("Status for task %s set to '%s'", self.taskName, tapeRecallStatus)
if automaticTapeRecallIsImplemented:
msg += "\nThis task will be automatically submitted as soon as the stage-out is completed."
self.uploadWarning(msg, self.userproxy, self.taskName)
raise TapeDatasetException(msg)
# fall here if could not setup for automatic submission after recall
msg += "\nPlease monitor recall progress via Rucio or DAS and try again once data are on disk."
raise TaskWorkerException(msg)
if system == 'None':
msg += '\nIt is not possible to request a recall from tape.'
msg += "\nPlease, check DAS (https://cmsweb.cern.ch/das) and make sure the dataset is accessible on DISK."
raise TaskWorkerException(msg)
if system == 'Dynamo':
raise NotImplementedError
def execute(self, *args, **kwargs):
"""
This is a convenience wrapper around the executeInternal function
"""
# DBS3 requires X509_USER_CERT to be set - but we don't want to leak that to other modules
# so use a context manager to set an ad hoc env and restore as soon as
# executeInternal is over, even if it raises exception
with self.config.TaskWorker.envForCMSWEB:
result = self.executeInternal(*args, **kwargs)
return result
def executeInternal(self, *args, **kwargs):
self.logger.info("Data discovery with DBS") ## to be changed into debug
if kwargs['task']['tm_dbs_url']:
dbsurl = kwargs['task']['tm_dbs_url']
else:
dbsurl = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' # a sensible default
if hasattr(self.config.Services, 'DBSHostName'):
hostname = dbsurl.split('//')[1].split('/')[0]
dbsurl = dbsurl.replace(hostname, self.config.Services.DBSHostName)
self.logger.info("will connect to DBS at URL: %s", dbsurl)
self.dbs = DBSReader(dbsurl)
self.dbsInstance = self.dbs.dbs.serverinfo()["dbs_instance"]
self.taskName = kwargs['task']['tm_taskname'] # pylint: disable=W0201
self.username = kwargs['task']['tm_username'] # pylint: disable=W0201
self.userproxy = kwargs['task']['user_proxy'] # pylint: disable=W0201
self.logger.debug("Data discovery through %s for %s", self.dbs, self.taskName)
inputDataset = kwargs['task']['tm_input_dataset']
secondaryDataset = kwargs['task'].get('tm_secondary_input_dataset', None)
# the isUserDataset flag is used to look for data location in DBS instead of Rucio
isUserDataset = (self.dbsInstance.split('/')[1] != 'global') and \
(inputDataset.split('/')[-1] == 'USER')
self.checkDatasetStatus(inputDataset, kwargs)
if secondaryDataset:
self.checkDatasetStatus(secondaryDataset, kwargs)
try:
# Get the list of blocks for the locations.
blocks = self.dbs.listFileBlocks(inputDataset)
if secondaryDataset:
secondaryBlocks = self.dbs.listFileBlocks(secondaryDataset)
except DBSReaderError as dbsexc:
# dataset not found in DBS is a known use case
if str(dbsexc).find('No matching data'):
raise TaskWorkerException("CRAB could not find dataset %s in this DBS instance: %s" % inputDataset, dbsurl)
raise
## Create a map for block's locations: for each block get the list of locations.
## Note: listFileBlockLocation() gets first the locations from PhEDEx, and if no
## locations are found it gets the original locations from DBS. So it should
## never be the case at this point that some blocks have no locations.
## locationsMap is a dictionary, key=blockName, value=list of PhedexNodes, example:
## {'/JetHT/Run2016B-PromptReco-v2/AOD#b10179dc-3723-11e6-9aa5-001e67abf228': [u'T1_IT_CNAF_Buffer', u'T2_US_Wisconsin', u'T1_IT_CNAF_MSS', u'T2_BE_UCL'],
## '/JetHT/Run2016B-PromptReco-v2/AOD#89b03ca6-1dc9-11e6-b567-001e67ac06a0': [u'T1_IT_CNAF_Buffer', u'T2_US_Wisconsin', u'T1_IT_CNAF_MSS', u'T2_BE_UCL']}
# remove following line when ready to allow user dataset to have locations tracked in Rucio
useRucioForLocations = not isUserDataset
# uncomment followint line to look in Rucio first for any dataset, and fall back to DBS origin for USER ones
# useRucioForLocations = True
locationsFoundWithRucio = False
if not useRucioForLocations:
self.logger.info("Will not use Rucio for this dataset")
if useRucioForLocations:
scope = "cms"
# If the dataset is a USER one, use the Rucio user scope to find it
# TODO: we need a way to enable users to indicate others user scopes as source
if isUserDataset:
scope = "user.%s" % self.username
self.logger.info("Looking up data location with Rucio in %s scope.", scope)
locationsMap = {}
try:
for blockName in list(blocks):
replicas = set()
response = self.rucioClient.list_dataset_replicas(scope, blockName)
for item in response:
# same as complete='y' used for PhEDEx
if item['state'].upper() == 'AVAILABLE':
replicas.add(item['rse'])
if replicas: # only fill map for blocks which have at least one location
locationsMap[blockName] = replicas
except Exception as exc:
msg = "Rucio lookup failed with\n%s" % str(exc)
self.logger.warning(msg)
locationsMap = None
if locationsMap:
locationsFoundWithRucio = True
else:
msg = "No locations found with Rucio for this dataset"
self.logger.warning(msg)
if not locationsFoundWithRucio:
self.logger.info("No locations found with Rucio for %s", inputDataset)
if isUserDataset:
self.logger.info("USER dataset. Looking up data locations using origin site in DBS")
try:
locationsMap = self.dbs.listFileBlockLocation(list(blocks))
except Exception as ex:
raise TaskWorkerException(
"CRAB server could not get file locations from DBS for a USER dataset.\n"+\
"This is could be a temporary DBS glitch, please try to submit a new task (resubmit will not work)"+\
" and contact the experts if the error persists.\nError reason: %s" % str(ex)
)
else:
# datasets other than USER *must* be in Rucio
raise TaskWorkerException(
"CRAB server could not get file locations from Rucio.\n" + \
"This is could be a temporary Rucio glitch, please try to submit a new task (resubmit will not work)" + \
" and contact the experts if the error persists."
)
if secondaryDataset:
if secondaryDataset.endswith('USER'):
self.logger.info("Secondary dataset is USER. Looking up data locations using origin site in DBS")
try:
secondaryLocationsMap = self.dbs.listFileBlockLocation(list(secondaryBlocks))
except Exception as ex:
raise TaskWorkerException(
"CRAB server could not get file locations from DBS for secondary dataset of USER tier.\n"+\
"This is could be a temporary DBS glitch, please try to submit a new task (resubmit will not work)"+\
" and contact the experts if the error persists.\nError reason: %s" % str(ex)
)
else:
self.logger.info("Trying data location of secondary dataset blocks with Rucio")
secondaryLocationsMap = {}
try:
for blockName in list(secondaryBlocks):
replicas = set()
response = self.rucioClient.list_dataset_replicas(scope, blockName)
for item in response:
# same as complete='y' used for PhEDEx
if item['state'].upper() == 'AVAILABLE':
replicas.add(item['rse'])
if replicas: # only fill map for blocks which have at least one location
secondaryLocationsMap[blockName] = replicas
except Exception as exc:
msg = "Rucio lookup failed with\n%s" % str(exc)
self.logger.warning(msg)
secondaryLocationsMap = None
if not secondaryLocationsMap:
msg = "No locations found for secondaryDataset %s." % secondaryDataset
raise TaskWorkerException(msg)
# From now on code is not dependent from having used Rucio or PhEDEx
blocksWithLocation = locationsMap.keys()
if secondaryDataset:
secondaryBlocksWithLocation = secondaryLocationsMap.keys()
# filter out TAPE locations
self.keepOnlyDiskRSEs(locationsMap)
if not locationsMap:
msg = "Task could not be submitted because there is no DISK replica for dataset %s" % inputDataset
if self.tapeLocations:
msg += "\nN.B.: the input dataset is stored at %s, but those are TAPE locations." % ', '.join(sorted(self.tapeLocations))
# following function will always raise error and stop flow here, but will first
# try to trigger a tape recall and place the task in tapeRecall status
msg += "\nWill try to request a disk copy for you. See: https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3FAQ#crab_submit_fails_with_Task_coul"
self.requestTapeRecall(blockList=blocksWithLocation, system='Rucio', msgHead=msg)
# will not need lumi info if user has asked for split by file with no run/lumi mask
splitAlgo = kwargs['task']['tm_split_algo']
lumiMask = kwargs['task']['tm_split_args']['lumis']
runRange = kwargs['task']['tm_split_args']['runs']
needLumiInfo = splitAlgo != 'FileBased' or lumiMask != [] or runRange != []
# secondary dataset access relies on run/lumi info
if secondaryDataset:
needLumiInfo = True
if needLumiInfo:
self.checkBlocksSize(blocksWithLocation) # Interested only in blocks with locations, 'blocks' may contain invalid ones and trigger an Exception
if secondaryDataset:
self.checkBlocksSize(secondaryBlocksWithLocation)
try:
filedetails = self.dbs.listDatasetFileDetails(inputDataset, getParents=True, getLumis=needLumiInfo, validFileOnly=0)
if secondaryDataset:
moredetails = self.dbs.listDatasetFileDetails(secondaryDataset, getParents=False, getLumis=needLumiInfo, validFileOnly=0)
for secfilename, secinfos in moredetails.items():
secinfos['lumiobj'] = LumiList(runsAndLumis=secinfos['Lumis'])
self.logger.info("Beginning to match files from secondary dataset")
for dummyFilename, infos in filedetails.items():
infos['Parents'] = []
lumis = LumiList(runsAndLumis=infos['Lumis'])
for secfilename, secinfos in moredetails.items():
if lumis & secinfos['lumiobj']:
infos['Parents'].append(secfilename)
self.logger.info("Done matching files from secondary dataset")
kwargs['task']['tm_use_parent'] = 1
except Exception as ex: #TODO should we catch HttpException instead?
self.logger.exception(ex)
raise TaskWorkerException("The CRAB3 server backend could not contact DBS to get the files details (Lumis, events, etc).\n"+\
"This is could be a temporary DBS glitch. Please try to submit a new task (resubmit will not work)"+\
" and contact the experts if the error persists.\nError reason: %s" % str(ex))
if not filedetails:
raise TaskWorkerException(("Cannot find any file inside the dataset. Please, check your dataset in DAS, %s.\n" +\
"Aborting submission. Resubmitting your task will not help.") %\
("https://cmsweb.cern.ch/das/request?instance=%s&input=dataset=%s") %\
(self.dbsInstance, inputDataset))
## Format the output creating the data structures required by WMCore. Filters out invalid files,
## files whose block has no location, and figures out the PSN
result = self.formatOutput(task=kwargs['task'], requestname=self.taskName,
datasetfiles=filedetails, locations=locationsMap,
tempDir=kwargs['tempDir'])
if not result.result:
raise TaskWorkerException(("Cannot find any valid file inside the dataset. Please, check your dataset in DAS, %s.\n" +
"Aborting submission. Resubmitting your task will not help.") %
("https://cmsweb.cern.ch/das/request?instance=%s&input=dataset=%s") %
(self.dbsInstance, inputDataset))
self.logger.debug("Got %s files", len(result.result.getFiles()))
return result
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint=dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset", datasetName, "parts", outputDatasetParts
try:
#retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" %
datasetName)
#traverse each block
for blockName in blockNames:
#deal with white and black list.
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
#existing blocks in phedex
replicaInfo = phedexReader.getReplicaInfoForBlocks(block=blockName,
subscribed='y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
blockLocations = set()
#load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
#for each file on the block
for blockFile in blockFiles:
parentLFNs = []
#get parent information about file
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
#populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
#Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection[
"RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(
lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {
"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs
}
return files
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl, fakeLocation=False):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset",datasetName,"parts",outputDatasetParts
try:
#retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
has_parent = False
try:
parents = dbsReader.listDatasetParents( datasetName )
if parents: has_parent=True
except:
print "Dataset with no parent"
pass
#traverse each block
for blockName in blockNames:
#deal with white and black list.
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
#existing blocks in phedex
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
#has_parent = dbsReader.listBlockParents(blockName)
if has_parent:
try:
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
except:
print blockName,"does not appear to have a parent, even though it should. Very suspicious"
blockFileParents = dbsReader.listFilesInBlock(blockName)
else:
blockFileParents = dbsReader.listFilesInBlock(blockName)
blockLocations = set()
#load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.PNNtoPSN(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
se = siteDB.cmsNametoSE(cmsName)
blockLocations.update(se)
logging.debug("cmsName %s mapped to se %s", cmsName, se)
logging.debug("PhEDEx node %s, cmsSites %s, blockLocations %s", node, cmsSites, blockLocations)
# We cannot upload docs without location, so force it in case it's empty
if not blockLocations:
if fakeLocation:
logging.info("\t\t %s\tno location", blockName)
blockLocations.update([u'cmssrmdisk.fnal.gov', u'srm-eoscms.cern.ch'])
elif not has_parent: ## this should be the source
logging.info("Blockname: %s\tno location, ABORT", blockName)
sys.exit(1)
logging.info("Blockname: %s\tLocations: %s", blockName, blockLocations)
#for each file on the block
for blockFile in blockFiles:
parentLFNs = []
#populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
## remove when https://github.com/dmwm/WMCore/issues/7128 gets fixed
#elif not 'RAW' in blockName:
# print "no parent info"
runInfo = {}
#Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
class DBSReaderTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
#self.endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
self.endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dbs = None
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listPrimaryDatasets('Jet*')
self.assertTrue('Jet' in results)
self.assertTrue('JetMET' in results)
self.assertTrue('JetMETTau' in results)
self.assertFalse(self.dbs.listPrimaryDatasets('DoesntExist'))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v1')
self.assertEqual(1, len(dataset))
self.assertEqual(['/Jet/Run2011A-v1/RAW'], dataset[0]['PathList'])
self.assertEqual('Run2011A-v1', dataset[0]['Name'])
self.assertFalse(self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v666'))
@attr("integration")
def testlistRuns(self):
"""listRuns returns known runs"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRuns(dataset = DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(dataset = DATASET, block = BLOCK)
self.assertEqual([173657], runs)
@attr("integration")
def testlistRunLumis(self):
"""listRunLumis returns known runs and lumicounts"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRunLumis(dataset = DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(173692 in runs)
self.assertEqual(runs[173692], 2782)
runs = self.dbs.listRuns(dataset = DATASET, block = BLOCK)
self.assertEqual({173657 : 94}, runs)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
self.dbs = DBSReader(self.endpoint)
datasets = self.dbs.listProcessedDatasets('Jet', 'RAW')
self.assertTrue('Run2011A-v1' in datasets)
self.assertTrue('Run2011B-v1' in datasets)
self.assertFalse(self.dbs.listProcessedDatasets('Jet', 'blah'))
self.assertFalse(self.dbs.listProcessedDatasets('blah', 'RAW'))
@attr("integration")
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
@attr("integration")
def testlistDatasetFileDetails(self):
"""testlistDatasetFilesDetails returns lumis, events, and parents of a dataset"""
TESTFILE = '/store/data/Run2011A/HighPileUp/RAW/v1/000/173/658/56484BAB-CBCB-E011-AF00-BCAEC518FF56.root'
for endpoint in [self.endpoint, 'test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py:']:
self.dbs = DBSReader(endpoint)
details = self.dbs.listDatasetFileDetails(DATASET)
self.assertEqual(len(details), 49)
self.assertTrue(TESTFILE in details)
self.assertEqual(details[TESTFILE]['NumberOfEvents'], 545)
self.assertEqual(details[TESTFILE]['Size'], 286021145)
self.assertEqual(details[TESTFILE]['BlockName'], '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace')
self.assertEqual(details[TESTFILE]['Checksums'],
{'Checksum': '22218315', 'Adler32': 'a41a1446', 'Md5': 'NOTSET'}
)
self.assertTrue( 173658 in details[TESTFILE]['Lumis'])
self.assertEqual( sorted(details[TESTFILE]['Lumis'][173658]),
sorted( map( long, [8, 12, 9, 14, 10, 6, 2, 1, 4, 3, 36, 49, 16, 11, 27, 35, 46, 39, 20, 24, 52, 23, 40, 42, 45, 21, 32, 37, \
25, 22, 5, 33, 17, 15, 26, 50, 18, 29, 51, 44, 69, 43, 30, 73, 19, 41, 13, 38, 7, 31, 75, 48, 59, 65, 55, \
57, 34, 28, 74, 47, 64, 61, 68, 77, 66, 71, 60, 76, 70, 67, 62, 78, 82, 79, 88, 56, 101, 92, 58, 72, 54, \
63, 96, 53, 84, 95, 89, 85, 99, 81, 91, 102, 80, 100, 107, 94, 93, 90, 86, 87, 83, 97, 104, 110, 111, 106,\
108, 98, 103, 109, 105]))
)
@attr("integration")
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset['path'], DATASET)
self.assertEqual(dataset['block'], '')
self.assertEqual(dataset['NumberOfEvents'], '22075')
self.assertEqual(dataset['NumberOfBlocks'], '46')
self.assertEqual(dataset['total_size'], '4001680824')
self.assertEqual(dataset['NumberOfFiles'], '49')
self.assertEqual(dataset['NumberOfLumis'], '7223')
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block['path'], '')
self.assertEqual(block['block'], BLOCK)
self.assertEqual(block['NumberOfEvents'], '377')
self.assertEqual(block['NumberOfBlocks'], '1')
self.assertEqual(block['total_size'], '150780132')
self.assertEqual(block['NumberOfFiles'], '2')
self.assertEqual(block['NumberOfLumis'], '94')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET, BLOCK + 'asas')
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName = BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block['Name'] in [x['Name'] for x in blocks])
self.assertEqual(BLOCK, block['Name'])
#self.assertEqual(377, block['NumberOfEvents'])
self.assertEqual(150780132, block['BlockSize'])
self.assertEqual(2, block['NumberOfFiles'])
# possibly fragile but assume block located at least at cern
sites = [x['Name'] for x in block['StorageElementList'] if x['Name'].find('cern.ch') > -1]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo, DATASET + 'blah')
self.assertFalse(self.dbs.getFileBlocksInfo(DATASET, blockName = BLOCK + 'asas'))
@attr("integration")
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.listFileBlocks(DATASET)
# block is closed
block = self.dbs.listFileBlocks(DATASET, blockName = BLOCK, onlyClosedBlocks = True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
@attr("integration")
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
@attr("integration")
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertFalse(self.dbs.blockExists(DATASET + '#somethingelse'))
@attr("integration")
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in [x['LogicalFileName'] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock, DATASET + '#blah')
@attr("integration")
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
# hope PromptReco doesn't get deleted
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listFilesInBlockWithParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60')
self.assertEqual(1, len(files))
self.assertEqual('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60', files[0]['Block']['Name'])
self.assertEqual('/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
files[0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents, BLOCK + 'asas')
@attr("integration")
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in self.dbs.lfnsInBlock(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + 'asas')
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
WRONG_BLOCK = BLOCK[:-4]+'abcd'
BLOCK2 = '/HighPileUp/Run2011A-v1/RAW#6021175e-cbfb-11e0-80a9-003048caaace'
DBS_BLOCK = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#fb978442-a61b-413a-b4f4-526e6cdb142e'
DBS_BLOCK2 = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#0b04d417-d734-4ef2-88b0-392c48254dab'
self.dbs = DBSReader('https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader/')
# assume one site is cern
sites = [x for x in self.dbs.listFileBlockLocation(BLOCK) if x and x.find('cern.ch') > -1]
self.assertTrue(sites)
#This block is only found on DBS
self.assertTrue(self.dbs.listFileBlockLocation(DBS_BLOCK))
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(WRONG_BLOCK))
#test bulk call:
## two blocks in phedex
self.assertEqual(2, len(self.dbs.listFileBlockLocation([BLOCK, BLOCK2])))
## one block in phedex one does not exist
self.assertEqual(1, len(self.dbs.listFileBlockLocation([BLOCK, WRONG_BLOCK])))
## one in phedex one in dbs
self.assertEqual(2, len(self.dbs.listFileBlockLocation([BLOCK, DBS_BLOCK])))
## two in dbs
self.assertEqual(2, len(self.dbs.listFileBlockLocation([DBS_BLOCK, DBS_BLOCK2])))
## one in DBS and one does not exist
self.assertEqual(1, len(self.dbs.listFileBlockLocation([DBS_BLOCK, WRONG_BLOCK])))
@attr("integration")
def testGetFileBlock(self):
"""getFileBlock returns block"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block['Files']))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock, BLOCK + 'asas')
@attr("integration")
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlockWithParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60')
self.assertEqual(len(block), 1)
block = block['/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60']
self.assertEqual('/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
block['Files'][0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents, BLOCK + 'asas')
@attr("integration")
def testGetFiles(self):
"""getFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
@attr("integration")
def testListBlockParents(self):
"""listBlockParents returns block parents"""
self.dbs = DBSReader(self.endpoint)
parents = self.dbs.listBlockParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60')
self.assertEqual(1, len(parents))
self.assertEqual('/Jet/Run2011A-v1/RAW#37cf2a40-4e0e-11e0-9833-00151755cb60',
parents[0]['Name'])
sites = [x for x in parents[0]['StorageElementList'] if x.find("cern.ch") > -1]
self.assertTrue(sites)
self.assertFalse(self.dbs.listBlockParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60dsl'))
@attr("integration")
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
@attr("integration")
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.dbs = DBSReader(self.endpoint)
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertFalse(self.dbs.blockToDatasetPath(BLOCK + 'asas'))
def migrateDatasetBlocks(self, inputDBSUrl, datasetPath, blocks):
"""
_migrateDatasetBlocks_
Migrate the list of fileblocks provided by blocks, belonging
to the dataset specified by the dataset path to this DBS instance
from the inputDBSUrl provided
- *inputDBSUrl* : URL for connection to input DBS
- *datasetPath* : Name of dataset in input DBS (must exist in input
DBS)
- *blocks* : list of block names to be migrated (must exist
in input DBS)
"""
if len(blocks) == 0:
msg = "FileBlocks not provided.\n"
msg += "You must provide the name of at least one fileblock\n"
msg += "to be migrated"
raise DBSWriterError(msg)
# //
# // Hook onto input DBSUrl and verify that the dataset & blocks
# // exist
reader = DBSReader(inputDBSUrl)
inputBlocks = reader.listFileBlocks(datasetPath)
for block in blocks:
# //
# // Test block exists at source
# //
if block not in inputBlocks:
msg = "Block name:\n ==> %s\n" % block
msg += "Not found in input dataset:\n ==> %s\n" % datasetPath
msg += "In DBS Instance:\n ==> %s\n" % inputDBSUrl
raise DBSWriterError(msg)
# //
# // Test block does not exist in target
# //
if self.reader.blockExists(block):
# //
# // block exists
# // If block is closed dont attempt transfer
if not self.reader.blockIsOpen(block):
msg = "Block already exists in target DBS and is closed:\n"
msg += " ==> %s\n" % block
msg += "Skipping Migration of that block"
logging.warning(msg)
continue
try:
xferData = reader.dbs.listDatasetContents(datasetPath, block)
except DbsException, ex:
msg = "Error in DBSWriter.migrateDatasetBlocks\n"
msg += "Could not read content of dataset:\n ==> %s\n" % (datasetPath,)
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
xferData = _remapBlockParentage(datasetPath, xferData)
try:
self.dbs.insertDatasetContents(xferData)
except DbsException, ex:
msg = "Error in DBSWriter.migrateDatasetBlocks\n"
msg += "Could not write content of dataset:\n ==> %s\n" % (datasetPath,)
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl, fakeLocation=False):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint=dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
class BlockBuster(threading.Thread):
def __init__(self, **args):
threading.Thread.__init__(self)
for k,v in args.items():
setattr(self,k,v)
self.major_failure = False
def run(self):
self.files = {}
logging = self.l
has_parent = self.hp
fakeLocation = self.fl
blockName = self.bn
blockBlacklist = self.bbl
blockWhitelist = self.bwl
if blockBlacklist and blockName in blockBlacklist:
return
if blockWhitelist and blockName not in blockWhitelist:
return
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
dbsReader = DBSReader(endpoint=self.dbs)
replicaInfo = phedexReader.getReplicaInfoForBlocks(block=blockName,
subscribed='y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
if has_parent:
try:
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
except:
print blockName, "does not appear to have a parent, even though it should. Very suspicious"
blockFileParents = dbsReader.listFilesInBlock(blockName)
else:
blockFileParents = dbsReader.listFilesInBlock(blockName)
blockLocations = set()
# load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
PNN = replica["node"]
PSNs = siteDB.PNNtoPSN(PNN)
blockLocations.add(PNN)
#logging.debug("PhEDEx Node Name: %s\tPSNs: %s", PNN, PSNs)
# We cannot upload docs without location, so force it in case it's empty
if not blockLocations:
if fakeLocation:
#logging.info("\t\t %s\tno location", blockName)
blockLocations.update([u'T1_US_FNAL_Disk', u'T2_CH_CERN'])
elif not has_parent: ## this should be the source
logging.info("Blockname: %s\tno location, ABORT", blockName)
self.major_failure = True
#sys.exit(1)
#logging.info("Blockname: %s\tLocations: %s", blockName, blockLocations)
# for each file on the block
for blockFile in blockFiles:
parentLFNs = []
# populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
# Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
self.files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset", datasetName, "parts", outputDatasetParts
try:
# retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
has_parent = False
try:
parents = dbsReader.listDatasetParents(datasetName)
if parents: has_parent = True
except:
print "Dataset with no parent"
pass
bthreads=[]
# traverse each block
for blockName in blockNames:
bthreads.append( BlockBuster( bn = blockName,
hp=has_parent,
fl = fakeLocation,
bbl = blockBlacklist,
bwl = blockWhitelist,
l = logging,
dbs=dbsUrl))
print len(bthreads),"block query created"
bthreads = ThreadBuster( bthreads, 40, 2., verbose=False)
for t in bthreads:
if t.major_failure:
print "There was a major failure in processing block files"
sys.exit(1)
files.update(t.files)
print len(files)
return files
def migrateDatasetBlocks(self, inputDBSUrl, datasetPath, blocks):
"""
_migrateDatasetBlocks_
Migrate the list of fileblocks provided by blocks, belonging
to the dataset specified by the dataset path to this DBS instance
from the inputDBSUrl provided
- *inputDBSUrl* : URL for connection to input DBS
- *datasetPath* : Name of dataset in input DBS (must exist in input
DBS)
- *blocks* : list of block names to be migrated (must exist
in input DBS)
"""
if len(blocks) == 0:
msg = "FileBlocks not provided.\n"
msg += "You must provide the name of at least one fileblock\n"
msg += "to be migrated"
raise DBSWriterError(msg)
# //
# // Hook onto input DBSUrl and verify that the dataset & blocks
#// exist
reader = DBSReader(inputDBSUrl)
inputBlocks = reader.listFileBlocks(datasetPath)
for block in blocks:
# //
# // Test block exists at source
#//
if block not in inputBlocks:
msg = "Block name:\n ==> %s\n" % block
msg += "Not found in input dataset:\n ==> %s\n" % datasetPath
msg += "In DBS Instance:\n ==> %s\n" % inputDBSUrl
raise DBSWriterError(msg)
# //
# // Test block does not exist in target
#//
if self.reader.blockExists(block):
# //
# // block exists
#// If block is closed dont attempt transfer
if not self.reader.blockIsOpen(block):
msg = "Block already exists in target DBS and is closed:\n"
msg += " ==> %s\n" % block
msg += "Skipping Migration of that block"
logging.warning(msg)
continue
try:
xferData = reader.dbs.listDatasetContents(datasetPath, block)
except DbsException as ex:
msg = "Error in DBSWriter.migrateDatasetBlocks\n"
msg += "Could not read content of dataset:\n ==> %s\n" % (
datasetPath, )
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
xferData = _remapBlockParentage(datasetPath, xferData)
try:
self.dbs.insertDatasetContents(xferData)
except DbsException as ex:
msg = "Error in DBSWriter.migrateDatasetBlocks\n"
msg += "Could not write content of dataset:\n ==> %s\n" % (
datasetPath, )
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
del xferData
return
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset",datasetName,"parts",outputDatasetParts
try:
#retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
#traverse each block
for blockName in blockNames:
#deal with white and black list.
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
#existing blocks in phedex
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
blockLocations = set()
#load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
#for each file on the block
for blockFile in blockFiles:
parentLFNs = []
#get parent information about file
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
#populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
#Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
class DBSReaderTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
#self.endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
self.endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dbs = None
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listPrimaryDatasets('Jet*')
self.assertTrue('Jet' in results)
self.assertTrue('JetMET' in results)
self.assertTrue('JetMETTau' in results)
self.assertFalse(self.dbs.listPrimaryDatasets('DoesntExist'))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v1')
self.assertEqual(1, len(dataset))
self.assertEqual(['/Jet/Run2011A-v1/RAW'], dataset[0]['PathList'])
self.assertEqual('Run2011A-v1', dataset[0]['Name'])
self.assertFalse(
self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v666'))
@attr("integration")
def testlistRuns(self):
"""listRuns returns known runs"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRuns(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(dataset=DATASET, block=BLOCK)
self.assertEqual([173657], runs)
@attr("integration")
def testlistRunLumis(self):
"""listRunLumis returns known runs and lumicounts"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRunLumis(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(173692 in runs)
self.assertEqual(runs[173692], 2782)
runs = self.dbs.listRuns(dataset=DATASET, block=BLOCK)
self.assertEqual({173657: 94}, runs)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
self.dbs = DBSReader(self.endpoint)
datasets = self.dbs.listProcessedDatasets('Jet', 'RAW')
self.assertTrue('Run2011A-v1' in datasets)
self.assertTrue('Run2011B-v1' in datasets)
self.assertFalse(self.dbs.listProcessedDatasets('Jet', 'blah'))
self.assertFalse(self.dbs.listProcessedDatasets('blah', 'RAW'))
@attr("integration")
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
@attr("integrtion")
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset['path'], DATASET)
self.assertEqual(dataset['block'], '')
self.assertEqual(dataset['NumberOfEvents'], '22075')
self.assertEqual(dataset['NumberOfBlocks'], '46')
self.assertEqual(dataset['total_size'], '4001680824')
self.assertEqual(dataset['NumberOfFiles'], '49')
self.assertEqual(dataset['NumberOfLumis'], '7223')
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block['path'], '')
self.assertEqual(block['block'], BLOCK)
self.assertEqual(block['NumberOfEvents'], '377')
self.assertEqual(block['NumberOfBlocks'], '1')
self.assertEqual(block['total_size'], '150780132')
self.assertEqual(block['NumberOfFiles'], '2')
self.assertEqual(block['NumberOfLumis'], '94')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo,
DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET,
BLOCK + 'asas')
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block['Name'] in [x['Name'] for x in blocks])
self.assertEqual(BLOCK, block['Name'])
#self.assertEqual(377, block['NumberOfEvents'])
self.assertEqual(150780132, block['BlockSize'])
self.assertEqual(2, block['NumberOfFiles'])
# possibly fragile but assume block located at least at cern
sites = [
x['Name'] for x in block['StorageElementList']
if x['Name'].find('cern.ch') > -1
]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo,
DATASET + 'blah')
self.assertFalse(
self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK + 'asas'))
@attr("integration")
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.listFileBlocks(DATASET)
# block is closed
block = self.dbs.listFileBlocks(DATASET,
blockName=BLOCK,
onlyClosedBlocks=True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
@attr("integration")
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
@attr("integration")
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertFalse(self.dbs.blockExists(DATASET + '#somethingelse'))
@attr("integration")
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(
FILE in
[x['LogicalFileName'] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock,
DATASET + '#blah')
@attr("integration")
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
# hope PromptReco doesn't get deleted
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listFilesInBlockWithParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60'
)
self.assertEqual(1, len(files))
self.assertEqual(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60',
files[0]['Block']['Name'])
self.assertEqual(
'/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
files[0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents,
BLOCK + 'asas')
@attr("integration")
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in self.dbs.lfnsInBlock(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + 'asas')
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
self.dbs = DBSReader(self.endpoint)
# assume one site is cern
sites = [
x for x in self.dbs.listFileBlockLocation(BLOCK)
if x.find('cern.ch') > -1
]
self.assertTrue(sites)
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(BLOCK + 'blah'))
@attr("integration")
def testGetFileBlock(self):
"""getFileBlock returns block"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block['Files']))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock,
BLOCK + 'asas')
@attr("integration")
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlockWithParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60'
)
self.assertEqual(len(block), 1)
block = block[
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60']
self.assertEqual(
'/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
block['Files'][0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents,
BLOCK + 'asas')
@attr("integration")
def testGetFiles(self):
"""getFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
@attr("integration")
def testListBlockParents(self):
"""listBlockParents returns block parents"""
self.dbs = DBSReader(self.endpoint)
parents = self.dbs.listBlockParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60'
)
self.assertEqual(1, len(parents))
self.assertEqual(
'/Jet/Run2011A-v1/RAW#37cf2a40-4e0e-11e0-9833-00151755cb60',
parents[0]['Name'])
sites = [
x for x in parents[0]['StorageElementList']
if x.find("cern.ch") > -1
]
self.assertTrue(sites)
self.assertFalse(
self.dbs.listBlockParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60dsl'
))
@attr("integration")
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
@attr("integration")
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.dbs = DBSReader(self.endpoint)
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertFalse(self.dbs.blockToDatasetPath(BLOCK + 'asas'))
class DBSReaderTest(EmulatedUnitTestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
#self.endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
self.endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dbs = None
super(DBSReaderTest, self).setUp()
return
def tearDown(self):
"""
_tearDown_
:return:
"""
super(DBSReaderTest, self).tearDown()
return
@attr("integration")
def testListDatatiers(self):
"""
listDatatiers returns all datatiers available
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listDatatiers()
self.assertTrue('RAW' in results)
self.assertTrue('GEN-SIM-RECO' in results)
self.assertTrue('GEN-SIM' in results)
self.assertFalse('RAW-ALAN' in results)
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listPrimaryDatasets('Jet*')
self.assertTrue('Jet' in results)
self.assertTrue('JetMET' in results)
self.assertTrue('JetMETTau' in results)
self.assertFalse(self.dbs.listPrimaryDatasets('DoesntExist'))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v1')
self.assertEqual(1, len(dataset))
self.assertEqual(['/Jet/Run2011A-v1/RAW'], dataset[0]['PathList'])
self.assertEqual('Run2011A-v1', dataset[0]['Name'])
self.assertFalse(self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v666'))
def testlistRuns(self):
"""listRuns returns known runs"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRuns(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(block=BLOCK)
self.assertEqual(1, len(runs))
self.assertEqual([173657], runs)
def testlistRunLumis(self):
"""listRunLumis returns known runs and lumicounts (None for DBS3)"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRunLumis(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(173692 in runs)
self.assertEqual(runs[173692], None)
runs = self.dbs.listRunLumis(block=BLOCK)
self.assertEqual(1, len(runs))
self.assertTrue(173657 in runs)
self.assertEqual(runs[173657], None)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
self.dbs = DBSReader(self.endpoint)
datasets = self.dbs.listProcessedDatasets('Jet', 'RAW')
self.assertTrue('Run2011A-v1' in datasets)
self.assertTrue('Run2011B-v1' in datasets)
self.assertFalse(self.dbs.listProcessedDatasets('Jet', 'blah'))
self.assertFalse(self.dbs.listProcessedDatasets('blah', 'RAW'))
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
def testlistDatasetFileDetails(self):
"""testlistDatasetFilesDetails returns lumis, events, and parents of a dataset"""
TESTFILE = '/store/data/Run2011A/HighPileUp/RAW/v1/000/173/658/56484BAB-CBCB-E011-AF00-BCAEC518FF56.root'
self.dbs = DBSReader(self.endpoint)
details = self.dbs.listDatasetFileDetails(DATASET)
self.assertEqual(len(details), 49)
self.assertTrue(TESTFILE in details)
self.assertEqual(details[TESTFILE]['NumberOfEvents'], 545)
self.assertEqual(details[TESTFILE]['file_size'], 286021145)
self.assertEqual(details[TESTFILE]['BlockName'], '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace')
self.assertEqual(details[TESTFILE]['Md5'], 'NOTSET')
self.assertEqual(details[TESTFILE]['md5'], 'NOTSET')
self.assertEqual(details[TESTFILE]['Adler32'], 'a41a1446')
self.assertEqual(details[TESTFILE]['adler32'], 'a41a1446')
self.assertEqual(details[TESTFILE]['Checksum'], '22218315')
self.assertEqual(details[TESTFILE]['check_sum'], '22218315')
self.assertTrue(173658 in details[TESTFILE]['Lumis'])
self.assertEqual(sorted(details[TESTFILE]['Lumis'][173658]),
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50,
51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74,
75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98,
99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111])
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset['path'], DATASET)
self.assertEqual(dataset['block'], '')
self.assertEqual(dataset['NumberOfEvents'], 22075)
self.assertEqual(dataset['NumberOfBlocks'], 46)
self.assertEqual(dataset['FileSize'], 4001680824)
self.assertEqual(dataset['file_size'], 4001680824)
self.assertEqual(dataset['NumberOfFiles'], 49)
self.assertEqual(dataset['NumberOfLumis'], 7223)
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block['path'], '')
self.assertEqual(block['block'], BLOCK)
self.assertEqual(block['NumberOfEvents'], 377)
self.assertEqual(block['NumberOfBlocks'], 1)
self.assertEqual(block['FileSize'], 150780132)
self.assertEqual(block['file_size'], 150780132)
self.assertEqual(block['NumberOfFiles'], 2)
self.assertEqual(block['NumberOfLumis'], 94)
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET, BLOCK + 'asas')
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block['Name'] in [x['Name'] for x in blocks])
self.assertEqual(BLOCK, block['Name'])
self.assertEqual(0, block['OpenForWriting'])
self.assertEqual(150780132, block['BlockSize'])
self.assertEqual(2, block['NumberOfFiles'])
# possibly fragile but assume block located at least at cern
sites = [x['Name'] for x in block['PhEDExNodeList'] if x['Name'].find('CH_CERN') > -1]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo, DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo, DATASET, blockName=BLOCK + 'asas')
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.listFileBlocks(DATASET)
self.assertTrue(BLOCK in blocks)
# block is closed
block = self.dbs.listFileBlocks(DATASET, blockName=BLOCK, onlyClosedBlocks=True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.blockExists, DATASET + '#somethingelse')
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in [x['LogicalFileName'] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock, DATASET + '#blah')
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listFilesInBlockWithParents('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0')
self.assertEqual(4, len(files))
self.assertEqual('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0', files[0]['block_name'])
self.assertEqual('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0', files[0]['BlockName'])
self.assertEqual('/store/data/Commissioning2015/Cosmics/RAW/v1/000/238/545/00000/1043E89F-2DCF-E411-9CAE-02163E013751.root',
files[0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents, BLOCK + 'asas')
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in [x['logical_file_name'] for x in self.dbs.lfnsInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + 'asas')
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
WRONG_BLOCK = BLOCK[:-4]+'abcd'
BLOCK2 = '/HighPileUp/Run2011A-v1/RAW#6021175e-cbfb-11e0-80a9-003048caaace'
DBS_BLOCK = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#fb978442-a61b-413a-b4f4-526e6cdb142e'
DBS_BLOCK2 = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#0b04d417-d734-4ef2-88b0-392c48254dab'
self.dbs = DBSReader('https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader/')
# assume one site is cern
sites = [x for x in self.dbs.listFileBlockLocation(BLOCK) if x and x.find('CH_CERN') > -1]
self.assertTrue(sites)
#This block is only found on DBS
self.assertTrue(self.dbs.listFileBlockLocation(DBS_BLOCK))
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(WRONG_BLOCK))
#test bulk call:
## two blocks in phedex
self.assertEqual(2, len(self.dbs.listFileBlockLocation([BLOCK, BLOCK2])))
## one block in phedex one does not exist
self.assertEqual(1, len(self.dbs.listFileBlockLocation([BLOCK, WRONG_BLOCK])))
## one in phedex one in dbs
self.assertEqual(2, len(self.dbs.listFileBlockLocation([BLOCK, DBS_BLOCK])))
## two in dbs
self.assertEqual(2, len(self.dbs.listFileBlockLocation([DBS_BLOCK, DBS_BLOCK2])))
## one in DBS and one does not exist
self.assertEqual(1, len(self.dbs.listFileBlockLocation([DBS_BLOCK, WRONG_BLOCK])))
def testGetFileBlock(self):
"""getFileBlock returns block"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block['Files']))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock, BLOCK + 'asas')
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlockWithParents('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0')
self.assertEqual(len(block), 1)
block = block['/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0']
self.assertEqual('/store/data/Commissioning2015/Cosmics/RAW/v1/000/238/545/00000/1043E89F-2DCF-E411-9CAE-02163E013751.root',
block['Files'][0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents, BLOCK + 'asas')
def testGetFiles(self):
"""getFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
def testListBlockParents(self):
"""listBlockParents returns block parents"""
self.dbs = DBSReader(self.endpoint)
parents = self.dbs.listBlockParents('/Cosmics/Commissioning2015-PromptReco-v1/RECO#004ac3ba-d09e-11e4-afad-001e67ac06a0')
self.assertEqual(1, len(parents))
self.assertEqual('/Cosmics/Commissioning2015-v1/RAW#942d76fe-cf0e-11e4-afad-001e67ac06a0', parents[0]['Name'])
sites = [x for x in parents[0]['PhEDExNodeList'] if x.find("CH_CERN") > -1]
self.assertTrue(sites)
self.assertFalse(self.dbs.listBlockParents('/Cosmics/Commissioning2015-v1/RAW#942d76fe-cf0e-11e4-afad-001e67ac06a0'))
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.dbs = DBSReader(self.endpoint)
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertRaises(DBSReaderError, self.dbs.blockToDatasetPath, BLOCK + 'asas')
class DBSReaderTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
#self.endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
self.endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dbs = None
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listPrimaryDatasets('Jet*')
self.assertTrue('Jet' in results)
self.assertTrue('JetMET' in results)
self.assertTrue('JetMETTau' in results)
self.assertFalse(self.dbs.listPrimaryDatasets('DoesntExist'))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v1')
self.assertEqual(1, len(dataset))
self.assertEqual(['/Jet/Run2011A-v1/RAW'], dataset[0]['PathList'])
self.assertEqual('Run2011A-v1', dataset[0]['Name'])
self.assertFalse(
self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v666'))
@attr("integration")
def testlistRuns(self):
"""listRuns returns known runs"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRuns(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(dataset=DATASET, block=BLOCK)
self.assertEqual([173657], runs)
@attr("integration")
def testlistRunLumis(self):
"""listRunLumis returns known runs and lumicounts"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRunLumis(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(173692 in runs)
self.assertEqual(runs[173692], 2782)
runs = self.dbs.listRuns(dataset=DATASET, block=BLOCK)
self.assertEqual({173657: 94}, runs)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
self.dbs = DBSReader(self.endpoint)
datasets = self.dbs.listProcessedDatasets('Jet', 'RAW')
self.assertTrue('Run2011A-v1' in datasets)
self.assertTrue('Run2011B-v1' in datasets)
self.assertFalse(self.dbs.listProcessedDatasets('Jet', 'blah'))
self.assertFalse(self.dbs.listProcessedDatasets('blah', 'RAW'))
@attr("integration")
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
@attr("integration")
def testlistDatasetFileDetails(self):
"""testlistDatasetFilesDetails returns lumis, events, and parents of a dataset"""
TESTFILE = '/store/data/Run2011A/HighPileUp/RAW/v1/000/173/658/56484BAB-CBCB-E011-AF00-BCAEC518FF56.root'
for endpoint in [
self.endpoint,
'test/python/WMCore_t/Services_t/DBS_t/DBSReader_t.py:'
]:
self.dbs = DBSReader(endpoint)
details = self.dbs.listDatasetFileDetails(DATASET)
self.assertEqual(len(details), 49)
self.assertTrue(TESTFILE in details)
self.assertEqual(details[TESTFILE]['NumberOfEvents'], 545)
self.assertEqual(details[TESTFILE]['Size'], 286021145)
self.assertEqual(
details[TESTFILE]['BlockName'],
'/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace'
)
self.assertEqual(details[TESTFILE]['Checksums'], {
'Checksum': '22218315',
'Adler32': 'a41a1446',
'Md5': 'NOTSET'
})
self.assertTrue(173658 in details[TESTFILE]['Lumis'])
self.assertEqual( sorted(details[TESTFILE]['Lumis'][173658]),
sorted( map( long, [8, 12, 9, 14, 10, 6, 2, 1, 4, 3, 36, 49, 16, 11, 27, 35, 46, 39, 20, 24, 52, 23, 40, 42, 45, 21, 32, 37, \
25, 22, 5, 33, 17, 15, 26, 50, 18, 29, 51, 44, 69, 43, 30, 73, 19, 41, 13, 38, 7, 31, 75, 48, 59, 65, 55, \
57, 34, 28, 74, 47, 64, 61, 68, 77, 66, 71, 60, 76, 70, 67, 62, 78, 82, 79, 88, 56, 101, 92, 58, 72, 54, \
63, 96, 53, 84, 95, 89, 85, 99, 81, 91, 102, 80, 100, 107, 94, 93, 90, 86, 87, 83, 97, 104, 110, 111, 106,\
108, 98, 103, 109, 105]))
)
@attr("integration")
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset['path'], DATASET)
self.assertEqual(dataset['block'], '')
self.assertEqual(dataset['NumberOfEvents'], '22075')
self.assertEqual(dataset['NumberOfBlocks'], '46')
self.assertEqual(dataset['total_size'], '4001680824')
self.assertEqual(dataset['NumberOfFiles'], '49')
self.assertEqual(dataset['NumberOfLumis'], '7223')
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block['path'], '')
self.assertEqual(block['block'], BLOCK)
self.assertEqual(block['NumberOfEvents'], '377')
self.assertEqual(block['NumberOfBlocks'], '1')
self.assertEqual(block['total_size'], '150780132')
self.assertEqual(block['NumberOfFiles'], '2')
self.assertEqual(block['NumberOfLumis'], '94')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo,
DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET,
BLOCK + 'asas')
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block['Name'] in [x['Name'] for x in blocks])
self.assertEqual(BLOCK, block['Name'])
#self.assertEqual(377, block['NumberOfEvents'])
self.assertEqual(150780132, block['BlockSize'])
self.assertEqual(2, block['NumberOfFiles'])
# possibly fragile but assume block located at least at cern
sites = [
x['Name'] for x in block['StorageElementList']
if x['Name'].find('cern.ch') > -1
]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo,
DATASET + 'blah')
self.assertFalse(
self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK + 'asas'))
@attr("integration")
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.listFileBlocks(DATASET)
# block is closed
block = self.dbs.listFileBlocks(DATASET,
blockName=BLOCK,
onlyClosedBlocks=True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
@attr("integration")
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
@attr("integration")
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertFalse(self.dbs.blockExists(DATASET + '#somethingelse'))
@attr("integration")
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(
FILE in
[x['LogicalFileName'] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock,
DATASET + '#blah')
@attr("integration")
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
# hope PromptReco doesn't get deleted
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listFilesInBlockWithParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60'
)
self.assertEqual(1, len(files))
self.assertEqual(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60',
files[0]['Block']['Name'])
self.assertEqual(
'/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
files[0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents,
BLOCK + 'asas')
@attr("integration")
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in self.dbs.lfnsInBlock(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + 'asas')
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
WRONG_BLOCK = BLOCK[:-4] + 'abcd'
BLOCK2 = '/HighPileUp/Run2011A-v1/RAW#6021175e-cbfb-11e0-80a9-003048caaace'
DBS_BLOCK = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#fb978442-a61b-413a-b4f4-526e6cdb142e'
DBS_BLOCK2 = '/GenericTTbar/hernan-140317_231446_crab_JH_ASO_test_T2_ES_CIEMAT_5000_100_140318_0014-'+\
'ea0972193530f531086947d06eb0f121/USER#0b04d417-d734-4ef2-88b0-392c48254dab'
self.dbs = DBSReader(
'https://cmsweb.cern.ch/dbs/prod/phys03/DBSReader/')
# assume one site is cern
sites = [
x for x in self.dbs.listFileBlockLocation(BLOCK)
if x and x.find('cern.ch') > -1
]
self.assertTrue(sites)
#This block is only found on DBS
self.assertTrue(self.dbs.listFileBlockLocation(DBS_BLOCK))
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(WRONG_BLOCK))
#test bulk call:
## two blocks in phedex
self.assertEqual(2, len(self.dbs.listFileBlockLocation([BLOCK,
BLOCK2])))
## one block in phedex one does not exist
self.assertEqual(
1, len(self.dbs.listFileBlockLocation([BLOCK, WRONG_BLOCK])))
## one in phedex one in dbs
self.assertEqual(
2, len(self.dbs.listFileBlockLocation([BLOCK, DBS_BLOCK])))
## two in dbs
self.assertEqual(
2, len(self.dbs.listFileBlockLocation([DBS_BLOCK, DBS_BLOCK2])))
## one in DBS and one does not exist
self.assertEqual(
1, len(self.dbs.listFileBlockLocation([DBS_BLOCK, WRONG_BLOCK])))
@attr("integration")
def testGetFileBlock(self):
"""getFileBlock returns block"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block['Files']))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock,
BLOCK + 'asas')
@attr("integration")
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlockWithParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60'
)
self.assertEqual(len(block), 1)
block = block[
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60']
self.assertEqual(
'/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
block['Files'][0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents,
BLOCK + 'asas')
@attr("integration")
def testGetFiles(self):
"""getFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
@attr("integration")
def testListBlockParents(self):
"""listBlockParents returns block parents"""
self.dbs = DBSReader(self.endpoint)
parents = self.dbs.listBlockParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60'
)
self.assertEqual(1, len(parents))
self.assertEqual(
'/Jet/Run2011A-v1/RAW#37cf2a40-4e0e-11e0-9833-00151755cb60',
parents[0]['Name'])
sites = [
x for x in parents[0]['StorageElementList']
if x.find("cern.ch") > -1
]
self.assertTrue(sites)
self.assertFalse(
self.dbs.listBlockParents(
'/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60dsl'
))
@attr("integration")
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
@attr("integration")
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.dbs = DBSReader(self.endpoint)
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertFalse(self.dbs.blockToDatasetPath(BLOCK + 'asas'))
class DBSReaderTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
#endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
endpoint = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
self.dbs = DBSReader(endpoint)
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
results = self.dbs.listPrimaryDatasets('Jet*')
self.assertTrue('Jet' in results)
self.assertTrue('JetMET' in results)
self.assertTrue('JetMETTau' in results)
self.assertFalse(self.dbs.listPrimaryDatasets('DoesntExist'))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
dataset = self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v1')
self.assertEqual(1, len(dataset))
self.assertEqual(['/Jet/Run2011A-v1/RAW'], dataset[0]['PathList'])
self.assertEqual('Run2011A-v1', dataset[0]['Name'])
self.assertFalse(self.dbs.matchProcessedDatasets('Jet', 'RAW', 'Run2011A-v666'))
@attr("integration")
def testlistRuns(self):
"""listRuns returns known runs"""
runs = self.dbs.listRuns(dataset = DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(dataset = DATASET, block = BLOCK)
self.assertEqual([173657], runs)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
datasets = self.dbs.listProcessedDatasets('Jet', 'RAW')
self.assertTrue('Run2011A-v1' in datasets)
self.assertTrue('Run2011B-v1' in datasets)
self.assertFalse(self.dbs.listProcessedDatasets('Jet', 'blah'))
self.assertFalse(self.dbs.listProcessedDatasets('blah', 'RAW'))
@attr("integration")
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
@attr("integrtion")
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset['path'], DATASET)
self.assertEqual(dataset['block'], '')
self.assertEqual(dataset['NumberOfEvents'], '22075')
self.assertEqual(dataset['NumberOfBlocks'], '46')
self.assertEqual(dataset['total_size'], '4001680824')
self.assertEqual(dataset['NumberOfFiles'], '49')
self.assertEqual(dataset['NumberOfLumis'], '7223')
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block['path'], '')
self.assertEqual(block['block'], BLOCK)
self.assertEqual(block['NumberOfEvents'], '377')
self.assertEqual(block['NumberOfBlocks'], '1')
self.assertEqual(block['total_size'], '150780132')
self.assertEqual(block['NumberOfFiles'], '2')
self.assertEqual(block['NumberOfLumis'], '94')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET + 'blah')
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET, BLOCK + 'asas')
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName = BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block['Name'] in [x['Name'] for x in blocks])
self.assertEqual(BLOCK, block['Name'])
#self.assertEqual(377, block['NumberOfEvents'])
self.assertEqual(150780132, block['BlockSize'])
self.assertEqual(2, block['NumberOfFiles'])
# possibly fragile but assume block located at least at cern
sites = [x['Name'] for x in block['StorageElementList'] if x['Name'].find('cern.ch') > -1]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo, DATASET + 'blah')
self.assertFalse(self.dbs.getFileBlocksInfo(DATASET, blockName = BLOCK + 'asas'))
@attr("integration")
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
blocks = self.dbs.listFileBlocks(DATASET)
# block is closed
block = self.dbs.listFileBlocks(DATASET, blockName = BLOCK, onlyClosedBlocks = True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
@attr("integration")
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
@attr("integration")
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertFalse(self.dbs.blockExists(DATASET + '#somethingelse'))
@attr("integration")
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.assertTrue(FILE in [x['LogicalFileName'] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock, DATASET + '#blah')
@attr("integration")
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
# hope PromptReco doesn't get deleted
files = self.dbs.listFilesInBlockWithParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60')
self.assertEqual(1, len(files))
self.assertEqual('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60', files[0]['Block']['Name'])
self.assertEqual('/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
files[0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents, BLOCK + 'asas')
@attr("integration")
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.assertTrue(FILE in self.dbs.lfnsInBlock(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + 'asas')
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
# assume one site is cern
sites = [x for x in self.dbs.listFileBlockLocation(BLOCK) if x.find('cern.ch') > -1]
self.assertTrue(sites)
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(BLOCK + 'blah'))
@attr("integration")
def testGetFileBlock(self):
"""getFileBlock returns block"""
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block['Files']))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock, BLOCK + 'asas')
@attr("integration")
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
block = self.dbs.getFileBlockWithParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60')
self.assertEqual(len(block), 1)
block = block['/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60']
self.assertEqual('/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root',
block['Files'][0]['ParentList'][0]['LogicalFileName'])
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents, BLOCK + 'asas')
@attr("integration")
def testGetFiles(self):
"""getFiles returns files in dataset"""
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
@attr("integration")
def testListBlockParents(self):
"""listBlockParents returns block parents"""
parents = self.dbs.listBlockParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60')
self.assertEqual(1, len(parents))
self.assertEqual('/Jet/Run2011A-v1/RAW#37cf2a40-4e0e-11e0-9833-00151755cb60',
parents[0]['Name'])
sites = [x for x in parents[0]['StorageElementList'] if x.find("cern.ch") > -1]
self.assertTrue(sites)
self.assertFalse(self.dbs.listBlockParents('/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60dsl'))
@attr("integration")
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
@attr("integration")
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertFalse(self.dbs.blockToDatasetPath(BLOCK + 'asas'))
def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl):
"""
pileupDict is a Python dictionary containing particular pileup
configuration information. Query DBS on given dataset contained
now in both input defaultArguments as well as in the pileupDict
and compare values.
"""
args = {}
args["version"] = "DBS_2_0_9"
args["mode"] = "GET"
reader = DBSReader(dbsUrl, **args)
inputArgs = defaultArguments["PileupConfig"]
self.assertEqual(len(inputArgs), len(pileupDict),
"Number of pileup types different.")
for pileupType in inputArgs:
m = ("pileup type '%s' not in PileupFetcher-produced pileup "
"configuration: '%s'" % (pileupType, pileupDict))
self.assertTrue(pileupType in pileupDict, m)
# now query DBS for compare actual results on files lists for each
# pileup type and dataset and location (storage element names)
# pileupDict is saved in the file and now comparing items of this
# configuration with actual DBS results, the structure of pileupDict:
# {"pileupTypeA": {"BlockA": {"FileList": [], "StorageElementNames": []},
# "BlockB": {"FileList": [], "StorageElementName": []}, ....}
for pileupType, datasets in inputArgs.items():
# this is from the pileup configuration produced by PileupFetcher
blockDict = pileupDict[pileupType]
for dataset in datasets:
dbsFileBlocks = reader.listFileBlocks(dataset=dataset)
for dbsFileBlockName in dbsFileBlocks:
fileList = [
] # list of files in the block (dbsFile["LogicalFileName"])
storageElemNames = set() # list of StorageElementName
# each DBS block has a list under 'StorageElementList', iterate over
storageElements = reader.listFileBlockLocation(
dbsFileBlockName)
for storElem in storageElements:
storageElemNames.add(storElem)
# now get list of files in the block
dbsFiles = reader.listFilesInBlock(dbsFileBlockName)
for dbsFile in dbsFiles:
fileList.append(dbsFile["LogicalFileName"])
# now compare the sets:
m = (
"StorageElementNames don't agree for pileup type '%s', "
"dataset '%s' in configuration: '%s'" %
(pileupType, dataset, pileupDict))
self.assertEqual(
set(blockDict[dbsFileBlockName]
["StorageElementNames"]), storageElemNames, m)
m = (
"FileList don't agree for pileup type '%s', dataset '%s' "
" in configuration: '%s'" %
(pileupType, dataset, pileupDict))
print(fileList)
print(blockDict[dbsFileBlockName]["FileList"])
self.assertEqual(
sorted(blockDict[dbsFileBlockName]["FileList"]),
sorted(fileList))
class DBSReaderTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
# self.endpoint = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet"
self.endpoint = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
self.dbs = None
return
@attr("integration")
def testListPrimaryDatasets(self):
"""
listPrimaryDatasets returns known primary datasets
"""
self.dbs = DBSReader(self.endpoint)
results = self.dbs.listPrimaryDatasets("Jet*")
self.assertTrue("Jet" in results)
self.assertTrue("JetMET" in results)
self.assertTrue("JetMETTau" in results)
self.assertFalse(self.dbs.listPrimaryDatasets("DoesntExist"))
return
@attr("integration")
def testMatchProcessedDatasets(self):
"""
matchProcessedDatasets returns known processed datasets
"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.matchProcessedDatasets("Jet", "RAW", "Run2011A-v1")
self.assertEqual(1, len(dataset))
self.assertEqual(["/Jet/Run2011A-v1/RAW"], dataset[0]["PathList"])
self.assertEqual("Run2011A-v1", dataset[0]["Name"])
self.assertFalse(self.dbs.matchProcessedDatasets("Jet", "RAW", "Run2011A-v666"))
@attr("integration")
def testlistRuns(self):
"""listRuns returns known runs"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRuns(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(174074 in runs)
runs = self.dbs.listRuns(dataset=DATASET, block=BLOCK)
self.assertEqual([173657], runs)
@attr("integration")
def testlistRunLumis(self):
"""listRunLumis returns known runs and lumicounts"""
self.dbs = DBSReader(self.endpoint)
runs = self.dbs.listRunLumis(dataset=DATASET)
self.assertEqual(46, len(runs))
self.assertTrue(173692 in runs)
self.assertEqual(runs[173692], 2782)
runs = self.dbs.listRuns(dataset=DATASET, block=BLOCK)
self.assertEqual({173657: 94}, runs)
@attr("integration")
def testListProcessedDatasets(self):
"""listProcessedDatasets returns known processed datasets"""
self.dbs = DBSReader(self.endpoint)
datasets = self.dbs.listProcessedDatasets("Jet", "RAW")
self.assertTrue("Run2011A-v1" in datasets)
self.assertTrue("Run2011B-v1" in datasets)
self.assertFalse(self.dbs.listProcessedDatasets("Jet", "blah"))
self.assertFalse(self.dbs.listProcessedDatasets("blah", "RAW"))
@attr("integration")
def testlistDatasetFiles(self):
"""listDatasetFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listDatasetFiles(DATASET)
self.assertEqual(49, len(files))
self.assertTrue(FILE in files)
@attr("integrtion")
def testGetDBSSummaryInfo(self):
"""getDBSSummaryInfo returns summary of dataset and block"""
self.dbs = DBSReader(self.endpoint)
dataset = self.dbs.getDBSSummaryInfo(DATASET)
self.assertEqual(dataset["path"], DATASET)
self.assertEqual(dataset["block"], "")
self.assertEqual(dataset["NumberOfEvents"], "22075")
self.assertEqual(dataset["NumberOfBlocks"], "46")
self.assertEqual(dataset["total_size"], "4001680824")
self.assertEqual(dataset["NumberOfFiles"], "49")
self.assertEqual(dataset["NumberOfLumis"], "7223")
block = self.dbs.getDBSSummaryInfo(DATASET, BLOCK)
self.assertEqual(block["path"], "")
self.assertEqual(block["block"], BLOCK)
self.assertEqual(block["NumberOfEvents"], "377")
self.assertEqual(block["NumberOfBlocks"], "1")
self.assertEqual(block["total_size"], "150780132")
self.assertEqual(block["NumberOfFiles"], "2")
self.assertEqual(block["NumberOfLumis"], "94")
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET + "blah")
self.assertRaises(DBSReaderError, self.dbs.getDBSSummaryInfo, DATASET, BLOCK + "asas")
@attr("integration")
def testGetFileBlocksInfo(self):
"""getFileBlocksInfo returns block info, including location lookup"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.getFileBlocksInfo(DATASET)
block = self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK)
self.assertEqual(1, len(block))
block = block[0]
self.assertEqual(46, len(blocks))
self.assertTrue(block["Name"] in [x["Name"] for x in blocks])
self.assertEqual(BLOCK, block["Name"])
# self.assertEqual(377, block['NumberOfEvents'])
self.assertEqual(150780132, block["BlockSize"])
self.assertEqual(2, block["NumberOfFiles"])
# possibly fragile but assume block located at least at cern
sites = [x["Name"] for x in block["StorageElementList"] if x["Name"].find("cern.ch") > -1]
self.assertTrue(sites)
# weird error handling - depends on whether block or dataset is missing
self.assertRaises(DBSReaderError, self.dbs.getFileBlocksInfo, DATASET + "blah")
self.assertFalse(self.dbs.getFileBlocksInfo(DATASET, blockName=BLOCK + "asas"))
@attr("integration")
def testListFileBlocks(self):
"""listFileBlocks returns block names in dataset"""
self.dbs = DBSReader(self.endpoint)
blocks = self.dbs.listFileBlocks(DATASET)
# block is closed
block = self.dbs.listFileBlocks(DATASET, blockName=BLOCK, onlyClosedBlocks=True)[0]
self.assertEqual(block, BLOCK)
self.assertTrue(BLOCK in block)
@attr("integration")
def testListOpenFileBlocks(self):
"""listOpenFileBlocks finds open blocks"""
# hard to find a dataset with open blocks, so don't bother
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.listOpenFileBlocks(DATASET))
@attr("integration")
def testBlockExists(self):
"""blockExists returns existence of blocks"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(self.dbs.blockExists(BLOCK))
self.assertFalse(self.dbs.blockExists(DATASET + "#somethingelse"))
@attr("integration")
def testListFilesInBlock(self):
"""listFilesInBlock returns files in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in [x["LogicalFileName"] for x in self.dbs.listFilesInBlock(BLOCK)])
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlock, DATASET + "#blah")
@attr("integration")
def testListFilesInBlockWithParents(self):
"""listFilesInBlockWithParents gets files with parents for a block"""
# hope PromptReco doesn't get deleted
self.dbs = DBSReader(self.endpoint)
files = self.dbs.listFilesInBlockWithParents(
"/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60"
)
self.assertEqual(1, len(files))
self.assertEqual(
"/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60", files[0]["Block"]["Name"]
)
self.assertEqual(
"/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root",
files[0]["ParentList"][0]["LogicalFileName"],
)
self.assertRaises(DBSReaderError, self.dbs.listFilesInBlockWithParents, BLOCK + "asas")
@attr("integration")
def testLfnsInBlock(self):
"""lfnsInBlock returns lfns in block"""
self.dbs = DBSReader(self.endpoint)
self.assertTrue(FILE in self.dbs.lfnsInBlock(BLOCK))
self.assertRaises(DBSReaderError, self.dbs.lfnsInBlock, BLOCK + "asas")
@attr("integration")
def testListFileBlockLocation(self):
"""listFileBlockLocation returns block location"""
self.dbs = DBSReader(self.endpoint)
# assume one site is cern
sites = [x for x in self.dbs.listFileBlockLocation(BLOCK) if x.find("cern.ch") > -1]
self.assertTrue(sites)
# doesn't raise on non-existant block
self.assertFalse(self.dbs.listFileBlockLocation(BLOCK + "blah"))
@attr("integration")
def testGetFileBlock(self):
"""getFileBlock returns block"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlock(BLOCK)
self.assertEqual(len(block), 1)
block = block[BLOCK]
self.assertEqual(2, len(block["Files"]))
self.assertRaises(DBSReaderError, self.dbs.getFileBlock, BLOCK + "asas")
@attr("integration")
def testGetFileBlockWithParents(self):
"""getFileBlockWithParents returns block and parents"""
self.dbs = DBSReader(self.endpoint)
block = self.dbs.getFileBlockWithParents(
"/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60"
)
self.assertEqual(len(block), 1)
block = block["/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60"]
self.assertEqual(
"/store/data/Run2011A/Jet/RAW/v1/000/160/433/24B46223-0D4E-E011-B573-0030487C778E.root",
block["Files"][0]["ParentList"][0]["LogicalFileName"],
)
self.assertRaises(DBSReaderError, self.dbs.getFileBlockWithParents, BLOCK + "asas")
@attr("integration")
def testGetFiles(self):
"""getFiles returns files in dataset"""
self.dbs = DBSReader(self.endpoint)
files = self.dbs.getFiles(DATASET)
self.assertEqual(len(files), 46)
@attr("integration")
def testListBlockParents(self):
"""listBlockParents returns block parents"""
self.dbs = DBSReader(self.endpoint)
parents = self.dbs.listBlockParents("/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60")
self.assertEqual(1, len(parents))
self.assertEqual("/Jet/Run2011A-v1/RAW#37cf2a40-4e0e-11e0-9833-00151755cb60", parents[0]["Name"])
sites = [x for x in parents[0]["StorageElementList"] if x.find("cern.ch") > -1]
self.assertTrue(sites)
self.assertFalse(
self.dbs.listBlockParents("/Jet/Run2011A-PromptReco-v1/RECO#f8d36af3-4fb6-11e0-9d39-00151755cb60dsl")
)
@attr("integration")
def testBlockIsOpen(self):
"""blockIsOpen checks if a block is open"""
self.dbs = DBSReader(self.endpoint)
self.assertFalse(self.dbs.blockIsOpen(BLOCK))
@attr("integration")
def testBlockToDatasetPath(self):
"""blockToDatasetPath extracts path from block name"""
self.dbs = DBSReader(self.endpoint)
self.assertEqual(self.dbs.blockToDatasetPath(BLOCK), DATASET)
self.assertFalse(self.dbs.blockToDatasetPath(BLOCK + "asas"))