class PhEDExTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.phedexApi = PhEDEx({"endpoint": phedexTestDS,
"method": "POST"})
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(result["phedex"]["injected"],
{"stats": {"closed_datasets": 0, "closed_blocks": 0,
"new_blocks": 0, "new_datasets": 1,
"new_files": 0}})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testSub.getDatasetPaths())
result = self.phedexApi.subscribe(testSub, xmlData)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue(requestIDs[0].has_key("id"),
"Error: Missing request ID")
return
@attr("integration")
def testNodeMap(self):
"""
_testNodeMap_
Verify that the node map can be retrieve from PhEDEx and that the
getNodeSE() and getNodeNames() methods work correctly.
"""
self.failUnless(self.phedexApi.getNodeSE("T2_FR_GRIF_LLR") == "polgrid4.in2p3.fr")
self.failUnless(self.phedexApi.getNodeNames("cmssrm.fnal.gov") == ["T1_US_FNAL_Buffer",
"T1_US_FNAL_MSS"])
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS",
"Saturn", requestOnly = "n")
datasetSpec = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testDatasetSub.getDatasetPaths())
self.phedexApi.subscribe(testDatasetSub, datasetSpec)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn",
level = "block", requestOnly = "n")
self.phedexApi.subscribe(testBlockSub, blockSpec)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], set(["T1_UK_RAL_MSS"]),
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
@attr('integration')
def testXMLJSON(self):
"""
Test XML and JSON in the same scope
"""
site = 'T1_US_FNAL_Buffer'
dict = {}
dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
phedexJSON = PhEDEx(responseType='json', dict=dict)
dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
phedexXML = PhEDEx(responseType='xml', dict=dict)
phedexXML.getNodeTFC(site)
tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node' : site})
tfc_map = {}
tfc_map[site] = readTFC(tfc_file)
pfn = tfc_map[site].matchLFN('srmv2', '/store/user/jblow/dir/test.root')
self.failUnless(pfn == 'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root')
self.failUnless(phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
@attr('integration')
def testAuth(self):
"""
_testAuth_
Verify that the auth method works correctly."
"""
self.assertFalse(self.phedexApi.getAuth("datasvc_whatever"))
self.assertTrue(self.phedexApi.getAuth("datasvc_subscribe"))
self.assertTrue(self.phedexApi.getAuth("datasvc_inject"))
return
class PhEDExTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"})
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(
result["phedex"]["injected"], {
"stats": {
"closed_datasets": 0,
"closed_blocks": 0,
"new_blocks": 0,
"new_datasets": 1,
"new_files": 0
}
})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
result = self.phedexApi.subscribe(testSub)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue("id" in requestIDs[0], "Error: Missing request ID")
return
@attr("integration")
def testBestNodeName(self):
"""
_testBestNodeName_
Verify that the node name is Buffer first
"""
self.assertTrue(
self.phedexApi.getBestNodeName("cmssrm.fnal.gov") ==
"T1_US_FNAL_Buffer")
return
@attr("integration")
def testNodeMap(self):
"""
_testNodeMap_
Verify that the node map can be retrieve from PhEDEx and that the
getNodeSE() and getNodeNames() methods work correctly.
"""
self.assertTrue(
self.phedexApi.getNodeSE("T2_FR_GRIF_LLR") == "polgrid4.in2p3.fr")
self.assertTrue(
self.phedexApi.getNodeNames("cmssrm.fnal.gov") ==
["T1_US_FNAL_Buffer", "T1_US_FNAL_MSS"])
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
request_only="n")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
request_only="n")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"},
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'],
['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(
['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'],
['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
@attr('integration')
def testXMLJSON(self):
"""
Test XML and JSON in the same scope
"""
site = 'T1_US_FNAL_Buffer'
httpDict = {
'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"
}
phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
httpDict = {
'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
}
phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)
phedexXML.getNodeTFC(site)
tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
tfc_map = {}
tfc_map[site] = readTFC(tfc_file)
pfn = tfc_map[site].matchLFN('srmv2',
'/store/user/jblow/dir/test.root')
self.assertTrue(
pfn ==
'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root'
)
self.assertTrue(
phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
@attr('integration')
def testAuth(self):
"""
_testAuth_
Verify that the auth method works correctly."
"""
self.assertFalse(self.phedexApi.getAuth("datasvc_whatever"))
self.assertTrue(self.phedexApi.getAuth("datasvc_subscribe"))
self.assertTrue(self.phedexApi.getAuth("datasvc_inject"))
return
class DBS3Reader:
"""
_DBSReader_
General API for reading data from DBS
"""
def __init__(self, url, **contact):
# instantiate dbs api object
try:
self.dbs = DbsApi(url, **contact)
except dbsClientException as ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
# connection to PhEDEx (Use default endpoint url)
self.phedex = PhEDEx(responseType = "json")
def _getLumiList(self, blockName = None, lfns = None):
"""
currently only take one lfn but dbs api need be updated
"""
try:
if blockName:
lumiLists = self.dbs.listFileLumis(block_name=blockName, validFileOnly = 1)
elif lfns:
lumiLists = []
for slfn in slicedIterator(lfns, 50):
lumiLists.extend(self.dbs.listFileLumiArray(logical_file_name = slfn))
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFileLumiArray(%s)\n" % lfns
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
lumiDict = {}
for lumisItem in lumiLists:
lumiDict.setdefault(lumisItem['logical_file_name'], [])
item = {}
item["RunNumber"] = lumisItem['run_num']
item['LumiSectionNumber'] = lumisItem['lumi_section_num']
lumiDict[lumisItem['logical_file_name']].append(item)
return lumiDict
def listPrimaryDatasets(self, match = '*'):
"""
_listPrimaryDatasets_
return a list of primary datasets, The full dataset name must be provided
pattern based mathcing is no longer supported.
If no expression is provided, all datasets are returned
"""
try:
result = self.dbs.listPrimaryDatasets(primary_ds_name = match)
except dbsClientException as ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
result = [ x['primary_ds_name'] for x in result ]
return result
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
result = []
try:
datasets = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = tier, detail = True)
except dbsClientException as ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
for dataset in datasets:
dataset = remapDBS3Keys(dataset, processed_ds_name = 'Name')
dataset['PathList'] = [dataset['dataset']]
if dataset['Name'] == process:
result.append(dataset)
return result
def listRuns(self, dataset = None, block = None):
"""
it gets list of DbsRun object but for our purpose
only list of number is collected.
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
runs = []
try:
if block:
results = self.dbs.listRuns(block_name = block)
else:
results = self.dbs.listRuns(dataset = dataset)
except dbsClientException as ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
[runs.extend(x['run_num']) for x in results]
return runs
def listRunLumis(self, dataset = None, block = None):
"""
It gets a list of DBSRun objects and returns the number of lumisections per run
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
try:
if block:
results = self.dbs.listRuns(block_name = block)
else:
results = self.dbs.listRuns(dataset = dataset)
except dbsClientException as ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
# send runDict format as result, this format is for sync with dbs2 call
# which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis
# So it returns {run_number: None}
# TODO: After DBS2 is completely removed change the return format more sensible one
runDict = {}
for x in results:
for runNumber in x["run_num"]:
runDict[runNumber] = None
return runDict
def listProcessedDatasets(self, primary, dataTier = '*'):
"""
_listProcessedDatasets_
return a list of Processed datasets for the primary and optional
data tier value
"""
try:
result = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = dataTier)
except dbsClientException as ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
result = [ x['dataset'].split('/')[2] for x in result ]
return result
def listDatasetFiles(self, datasetPath):
"""
_listDatasetFiles_
Get list of files for dataset
"""
return [ x['logical_file_name'] for x in self.dbs.listFileArray(dataset = datasetPath)]
def listDatatiers(self):
"""
_listDatatiers_
Get a list of datatiers known by DBS.
"""
return [ tier['data_tier_name'] for tier in self.dbs.listDataTiers() ]
def listDatasetFileDetails(self, datasetPath, getParents=False, validFileOnly=1):
"""
TODO: This is completely wrong need to be redone. or be removed - getting dataset altogether
might be to costly
_listDatasetFileDetails_
Get list of lumis, events, and parents for each file in a dataset
Return a dict where the keys are the files, and for each file we have something like:
{ 'NumberOfEvents': 545,
'BlockName': '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace',
'Lumis': {173658: [8, 12, 9, 14, 19, 109, 105]},
'Parents': [],
'Checksum': '22218315',
'Adler32': 'a41a1446',
'FileSize': 286021145,
'ValidFile': 1
}
"""
fileDetails = self.dbs.listFileArray(dataset = datasetPath, validFileOnly = validFileOnly, detail=True)
blocks = set() #the set of blocks of the dataset
#Iterate over the files and prepare the set of blocks and a dict where the keys are the files
files = {}
for f in fileDetails:
blocks.add(f['block_name'])
files[f['logical_file_name']] = remapDBS3Keys(f, stringify = True)
files[f['logical_file_name']]['ValidFile'] = f['is_file_valid']
files[f['logical_file_name']]['Lumis'] = {}
files[f['logical_file_name']]['Parents'] = []
#Iterate over the blocks and get parents and lumis
for blockName in blocks:
#get the parents
if getParents:
parents = self.dbs.listFileParents(block_name=blockName)
for p in parents:
if p['logical_file_name'] in files: #invalid files are not there if validFileOnly=1
files[p['logical_file_name']]['Parents'].extend(p['parent_logical_file_name'])
#get the lumis
file_lumis = self.dbs.listFileLumis(block_name=blockName)
for f in file_lumis:
if f['logical_file_name'] in files: #invalid files are not there if validFileOnly=1
if f['run_num'] in files[f['logical_file_name']]['Lumis']:
files[f['logical_file_name']]['Lumis'][f['run_num']].extend(f['lumi_section_num'])
else:
files[f['logical_file_name']]['Lumis'][f['run_num']] = f['lumi_section_num']
return files
def crossCheck(self, datasetPath, *lfns):
"""
_crossCheck_
For the dataset provided, check that the lfns listed all exist
in the dataset.
Return the list of lfns that are in the dataset
"""
allLfns = self.dbs.listFileArray(dataset = datasetPath, validFileOnly = 1, detail = False)
setOfAllLfns = set(allLfns)
setOfKnownLfns = set(lfns)
return list(setOfAllLfns.intersection(setOfKnownLfns))
def crossCheckMissing(self, datasetPath, *lfns):
"""
_crossCheckMissing_
As cross check, but return value is a list of files that
are *not* known by DBS
"""
allLfns = self.dbs.listFileArray(dataset = datasetPath, validFileOnly = 1, detail = False)
setOfAllLfns = set(allLfns)
setOfKnownLfns = set(lfns)
knownFiles = setOfAllLfns.intersection(setOfKnownLfns)
unknownFiles = setOfKnownLfns.difference(knownFiles)
return list(unknownFiles)
def getDBSSummaryInfo(self, dataset = None, block = None):
"""
Get dataset summary includes # of files, events, blocks and total size
"""
#FIXME: Doesnt raise exceptions on missing data as old api did
if dataset:
self.checkDatasetPath(dataset)
try:
if block:
summary = self.dbs.listFileSummaries(block_name = block, validFileOnly = 1)
else: # dataset case dataset shouldn't be None
summary = self.dbs.listFileSummaries(dataset = dataset, validFileOnly = 1)
except Exception as ex:
msg = "Error in DBSReader.getDBSSummaryInfo(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not summary or summary[0].get('file_size') is None: # appears to indicate missing dataset
msg = "DBSReader.listDatasetSummary(%s, %s): No matching data"
raise DBSReaderError(msg % (dataset, block))
result = remapDBS3Keys(summary[0], stringify = True)
result['path'] = dataset if not block else ''
result['block'] = block if block else ''
return result
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = False,
blockName = None, locations = True):
"""
"""
self.checkDatasetPath(dataset)
args = {'dataset' : dataset, 'detail' : True}
if blockName:
args['block_name'] = blockName
try:
blocks = self.dbs.listBlocks(**args)
except Exception as ex:
msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
blocks = [remapDBS3Keys(block, stringify = True, block_name = 'Name') for block in blocks]
# only raise if blockName not specified - mimic dbs2 error handling
if not blocks and not blockName:
msg = "DBSReader.getFileBlocksInfo(%s, %s): No matching data"
raise DBSReaderError(msg % (dataset, blockName))
if locations:
for block in blocks:
block['StorageElementList'] = [{'Name' : x} for x in self.listFileBlockLocation(block['Name'])]
if onlyClosedBlocks:
return [x for x in blocks if str(x['OpenForWriting']) != "1"]
return blocks
def listFileBlocks(self, dataset, onlyClosedBlocks = False,
blockName = None):
"""
_listFileBlocks_
Retrieve a list of fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
args = {'dataset' : dataset, 'detail' : False}
if blockName:
args['block_name'] = blockName
if onlyClosedBlocks:
args['detail'] = True
try:
blocks = self.dbs.listBlocks(**args)
except dbsClientException as ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if onlyClosedBlocks:
result = [
x['block_name'] for x in blocks \
if str(x['open_for_writing']) != "1"
]
else:
result = [ x['block_name'] for x in blocks ]
return result
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset = dataset, detail = True)
except dbsClientException as ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
result = [
x['block_name'] for x in blocks \
if str(x['open_for_writing']) == "1"
]
return result
def blockExists(self, fileBlockName):
"""
_blockExists_
Check to see if block with name provided exists in the DBS
Instance.
Return True if exists, False if not
"""
self.checkBlockName(fileBlockName)
try:
blocks = self.dbs.listBlocks(block_name = fileBlockName)
except Exception as ex:
msg = "Error in "
msg += "DBSReader.blockExists(%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if len(blocks) == 0:
return False
return True
def listFilesInBlock(self, fileBlockName, lumis = True):
"""
_listFilesInBlock_
Get a list of files in the named fileblock
TODO: lumis can be false when lumi splitting is not required
However WMBSHelper expect file['LumiList'] to get the run number
so for now it will be always true.
We need to clean code up when dbs2 is completely deprecated.
calling lumis for run number is expensive.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFileArray(block_name = fileBlockName, validFileOnly = 1, detail = True)
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if lumis:
lumiDict = self._getLumiList(blockName = fileBlockName)
result = []
for fileInfo in files:
if lumis:
fileInfo["LumiList"] = lumiDict[fileInfo['logical_file_name']]
result.append(remapDBS3Keys(fileInfo, stringify = True))
return result
def listFilesInBlockWithParents(self, fileBlockName, lumis = True):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
TODO: lumis can be false when lumi splitting is not required
However WMBSHelper expect file['LumiList'] to get the run number
so for now it will be always true.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
#TODO: shoud we get only valid block for this?
files = self.dbs.listFileParents(block_name = fileBlockName)
fileDetails = self.listFilesInBlock(fileBlockName, lumis)
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName,)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
childByParents = defaultdict(list)
for f in files:
# Probably a child can have more than 1 parent file
for fp in f['parent_logical_file_name']:
childByParents[fp].append(f['logical_file_name'])
parentsLFNs = childByParents.keys()
parentFilesDetail = []
#TODO: slicing parentLFNs util DBS api is handling that.
#Remove slicing if DBS api handles
for pLFNs in slicedIterator(parentsLFNs, 50):
parentFilesDetail.extend(self.dbs.listFileArray(logical_file_name = pLFNs, detail = True))
if lumis:
parentLumis = self._getLumiList(lfns = parentsLFNs)
parentsByLFN = defaultdict(list)
for pf in parentFilesDetail:
parentLFN = pf['logical_file_name']
dbsFile = remapDBS3Keys(pf, stringify = True)
if lumis:
dbsFile["LumiList"] = parentLumis[parentLFN]
for childLFN in childByParents[parentLFN]:
parentsByLFN[childLFN].append(dbsFile)
for fileInfo in fileDetails:
fileInfo["ParentList"] = parentsByLFN[fileInfo['logical_file_name']]
return fileDetails
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.lfnsInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
lfns = self.dbs.listFileArray(block_name = fileBlockName, validFileOnly = 1, detail = False)
return lfns
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
def listFileBlockLocation(self, fileBlockName, dbsOnly = False, phedexNodes=False):
"""
_listFileBlockLocation_
Get origin_site_name of a block
"""
blockNames = [fileBlockName] if isinstance(fileBlockName, basestring) else fileBlockName
for block in blockNames:
self.checkBlockName(block)
blockInfo = {}
if not dbsOnly:
try:
blockInfo = self.phedex.getReplicaSEForBlocks(phedexNodes=phedexNodes, block=blockNames, complete='y')
except Exception as ex:
msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockName
msg += "%s\n" % str(ex)
raise Exception(msg)
if not blockInfo or len(blockInfo) != len(blockNames): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
dbsOnly = True
blockNames = set(blockNames) - set(blockInfo) #get the blocks we did not find information in phedex
if dbsOnly:
try:
for block in blockNames:
res = self.dbs.listBlockOrigin(block_name = block)
if res:
blockInfo[block] = [res[0]['origin_site_name']]
except dbsClientException as ex:
msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not any(blockInfo.values()): # no data location from dbs
return list()
#removing duplicates and 'UNKNOWN entries
locations = {}
node_filter_list = set(['UNKNOWN', None])
for name, nodes in blockInfo.iteritems():
final_nodes = set()
for n in nodes:
if n in node_filter_list:
continue
try:
cmsname(n)
except AssertionError: ## is SE
n = self.phedex.getNodeNames(n) if phedexNodes else [n]
else: ## not SE i.e. phedexNode
n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
final_nodes = final_nodes.union(n)
locations[name] = list(final_nodes - node_filter_list)
#returning single list if a single block is passed
if isinstance(fileBlockName, basestring):
locations = locations[fileBlockName]
return locations
def getFileBlock(self, fileBlockName):
"""
_getFileBlock_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : { LFN : Events },
}
}
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.getFileBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
result = {fileBlockName: {
"StorageElements" : self.listFileBlockLocation(fileBlockName),
"Files" : self.listFilesInBlock(fileBlockName),
"IsOpen" : self.blockIsOpen(fileBlockName)
}
}
return result
def getFileBlockWithParents(self, fileBlockName):
"""
_getFileBlockWithParents_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : dictionaries representing each file
}
}
files
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.getFileBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
result = {fileBlockName: {
"StorageElements" : self.listFileBlockLocation(fileBlockName),
"Files" : self.listFilesInBlockWithParents(fileBlockName),
"IsOpen" : self.blockIsOpen(fileBlockName)
}
}
return result
def getFiles(self, dataset, onlyClosedBlocks = False):
"""
_getFiles_
Returns a dictionary of block names for the dataset where
each block constists of a dictionary containing the StorageElements
for that block and the files in that block by LFN mapped to NEvents
"""
result = {}
blocks = self.listFileBlocks(dataset, onlyClosedBlocks)
[ result.update(self.getFileBlock(x)) for x in blocks ]
return result
def listBlockParents(self, blockName):
"""Get parent blocks for block"""
result = []
self.checkBlockName(blockName)
blocks = self.dbs.listBlockParents(block_name = blockName)
for block in blocks:
toreturn = {'Name' : block['parent_block_name']}
toreturn['StorageElementList'] = self.listFileBlockLocation(toreturn['Name'])
result.append(toreturn)
return result
def blockIsOpen(self, blockName):
"""
_blockIsOpen_
Return True if named block is open, false if not, or if block
doenst exist
"""
self.checkBlockName(blockName)
blockInstance = self.dbs.listBlocks(block_name = blockName, detail = True)
if len(blockInstance) == 0:
return False
blockInstance = blockInstance[0]
isOpen = blockInstance.get('open_for_writing', 1)
if isOpen == 0:
return False
return True
def blockToDatasetPath(self, blockName):
"""
_blockToDatasetPath_
Given a block name, get the dataset Path associated with that
Block.
Returns the dataset path, or None if not found
"""
self.checkBlockName(blockName)
try:
blocks = self.dbs.listBlocks(block_name = blockName, detail = True)
except Exception as ex:
msg = "Error in "
msg += "DBSReader.blockToDatasetPath(%s)\n" % blockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if blocks == []:
return None
pathname = blocks[-1].get('dataset', None)
return pathname
def listDatasetLocation(self, datasetName, dbsOnly = False):
"""
_listDatasetLocation_
List the origin SEs where there is at least a block of the given
dataset.
"""
self.checkDatasetPath(datasetName)
if not dbsOnly:
try:
blocksInfo = self.phedex.getReplicaSEForBlocks(dataset=[datasetName], complete='y')
except Exception as ex:
msg = "Error while getting block location from PhEDEx for dataset=%s)\n" % datasetName
msg += "%s\n" % str(ex)
raise Exception(msg)
if not blocksInfo: # if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
dbsOnly = True
else:
locations = set(blocksInfo.values()[0])
for blockSites in blocksInfo.values():
locations.intersection_update(blockSites)
if dbsOnly:
try:
blocksInfo = self.dbs.listBlockOrigin(dataset = datasetName)
except dbsClientException as ex:
msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not blocksInfo: # no data location from dbs
return list()
locations = set()
for blockInfo in blocksInfo:
locations.update([blockInfo['origin_site_name']])
locations.difference_update(['UNKNOWN']) # remove entry when SE name is 'UNKNOWN'
return list(locations)
def checkDatasetPath(self, pathName):
"""
_checkDatasetPath_
"""
if pathName in ("", None):
raise DBSReaderError("Invalid Dataset Path name: => %s <=" % pathName)
def checkBlockName(self, blockName):
"""
_checkBlockName_
"""
if blockName in ("", "*", None):
raise DBSReaderError("Invalid Block name: => %s <=" % blockName)
class DBS3Reader:
"""
_DBSReader_
General API for reading data from DBS
"""
def __init__(self, url, **contact):
# instantiate dbs api object
try:
self.dbs = DbsApi(url, **contact)
except dbsClientException as ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
# connection to PhEDEx (Use default endpoint url)
self.phedex = PhEDEx(responseType="json")
def _getLumiList(self, blockName=None, lfns=None):
"""
currently only take one lfn but dbs api need be updated
"""
try:
if blockName:
lumiLists = self.dbs.listFileLumis(block_name=blockName,
validFileOnly=1)
elif lfns:
lumiLists = []
for slfn in slicedIterator(lfns, 50):
lumiLists.extend(
self.dbs.listFileLumiArray(logical_file_name=slfn))
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFileLumiArray(%s)\n" % lfns
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
lumiDict = {}
for lumisItem in lumiLists:
lumiDict.setdefault(lumisItem['logical_file_name'], [])
item = {}
item["RunNumber"] = lumisItem['run_num']
item['LumiSectionNumber'] = lumisItem['lumi_section_num']
lumiDict[lumisItem['logical_file_name']].append(item)
return lumiDict
def listPrimaryDatasets(self, match='*'):
"""
_listPrimaryDatasets_
return a list of primary datasets, The full dataset name must be provided
pattern based mathcing is no longer supported.
If no expression is provided, all datasets are returned
"""
try:
result = self.dbs.listPrimaryDatasets(primary_ds_name=match)
except dbsClientException as ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
result = [x['primary_ds_name'] for x in result]
return result
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
result = []
try:
datasets = self.dbs.listDatasets(primary_ds_name=primary,
data_tier_name=tier,
detail=True)
except dbsClientException as ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
for dataset in datasets:
dataset = remapDBS3Keys(dataset, processed_ds_name='Name')
dataset['PathList'] = [dataset['dataset']]
if dataset['Name'] == process:
result.append(dataset)
return result
def listRuns(self, dataset=None, block=None):
"""
it gets list of DbsRun object but for our purpose
only list of number is collected.
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
runs = []
try:
if block:
results = self.dbs.listRuns(block_name=block)
else:
results = self.dbs.listRuns(dataset=dataset)
except dbsClientException as ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
[runs.extend(x['run_num']) for x in results]
return runs
def listRunLumis(self, dataset=None, block=None):
"""
It gets a list of DBSRun objects and returns the number of lumisections per run
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
try:
if block:
results = self.dbs.listRuns(block_name=block)
else:
results = self.dbs.listRuns(dataset=dataset)
except dbsClientException as ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
# send runDict format as result, this format is for sync with dbs2 call
# which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis
# So it returns {run_number: None}
# TODO: After DBS2 is completely removed change the return format more sensible one
runDict = {}
for x in results:
for runNumber in x["run_num"]:
runDict[runNumber] = None
return runDict
def listProcessedDatasets(self, primary, dataTier='*'):
"""
_listProcessedDatasets_
return a list of Processed datasets for the primary and optional
data tier value
"""
try:
result = self.dbs.listDatasets(primary_ds_name=primary,
data_tier_name=dataTier)
except dbsClientException as ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
result = [x['dataset'].split('/')[2] for x in result]
return result
def listDatasetFiles(self, datasetPath):
"""
_listDatasetFiles_
Get list of files for dataset
"""
return [
x['logical_file_name']
for x in self.dbs.listFileArray(dataset=datasetPath)
]
def listDatasetFileDetails(self, datasetPath, getParents=False):
"""
TODO: This is completely wrong need to be redone. or be removed - getting dataset altogether
might be to costly
_listDatasetFileDetails_
Get list of lumis, events, and parents for each file in a dataset
Return a dict where the keys are the files, and for each file we have something like:
{ 'NumberOfEvents': 545,
'BlockName': '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace',
'Lumis': {173658: [8, 12, 9, 14, 19, 109, 105]},
'Parents': [],
'Checksum': '22218315',
'Adler32': 'a41a1446',
'FileSize': 286021145
}
"""
fileDetails = self.dbs.listFileArray(dataset=datasetPath,
validFileOnly=1,
detail=True)
blocks = set() #the set of blocks of the dataset
#Iterate over the files and prepare the set of blocks and a dict where the keys are the files
files = {}
for f in fileDetails:
blocks.add(f['block_name'])
files[f['logical_file_name']] = remapDBS3Keys(f, stringify=True)
files[f['logical_file_name']]['Lumis'] = {}
files[f['logical_file_name']]['Parents'] = []
#Iterate over the blocks and get parents and lumis
#TODO this part is completely wrong need to be redone
for blockName in blocks:
#get the parents
if getParents:
parents = self.dbs.listFileParents(block_name=blockName)
for p in parents:
if p['logical_file_name'] in files: #invalid files are not there
files[p['logical_file_name']]['Parents'].extend(
p['parent_logical_file_name'])
#get the lumis
file_lumis = self.dbs.listFileLumis(block_name=blockName,
validFileOnly=1)
for f in file_lumis:
if f['run_num'] in files[f['logical_file_name']]['Lumis']:
files[f['logical_file_name']]['Lumis'][
f['run_num']].extend(f['lumi_section_num'])
else:
files[f['logical_file_name']]['Lumis'][
f['run_num']] = f['lumi_section_num']
return files
def crossCheck(self, datasetPath, *lfns):
"""
_crossCheck_
For the dataset provided, check that the lfns listed all exist
in the dataset.
Return the list of lfns that are in the dataset
"""
allLfns = self.dbs.listFileArray(dataset=datasetPath,
validFileOnly=1,
detail=False)
setOfAllLfns = set(allLfns)
setOfKnownLfns = set(lfns)
return list(setOfAllLfns.intersection(setOfKnownLfns))
def crossCheckMissing(self, datasetPath, *lfns):
"""
_crossCheckMissing_
As cross check, but return value is a list of files that
are *not* known by DBS
"""
allLfns = self.dbs.listFileArray(dataset=datasetPath,
validFileOnly=1,
detail=False)
setOfAllLfns = set(allLfns)
setOfKnownLfns = set(lfns)
knownFiles = setOfAllLfns.intersection(setOfKnownLfns)
unknownFiles = setOfKnownLfns.difference(knownFiles)
return list(unknownFiles)
def getDBSSummaryInfo(self, dataset=None, block=None):
"""
Get dataset summary includes # of files, events, blocks and total size
"""
#FIXME: Doesnt raise exceptions on missing data as old api did
if dataset:
self.checkDatasetPath(dataset)
try:
if block:
summary = self.dbs.listFileSummaries(block_name=block,
validFileOnly=1)
else: # dataset case dataset shouldn't be None
summary = self.dbs.listFileSummaries(dataset=dataset,
validFileOnly=1)
except dbsClientException as ex:
msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset,
block)
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not summary or summary[0].get(
'file_size') is None: # appears to indicate missing dataset
msg = "DBSReader.listDatasetSummary(%s, %s): No matching data"
raise DBSReaderError(msg % (dataset, block))
result = remapDBS3Keys(summary[0], stringify=True)
result['path'] = dataset if not block else ''
result['block'] = block if block else ''
return result
def getFileBlocksInfo(self,
dataset,
onlyClosedBlocks=False,
blockName=None,
locations=True):
"""
"""
self.checkDatasetPath(dataset)
args = {'dataset': dataset, 'detail': True}
if blockName:
args['block_name'] = blockName
try:
blocks = self.dbs.listBlocks(**args)
except dbsClientException as ex:
msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
blocks = [
remapDBS3Keys(block, stringify=True, block_name='Name')
for block in blocks
]
# only raise if blockName not specified - mimic dbs2 error handling
if not blocks and not blockName:
msg = "DBSReader.getFileBlocksInfo(%s, %s): No matching data"
raise DBSReaderError(msg % (dataset, blockName))
if locations:
for block in blocks:
block['StorageElementList'] = [{
'Name': x
} for x in self.listFileBlockLocation(block['Name'])]
if onlyClosedBlocks:
return [x for x in blocks if str(x['OpenForWriting']) != "1"]
return blocks
def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None):
"""
_listFileBlocks_
Retrieve a list of fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
args = {'dataset': dataset, 'detail': False}
if blockName:
args['block_name'] = blockName
if onlyClosedBlocks:
args['detail'] = True
try:
blocks = self.dbs.listBlocks(**args)
except dbsClientException as ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if onlyClosedBlocks:
result = [
x['block_name'] for x in blocks \
if str(x['open_for_writing']) != "1"
]
else:
result = [x['block_name'] for x in blocks]
return result
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset=dataset, detail=True)
except dbsClientException as ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
result = [
x['block_name'] for x in blocks \
if str(x['open_for_writing']) == "1"
]
return result
def blockExists(self, fileBlockName):
"""
_blockExists_
Check to see if block with name provided exists in the DBS
Instance.
Return True if exists, False if not
"""
self.checkBlockName(fileBlockName)
try:
blocks = self.dbs.listBlocks(block_name=fileBlockName)
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.blockExists(%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if len(blocks) == 0:
return False
return True
def listFilesInBlock(self, fileBlockName, lumis=True):
"""
_listFilesInBlock_
Get a list of files in the named fileblock
TODO: lumis can be false when lumi splitting is not required
However WMBSHelper expect file['LumiList'] to get the run number
so for now it will be always true.
We need to clean code up when dbs2 is completely deprecated.
calling lumis for run number is expensive.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFileArray(block_name=fileBlockName,
validFileOnly=1,
detail=True)
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if lumis:
lumiDict = self._getLumiList(blockName=fileBlockName)
result = []
for fileInfo in files:
if lumis:
fileInfo["LumiList"] = lumiDict[fileInfo['logical_file_name']]
result.append(remapDBS3Keys(fileInfo, stringify=True))
return result
def listFilesInBlockWithParents(self, fileBlockName, lumis=True):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
TODO: lumis can be false when lumi splitting is not required
However WMBSHelper expect file['LumiList'] to get the run number
so for now it will be always true.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
#TODO: shoud we get only valid block for this?
files = self.dbs.listFileParents(block_name=fileBlockName)
fileDetails = self.listFilesInBlock(fileBlockName, lumis)
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName, )
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
childByParents = defaultdict(list)
for f in files:
# Probably a child can have more than 1 parent file
for fp in f['parent_logical_file_name']:
childByParents[fp].append(f['logical_file_name'])
parentsLFNs = childByParents.keys()
parentFilesDetail = []
#TODO: slicing parentLFNs util DBS api is handling that.
#Remove slicing if DBS api handles
for pLFNs in slicedIterator(parentsLFNs, 50):
parentFilesDetail.extend(
self.dbs.listFileArray(logical_file_name=pLFNs, detail=True))
if lumis:
parentLumis = self._getLumiList(lfns=parentsLFNs)
parentsByLFN = defaultdict(list)
for pf in parentFilesDetail:
parentLFN = pf['logical_file_name']
dbsFile = remapDBS3Keys(pf, stringify=True)
if lumis:
dbsFile["LumiList"] = parentLumis[parentLFN]
for childLFN in childByParents[parentLFN]:
parentsByLFN[childLFN].append(dbsFile)
for fileInfo in fileDetails:
fileInfo["ParentList"] = parentsByLFN[
fileInfo['logical_file_name']]
return fileDetails
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.lfnsInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
lfns = self.dbs.listFileArray(block_name=fileBlockName,
validFileOnly=1,
detail=False)
return lfns
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
def listFileBlockLocation(self,
fileBlockName,
dbsOnly=False,
phedexNodes=False):
"""
_listFileBlockLocation_
Get origin_site_name of a block
"""
blockNames = [fileBlockName] if isinstance(
fileBlockName, basestring) else fileBlockName
for block in blockNames:
self.checkBlockName(block)
blockInfo = {}
if not dbsOnly:
try:
blockInfo = self.phedex.getReplicaSEForBlocks(
phedexNodes=phedexNodes, block=blockNames, complete='y')
except Exception as ex:
msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockName
msg += "%s\n" % str(ex)
raise Exception(msg)
if not blockInfo or len(blockInfo) != len(
blockNames
): #if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
dbsOnly = True
blockNames = set(blockNames) - set(
blockInfo
) #get the blocks we did not find information in phedex
if dbsOnly:
try:
for block in blockNames:
res = self.dbs.listBlockOrigin(block_name=block)
if res:
blockInfo[block] = [res[0]['origin_site_name']]
except dbsClientException as ex:
msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not any(blockInfo.values()): # no data location from dbs
return list()
#removing duplicates and 'UNKNOWN entries
locations = {}
node_filter_list = set(['UNKNOWN', None])
for name, nodes in blockInfo.iteritems():
final_nodes = set()
for n in nodes:
if n in node_filter_list:
continue
try:
cmsname(n)
except AssertionError: ## is SE
n = self.phedex.getNodeNames(n) if phedexNodes else [n]
else: ## not SE i.e. phedexNode
n = [self.phedex.getNodeSE(n)] if not phedexNodes else [n]
final_nodes = final_nodes.union(n)
locations[name] = list(final_nodes - node_filter_list)
#returning single list if a single block is passed
if isinstance(fileBlockName, basestring):
locations = locations[fileBlockName]
return locations
def getFileBlock(self, fileBlockName):
"""
_getFileBlock_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : { LFN : Events },
}
}
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.getFileBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
result = {
fileBlockName: {
"StorageElements": self.listFileBlockLocation(fileBlockName),
"Files": self.listFilesInBlock(fileBlockName),
"IsOpen": self.blockIsOpen(fileBlockName),
}
}
return result
def getFileBlockWithParents(self, fileBlockName):
"""
_getFileBlockWithParents_
return a dictionary:
{ blockName: {
"StorageElements" : [<se list>],
"Files" : dictionaries representing each file
}
}
files
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.getFileBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
result = {
fileBlockName: {
"StorageElements": self.listFileBlockLocation(fileBlockName),
"Files": self.listFilesInBlockWithParents(fileBlockName),
"IsOpen": self.blockIsOpen(fileBlockName),
}
}
return result
def getFiles(self, dataset, onlyClosedBlocks=False):
"""
_getFiles_
Returns a dictionary of block names for the dataset where
each block constists of a dictionary containing the StorageElements
for that block and the files in that block by LFN mapped to NEvents
"""
result = {}
blocks = self.listFileBlocks(dataset, onlyClosedBlocks)
[result.update(self.getFileBlock(x)) for x in blocks]
return result
def listBlockParents(self, blockName):
"""Get parent blocks for block"""
result = []
self.checkBlockName(blockName)
blocks = self.dbs.listBlockParents(block_name=blockName)
for block in blocks:
toreturn = {'Name': block['parent_block_name']}
toreturn['StorageElementList'] = self.listFileBlockLocation(
toreturn['Name'])
result.append(toreturn)
return result
def blockIsOpen(self, blockName):
"""
_blockIsOpen_
Return True if named block is open, false if not, or if block
doenst exist
"""
self.checkBlockName(blockName)
blockInstance = self.dbs.listBlocks(block_name=blockName, detail=True)
if len(blockInstance) == 0:
return False
blockInstance = blockInstance[0]
isOpen = blockInstance.get('open_for_writing', 1)
if isOpen == 0:
return False
return True
def blockToDatasetPath(self, blockName):
"""
_blockToDatasetPath_
Given a block name, get the dataset Path associated with that
Block.
Returns the dataset path, or None if not found
"""
self.checkBlockName(blockName)
try:
blocks = self.dbs.listBlocks(block_name=blockName, detail=True)
except dbsClientException as ex:
msg = "Error in "
msg += "DBSReader.blockToDataset(%s)\n" % blockName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if blocks == []:
return None
pathname = blocks[-1].get('dataset', None)
return pathname
def listDatasetLocation(self, datasetName, dbsOnly=False):
"""
_listDatasetLocation_
List the origin SEs where there is at least a block of the given
dataset.
"""
self.checkDatasetPath(datasetName)
if not dbsOnly:
try:
blocksInfo = self.phedex.getReplicaSEForBlocks(
dataset=[datasetName], complete='y')
except Exception as ex:
msg = "Error while getting block location from PhEDEx for dataset=%s)\n" % datasetName
msg += "%s\n" % str(ex)
raise Exception(msg)
if not blocksInfo: # if we couldnt get data location from PhEDEx, try to look into origin site location from dbs
dbsOnly = True
else:
locations = set(blocksInfo.values()[0])
for blockSites in blocksInfo.values():
locations.intersection_update(blockSites)
if dbsOnly:
try:
blocksInfo = self.dbs.listBlockOrigin(dataset=datasetName)
except dbsClientException as ex:
msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName
msg += "%s\n" % formatEx3(ex)
raise DBSReaderError(msg)
if not blocksInfo: # no data location from dbs
return list()
locations = set()
for blockInfo in blocksInfo:
locations.update([blockInfo['origin_site_name']])
locations.difference_update(
['UNKNOWN']) # remove entry when SE name is 'UNKNOWN'
return list(locations)
def checkDatasetPath(self, pathName):
"""
_checkDatasetPath_
"""
if pathName in ("", None):
raise DBSReaderError("Invalid Dataset Path name: => %s <=" %
pathName)
def checkBlockName(self, blockName):
"""
_checkBlockName_
"""
if blockName in ("", "*", None):
raise DBSReaderError("Invalid Block name: => %s <=" % blockName)
