class PhEDExTest(EmulatedUnitTestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
self.dbsTestUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
self.phedexApi = PhEDEx()
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(
result["phedex"]["injected"], {
"stats": {
"closed_datasets": 0,
"closed_blocks": 0,
"new_blocks": 0,
"new_datasets": 1,
"new_files": 0
}
})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
result = self.phedexApi.subscribe(testSub)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue("id" in requestIDs[0], "Error: Missing request ID")
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
# NOTE: leaving it broken on purpose, we do NOT want to subscribe
# data via unit tests :-)
#injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
request_only="y")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
request_only="y")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"},
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'],
['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(
['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'],
['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
def testGetReplicaInfoForBlocks(self):
"""
Test `getReplicaInfoForBlocks` method, the ability to retrieve replica
locations provided a (or a list of) datasets and blocks
"""
def _checkOutcome(numFiles, replica):
"run the checks"
if rep['complete'] == 'y':
self.assertEqual(rep['files'], numFiles)
if rep['custodial'] == 'y':
self.assertTrue(rep['node'].endswith("_MSS"))
self.assertTrue(rep['subscribed'], 'y')
replicaDict = {
'bytes', 'complete', 'custodial', 'files', 'group', 'node',
'node_id', 'se', 'subscribed', 'time_create', 'time_update'
}
res = self.phedexApi.getReplicaInfoForBlocks(block=BLOCK)['phedex']
self.assertEqual(len(res['block']), 1)
self.assertEqual(res['block'][0]['name'], BLOCK)
self.assertTrue(len(res['block'][0]['replica']) > 1)
self.assertItemsEqual(res['block'][0]['replica'][0].keys(),
replicaDict)
numFiles = res['block'][0]['files']
for rep in res['block'][0]['replica']:
_checkOutcome(numFiles, rep)
# same test, but providing a dataset as input (which has only the block above)
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET)['phedex']
self.assertEqual(len(res['block']), 4)
self.assertTrue(BLOCK in [blk['name'] for blk in res['block']])
for block in res['block']:
numFiles = block['files']
for rep in block['replica']:
self.assertTrue(len(block['replica']) > 1)
_checkOutcome(numFiles, rep)
# same test again, but providing both block and dataset
# NOTE the PhEDEx service only process the block input, the
# dataset argument is completely ignored
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET,
block=BLOCK)['phedex']
self.assertEqual(len(res['block']), 1)
self.assertEqual(res['block'][0]['name'], BLOCK)
self.assertTrue(len(res['block'][0]['replica']) > 1)
self.assertItemsEqual(res['block'][0]['replica'][0].keys(),
replicaDict)
numFiles = res['block'][0]['files']
for rep in res['block'][0]['replica']:
_checkOutcome(numFiles, rep)
# provide a block that does not exist
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET,
block=BLOCK +
"BLAH")['phedex']
self.assertTrue(res['block'] == [])
def testGroupUsage(self):
"""
_testGroupUsage_
Verify that the `getGroupUsage` API works correctly.
"""
node = "T2_DE_DESY"
group = "DataOps"
res = self.phedexApi.getGroupUsage(group=group, node=node)['phedex']
self.assertEqual(len(res['node']), 1)
self.assertEqual(len(res['node'][0]['group']), 1)
self.assertEqual(res['node'][0]['group'][0]['name'], group)
self.assertEqual(res['node'][0]['name'], node)
self.assertTrue(res['node'][0]['group'][0]['dest_bytes'] > 100)
res = self.phedexApi.getGroupUsage(group=group)['phedex']
self.assertTrue(len(res['node']) > 50)
self.assertEqual(len(res['node'][10]['group']), 1)
self.assertEqual(res['node'][10]['group'][0]['name'], group)
return
