def checkForMissingFiles(options):
#Initialize stuff
phedexAPI = PhEDEx({'cachepath' : options.cachepath})
acdcCouch = Database('wmagent_acdc', options.acdcUrl)
#Let's get the IDs of the ACDC documents for the task/request/group/user
array = [options.group, options.user, options.request, options.task]
result = acdcCouch.loadView('ACDC', 'owner_coll_fileset_docs', {'reduce' : False}, [array])
documentsIDs = [x['id'] for x in result['rows']]
badFiles = {}
#Go through the documents
for docID in documentsIDs:
doc = acdcCouch.document(docID)
#Are we going to change this doc? Better back it up
if options.change:
backupFile = os.open(os.path.join(options.backup, "%s.bkp" % doc["_id"]), 'w')
json.dump(doc, backupFile)
backupFile.close()
#Go through the files
files = doc["files"]
for inputFile in files:
#Use PhEDEx API to get site based on the SE
se = files[inputFile]["locations"][0]
siteLocation = phedexAPI.getBestNodeName(se)
#Now get the PFN
pfnDict = phedexAPI.getPFN(siteLocation, inputFile)
inputPfn = pfnDict[(siteLocation, inputFile)]
#Run lcg-ls commands and see what we get
command = 'lcg-ls -b -D srmv2 --srm-timeout 60 %s' % inputPfn
commandList = shlex.split(command)
try:
(stdout, stderr, exitCode) = runCommand(commandList, False, 70)
except Exception, ex:
exitCode = 99999
stdout = ''
stderr = str(ex)
if exitCode:
#Something went wrong with the command
#Mark the file as bad
if docID not in badFiles:
badFiles[docID] = []
badFiles[docID].append(inputFile)
print 'File %s is thought to be bad' % inputFile
print 'Command was %s' % command
print 'Return code was %i' % exitCode
print 'Stdout was %s' % stdout
print 'Stderr was %s' % stderr
def remoteLFNPrefix(site, lfn=''):
"""
Convert a site name to the relevant remote LFN prefix
"""
from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
phedexJSON = PhEDEx(responseType='json')
seName = phedexJSON.getNodeSE(site)
uri = phedexJSON.getPFN(nodes=[site], lfns=[lfn])[(site,lfn)]
return uri.replace(lfn, ''), seName # Don't want the actual LFN, just prefix
def remoteLFNPrefix(site, lfn=''):
"""
Convert a site name to the relevant remote LFN prefix
"""
from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
phedexJSON = PhEDEx(responseType='json')
seName = phedexJSON.getNodeSE(site)
uri = phedexJSON.getPFN(nodes=[site], lfns=[lfn])[(site, lfn)]
return uri.replace(lfn,
''), seName # Don't want the actual LFN, just prefix
def getPFN(proxy, lfnsaddprefix, filename, sitename, logger):
from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
phedex = PhEDEx({"cert": proxy, "key": proxy, "logger": logger})
lfnsadd = os.path.join(lfnsaddprefix, filename)
try:
pfndict = phedex.getPFN(nodes=[sitename], lfns=[lfnsadd])
pfn = pfndict[(sitename, lfnsadd)]
if not pfn:
logger.info('Error: Failed to get PFN from the site. Please check the site status')
return False
except HTTPException as errormsg:
logger.info('Error: Failed to contact PhEDEx or wrong PhEDEx node name is used')
logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url))
raise HTTPException(errormsg)
return pfn
def getPFN(proxy, lfnsaddprefix, filename, sitename, logger):
from WMCore.Services.PhEDEx.PhEDEx import PhEDEx
phedex = PhEDEx({"cert": proxy, "key": proxy, "logger": logger})
lfnsadd = os.path.join(lfnsaddprefix, filename)
try:
pfndict = phedex.getPFN(nodes = [sitename], lfns = [lfnsadd])
pfn = pfndict[(sitename, lfnsadd)]
if not pfn:
logger.info('Error: Failed to get PFN from the site. Please check the site status')
return False
except HTTPException as errormsg:
logger.info('Error: Failed to contact PhEDEx or wrong PhEDEx node name is used')
logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url))
raise HTTPException(errormsg)
return pfn
def insertPfns(self, fileInfoList):
"""
Query phedex to retrieve the pfn for each file and store it in the passed fileInfoList.
"""
phedex = PhEDEx({'cert': self.proxyfilename, 'key': self.proxyfilename, 'logger': self.logger, 'pycurl': True})
# Pick out the correct lfns and sites
if len(fileInfoList) > 0:
for fileInfo in fileInfoList:
if str(fileInfo['jobid']) in self.transferringIds:
lfn = fileInfo['tmplfn']
site = fileInfo['tmpsite']
else:
lfn = fileInfo['lfn']
site = fileInfo['site']
pfn = phedex.getPFN(site, lfn)[(site, lfn)]
fileInfo['pfn'] = pfn
def testDataServiceXML(self):
# asks for PEM pass phrase ...
raise nose.SkipTest
phedex = PhEDEx(responseType='xml')
site = 'T2_UK_SGrid_Bristol'
lfn = '/store/users/metson/file'
protocol = 'srmv2'
phedex.getNodeTFC(site)
tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
tfc = readTFC(tfc_file)
pfn_dict = phedex.getPFN(site, lfn, protocol)
phedex_pfn = pfn_dict[(site, lfn)]
pfn = tfc.matchLFN(protocol, lfn)
msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn)
self.assertEqual(phedex_pfn, pfn, msg)
def testDataServiceXML(self):
# asks for PEM pass phrase ...
raise nose.SkipTest
phedex = PhEDEx(responseType='xml')
site = 'T2_UK_SGrid_Bristol'
lfn = '/store/users/metson/file'
protocol = 'srmv2'
phedex.getNodeTFC(site)
tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
tfc = readTFC(tfc_file)
pfn_dict = phedex.getPFN(site, lfn, protocol)
phedex_pfn = pfn_dict[(site, lfn)]
pfn = tfc.matchLFN(protocol, lfn)
msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn)
self.assertEqual(phedex_pfn, pfn, msg)
def __call__(self):
self.filename = 'crab3checkwrite.tmp'
self.username = self.proxy.getHyperNewsName()
phedex = PhEDEx({"cert": self.proxyfilename, "key": self.proxyfilename})
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
lfnsadd = self.options.userlfn + '/' + self.filename
else:
lfnsadd = '/store/user/' + self.username + '/' + self.filename
try:
pfndict = phedex.getPFN(nodes = [self.options.sitename], lfns = [lfnsadd])
pfn = pfndict[(self.options.sitename, lfnsadd)]
if not pfn:
self.logger.info('%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL))
raise ConfigurationException
except HTTPException, errormsg :
self.logger.info('%sError:%s Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL))
self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url))
raise HTTPException, errormsg
class PhEDExTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"})
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(
result["phedex"]["injected"], {
"stats": {
"closed_datasets": 0,
"closed_blocks": 0,
"new_blocks": 0,
"new_datasets": 1,
"new_files": 0
}
})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
result = self.phedexApi.subscribe(testSub)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue("id" in requestIDs[0], "Error: Missing request ID")
return
@attr("integration")
def testBestNodeName(self):
"""
_testBestNodeName_
Verify that the node name is Buffer first
"""
self.assertTrue(
self.phedexApi.getBestNodeName("cmssrm.fnal.gov") ==
"T1_US_FNAL_Buffer")
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
request_only="n")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
request_only="n")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"},
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'],
['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(
['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'],
['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
class PhEDExTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.phedexApi = PhEDEx({"endpoint": phedexTestDS,
"method": "POST"})
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(result["phedex"]["injected"],
{"stats": {"closed_datasets": 0, "closed_blocks": 0,
"new_blocks": 0, "new_datasets": 1,
"new_files": 0}})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testSub.getDatasetPaths())
result = self.phedexApi.subscribe(testSub, xmlData)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue(requestIDs[0].has_key("id"),
"Error: Missing request ID")
return
@attr("integration")
def testNodeMap(self):
"""
_testNodeMap_
Verify that the node map can be retrieve from PhEDEx and that the
getNodeSE() and getNodeNames() methods work correctly.
"""
self.failUnless(self.phedexApi.getNodeSE("T2_FR_GRIF_LLR") == "polgrid4.in2p3.fr")
self.failUnless(self.phedexApi.getNodeNames("cmssrm.fnal.gov") == ["T1_US_FNAL_Buffer",
"T1_US_FNAL_MSS"])
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS",
"Saturn", requestOnly = "n")
datasetSpec = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testDatasetSub.getDatasetPaths())
self.phedexApi.subscribe(testDatasetSub, datasetSpec)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn",
level = "block", requestOnly = "n")
self.phedexApi.subscribe(testBlockSub, blockSpec)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], set(["T1_UK_RAL_MSS"]),
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
@attr('integration')
def testXMLJSON(self):
"""
Test XML and JSON in the same scope
"""
site = 'T1_US_FNAL_Buffer'
dict = {}
dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
phedexJSON = PhEDEx(responseType='json', dict=dict)
dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
phedexXML = PhEDEx(responseType='xml', dict=dict)
phedexXML.getNodeTFC(site)
tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node' : site})
tfc_map = {}
tfc_map[site] = readTFC(tfc_file)
pfn = tfc_map[site].matchLFN('srmv2', '/store/user/jblow/dir/test.root')
self.failUnless(pfn == 'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root')
self.failUnless(phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
@attr('integration')
def testAuth(self):
"""
_testAuth_
Verify that the auth method works correctly."
"""
self.assertFalse(self.phedexApi.getAuth("datasvc_whatever"))
self.assertTrue(self.phedexApi.getAuth("datasvc_subscribe"))
self.assertTrue(self.phedexApi.getAuth("datasvc_inject"))
return
class PhEDExTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
self.dbsTestUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
self.phedexApi = PhEDEx()
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(result["phedex"]["injected"],
{"stats": {"closed_datasets": 0, "closed_blocks": 0,
"new_blocks": 0, "new_datasets": 1,
"new_files": 0}})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
result = self.phedexApi.subscribe(testSub)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue("id" in requestIDs[0],
"Error: Missing request ID")
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS",
"Saturn", request_only="n")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn",
level="block", request_only="n")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"}, "Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
def testGetReplicaInfoForBlocks(self):
"""
Test `getReplicaInfoForBlocks` method, the ability to retrieve replica
locations provided a (or a list of) datasets and blocks
"""
def _checkOutcome(numFiles, replica):
"run the checks"
if rep['complete'] == 'y':
self.assertEqual(rep['files'], numFiles)
if rep['custodial'] == 'y':
self.assertTrue(rep['node'].endswith("_MSS"))
self.assertTrue(rep['subscribed'], 'y')
replicaDict = {'bytes', 'complete', 'custodial', 'files', 'group',
'node', 'node_id', 'se', 'subscribed',
'time_create', 'time_update'}
res = self.phedexApi.getReplicaInfoForBlocks(block=BLOCK)['phedex']
self.assertEqual(len(res['block']), 1)
self.assertEqual(res['block'][0]['name'], BLOCK)
self.assertTrue(len(res['block'][0]['replica']) > 1)
self.assertItemsEqual(res['block'][0]['replica'][0].keys(), replicaDict)
numFiles = res['block'][0]['files']
for rep in res['block'][0]['replica']:
_checkOutcome(numFiles, rep)
# same test, but providing a dataset as input (which has only the block above)
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET)['phedex']
self.assertEqual(len(res['block']), 4)
self.assertTrue(BLOCK in [blk['name'] for blk in res['block']])
for block in res['block']:
numFiles = block['files']
for rep in block['replica']:
self.assertTrue(len(block['replica']) > 1)
_checkOutcome(numFiles, rep)
# same test again, but providing both block and dataset
# NOTE the PhEDEx service only process the block input, the
# dataset argument is completely ignored
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET, block=BLOCK)['phedex']
self.assertEqual(len(res['block']), 1)
self.assertEqual(res['block'][0]['name'], BLOCK)
self.assertTrue(len(res['block'][0]['replica']) > 1)
self.assertItemsEqual(res['block'][0]['replica'][0].keys(), replicaDict)
numFiles = res['block'][0]['files']
for rep in res['block'][0]['replica']:
_checkOutcome(numFiles, rep)
# provide a block that does not exist
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET, block=BLOCK + "BLAH")['phedex']
self.assertTrue(res['block'] == [])
class checkwrite(SubCommand):
"""
Let user to test if he/she have permission to write on specify site
"""
name = 'checkwrite'
shortnames = ['chk']
def __init__(self, logger, cmdargs=None):
SubCommand.__init__(self, logger, cmdargs)
self.phedex = PhEDEx({
"cert": self.proxyfilename,
"key": self.proxyfilename,
"logger": self.logger
})
self.lfnsaddprefix = None
self.filename = None
def __call__(self):
username = None
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnsaddprefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>. Retrieve his/her
## username from SiteDB.
self.logger.info(
'Will check write permission in the default location /store/user/<username>'
)
username = getUserDNandUsernameFromSiteDB(
self.logger).get('username')
if username:
self.lfnsaddprefix = '/store/user/' + username
else:
return {'status': 'FAILED'}
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix))
msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (
self.lfnsaddprefix)
msg += "\nThe LFN must start with either '/store/user/<username>/', '/store/group/<groupname>[/<subgroupname>]*/<username>/' or '/store/local/<dirname>',"
msg += " where username is your username as registered in SiteDB (i.e. the username of your CERN primary account)."
msg += "\nLFN %s is not valid." % (self.lfnsaddprefix)
if not username:
username = getUserDNandUsernameFromSiteDB(
self.logger).get('username')
if not checkOutLFN(self.lfnsaddprefix, username):
self.logger.info(msg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid." % (self.lfnsaddprefix))
cp_cmd = ""
del_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist("gfal-rm"):
self.logger.info(
"Will use `gfal-copy`, `gfal-rm` commands for checking write permissions"
)
cp_cmd = "env -i gfal-copy -v -t 180 "
del_cmd = "env -i gfal-rm -v -t 180 "
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info(
"Will use `lcg-cp`, `lcg-del` commands for checking write permissions"
)
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
del_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s' %
(self.lfnsaddprefix, self.options.sitename))
retry = 0
stop = False
use_new_file = True
while not stop:
if use_new_file:
self.filename = 'crab3checkwrite.' + str(retry) + '.tmp'
self.createFile()
pfn = self.getPFN()
self.logger.info(
'Attempting to copy (dummy) file %s to %s on site %s' %
(self.filename, self.lfnsaddprefix, self.options.sitename))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info(
'Successfully copied file %s to %s on site %s' %
(self.filename, self.lfnsaddprefix, self.options.sitename))
self.logger.info('Attempting to delete file %s from site %s' %
(pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info(
'%sWarning%s: Failed to delete file %s from site %s' %
(colors.RED, colors.NORMAL, pfn,
self.options.sitename))
else:
self.logger.info(
'Successfully deleted file %s from site %s' %
(pfn, self.options.sitename))
self.logger.info(
'%sSuccess%s: Able to write in %s on site %s' %
(colors.GREEN, colors.NORMAL, self.lfnsaddprefix,
self.options.sitename))
returndict = {'status': 'SUCCESS'}
stop = True
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
msg = '%sError%s: Unable to write in %s on site %s' % (
colors.RED, colors.NORMAL, self.lfnsaddprefix,
self.options.sitename)
msg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
msg = 'Unable to check write permission in %s on site %s' % (
self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'exist' in cpout or 'exist' in cperr and retry == 0:
self.logger.info(
'Error copying file %s to %s on site %s; it may be that file already exists.'
% (self.filename, self.lfnsaddprefix,
self.options.sitename))
self.logger.info(
'Attempting to delete file %s from site %s' %
(pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info(
'Failed to delete file %s from site %s' %
(pfn, self.options.sitename))
use_new_file = True
else:
self.logger.info(
'Successfully deleted file %s from site %s' %
(pfn, self.options.sitename))
use_new_file = False
retry += 1
else:
msg = 'Unable to check write permission in %s on site %s' % (
self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
if stop or use_new_file:
self.removeFile()
self.logger.info(
'%sNote%s: You cannot write to a site if you did not ask permission.'
% (colors.BOLD, colors.NORMAL))
return returndict
def createFile(self):
abspath = path.abspath(self.filename)
try:
with open(abspath, 'w') as fd:
fd.write(
'This is a dummy file created by the crab checkwrite command on %s'
% str(datetime.datetime.now().strftime(
'%d/%m/%Y at %H:%M:%S')))
except IOError:
self.logger.info('%sError%s: Failed to create file %s' %
(colors.RED, colors.NORMAL, self.filename))
raise Exception
def removeFile(self):
abspath = path.abspath(self.filename)
try:
remove(abspath)
except:
self.logger.info('%sWarning%s: Failed to delete file %s' %
(colors.RED, colors.NORMAL, self.filename))
def getPFN(self):
lfnsadd = self.lfnsaddprefix + '/' + self.filename
try:
pfndict = self.phedex.getPFN(nodes=[self.options.sitename],
lfns=[lfnsadd])
pfn = pfndict[(self.options.sitename, lfnsadd)]
if not pfn:
self.logger.info(
'%sError%s: Failed to get PFN from the site. Please check the site status'
% (colors.RED, colors.NORMAL))
raise ConfigurationException
except HTTPException, errormsg:
self.logger.info(
'%sError%s: Failed to contact PhEDEx or wrong PhEDEx node name is used'
% (colors.RED, colors.NORMAL))
self.logger.info('Result: %s\nStatus :%s\nURL :%s' %
(errormsg.result, errormsg.status, errormsg.url))
raise HTTPException, errormsg
return pfn
def submit(trans_tuple, job_data, log, direct=False):
"""Manage threads for transfers submission through Rucio
:param trans_tuple: ordered list of needed xfer info (transfers, to_submit_columns)
:type trans_tuple: tuple
:param job_data: general CRAB job metadata
:type job_data: dict
:param log: log object
:type log: logging
:param direct: job output stored on temp or directly, defaults to False
:param direct: bool, optional
"""
threadLock = threading.Lock()
threads = []
to_update = []
toTrans = trans_tuple[0]
columns = trans_tuple[1]
proxy = job_data['proxy']
rest_filetransfers = job_data['rest']
user = job_data['username']
destination = job_data['destination']
taskname = job_data['taskname']
try:
phedex = PhEDEx(responseType='xml',
httpDict={'key': proxy,
'cert': proxy,
'pycurl': True})
except Exception:
log.exception('PhEDEx exception.')
return
# Split threads by source RSEs
sources = list(set([x[columns.index('source')] for x in toTrans]))
os.environ["X509_CERT_DIR"] = os.getcwd()
log.info("Connection to %s with proxy in:\n %s" % (rest_filetransfers,proxy))
oracleDB = HTTPRequests(rest_filetransfers,
proxy,
proxy)
#verbose=True)
# mapping lfn <--> pfn
for source in sources:
ids = [x[columns.index('id')] for x in toTrans if x[columns.index('source')] == source]
src_lfns = [x[columns.index('source_lfn')] for x in toTrans if x[columns.index('source')] == source]
dst_lfns = [x[columns.index('destination_lfn')] for x in toTrans if x[columns.index('source')] == source]
sorted_source_pfns = []
sorted_dest_lfns = []
sorted_dest_pfns = []
# workaround for phedex.getPFN issue --> shuffling output order w.r.t. the list in input
try:
for chunk in chunks(src_lfns, 10):
unsorted_source_pfns = [[k[1], str(x)] for k, x in phedex.getPFN(source, chunk).items()]
for order_lfn in chunk:
for lfn, pfn in unsorted_source_pfns:
if order_lfn == lfn:
sorted_source_pfns.append(pfn)
break
for chunk in chunks(dst_lfns, 10):
unsorted_dest_pfns = [[k[1], str(x)] for k, x in phedex.getPFN(toTrans[0][4], chunk).items()]
for order_lfn in chunk:
for lfn, pfn in unsorted_dest_pfns:
if order_lfn == lfn:
sorted_dest_pfns.append(pfn)
sorted_dest_lfns.append(lfn)
break
except Exception as ex:
log.error("Failed to map lfns to pfns: %s", ex)
mark_failed(ids, ["Failed to map lfn to pfn: " + str(ex) for _ in ids], oracleDB)
source_pfns = sorted_source_pfns
dest_lfns = sorted_dest_lfns
# saving file sizes and checksums
filesizes = [x[columns.index('filesize')] for x in toTrans if x[columns.index('source')] == source]
checksums = [x[columns.index('checksums')] for x in toTrans if x[columns.index('source')] == source]
pubnames = [x[columns.index('publishname')] for x in toTrans if x[columns.index('source')] == source]
# ordered list of replicas information
jobs = zip(source_pfns, dest_lfns, ids, checksums, filesizes, pubnames)
job_columns = ['source_pfns', 'dest_lfns', 'ids', 'checksums', 'filesizes', 'pubnames']
# ordered list of transfers details
tx_from_source = [[job, source, taskname, user, destination] for job in jobs]
tx_columns = ['job', 'source', 'taskname', 'user', 'destination']
# split submission process in chunks of max 200 files
for files in chunks(tx_from_source, 200):
if not direct:
log.info("Submitting: %s", files)
thread = submit_thread(threadLock,
log,
(files, tx_columns),
job_columns,
proxy,
to_update)
thread.start()
threads.append(thread)
elif direct:
log.info("Registering direct stageout: %s", files)
thread = submit_thread(threadLock,
log,
(files, tx_columns),
job_columns,
proxy,
to_update,
direct=True)
thread.start()
threads.append(thread)
for t in threads:
t.join()
if len(to_update) == 0:
return False
# update statuses in oracle table as per threads result
for fileDoc in to_update:
try:
log.debug("%s/filetransfers?%s" % (rest_filetransfers, encodeRequest(fileDoc)))
oracleDB.post('/filetransfers', data=encodeRequest(fileDoc))
log.info("Marked submitted %s files" % (fileDoc['list_of_ids']))
except Exception:
log.exception('Failed to mark files as submitted on DBs')
return True
class checkwrite(SubCommand):
"""
Let user to test if he/she have permission to write on specify site
"""
name = 'checkwrite'
shortnames = ['chk']
def __init__(self, logger, cmdargs = None):
SubCommand.__init__(self, logger, cmdargs)
self.phedex = PhEDEx({"cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger, "pycurl" : True})
self.lfnsaddprefix = None
self.filename = None
def __call__(self):
username = None
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnsaddprefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>. Retrieve his/her
## username from SiteDB.
self.logger.info('Will check write permission in the default location /store/user/<username>')
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if username:
self.lfnsaddprefix = '/store/user/' + username
else:
return {'status': 'FAILED'}
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix))
msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (self.lfnsaddprefix)
msg += "\nThe LFN must start with either"
msg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
msg += " (or '/store/local/<something>/' if publication is off),"
msg += " where username is your username as registered in SiteDB"
msg += " (i.e. the username of your CERN primary account)."
msg += "\nLFN %s is not valid." % (self.lfnsaddprefix)
if not username and self.lfnsaddprefix.startswith('/store/user/'):
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if not checkOutLFN(self.lfnsaddprefix, username):
self.logger.info(msg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid." % (self.lfnsaddprefix))
cp_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist("gfal-rm") and self.command in [None, "GFAL"]:
self.logger.info("Will use `gfal-copy`, `gfal-rm` commands for checking write permissions")
cp_cmd = "env -i X509_USER_PROXY=%s gfal-copy -p -v -t 180 " % os.path.abspath(self.proxyfilename)
delfile_cmd = "env -i X509_USER_PROXY=%s gfal-rm -v -t 180 " % os.path.abspath(self.proxyfilename)
deldir_cmd = "env -i X509_USER_PROXY=%s gfal-rm -r -v -t 180 " % os.path.abspath(self.proxyfilename)
if self.checksum:
cp_cmd += "-K %s " % self.checksum
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info("Will use `lcg-cp`, `lcg-del` commands for checking write permissions")
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 "
if self.checksum:
cp_cmd += "--checksum-type %s " % self.checksum
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename))
timestamp = str(time.strftime("%Y%m%d_%H%M%S"))
self.filename = 'crab3checkwrite_' + timestamp + '.tmp'
self.subdir = 'crab3checkwrite_' + timestamp
self.createFile()
pfn = self.getPFN()
dirpfn = pfn[:len(pfn)-len(self.filename)]
self.logger.info('\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n' % (self.subdir, self.filename, self.lfnsaddprefix))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info('\nSuccessfully created directory %s and copied file %s to %s' % (self.subdir, self.filename, self.lfnsaddprefix))
self.logger.info('\nAttempting to delete file %s\n' % (pfn))
delexitcode = self.delete(pfn, delfile_cmd)
if delexitcode:
self.logger.info('\nFailed to delete file %s' % (pfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted file %s' % (pfn))
self.logger.info('\nAttempting to delete directory %s\n' % (dirpfn))
delexitcode = self.delete(dirpfn, deldir_cmd)
if delexitcode:
self.logger.info('\nFailed to delete directory %s' % (dirpfn))
finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
self.logger.info('\nSuccessfully deleted directory %s' % (dirpfn))
finalmsg = '%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
returndict = {'status': 'SUCCESS'}
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
finalmsg = '%sError%s: Unable to write in %s on site %s' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
finalmsg = '\nUnable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status': 'FAILED'}
else:
finalmsg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
returndict = {'status' : 'FAILED'}
self.removeFile()
self.logger.info('\nCheckwrite Result:')
self.logger.info(finalmsg)
if returndict['status'] == 'FAILED':
self.logger.info('%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL))
return returndict
def createFile(self):
abspath = os.path.abspath(self.filename)
try:
with open(abspath, 'w') as fd:
fd.write('This is a dummy file created by the crab checkwrite command on %s' % str(datetime.datetime.now().strftime('%d/%m/%Y at %H:%M:%S')))
except IOError:
self.logger.info('%sError%s: Failed to create file %s' % (colors.RED, colors.NORMAL, self.filename))
raise Exception
def removeFile(self):
abspath = os.path.abspath(self.filename)
try:
os.remove(abspath)
except:
self.logger.info('%sWarning%s: Failed to delete file %s' % (colors.RED, colors.NORMAL, self.filename))
def getPFN(self):
lfnsadd = self.lfnsaddprefix + '/' + self.subdir + '/' + self.filename
try:
pfndict = self.phedex.getPFN(nodes = [self.options.sitename], lfns = [lfnsadd])
pfn = pfndict[(self.options.sitename, lfnsadd)]
if not pfn:
self.logger.info('%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL))
raise ConfigurationException
except HTTPException as errormsg:
self.logger.info('%sError%s: Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL))
self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url))
raise
return pfn
def cp(self, pfn, command):
abspath = os.path.abspath(self.filename)
if cmd_exist("gfal-copy") and self.command in [None, "GFAL"]:
abspath = "file://" + abspath
cpcmd = command + abspath + " '" + pfn + "'"
self.logger.info('Executing command: %s' % cpcmd)
self.logger.info('Please wait...')
cpprocess = subprocess.Popen(cpcmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
cpout, cperr = cpprocess.communicate()
cpexitcode = cpprocess.returncode
if cpexitcode:
self.logger.info('Failed running copy command')
if cpout:
self.logger.info(' Stdout:\n %s' % str(cpout).replace('\n', '\n '))
if cperr:
self.logger.info(' Stderr:\n %s' % str(cperr).replace('\n', '\n '))
return cpout, cperr, cpexitcode
def delete(self, pfn, command):
rmcmd = command + "'" + pfn + "'"
self.logger.info('Executing command: %s' % rmcmd)
self.logger.info('Please wait...')
delprocess = subprocess.Popen(rmcmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
delout, delerr = delprocess.communicate()
delexitcode = delprocess.returncode
if delexitcode:
self.logger.info('Failed running delete command')
if delout:
self.logger.info(' Stdout:\n %s' % str(delout).replace('\n', '\n '))
if delerr:
self.logger.info(' Stderr:\n %s' % str(delerr).replace('\n', '\n '))
return delexitcode
def terminate(self, exitcode):
pass
def setOptions(self):
"""
__setOptions__
This allows to set specific command options
"""
self.parser.add_option('--site',
dest = 'sitename',
default = None,
help = 'The PhEDEx node name of the site to be checked.')
self.parser.add_option('--lfn',
dest = 'userlfn',
default = None,
help = 'A user lfn address.')
self.parser.add_option('--checksum',
dest = 'checksum',
default = 'yes',
help = 'Set it to true if needed. If true will use ADLER32 checksum' +\
'Allowed values are yes/no. Default is yes.')
self.parser.add_option('--command',
dest = 'command',
default = None,
help = 'A command which to use. Available commands are LCG or GFAL.')
def validateOptions(self):
SubCommand.validateOptions(self)
if self.options.sitename is None:
msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL)
msg += " Use the --site option."
ex = MissingOptionException(msg)
ex.missingOption = "sitename"
raise ex
if hasattr(self.options, 'command') and self.options.command != None:
AvailableCommands = ['LCG', 'GFAL']
self.command = self.options.command.upper()
if self.command not in AvailableCommands:
msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands))
ex = ConfigurationException(msg)
raise ex
else:
self.command = None
if hasattr(self.options, 'checksum'):
if re.match('^yes$|^no$', self.options.checksum):
self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None
else:
msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum
ex = ConfigurationException(msg)
raise ex
class PhEDExTest(EmulatedUnitTestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
self.dbsTestUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader"
self.phedexApi = PhEDEx()
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(
result["phedex"]["injected"], {
"stats": {
"closed_datasets": 0,
"closed_blocks": 0,
"new_blocks": 0,
"new_datasets": 1,
"new_files": 0
}
})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
result = self.phedexApi.subscribe(testSub)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue("id" in requestIDs[0], "Error: Missing request ID")
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
# NOTE: leaving it broken on purpose, we do NOT want to subscribe
# data via unit tests :-)
#injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
request_only="y")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
request_only="y")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"},
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'],
['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(
['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'],
['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
def testGetReplicaInfoForBlocks(self):
"""
Test `getReplicaInfoForBlocks` method, the ability to retrieve replica
locations provided a (or a list of) datasets and blocks
"""
def _checkOutcome(numFiles, replica):
"run the checks"
if rep['complete'] == 'y':
self.assertEqual(rep['files'], numFiles)
if rep['custodial'] == 'y':
self.assertTrue(rep['node'].endswith("_MSS"))
self.assertTrue(rep['subscribed'], 'y')
replicaDict = {
'bytes', 'complete', 'custodial', 'files', 'group', 'node',
'node_id', 'se', 'subscribed', 'time_create', 'time_update'
}
res = self.phedexApi.getReplicaInfoForBlocks(block=BLOCK)['phedex']
self.assertEqual(len(res['block']), 1)
self.assertEqual(res['block'][0]['name'], BLOCK)
self.assertTrue(len(res['block'][0]['replica']) > 1)
self.assertItemsEqual(res['block'][0]['replica'][0].keys(),
replicaDict)
numFiles = res['block'][0]['files']
for rep in res['block'][0]['replica']:
_checkOutcome(numFiles, rep)
# same test, but providing a dataset as input (which has only the block above)
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET)['phedex']
self.assertEqual(len(res['block']), 4)
self.assertTrue(BLOCK in [blk['name'] for blk in res['block']])
for block in res['block']:
numFiles = block['files']
for rep in block['replica']:
self.assertTrue(len(block['replica']) > 1)
_checkOutcome(numFiles, rep)
# same test again, but providing both block and dataset
# NOTE the PhEDEx service only process the block input, the
# dataset argument is completely ignored
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET,
block=BLOCK)['phedex']
self.assertEqual(len(res['block']), 1)
self.assertEqual(res['block'][0]['name'], BLOCK)
self.assertTrue(len(res['block'][0]['replica']) > 1)
self.assertItemsEqual(res['block'][0]['replica'][0].keys(),
replicaDict)
numFiles = res['block'][0]['files']
for rep in res['block'][0]['replica']:
_checkOutcome(numFiles, rep)
# provide a block that does not exist
res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET,
block=BLOCK +
"BLAH")['phedex']
self.assertTrue(res['block'] == [])
def testGroupUsage(self):
"""
_testGroupUsage_
Verify that the `getGroupUsage` API works correctly.
"""
node = "T2_DE_DESY"
group = "DataOps"
res = self.phedexApi.getGroupUsage(group=group, node=node)['phedex']
self.assertEqual(len(res['node']), 1)
self.assertEqual(len(res['node'][0]['group']), 1)
self.assertEqual(res['node'][0]['group'][0]['name'], group)
self.assertEqual(res['node'][0]['name'], node)
self.assertTrue(res['node'][0]['group'][0]['dest_bytes'] > 100)
res = self.phedexApi.getGroupUsage(group=group)['phedex']
self.assertTrue(len(res['node']) > 50)
self.assertEqual(len(res['node'][10]['group']), 1)
self.assertEqual(res['node'][10]['group'][0]['name'], group)
return
def swapLocations(options):
#Initialize stuff
phedexAPI = PhEDEx({'cachepath' : options.cachepath})
acdcCouch = Database('wmagent_acdc', options.acdcUrl)
#Let's get the IDs of the ACDC documents for the task/request/group/user
array = [options.group, options.user, options.request, options.task]
result = acdcCouch.loadView('ACDC', 'owner_coll_fileset_docs', {'reduce' : False}, [array])
documentsIDs = [x['id'] for x in result['rows']]
#Load the map file saying what we want to change of location
mapFile = open(options.map, 'r')
locationMap = json.load(mapFile)
mapFile.close()
#Go through the documents
for docID in documentsIDs:
doc = acdcCouch.document(docID)
#Are we going to change this doc? Better back it up
if options.change:
backupFile = os.open(os.path.join(options.backup, "%s.bkp" % doc["_id"]), 'w')
json.dump(doc, backupFile)
backupFile.close()
#Go through the files
files = doc["files"]
for inputFile in files:
#Use PhEDEx API to get site based on the SE
#Then map that to the desired target
se = files[inputFile]["locations"][0]
siteLocation = phedexAPI.getBestNodeName(se)
targetLocation = locationMap.get(siteLocation, siteLocation)
if siteLocation == targetLocation:
#Nothing to do with this one, move on
continue
if not options.change:
#No changes, then give the commands to move the files
#Get the PFN for both the current location and the target location
pfnDict = phedexAPI.getPFN(siteLocation, inputFile)
inputPfn = pfnDict[(siteLocation, inputFile)]
pfnDict = phedexAPI.getPFN(targetLocation, inputFile)
targetPfn = pfnDict[(targetLocation, inputFile)]
#Print it to stdout
print "lcg-cp -D srmv2 -b %s %s" % (inputPfn, targetPfn)
else:
#This is changes time, let's move the stuff
targetSE = phedexAPI.getNodeSE(targetLocation)
files[inputFile]["locations"][0] = targetSE
print "Changing location of %s from %s to %s" % (inputFile, se, targetSE)
#If specified, commit the changes
if options.change:
acdcCouch.commitOne(doc)
return 0
class checkwrite(SubCommand):
"""
Let user to test if he/she have permission to write on specify site
"""
name = 'checkwrite'
shortnames = ['chk']
def __init__(self, logger, cmdargs = None):
SubCommand.__init__(self, logger, cmdargs)
self.phedex = PhEDEx({"cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger})
self.lfnsaddprefix = None
self.filename = None
def __call__(self):
username = None
if hasattr(self.options, 'userlfn') and self.options.userlfn != None:
self.lfnsaddprefix = self.options.userlfn
else:
## If the user didn't provide an LFN path where to check the write permission,
## assume he/she wants to check in /store/user/<username>. Retrieve his/her
## username from SiteDB.
self.logger.info('Will check write permission in the default location /store/user/<username>')
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if username:
self.lfnsaddprefix = '/store/user/' + username
else:
return {'status': 'FAILED'}
## Check that the location where we want to check write permission
## is one where the user will be allowed to stageout.
self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix))
msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (self.lfnsaddprefix)
msg += "\nThe LFN must start with either"
msg += " '/store/user/<username>/' or '/store/group/<groupname>/'"
msg += " (or '/store/local/<something>/' if publication is off),"
msg += " where username is your username as registered in SiteDB"
msg += " (i.e. the username of your CERN primary account)."
msg += "\nLFN %s is not valid." % (self.lfnsaddprefix)
if not username and self.lfnsaddprefix.startswith('/store/user/'):
username = getUserDNandUsernameFromSiteDB(self.logger).get('username')
if not checkOutLFN(self.lfnsaddprefix, username):
self.logger.info(msg)
return {'status': 'FAILED'}
else:
self.logger.info("LFN %s is valid." % (self.lfnsaddprefix))
cp_cmd = ""
del_cmd = ""
if cmd_exist("gfal-copy") and cmd_exist("gfal-rm"):
self.logger.info("Will use `gfal-copy`, `gfal-rm` commands for checking write permissions")
cp_cmd = "env -i gfal-copy -v -t 180 "
del_cmd = "env -i gfal-rm -v -t 180 "
elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"):
self.logger.info("Will use `lcg-cp`, `lcg-del` commands for checking write permissions")
cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 "
del_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 "
else:
self.logger.info("Neither gfal nor lcg command was found")
return {'status': 'FAILED'}
self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename))
retry = 0
stop = False
use_new_file = True
while not stop:
if use_new_file:
self.filename = 'crab3checkwrite.' + str(retry) + '.tmp'
self.createFile()
pfn = self.getPFN()
self.logger.info('Attempting to copy (dummy) file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename))
cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd)
if cpexitcode == 0:
self.logger.info('Successfully copied file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename))
self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info('%sWarning%s: Failed to delete file %s from site %s' % (colors.RED, colors.NORMAL, pfn, self.options.sitename))
else:
self.logger.info('Successfully deleted file %s from site %s' % (pfn, self.options.sitename))
self.logger.info('%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename))
returndict = {'status': 'SUCCESS'}
stop = True
else:
if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr:
msg = '%sError%s: Unable to write in %s on site %s' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)
msg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'timeout' in cpout or 'timeout' in cperr:
self.logger.info('Connection time out.')
msg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status': 'FAILED'}
stop = True
elif 'exist' in cpout or 'exist' in cperr and retry == 0:
self.logger.info('Error copying file %s to %s on site %s; it may be that file already exists.' % (self.filename, self.lfnsaddprefix, self.options.sitename))
self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename))
delexitcode = self.delete(pfn, del_cmd)
if delexitcode:
self.logger.info('Failed to delete file %s from site %s' % (pfn, self.options.sitename))
use_new_file = True
else:
self.logger.info('Successfully deleted file %s from site %s' % (pfn, self.options.sitename))
use_new_file = False
retry += 1
else:
msg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)
msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.'
self.logger.info(msg)
returndict = {'status' : 'FAILED'}
stop = True
if stop or use_new_file:
self.removeFile()
self.logger.info('%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL))
return returndict
def createFile(self):
abspath = path.abspath(self.filename)
try:
with open(abspath, 'w') as fd:
fd.write('This is a dummy file created by the crab checkwrite command on %s' % str(datetime.datetime.now().strftime('%d/%m/%Y at %H:%M:%S')))
except IOError:
self.logger.info('%sError%s: Failed to create file %s' % (colors.RED, colors.NORMAL, self.filename))
raise Exception
def removeFile(self):
abspath = path.abspath(self.filename)
try:
remove(abspath)
except:
self.logger.info('%sWarning%s: Failed to delete file %s' % (colors.RED, colors.NORMAL, self.filename))
def getPFN(self):
lfnsadd = self.lfnsaddprefix + '/' + self.filename
try:
pfndict = self.phedex.getPFN(nodes = [self.options.sitename], lfns = [lfnsadd])
pfn = pfndict[(self.options.sitename, lfnsadd)]
if not pfn:
self.logger.info('%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL))
raise ConfigurationException
except HTTPException, errormsg:
self.logger.info('%sError%s: Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL))
self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url))
raise HTTPException, errormsg
return pfn
class PhEDExTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Initialize the PhEDEx API to point at the test server.
"""
phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"})
return
@attr("integration")
def testInjection(self):
"""
_testInjection_
Verify that we can inject data into PhEDEx.
"""
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID())
result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
self.assertEqual(
result["phedex"]["injected"], {
"stats": {
"closed_datasets": 0,
"closed_blocks": 0,
"new_blocks": 0,
"new_datasets": 1,
"new_files": 0
}
})
return
@attr("integration")
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
result = self.phedexApi.subscribe(testSub)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue("id" in requestIDs[0], "Error: Missing request ID")
return
@attr("integration")
def testBestNodeName(self):
"""
_testBestNodeName_
Verify that the node name is Buffer first
"""
self.assertTrue(
self.phedexApi.getBestNodeName("cmssrm.fnal.gov") ==
"T1_US_FNAL_Buffer")
return
@attr("integration")
def testNodeMap(self):
"""
_testNodeMap_
Verify that the node map can be retrieve from PhEDEx and that the
getNodeSE() and getNodeNames() methods work correctly.
"""
self.assertTrue(
self.phedexApi.getNodeSE("T2_FR_GRIF_LLR") == "polgrid4.in2p3.fr")
self.assertTrue(
self.phedexApi.getNodeNames("cmssrm.fnal.gov") ==
["T1_US_FNAL_Buffer", "T1_US_FNAL_MSS"])
return
@attr('integration')
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
request_only="n")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
request_only="n")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"},
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testPFNLookup(self):
"""
_testPFNLookup_
Verify that the PFN lookup in PhEDEx works correctly.
"""
call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'],
['/store/user/metson/file'])
# Should get one mapping back (one lfn, one node)
self.assertTrue(len(call1.keys()) == 1)
call1_key = call1.keys()[0]
call2 = self.phedexApi.getPFN(
['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'],
['/store/user/metson/file'])
# Should get back two mappings (two nodes)
self.assertTrue(call1_key in call2.keys())
# and one of the mappings should be the same as from the previous call
self.assertTrue(call1[call1_key] == call2[call1_key])
return
@attr('integration')
def testXMLJSON(self):
"""
Test XML and JSON in the same scope
"""
site = 'T1_US_FNAL_Buffer'
httpDict = {
'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"
}
phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
httpDict = {
'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
}
phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)
phedexXML.getNodeTFC(site)
tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
tfc_map = {}
tfc_map[site] = readTFC(tfc_file)
pfn = tfc_map[site].matchLFN('srmv2',
'/store/user/jblow/dir/test.root')
self.assertTrue(
pfn ==
'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root'
)
self.assertTrue(
phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
@attr('integration')
def testAuth(self):
"""
_testAuth_
Verify that the auth method works correctly."
"""
self.assertFalse(self.phedexApi.getAuth("datasvc_whatever"))
self.assertTrue(self.phedexApi.getAuth("datasvc_subscribe"))
self.assertTrue(self.phedexApi.getAuth("datasvc_inject"))
return