def checkForMissingFiles(options): #Initialize stuff phedexAPI = PhEDEx({'cachepath' : options.cachepath}) acdcCouch = Database('wmagent_acdc', options.acdcUrl) #Let's get the IDs of the ACDC documents for the task/request/group/user array = [options.group, options.user, options.request, options.task] result = acdcCouch.loadView('ACDC', 'owner_coll_fileset_docs', {'reduce' : False}, [array]) documentsIDs = [x['id'] for x in result['rows']] badFiles = {} #Go through the documents for docID in documentsIDs: doc = acdcCouch.document(docID) #Are we going to change this doc? Better back it up if options.change: backupFile = os.open(os.path.join(options.backup, "%s.bkp" % doc["_id"]), 'w') json.dump(doc, backupFile) backupFile.close() #Go through the files files = doc["files"] for inputFile in files: #Use PhEDEx API to get site based on the SE se = files[inputFile]["locations"][0] siteLocation = phedexAPI.getBestNodeName(se) #Now get the PFN pfnDict = phedexAPI.getPFN(siteLocation, inputFile) inputPfn = pfnDict[(siteLocation, inputFile)] #Run lcg-ls commands and see what we get command = 'lcg-ls -b -D srmv2 --srm-timeout 60 %s' % inputPfn commandList = shlex.split(command) try: (stdout, stderr, exitCode) = runCommand(commandList, False, 70) except Exception, ex: exitCode = 99999 stdout = '' stderr = str(ex) if exitCode: #Something went wrong with the command #Mark the file as bad if docID not in badFiles: badFiles[docID] = [] badFiles[docID].append(inputFile) print 'File %s is thought to be bad' % inputFile print 'Command was %s' % command print 'Return code was %i' % exitCode print 'Stdout was %s' % stdout print 'Stderr was %s' % stderr
def remoteLFNPrefix(site, lfn=''): """ Convert a site name to the relevant remote LFN prefix """ from WMCore.Services.PhEDEx.PhEDEx import PhEDEx phedexJSON = PhEDEx(responseType='json') seName = phedexJSON.getNodeSE(site) uri = phedexJSON.getPFN(nodes=[site], lfns=[lfn])[(site,lfn)] return uri.replace(lfn, ''), seName # Don't want the actual LFN, just prefix
def remoteLFNPrefix(site, lfn=''): """ Convert a site name to the relevant remote LFN prefix """ from WMCore.Services.PhEDEx.PhEDEx import PhEDEx phedexJSON = PhEDEx(responseType='json') seName = phedexJSON.getNodeSE(site) uri = phedexJSON.getPFN(nodes=[site], lfns=[lfn])[(site, lfn)] return uri.replace(lfn, ''), seName # Don't want the actual LFN, just prefix
def getPFN(proxy, lfnsaddprefix, filename, sitename, logger): from WMCore.Services.PhEDEx.PhEDEx import PhEDEx phedex = PhEDEx({"cert": proxy, "key": proxy, "logger": logger}) lfnsadd = os.path.join(lfnsaddprefix, filename) try: pfndict = phedex.getPFN(nodes=[sitename], lfns=[lfnsadd]) pfn = pfndict[(sitename, lfnsadd)] if not pfn: logger.info('Error: Failed to get PFN from the site. Please check the site status') return False except HTTPException as errormsg: logger.info('Error: Failed to contact PhEDEx or wrong PhEDEx node name is used') logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url)) raise HTTPException(errormsg) return pfn
def getPFN(proxy, lfnsaddprefix, filename, sitename, logger): from WMCore.Services.PhEDEx.PhEDEx import PhEDEx phedex = PhEDEx({"cert": proxy, "key": proxy, "logger": logger}) lfnsadd = os.path.join(lfnsaddprefix, filename) try: pfndict = phedex.getPFN(nodes = [sitename], lfns = [lfnsadd]) pfn = pfndict[(sitename, lfnsadd)] if not pfn: logger.info('Error: Failed to get PFN from the site. Please check the site status') return False except HTTPException as errormsg: logger.info('Error: Failed to contact PhEDEx or wrong PhEDEx node name is used') logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url)) raise HTTPException(errormsg) return pfn
def insertPfns(self, fileInfoList): """ Query phedex to retrieve the pfn for each file and store it in the passed fileInfoList. """ phedex = PhEDEx({'cert': self.proxyfilename, 'key': self.proxyfilename, 'logger': self.logger, 'pycurl': True}) # Pick out the correct lfns and sites if len(fileInfoList) > 0: for fileInfo in fileInfoList: if str(fileInfo['jobid']) in self.transferringIds: lfn = fileInfo['tmplfn'] site = fileInfo['tmpsite'] else: lfn = fileInfo['lfn'] site = fileInfo['site'] pfn = phedex.getPFN(site, lfn)[(site, lfn)] fileInfo['pfn'] = pfn
def testDataServiceXML(self): # asks for PEM pass phrase ... raise nose.SkipTest phedex = PhEDEx(responseType='xml') site = 'T2_UK_SGrid_Bristol' lfn = '/store/users/metson/file' protocol = 'srmv2' phedex.getNodeTFC(site) tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site}) tfc = readTFC(tfc_file) pfn_dict = phedex.getPFN(site, lfn, protocol) phedex_pfn = pfn_dict[(site, lfn)] pfn = tfc.matchLFN(protocol, lfn) msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn) self.assertEqual(phedex_pfn, pfn, msg)
def __call__(self): self.filename = 'crab3checkwrite.tmp' self.username = self.proxy.getHyperNewsName() phedex = PhEDEx({"cert": self.proxyfilename, "key": self.proxyfilename}) if hasattr(self.options, 'userlfn') and self.options.userlfn != None: lfnsadd = self.options.userlfn + '/' + self.filename else: lfnsadd = '/store/user/' + self.username + '/' + self.filename try: pfndict = phedex.getPFN(nodes = [self.options.sitename], lfns = [lfnsadd]) pfn = pfndict[(self.options.sitename, lfnsadd)] if not pfn: self.logger.info('%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL)) raise ConfigurationException except HTTPException, errormsg : self.logger.info('%sError:%s Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL)) self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url)) raise HTTPException, errormsg
class PhEDExTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the PhEDEx API to point at the test server. """ phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test" self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"}) return @attr("integration") def testInjection(self): """ _testInjection_ Verify that we can inject data into PhEDEx. """ xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID()) result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) self.assertEqual( result["phedex"]["injected"], { "stats": { "closed_datasets": 0, "closed_blocks": 0, "new_blocks": 0, "new_datasets": 1, "new_files": 0 } }) return @attr("integration") def testSubscription(self): """ _testSubscription_ Verify that the subscription API works. """ datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID() datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID() xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS", "Saturn") result = self.phedexApi.subscribe(testSub) requestIDs = result["phedex"]["request_created"] self.assertEqual(len(requestIDs), 1, "Error: Wrong number of request IDs") self.assertTrue("id" in requestIDs[0], "Error: Missing request ID") return @attr("integration") def testBestNodeName(self): """ _testBestNodeName_ Verify that the node name is Buffer first """ self.assertTrue( self.phedexApi.getBestNodeName("cmssrm.fnal.gov") == "T1_US_FNAL_Buffer") return @attr('integration') def testGetSubscriptionMapping(self): """ _testGetSubscriptionMapping_ Verify that the subscription mapping API works correctly. """ testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID() blockA = "%s#%s" % (testDataset, makeUUID()) blockB = "%s#%s" % (testDataset, makeUUID()) datasetSpec = injectionSpec.getDataset(testDataset) datasetSpec.getFileblock(blockA, 'y') datasetSpec.getFileblock(blockB, 'y') blockSpec = injectionSpec.save() self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec) # Create a dataset level subscription to a node testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS", "Saturn", request_only="n") self.phedexApi.subscribe(testDatasetSub) # Create a block level subscrtion to a different node testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn", level="block", request_only="n") self.phedexApi.subscribe(testBlockSub) subs = self.phedexApi.getSubscriptionMapping(testDataset) self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"}, "Error: Dataset subscription is wrong.") subs = self.phedexApi.getSubscriptionMapping(blockA) self.assertEqual( len(subs[blockA]), 2, "Error: Wrong number of nodes in block subscription.") self.assertTrue("T1_UK_RAL_MSS" in subs[blockA], "Error: RAL missing from block sub.") self.assertTrue("T1_DE_KIT_MSS" in subs[blockA], "Error: KIT missing from block sub.") return def testPFNLookup(self): """ _testPFNLookup_ Verify that the PFN lookup in PhEDEx works correctly. """ call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file']) # Should get one mapping back (one lfn, one node) self.assertTrue(len(call1.keys()) == 1) call1_key = call1.keys()[0] call2 = self.phedexApi.getPFN( ['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file']) # Should get back two mappings (two nodes) self.assertTrue(call1_key in call2.keys()) # and one of the mappings should be the same as from the previous call self.assertTrue(call1[call1_key] == call2[call1_key]) return
class PhEDExTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the PhEDEx API to point at the test server. """ phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test" self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"}) return @attr("integration") def testInjection(self): """ _testInjection_ Verify that we can inject data into PhEDEx. """ xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID()) result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) self.assertEqual(result["phedex"]["injected"], {"stats": {"closed_datasets": 0, "closed_blocks": 0, "new_blocks": 0, "new_datasets": 1, "new_files": 0}}) return @attr("integration") def testSubscription(self): """ _testSubscription_ Verify that the subscription API works. """ datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID() datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID() xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS", "Saturn") xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl, testSub.getDatasetPaths()) result = self.phedexApi.subscribe(testSub, xmlData) requestIDs = result["phedex"]["request_created"] self.assertEqual(len(requestIDs), 1, "Error: Wrong number of request IDs") self.assertTrue(requestIDs[0].has_key("id"), "Error: Missing request ID") return @attr("integration") def testNodeMap(self): """ _testNodeMap_ Verify that the node map can be retrieve from PhEDEx and that the getNodeSE() and getNodeNames() methods work correctly. """ self.failUnless(self.phedexApi.getNodeSE("T2_FR_GRIF_LLR") == "polgrid4.in2p3.fr") self.failUnless(self.phedexApi.getNodeNames("cmssrm.fnal.gov") == ["T1_US_FNAL_Buffer", "T1_US_FNAL_MSS"]) return @attr('integration') def testGetSubscriptionMapping(self): """ _testGetSubscriptionMapping_ Verify that the subscription mapping API works correctly. """ testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID() blockA = "%s#%s" % (testDataset, makeUUID()) blockB = "%s#%s" % (testDataset, makeUUID()) injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl) datasetSpec = injectionSpec.getDataset(testDataset) datasetSpec.getFileblock(blockA, 'y') datasetSpec.getFileblock(blockB, 'y') blockSpec = injectionSpec.save() self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec) # Create a dataset level subscription to a node testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS", "Saturn", requestOnly = "n") datasetSpec = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl, testDatasetSub.getDatasetPaths()) self.phedexApi.subscribe(testDatasetSub, datasetSpec) # Create a block level subscrtion to a different node testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn", level = "block", requestOnly = "n") self.phedexApi.subscribe(testBlockSub, blockSpec) subs = self.phedexApi.getSubscriptionMapping(testDataset) self.assertEqual(subs[testDataset], set(["T1_UK_RAL_MSS"]), "Error: Dataset subscription is wrong.") subs = self.phedexApi.getSubscriptionMapping(blockA) self.assertEqual(len(subs[blockA]), 2, "Error: Wrong number of nodes in block subscription.") self.assertTrue("T1_UK_RAL_MSS" in subs[blockA], "Error: RAL missing from block sub.") self.assertTrue("T1_DE_KIT_MSS" in subs[blockA], "Error: KIT missing from block sub.") return def testPFNLookup(self): """ _testPFNLookup_ Verify that the PFN lookup in PhEDEx works correctly. """ call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file']) # Should get one mapping back (one lfn, one node) self.assertTrue(len(call1.keys()) == 1) call1_key = call1.keys()[0] call2 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file']) # Should get back two mappings (two nodes) self.assertTrue(call1_key in call2.keys()) # and one of the mappings should be the same as from the previous call self.assertTrue(call1[call1_key] == call2[call1_key]) return @attr('integration') def testXMLJSON(self): """ Test XML and JSON in the same scope """ site = 'T1_US_FNAL_Buffer' dict = {} dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/json/test" phedexJSON = PhEDEx(responseType='json', dict=dict) dict['endpoint'] = "https://cmsweb.cern.ch/phedex/datasvc/xml/test" phedexXML = PhEDEx(responseType='xml', dict=dict) phedexXML.getNodeTFC(site) tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node' : site}) tfc_map = {} tfc_map[site] = readTFC(tfc_file) pfn = tfc_map[site].matchLFN('srmv2', '/store/user/jblow/dir/test.root') self.failUnless(pfn == 'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root') self.failUnless(phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov') @attr('integration') def testAuth(self): """ _testAuth_ Verify that the auth method works correctly." """ self.assertFalse(self.phedexApi.getAuth("datasvc_whatever")) self.assertTrue(self.phedexApi.getAuth("datasvc_subscribe")) self.assertTrue(self.phedexApi.getAuth("datasvc_inject")) return
class PhEDExTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the PhEDEx API to point at the test server. """ self.dbsTestUrl = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" self.phedexApi = PhEDEx() return @attr("integration") def testInjection(self): """ _testInjection_ Verify that we can inject data into PhEDEx. """ xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID()) result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) self.assertEqual(result["phedex"]["injected"], {"stats": {"closed_datasets": 0, "closed_blocks": 0, "new_blocks": 0, "new_datasets": 1, "new_files": 0}}) return @attr("integration") def testSubscription(self): """ _testSubscription_ Verify that the subscription API works. """ datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID() datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID() xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS", "Saturn") result = self.phedexApi.subscribe(testSub) requestIDs = result["phedex"]["request_created"] self.assertEqual(len(requestIDs), 1, "Error: Wrong number of request IDs") self.assertTrue("id" in requestIDs[0], "Error: Missing request ID") return @attr('integration') def testGetSubscriptionMapping(self): """ _testGetSubscriptionMapping_ Verify that the subscription mapping API works correctly. """ testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID() blockA = "%s#%s" % (testDataset, makeUUID()) blockB = "%s#%s" % (testDataset, makeUUID()) datasetSpec = injectionSpec.getDataset(testDataset) datasetSpec.getFileblock(blockA, 'y') datasetSpec.getFileblock(blockB, 'y') blockSpec = injectionSpec.save() self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec) # Create a dataset level subscription to a node testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS", "Saturn", request_only="n") self.phedexApi.subscribe(testDatasetSub) # Create a block level subscrtion to a different node testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn", level="block", request_only="n") self.phedexApi.subscribe(testBlockSub) subs = self.phedexApi.getSubscriptionMapping(testDataset) self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"}, "Error: Dataset subscription is wrong.") subs = self.phedexApi.getSubscriptionMapping(blockA) self.assertEqual(len(subs[blockA]), 2, "Error: Wrong number of nodes in block subscription.") self.assertTrue("T1_UK_RAL_MSS" in subs[blockA], "Error: RAL missing from block sub.") self.assertTrue("T1_DE_KIT_MSS" in subs[blockA], "Error: KIT missing from block sub.") return def testPFNLookup(self): """ _testPFNLookup_ Verify that the PFN lookup in PhEDEx works correctly. """ call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file']) # Should get one mapping back (one lfn, one node) self.assertTrue(len(call1.keys()) == 1) call1_key = call1.keys()[0] call2 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file']) # Should get back two mappings (two nodes) self.assertTrue(call1_key in call2.keys()) # and one of the mappings should be the same as from the previous call self.assertTrue(call1[call1_key] == call2[call1_key]) return def testGetReplicaInfoForBlocks(self): """ Test `getReplicaInfoForBlocks` method, the ability to retrieve replica locations provided a (or a list of) datasets and blocks """ def _checkOutcome(numFiles, replica): "run the checks" if rep['complete'] == 'y': self.assertEqual(rep['files'], numFiles) if rep['custodial'] == 'y': self.assertTrue(rep['node'].endswith("_MSS")) self.assertTrue(rep['subscribed'], 'y') replicaDict = {'bytes', 'complete', 'custodial', 'files', 'group', 'node', 'node_id', 'se', 'subscribed', 'time_create', 'time_update'} res = self.phedexApi.getReplicaInfoForBlocks(block=BLOCK)['phedex'] self.assertEqual(len(res['block']), 1) self.assertEqual(res['block'][0]['name'], BLOCK) self.assertTrue(len(res['block'][0]['replica']) > 1) self.assertItemsEqual(res['block'][0]['replica'][0].keys(), replicaDict) numFiles = res['block'][0]['files'] for rep in res['block'][0]['replica']: _checkOutcome(numFiles, rep) # same test, but providing a dataset as input (which has only the block above) res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET)['phedex'] self.assertEqual(len(res['block']), 4) self.assertTrue(BLOCK in [blk['name'] for blk in res['block']]) for block in res['block']: numFiles = block['files'] for rep in block['replica']: self.assertTrue(len(block['replica']) > 1) _checkOutcome(numFiles, rep) # same test again, but providing both block and dataset # NOTE the PhEDEx service only process the block input, the # dataset argument is completely ignored res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET, block=BLOCK)['phedex'] self.assertEqual(len(res['block']), 1) self.assertEqual(res['block'][0]['name'], BLOCK) self.assertTrue(len(res['block'][0]['replica']) > 1) self.assertItemsEqual(res['block'][0]['replica'][0].keys(), replicaDict) numFiles = res['block'][0]['files'] for rep in res['block'][0]['replica']: _checkOutcome(numFiles, rep) # provide a block that does not exist res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET, block=BLOCK + "BLAH")['phedex'] self.assertTrue(res['block'] == [])
class checkwrite(SubCommand): """ Let user to test if he/she have permission to write on specify site """ name = 'checkwrite' shortnames = ['chk'] def __init__(self, logger, cmdargs=None): SubCommand.__init__(self, logger, cmdargs) self.phedex = PhEDEx({ "cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger }) self.lfnsaddprefix = None self.filename = None def __call__(self): username = None if hasattr(self.options, 'userlfn') and self.options.userlfn != None: self.lfnsaddprefix = self.options.userlfn else: ## If the user didn't provide an LFN path where to check the write permission, ## assume he/she wants to check in /store/user/<username>. Retrieve his/her ## username from SiteDB. self.logger.info( 'Will check write permission in the default location /store/user/<username>' ) username = getUserDNandUsernameFromSiteDB( self.logger).get('username') if username: self.lfnsaddprefix = '/store/user/' + username else: return {'status': 'FAILED'} ## Check that the location where we want to check write permission ## is one where the user will be allowed to stageout. self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix)) msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % ( self.lfnsaddprefix) msg += "\nThe LFN must start with either '/store/user/<username>/', '/store/group/<groupname>[/<subgroupname>]*/<username>/' or '/store/local/<dirname>'," msg += " where username is your username as registered in SiteDB (i.e. the username of your CERN primary account)." msg += "\nLFN %s is not valid." % (self.lfnsaddprefix) if not username: username = getUserDNandUsernameFromSiteDB( self.logger).get('username') if not checkOutLFN(self.lfnsaddprefix, username): self.logger.info(msg) return {'status': 'FAILED'} else: self.logger.info("LFN %s is valid." % (self.lfnsaddprefix)) cp_cmd = "" del_cmd = "" if cmd_exist("gfal-copy") and cmd_exist("gfal-rm"): self.logger.info( "Will use `gfal-copy`, `gfal-rm` commands for checking write permissions" ) cp_cmd = "env -i gfal-copy -v -t 180 " del_cmd = "env -i gfal-rm -v -t 180 " elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"): self.logger.info( "Will use `lcg-cp`, `lcg-del` commands for checking write permissions" ) cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 " del_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 " else: self.logger.info("Neither gfal nor lcg command was found") return {'status': 'FAILED'} self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)) retry = 0 stop = False use_new_file = True while not stop: if use_new_file: self.filename = 'crab3checkwrite.' + str(retry) + '.tmp' self.createFile() pfn = self.getPFN() self.logger.info( 'Attempting to copy (dummy) file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename)) cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd) if cpexitcode == 0: self.logger.info( 'Successfully copied file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename)) self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename)) delexitcode = self.delete(pfn, del_cmd) if delexitcode: self.logger.info( '%sWarning%s: Failed to delete file %s from site %s' % (colors.RED, colors.NORMAL, pfn, self.options.sitename)) else: self.logger.info( 'Successfully deleted file %s from site %s' % (pfn, self.options.sitename)) self.logger.info( '%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)) returndict = {'status': 'SUCCESS'} stop = True else: if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr: msg = '%sError%s: Unable to write in %s on site %s' % ( colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename) msg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' self.logger.info(msg) returndict = {'status': 'FAILED'} stop = True elif 'timeout' in cpout or 'timeout' in cperr: self.logger.info('Connection time out.') msg = 'Unable to check write permission in %s on site %s' % ( self.lfnsaddprefix, self.options.sitename) msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' self.logger.info(msg) returndict = {'status': 'FAILED'} stop = True elif 'exist' in cpout or 'exist' in cperr and retry == 0: self.logger.info( 'Error copying file %s to %s on site %s; it may be that file already exists.' % (self.filename, self.lfnsaddprefix, self.options.sitename)) self.logger.info( 'Attempting to delete file %s from site %s' % (pfn, self.options.sitename)) delexitcode = self.delete(pfn, del_cmd) if delexitcode: self.logger.info( 'Failed to delete file %s from site %s' % (pfn, self.options.sitename)) use_new_file = True else: self.logger.info( 'Successfully deleted file %s from site %s' % (pfn, self.options.sitename)) use_new_file = False retry += 1 else: msg = 'Unable to check write permission in %s on site %s' % ( self.lfnsaddprefix, self.options.sitename) msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' self.logger.info(msg) returndict = {'status': 'FAILED'} stop = True if stop or use_new_file: self.removeFile() self.logger.info( '%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL)) return returndict def createFile(self): abspath = path.abspath(self.filename) try: with open(abspath, 'w') as fd: fd.write( 'This is a dummy file created by the crab checkwrite command on %s' % str(datetime.datetime.now().strftime( '%d/%m/%Y at %H:%M:%S'))) except IOError: self.logger.info('%sError%s: Failed to create file %s' % (colors.RED, colors.NORMAL, self.filename)) raise Exception def removeFile(self): abspath = path.abspath(self.filename) try: remove(abspath) except: self.logger.info('%sWarning%s: Failed to delete file %s' % (colors.RED, colors.NORMAL, self.filename)) def getPFN(self): lfnsadd = self.lfnsaddprefix + '/' + self.filename try: pfndict = self.phedex.getPFN(nodes=[self.options.sitename], lfns=[lfnsadd]) pfn = pfndict[(self.options.sitename, lfnsadd)] if not pfn: self.logger.info( '%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL)) raise ConfigurationException except HTTPException, errormsg: self.logger.info( '%sError%s: Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL)) self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url)) raise HTTPException, errormsg return pfn
def submit(trans_tuple, job_data, log, direct=False): """Manage threads for transfers submission through Rucio :param trans_tuple: ordered list of needed xfer info (transfers, to_submit_columns) :type trans_tuple: tuple :param job_data: general CRAB job metadata :type job_data: dict :param log: log object :type log: logging :param direct: job output stored on temp or directly, defaults to False :param direct: bool, optional """ threadLock = threading.Lock() threads = [] to_update = [] toTrans = trans_tuple[0] columns = trans_tuple[1] proxy = job_data['proxy'] rest_filetransfers = job_data['rest'] user = job_data['username'] destination = job_data['destination'] taskname = job_data['taskname'] try: phedex = PhEDEx(responseType='xml', httpDict={'key': proxy, 'cert': proxy, 'pycurl': True}) except Exception: log.exception('PhEDEx exception.') return # Split threads by source RSEs sources = list(set([x[columns.index('source')] for x in toTrans])) os.environ["X509_CERT_DIR"] = os.getcwd() log.info("Connection to %s with proxy in:\n %s" % (rest_filetransfers,proxy)) oracleDB = HTTPRequests(rest_filetransfers, proxy, proxy) #verbose=True) # mapping lfn <--> pfn for source in sources: ids = [x[columns.index('id')] for x in toTrans if x[columns.index('source')] == source] src_lfns = [x[columns.index('source_lfn')] for x in toTrans if x[columns.index('source')] == source] dst_lfns = [x[columns.index('destination_lfn')] for x in toTrans if x[columns.index('source')] == source] sorted_source_pfns = [] sorted_dest_lfns = [] sorted_dest_pfns = [] # workaround for phedex.getPFN issue --> shuffling output order w.r.t. the list in input try: for chunk in chunks(src_lfns, 10): unsorted_source_pfns = [[k[1], str(x)] for k, x in phedex.getPFN(source, chunk).items()] for order_lfn in chunk: for lfn, pfn in unsorted_source_pfns: if order_lfn == lfn: sorted_source_pfns.append(pfn) break for chunk in chunks(dst_lfns, 10): unsorted_dest_pfns = [[k[1], str(x)] for k, x in phedex.getPFN(toTrans[0][4], chunk).items()] for order_lfn in chunk: for lfn, pfn in unsorted_dest_pfns: if order_lfn == lfn: sorted_dest_pfns.append(pfn) sorted_dest_lfns.append(lfn) break except Exception as ex: log.error("Failed to map lfns to pfns: %s", ex) mark_failed(ids, ["Failed to map lfn to pfn: " + str(ex) for _ in ids], oracleDB) source_pfns = sorted_source_pfns dest_lfns = sorted_dest_lfns # saving file sizes and checksums filesizes = [x[columns.index('filesize')] for x in toTrans if x[columns.index('source')] == source] checksums = [x[columns.index('checksums')] for x in toTrans if x[columns.index('source')] == source] pubnames = [x[columns.index('publishname')] for x in toTrans if x[columns.index('source')] == source] # ordered list of replicas information jobs = zip(source_pfns, dest_lfns, ids, checksums, filesizes, pubnames) job_columns = ['source_pfns', 'dest_lfns', 'ids', 'checksums', 'filesizes', 'pubnames'] # ordered list of transfers details tx_from_source = [[job, source, taskname, user, destination] for job in jobs] tx_columns = ['job', 'source', 'taskname', 'user', 'destination'] # split submission process in chunks of max 200 files for files in chunks(tx_from_source, 200): if not direct: log.info("Submitting: %s", files) thread = submit_thread(threadLock, log, (files, tx_columns), job_columns, proxy, to_update) thread.start() threads.append(thread) elif direct: log.info("Registering direct stageout: %s", files) thread = submit_thread(threadLock, log, (files, tx_columns), job_columns, proxy, to_update, direct=True) thread.start() threads.append(thread) for t in threads: t.join() if len(to_update) == 0: return False # update statuses in oracle table as per threads result for fileDoc in to_update: try: log.debug("%s/filetransfers?%s" % (rest_filetransfers, encodeRequest(fileDoc))) oracleDB.post('/filetransfers', data=encodeRequest(fileDoc)) log.info("Marked submitted %s files" % (fileDoc['list_of_ids'])) except Exception: log.exception('Failed to mark files as submitted on DBs') return True
class checkwrite(SubCommand): """ Let user to test if he/she have permission to write on specify site """ name = 'checkwrite' shortnames = ['chk'] def __init__(self, logger, cmdargs = None): SubCommand.__init__(self, logger, cmdargs) self.phedex = PhEDEx({"cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger, "pycurl" : True}) self.lfnsaddprefix = None self.filename = None def __call__(self): username = None if hasattr(self.options, 'userlfn') and self.options.userlfn != None: self.lfnsaddprefix = self.options.userlfn else: ## If the user didn't provide an LFN path where to check the write permission, ## assume he/she wants to check in /store/user/<username>. Retrieve his/her ## username from SiteDB. self.logger.info('Will check write permission in the default location /store/user/<username>') username = getUserDNandUsernameFromSiteDB(self.logger).get('username') if username: self.lfnsaddprefix = '/store/user/' + username else: return {'status': 'FAILED'} ## Check that the location where we want to check write permission ## is one where the user will be allowed to stageout. self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix)) msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (self.lfnsaddprefix) msg += "\nThe LFN must start with either" msg += " '/store/user/<username>/' or '/store/group/<groupname>/'" msg += " (or '/store/local/<something>/' if publication is off)," msg += " where username is your username as registered in SiteDB" msg += " (i.e. the username of your CERN primary account)." msg += "\nLFN %s is not valid." % (self.lfnsaddprefix) if not username and self.lfnsaddprefix.startswith('/store/user/'): username = getUserDNandUsernameFromSiteDB(self.logger).get('username') if not checkOutLFN(self.lfnsaddprefix, username): self.logger.info(msg) return {'status': 'FAILED'} else: self.logger.info("LFN %s is valid." % (self.lfnsaddprefix)) cp_cmd = "" if cmd_exist("gfal-copy") and cmd_exist("gfal-rm") and self.command in [None, "GFAL"]: self.logger.info("Will use `gfal-copy`, `gfal-rm` commands for checking write permissions") cp_cmd = "env -i X509_USER_PROXY=%s gfal-copy -p -v -t 180 " % os.path.abspath(self.proxyfilename) delfile_cmd = "env -i X509_USER_PROXY=%s gfal-rm -v -t 180 " % os.path.abspath(self.proxyfilename) deldir_cmd = "env -i X509_USER_PROXY=%s gfal-rm -r -v -t 180 " % os.path.abspath(self.proxyfilename) if self.checksum: cp_cmd += "-K %s " % self.checksum elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"): self.logger.info("Will use `lcg-cp`, `lcg-del` commands for checking write permissions") cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 " delfile_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 " deldir_cmd = "lcg-del -d --connect-timeout 180 -b -l -D srmv2 " if self.checksum: cp_cmd += "--checksum-type %s " % self.checksum else: self.logger.info("Neither gfal nor lcg command was found") return {'status': 'FAILED'} self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)) timestamp = str(time.strftime("%Y%m%d_%H%M%S")) self.filename = 'crab3checkwrite_' + timestamp + '.tmp' self.subdir = 'crab3checkwrite_' + timestamp self.createFile() pfn = self.getPFN() dirpfn = pfn[:len(pfn)-len(self.filename)] self.logger.info('\nAttempting to create (dummy) directory %s and copy (dummy) file %s to %s\n' % (self.subdir, self.filename, self.lfnsaddprefix)) cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd) if cpexitcode == 0: self.logger.info('\nSuccessfully created directory %s and copied file %s to %s' % (self.subdir, self.filename, self.lfnsaddprefix)) self.logger.info('\nAttempting to delete file %s\n' % (pfn)) delexitcode = self.delete(pfn, delfile_cmd) if delexitcode: self.logger.info('\nFailed to delete file %s' % (pfn)) finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete file in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename) finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} else: self.logger.info('\nSuccessfully deleted file %s' % (pfn)) self.logger.info('\nAttempting to delete directory %s\n' % (dirpfn)) delexitcode = self.delete(dirpfn, deldir_cmd) if delexitcode: self.logger.info('\nFailed to delete directory %s' % (dirpfn)) finalmsg = '%sError%s: CRAB3 is able to copy but unable to delete directory in %s on site %s. Asynchronous Stage Out with CRAB3 will fail.' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename) finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} else: self.logger.info('\nSuccessfully deleted directory %s' % (dirpfn)) finalmsg = '%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename) returndict = {'status': 'SUCCESS'} else: if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr: finalmsg = '%sError%s: Unable to write in %s on site %s' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename) finalmsg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} elif 'timeout' in cpout or 'timeout' in cperr: self.logger.info('Connection time out.') finalmsg = '\nUnable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename) finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status': 'FAILED'} else: finalmsg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename) finalmsg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' returndict = {'status' : 'FAILED'} self.removeFile() self.logger.info('\nCheckwrite Result:') self.logger.info(finalmsg) if returndict['status'] == 'FAILED': self.logger.info('%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL)) return returndict def createFile(self): abspath = os.path.abspath(self.filename) try: with open(abspath, 'w') as fd: fd.write('This is a dummy file created by the crab checkwrite command on %s' % str(datetime.datetime.now().strftime('%d/%m/%Y at %H:%M:%S'))) except IOError: self.logger.info('%sError%s: Failed to create file %s' % (colors.RED, colors.NORMAL, self.filename)) raise Exception def removeFile(self): abspath = os.path.abspath(self.filename) try: os.remove(abspath) except: self.logger.info('%sWarning%s: Failed to delete file %s' % (colors.RED, colors.NORMAL, self.filename)) def getPFN(self): lfnsadd = self.lfnsaddprefix + '/' + self.subdir + '/' + self.filename try: pfndict = self.phedex.getPFN(nodes = [self.options.sitename], lfns = [lfnsadd]) pfn = pfndict[(self.options.sitename, lfnsadd)] if not pfn: self.logger.info('%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL)) raise ConfigurationException except HTTPException as errormsg: self.logger.info('%sError%s: Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL)) self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url)) raise return pfn def cp(self, pfn, command): abspath = os.path.abspath(self.filename) if cmd_exist("gfal-copy") and self.command in [None, "GFAL"]: abspath = "file://" + abspath cpcmd = command + abspath + " '" + pfn + "'" self.logger.info('Executing command: %s' % cpcmd) self.logger.info('Please wait...') cpprocess = subprocess.Popen(cpcmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True) cpout, cperr = cpprocess.communicate() cpexitcode = cpprocess.returncode if cpexitcode: self.logger.info('Failed running copy command') if cpout: self.logger.info(' Stdout:\n %s' % str(cpout).replace('\n', '\n ')) if cperr: self.logger.info(' Stderr:\n %s' % str(cperr).replace('\n', '\n ')) return cpout, cperr, cpexitcode def delete(self, pfn, command): rmcmd = command + "'" + pfn + "'" self.logger.info('Executing command: %s' % rmcmd) self.logger.info('Please wait...') delprocess = subprocess.Popen(rmcmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True) delout, delerr = delprocess.communicate() delexitcode = delprocess.returncode if delexitcode: self.logger.info('Failed running delete command') if delout: self.logger.info(' Stdout:\n %s' % str(delout).replace('\n', '\n ')) if delerr: self.logger.info(' Stderr:\n %s' % str(delerr).replace('\n', '\n ')) return delexitcode def terminate(self, exitcode): pass def setOptions(self): """ __setOptions__ This allows to set specific command options """ self.parser.add_option('--site', dest = 'sitename', default = None, help = 'The PhEDEx node name of the site to be checked.') self.parser.add_option('--lfn', dest = 'userlfn', default = None, help = 'A user lfn address.') self.parser.add_option('--checksum', dest = 'checksum', default = 'yes', help = 'Set it to true if needed. If true will use ADLER32 checksum' +\ 'Allowed values are yes/no. Default is yes.') self.parser.add_option('--command', dest = 'command', default = None, help = 'A command which to use. Available commands are LCG or GFAL.') def validateOptions(self): SubCommand.validateOptions(self) if self.options.sitename is None: msg = "%sError%s: Please specify the site where to check the write permissions." % (colors.RED, colors.NORMAL) msg += " Use the --site option." ex = MissingOptionException(msg) ex.missingOption = "sitename" raise ex if hasattr(self.options, 'command') and self.options.command != None: AvailableCommands = ['LCG', 'GFAL'] self.command = self.options.command.upper() if self.command not in AvailableCommands: msg = "You specified to use %s command and it is not allowed. Available commands are: %s " % (self.command, str(AvailableCommands)) ex = ConfigurationException(msg) raise ex else: self.command = None if hasattr(self.options, 'checksum'): if re.match('^yes$|^no$', self.options.checksum): self.checksum = 'ADLER32' if self.options.checksum == 'yes' else None else: msg = "You specified to use %s checksum. Only lowercase yes/no is accepted to turn ADLER32 checksum" % self.options.checksum ex = ConfigurationException(msg) raise ex
class PhEDExTest(EmulatedUnitTestCase): def setUp(self): """ _setUp_ Initialize the PhEDEx API to point at the test server. """ self.dbsTestUrl = "https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader" self.phedexApi = PhEDEx() return @attr("integration") def testInjection(self): """ _testInjection_ Verify that we can inject data into PhEDEx. """ xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID()) result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) self.assertEqual( result["phedex"]["injected"], { "stats": { "closed_datasets": 0, "closed_blocks": 0, "new_blocks": 0, "new_datasets": 1, "new_files": 0 } }) return @attr("integration") def testSubscription(self): """ _testSubscription_ Verify that the subscription API works. """ datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID() datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID() xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS", "Saturn") result = self.phedexApi.subscribe(testSub) requestIDs = result["phedex"]["request_created"] self.assertEqual(len(requestIDs), 1, "Error: Wrong number of request IDs") self.assertTrue("id" in requestIDs[0], "Error: Missing request ID") return @attr('integration') def testGetSubscriptionMapping(self): """ _testGetSubscriptionMapping_ Verify that the subscription mapping API works correctly. """ testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID() blockA = "%s#%s" % (testDataset, makeUUID()) blockB = "%s#%s" % (testDataset, makeUUID()) # NOTE: leaving it broken on purpose, we do NOT want to subscribe # data via unit tests :-) #injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl) datasetSpec = injectionSpec.getDataset(testDataset) datasetSpec.getFileblock(blockA, 'y') datasetSpec.getFileblock(blockB, 'y') blockSpec = injectionSpec.save() self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec) # Create a dataset level subscription to a node testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS", "Saturn", request_only="y") self.phedexApi.subscribe(testDatasetSub) # Create a block level subscrtion to a different node testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn", level="block", request_only="y") self.phedexApi.subscribe(testBlockSub) subs = self.phedexApi.getSubscriptionMapping(testDataset) self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"}, "Error: Dataset subscription is wrong.") subs = self.phedexApi.getSubscriptionMapping(blockA) self.assertEqual( len(subs[blockA]), 2, "Error: Wrong number of nodes in block subscription.") self.assertTrue("T1_UK_RAL_MSS" in subs[blockA], "Error: RAL missing from block sub.") self.assertTrue("T1_DE_KIT_MSS" in subs[blockA], "Error: KIT missing from block sub.") return def testPFNLookup(self): """ _testPFNLookup_ Verify that the PFN lookup in PhEDEx works correctly. """ call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file']) # Should get one mapping back (one lfn, one node) self.assertTrue(len(call1.keys()) == 1) call1_key = call1.keys()[0] call2 = self.phedexApi.getPFN( ['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file']) # Should get back two mappings (two nodes) self.assertTrue(call1_key in call2.keys()) # and one of the mappings should be the same as from the previous call self.assertTrue(call1[call1_key] == call2[call1_key]) return def testGetReplicaInfoForBlocks(self): """ Test `getReplicaInfoForBlocks` method, the ability to retrieve replica locations provided a (or a list of) datasets and blocks """ def _checkOutcome(numFiles, replica): "run the checks" if rep['complete'] == 'y': self.assertEqual(rep['files'], numFiles) if rep['custodial'] == 'y': self.assertTrue(rep['node'].endswith("_MSS")) self.assertTrue(rep['subscribed'], 'y') replicaDict = { 'bytes', 'complete', 'custodial', 'files', 'group', 'node', 'node_id', 'se', 'subscribed', 'time_create', 'time_update' } res = self.phedexApi.getReplicaInfoForBlocks(block=BLOCK)['phedex'] self.assertEqual(len(res['block']), 1) self.assertEqual(res['block'][0]['name'], BLOCK) self.assertTrue(len(res['block'][0]['replica']) > 1) self.assertItemsEqual(res['block'][0]['replica'][0].keys(), replicaDict) numFiles = res['block'][0]['files'] for rep in res['block'][0]['replica']: _checkOutcome(numFiles, rep) # same test, but providing a dataset as input (which has only the block above) res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET)['phedex'] self.assertEqual(len(res['block']), 4) self.assertTrue(BLOCK in [blk['name'] for blk in res['block']]) for block in res['block']: numFiles = block['files'] for rep in block['replica']: self.assertTrue(len(block['replica']) > 1) _checkOutcome(numFiles, rep) # same test again, but providing both block and dataset # NOTE the PhEDEx service only process the block input, the # dataset argument is completely ignored res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET, block=BLOCK)['phedex'] self.assertEqual(len(res['block']), 1) self.assertEqual(res['block'][0]['name'], BLOCK) self.assertTrue(len(res['block'][0]['replica']) > 1) self.assertItemsEqual(res['block'][0]['replica'][0].keys(), replicaDict) numFiles = res['block'][0]['files'] for rep in res['block'][0]['replica']: _checkOutcome(numFiles, rep) # provide a block that does not exist res = self.phedexApi.getReplicaInfoForBlocks(dataset=DSET, block=BLOCK + "BLAH")['phedex'] self.assertTrue(res['block'] == []) def testGroupUsage(self): """ _testGroupUsage_ Verify that the `getGroupUsage` API works correctly. """ node = "T2_DE_DESY" group = "DataOps" res = self.phedexApi.getGroupUsage(group=group, node=node)['phedex'] self.assertEqual(len(res['node']), 1) self.assertEqual(len(res['node'][0]['group']), 1) self.assertEqual(res['node'][0]['group'][0]['name'], group) self.assertEqual(res['node'][0]['name'], node) self.assertTrue(res['node'][0]['group'][0]['dest_bytes'] > 100) res = self.phedexApi.getGroupUsage(group=group)['phedex'] self.assertTrue(len(res['node']) > 50) self.assertEqual(len(res['node'][10]['group']), 1) self.assertEqual(res['node'][10]['group'][0]['name'], group) return
def swapLocations(options): #Initialize stuff phedexAPI = PhEDEx({'cachepath' : options.cachepath}) acdcCouch = Database('wmagent_acdc', options.acdcUrl) #Let's get the IDs of the ACDC documents for the task/request/group/user array = [options.group, options.user, options.request, options.task] result = acdcCouch.loadView('ACDC', 'owner_coll_fileset_docs', {'reduce' : False}, [array]) documentsIDs = [x['id'] for x in result['rows']] #Load the map file saying what we want to change of location mapFile = open(options.map, 'r') locationMap = json.load(mapFile) mapFile.close() #Go through the documents for docID in documentsIDs: doc = acdcCouch.document(docID) #Are we going to change this doc? Better back it up if options.change: backupFile = os.open(os.path.join(options.backup, "%s.bkp" % doc["_id"]), 'w') json.dump(doc, backupFile) backupFile.close() #Go through the files files = doc["files"] for inputFile in files: #Use PhEDEx API to get site based on the SE #Then map that to the desired target se = files[inputFile]["locations"][0] siteLocation = phedexAPI.getBestNodeName(se) targetLocation = locationMap.get(siteLocation, siteLocation) if siteLocation == targetLocation: #Nothing to do with this one, move on continue if not options.change: #No changes, then give the commands to move the files #Get the PFN for both the current location and the target location pfnDict = phedexAPI.getPFN(siteLocation, inputFile) inputPfn = pfnDict[(siteLocation, inputFile)] pfnDict = phedexAPI.getPFN(targetLocation, inputFile) targetPfn = pfnDict[(targetLocation, inputFile)] #Print it to stdout print "lcg-cp -D srmv2 -b %s %s" % (inputPfn, targetPfn) else: #This is changes time, let's move the stuff targetSE = phedexAPI.getNodeSE(targetLocation) files[inputFile]["locations"][0] = targetSE print "Changing location of %s from %s to %s" % (inputFile, se, targetSE) #If specified, commit the changes if options.change: acdcCouch.commitOne(doc) return 0
class checkwrite(SubCommand): """ Let user to test if he/she have permission to write on specify site """ name = 'checkwrite' shortnames = ['chk'] def __init__(self, logger, cmdargs = None): SubCommand.__init__(self, logger, cmdargs) self.phedex = PhEDEx({"cert": self.proxyfilename, "key": self.proxyfilename, "logger": self.logger}) self.lfnsaddprefix = None self.filename = None def __call__(self): username = None if hasattr(self.options, 'userlfn') and self.options.userlfn != None: self.lfnsaddprefix = self.options.userlfn else: ## If the user didn't provide an LFN path where to check the write permission, ## assume he/she wants to check in /store/user/<username>. Retrieve his/her ## username from SiteDB. self.logger.info('Will check write permission in the default location /store/user/<username>') username = getUserDNandUsernameFromSiteDB(self.logger).get('username') if username: self.lfnsaddprefix = '/store/user/' + username else: return {'status': 'FAILED'} ## Check that the location where we want to check write permission ## is one where the user will be allowed to stageout. self.logger.info("Validating LFN %s..." % (self.lfnsaddprefix)) msg = "Refusing to check write permission in %s, because this is not an allowed LFN for stageout." % (self.lfnsaddprefix) msg += "\nThe LFN must start with either" msg += " '/store/user/<username>/' or '/store/group/<groupname>/'" msg += " (or '/store/local/<something>/' if publication is off)," msg += " where username is your username as registered in SiteDB" msg += " (i.e. the username of your CERN primary account)." msg += "\nLFN %s is not valid." % (self.lfnsaddprefix) if not username and self.lfnsaddprefix.startswith('/store/user/'): username = getUserDNandUsernameFromSiteDB(self.logger).get('username') if not checkOutLFN(self.lfnsaddprefix, username): self.logger.info(msg) return {'status': 'FAILED'} else: self.logger.info("LFN %s is valid." % (self.lfnsaddprefix)) cp_cmd = "" del_cmd = "" if cmd_exist("gfal-copy") and cmd_exist("gfal-rm"): self.logger.info("Will use `gfal-copy`, `gfal-rm` commands for checking write permissions") cp_cmd = "env -i gfal-copy -v -t 180 " del_cmd = "env -i gfal-rm -v -t 180 " elif cmd_exist("lcg-cp") and cmd_exist("lcg-del"): self.logger.info("Will use `lcg-cp`, `lcg-del` commands for checking write permissions") cp_cmd = "lcg-cp -v -b -D srmv2 --connect-timeout 180 " del_cmd = "lcg-del --connect-timeout 180 -b -l -D srmv2 " else: self.logger.info("Neither gfal nor lcg command was found") return {'status': 'FAILED'} self.logger.info('Will check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename)) retry = 0 stop = False use_new_file = True while not stop: if use_new_file: self.filename = 'crab3checkwrite.' + str(retry) + '.tmp' self.createFile() pfn = self.getPFN() self.logger.info('Attempting to copy (dummy) file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename)) cpout, cperr, cpexitcode = self.cp(pfn, cp_cmd) if cpexitcode == 0: self.logger.info('Successfully copied file %s to %s on site %s' % (self.filename, self.lfnsaddprefix, self.options.sitename)) self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename)) delexitcode = self.delete(pfn, del_cmd) if delexitcode: self.logger.info('%sWarning%s: Failed to delete file %s from site %s' % (colors.RED, colors.NORMAL, pfn, self.options.sitename)) else: self.logger.info('Successfully deleted file %s from site %s' % (pfn, self.options.sitename)) self.logger.info('%sSuccess%s: Able to write in %s on site %s' % (colors.GREEN, colors.NORMAL, self.lfnsaddprefix, self.options.sitename)) returndict = {'status': 'SUCCESS'} stop = True else: if 'Permission denied' in cperr or 'mkdir: cannot create directory' in cperr: msg = '%sError%s: Unable to write in %s on site %s' % (colors.RED, colors.NORMAL, self.lfnsaddprefix, self.options.sitename) msg += '\n You may want to contact the site administrators sending them the \'crab checkwrite\' output as printed above.' self.logger.info(msg) returndict = {'status': 'FAILED'} stop = True elif 'timeout' in cpout or 'timeout' in cperr: self.logger.info('Connection time out.') msg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename) msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' self.logger.info(msg) returndict = {'status': 'FAILED'} stop = True elif 'exist' in cpout or 'exist' in cperr and retry == 0: self.logger.info('Error copying file %s to %s on site %s; it may be that file already exists.' % (self.filename, self.lfnsaddprefix, self.options.sitename)) self.logger.info('Attempting to delete file %s from site %s' % (pfn, self.options.sitename)) delexitcode = self.delete(pfn, del_cmd) if delexitcode: self.logger.info('Failed to delete file %s from site %s' % (pfn, self.options.sitename)) use_new_file = True else: self.logger.info('Successfully deleted file %s from site %s' % (pfn, self.options.sitename)) use_new_file = False retry += 1 else: msg = 'Unable to check write permission in %s on site %s' % (self.lfnsaddprefix, self.options.sitename) msg += '\nPlease try again later or contact the site administrators sending them the \'crab checkwrite\' output as printed above.' self.logger.info(msg) returndict = {'status' : 'FAILED'} stop = True if stop or use_new_file: self.removeFile() self.logger.info('%sNote%s: You cannot write to a site if you did not ask permission.' % (colors.BOLD, colors.NORMAL)) return returndict def createFile(self): abspath = path.abspath(self.filename) try: with open(abspath, 'w') as fd: fd.write('This is a dummy file created by the crab checkwrite command on %s' % str(datetime.datetime.now().strftime('%d/%m/%Y at %H:%M:%S'))) except IOError: self.logger.info('%sError%s: Failed to create file %s' % (colors.RED, colors.NORMAL, self.filename)) raise Exception def removeFile(self): abspath = path.abspath(self.filename) try: remove(abspath) except: self.logger.info('%sWarning%s: Failed to delete file %s' % (colors.RED, colors.NORMAL, self.filename)) def getPFN(self): lfnsadd = self.lfnsaddprefix + '/' + self.filename try: pfndict = self.phedex.getPFN(nodes = [self.options.sitename], lfns = [lfnsadd]) pfn = pfndict[(self.options.sitename, lfnsadd)] if not pfn: self.logger.info('%sError%s: Failed to get PFN from the site. Please check the site status' % (colors.RED, colors.NORMAL)) raise ConfigurationException except HTTPException, errormsg: self.logger.info('%sError%s: Failed to contact PhEDEx or wrong PhEDEx node name is used' % (colors.RED, colors.NORMAL)) self.logger.info('Result: %s\nStatus :%s\nURL :%s' % (errormsg.result, errormsg.status, errormsg.url)) raise HTTPException, errormsg return pfn
class PhEDExTest(unittest.TestCase): def setUp(self): """ _setUp_ Initialize the PhEDEx API to point at the test server. """ phedexTestDS = "https://cmsweb.cern.ch/phedex/datasvc/json/test" self.dbsTestUrl = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" self.phedexApi = PhEDEx({"endpoint": phedexTestDS, "method": "POST"}) return @attr("integration") def testInjection(self): """ _testInjection_ Verify that we can inject data into PhEDEx. """ xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, makeUUID()) result = self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) self.assertEqual( result["phedex"]["injected"], { "stats": { "closed_datasets": 0, "closed_blocks": 0, "new_blocks": 0, "new_datasets": 1, "new_files": 0 } }) return @attr("integration") def testSubscription(self): """ _testSubscription_ Verify that the subscription API works. """ datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID() datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID() xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB) self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData) testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS", "Saturn") result = self.phedexApi.subscribe(testSub) requestIDs = result["phedex"]["request_created"] self.assertEqual(len(requestIDs), 1, "Error: Wrong number of request IDs") self.assertTrue("id" in requestIDs[0], "Error: Missing request ID") return @attr("integration") def testBestNodeName(self): """ _testBestNodeName_ Verify that the node name is Buffer first """ self.assertTrue( self.phedexApi.getBestNodeName("cmssrm.fnal.gov") == "T1_US_FNAL_Buffer") return @attr("integration") def testNodeMap(self): """ _testNodeMap_ Verify that the node map can be retrieve from PhEDEx and that the getNodeSE() and getNodeNames() methods work correctly. """ self.assertTrue( self.phedexApi.getNodeSE("T2_FR_GRIF_LLR") == "polgrid4.in2p3.fr") self.assertTrue( self.phedexApi.getNodeNames("cmssrm.fnal.gov") == ["T1_US_FNAL_Buffer", "T1_US_FNAL_MSS"]) return @attr('integration') def testGetSubscriptionMapping(self): """ _testGetSubscriptionMapping_ Verify that the subscription mapping API works correctly. """ testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID() blockA = "%s#%s" % (testDataset, makeUUID()) blockB = "%s#%s" % (testDataset, makeUUID()) datasetSpec = injectionSpec.getDataset(testDataset) datasetSpec.getFileblock(blockA, 'y') datasetSpec.getFileblock(blockB, 'y') blockSpec = injectionSpec.save() self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec) # Create a dataset level subscription to a node testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS", "Saturn", request_only="n") self.phedexApi.subscribe(testDatasetSub) # Create a block level subscrtion to a different node testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn", level="block", request_only="n") self.phedexApi.subscribe(testBlockSub) subs = self.phedexApi.getSubscriptionMapping(testDataset) self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"}, "Error: Dataset subscription is wrong.") subs = self.phedexApi.getSubscriptionMapping(blockA) self.assertEqual( len(subs[blockA]), 2, "Error: Wrong number of nodes in block subscription.") self.assertTrue("T1_UK_RAL_MSS" in subs[blockA], "Error: RAL missing from block sub.") self.assertTrue("T1_DE_KIT_MSS" in subs[blockA], "Error: KIT missing from block sub.") return def testPFNLookup(self): """ _testPFNLookup_ Verify that the PFN lookup in PhEDEx works correctly. """ call1 = self.phedexApi.getPFN(['T2_UK_SGrid_Bristol'], ['/store/user/metson/file']) # Should get one mapping back (one lfn, one node) self.assertTrue(len(call1.keys()) == 1) call1_key = call1.keys()[0] call2 = self.phedexApi.getPFN( ['T2_UK_SGrid_Bristol', 'T1_US_FNAL_Buffer'], ['/store/user/metson/file']) # Should get back two mappings (two nodes) self.assertTrue(call1_key in call2.keys()) # and one of the mappings should be the same as from the previous call self.assertTrue(call1[call1_key] == call2[call1_key]) return @attr('integration') def testXMLJSON(self): """ Test XML and JSON in the same scope """ site = 'T1_US_FNAL_Buffer' httpDict = { 'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test" } phedexJSON = PhEDEx(responseType='json', httpDict=httpDict) httpDict = { 'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test" } phedexXML = PhEDEx(responseType='xml', httpDict=httpDict) phedexXML.getNodeTFC(site) tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site}) tfc_map = {} tfc_map[site] = readTFC(tfc_file) pfn = tfc_map[site].matchLFN('srmv2', '/store/user/jblow/dir/test.root') self.assertTrue( pfn == 'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root' ) self.assertTrue( phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov') @attr('integration') def testAuth(self): """ _testAuth_ Verify that the auth method works correctly." """ self.assertFalse(self.phedexApi.getAuth("datasvc_whatever")) self.assertTrue(self.phedexApi.getAuth("datasvc_subscribe")) self.assertTrue(self.phedexApi.getAuth("datasvc_inject")) return