def testTrivialFileCatalog(self):
"""
Run some simple tests on reading a trivialFileCatalog
"""
tfcFilename = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
if not os.path.exists(tfcFilename):
raise Exception("No TrivialFileCatalog found!")
tfcInstance = readTFC(tfcFilename)
self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))
# Look for similarities in each node of the TFC file
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for x in tfcInstance[mapping]:
self.assertEqual('path-match-expr' in x, True)
self.assertEqual('path-match' in x, True)
self.assertEqual('protocol' in x, True)
self.assertEqual('result' in x, True)
self.assertEqual('chain' in x, True)
self.assertEqual(x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'],
True, 'Could not find protocol %s' % (x['protocol']))
self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def testXMLJSON(self):
"""
Test XML and JSON in the same scope
"""
site = 'T1_US_FNAL_Buffer'
httpDict = {
'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"
}
phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
httpDict = {
'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
}
phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)
phedexXML.getNodeTFC(site)
tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
tfc_map = {}
tfc_map[site] = readTFC(tfc_file)
pfn = tfc_map[site].matchLFN('srmv2',
'/store/user/jblow/dir/test.root')
self.assertTrue(
pfn ==
'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root'
)
self.assertTrue(
phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
def testTrivialFileCatalog(self):
"""
Run some simple tests on reading a trivialFileCatalog
"""
tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
if not os.path.exists(tfcFilename):
raise Exception("No TrivialFileCatalog found!")
tfcInstance = readTFC(tfcFilename)
self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))
# Look for similarities in each node of the TFC file
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for x in tfcInstance[mapping]:
self.assertEqual('path-match-expr' in x, True)
self.assertEqual('path-match' in x, True)
self.assertEqual('protocol' in x, True)
self.assertEqual('result' in x, True)
self.assertEqual('chain' in x, True)
self.assertEqual(
x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True,
'Could not find protocol %s' % (x['protocol']))
self.assertEqual(x['chain'], None,
'Invalid chain %s' % (x['chain']))
def get_tfc_rules(self, site):
"""
Get the TFC regexp for a given site.
"""
self.phedex.getNodeTFC(site)
tfc_file = self.phedex.cacheFileName('tfc', inputdata={'node': site})
return readTFC(tfc_file)
def get_tfc_rules(site):
"""
Get the TFC regexp for a given site.
"""
phedex = PhEDEx(responseType='xml')
phedex.getNodeTFC(site)
tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
return readTFC(tfc_file)
def get_tfc_rules(site):
"""
Get the TFC regexp for a given site.
"""
phedex = PhEDEx(responseType='xml')
phedex.getNodeTFC(site)
tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
return readTFC(tfc_file)
def testMultipleRegexMatch(self):
# Check that an lfn is converted to the right pfn
in_lfn = '/store/user/fred/data'
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_CH_CERNBOX_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Florida_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_ES_IFCA_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Nebraska_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
def testMultipleRegexMatch(self):
# Check that an lfn is converted to the right pfn
in_lfn = '/store/user/fred/data'
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_CH_CERNBOX_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Florida_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_ES_IFCA_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Nebraska_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
def get_tfc_rules(self, site):
"""
Get the TFC regexp for a given site.
"""
self.phedex.getNodeTFC(site)
try:
tfc_file = self.phedex.cacheFileName('tfc', inputdata={'node': site})
except Exception:
self.logger.exception('A problem occured when getting the TFC regexp: %s')
return None
return readTFC(tfc_file)
def get_tfc_rules(self, site):
"""
Get the TFC regexp for a given site.
"""
tfc_file = None
try:
self.phedex.getNodeTFC(site)
except Exception as e:
self.logger.exception('PhEDEx exception: %s' % e)
try:
tfc_file = self.phedex.cacheFileName('tfc', inputdata={'node': site})
except Exception as e:
self.logger.exception('PhEDEx cache exception: %s' % e)
return readTFC(tfc_file)
def get_tfc_rules(self, site):
"""
Get the TFC regexp for a given site.
"""
tfc_file = None
try:
self.phedex.getNodeTFC(site)
except Exception as e:
self.logger.exception('PhEDEx exception: %s' % e)
try:
tfc_file = self.phedex.cacheFileName('tfc',
inputdata={'node': site})
except Exception as e:
self.logger.exception('PhEDEx cache exception: %s' % e)
return readTFC(tfc_file)
def testDataServiceXML(self):
# asks for PEM pass phrase ...
raise nose.SkipTest
phedex = PhEDEx(responseType='xml')
site = 'T2_UK_SGrid_Bristol'
lfn = '/store/users/metson/file'
protocol = 'srmv2'
phedex.getNodeTFC(site)
tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
tfc = readTFC(tfc_file)
pfn_dict = phedex.getPFN(site, lfn, protocol)
phedex_pfn = pfn_dict[(site, lfn)]
pfn = tfc.matchLFN(protocol, lfn)
msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn)
self.assertEqual(phedex_pfn, pfn, msg)
def testDataServiceXML(self):
# asks for PEM pass phrase ...
raise nose.SkipTest
phedex = PhEDEx(responseType='xml')
site = 'T2_UK_SGrid_Bristol'
lfn = '/store/users/metson/file'
protocol = 'srmv2'
phedex.getNodeTFC(site)
tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
tfc = readTFC(tfc_file)
pfn_dict = phedex.getPFN(site, lfn, protocol)
phedex_pfn = pfn_dict[(site, lfn)]
pfn = tfc.matchLFN(protocol, lfn)
msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn)
self.assertEqual(phedex_pfn, pfn, msg)
def testRoundTrip(self):
"""
Test PFN, LFN matching upon an existing TFC XML file
"""
tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
# Check that an lfn goes to an srmv2 pfn and comes back as the same lfn
in_lfn = '/store/data/my/data'
in_pfn = "srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/data/my/data"
out_pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, in_pfn)
out_lfn = tfc.matchPFN('srmv2', in_pfn)
self.assertEqual(in_lfn, out_lfn)
out_pfn = tfc.matchLFN('srmv2', out_lfn)
self.assertEqual(in_pfn, out_pfn)
def testXMLJSON(self):
"""
Test XML and JSON in the same scope
"""
site = 'T1_US_FNAL_Buffer'
httpDict = {'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"}
phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
httpDict = {'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"}
phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)
phedexXML.getNodeTFC(site)
tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
tfc_map = {}
tfc_map[site] = readTFC(tfc_file)
pfn = tfc_map[site].matchLFN('srmv2', '/store/user/jblow/dir/test.root')
self.assertTrue(pfn == 'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root')
self.assertTrue(phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
def testSaveTFC(self):
tfc = TrivialFileCatalog()
lfn = "some_lfn"
pfn = "some_pfn"
tfc.addMapping("direct", lfn, pfn, mapping_type='lfn-to-pfn')
tfc.addMapping("direct", pfn, lfn, mapping_type='pfn-to-lfn')
f = tempfile.NamedTemporaryFile("w+", delete=True) # read / write
tfcStr = str(tfc.getXML())
f.write(tfcStr)
f.flush()
f.seek(0)
tfcInstance = readTFC(f.name)
out_pfn = tfc.matchLFN("direct", lfn)
out_lfn = tfc.matchPFN("direct", pfn)
self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
f.close()
def trivialFileCatalog(self):
"""
_trivialFileCatalog_
Return an instance of FwkJobRep.TrivialFileCatalog
"""
tfcUrl = self.localStageOut.get('catalog', None)
if tfcUrl == None:
return None
try:
tfcFile = tfcFilename(tfcUrl)
tfcProto = tfcProtocol(tfcUrl)
tfcInstance = readTFC(tfcFile)
tfcInstance.preferredProtocol = tfcProto
except StandardError, ex:
msg = "Unable to load TrivialFileCatalog:\n"
msg += "URL = %s\n" % tfcUrl
raise SiteConfigError, msg
def testRoundTrip(self):
"""
Test PFN, LFN matching upon an existing TFC XML file
"""
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
# Check that an lfn goes to an srmv2 pfn and comes back as the same lfn
in_lfn = '/store/data/my/data'
in_pfn = "srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/data/my/data"
out_pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, in_pfn)
out_lfn = tfc.matchPFN('srmv2', in_pfn)
self.assertEqual(in_lfn, out_lfn)
out_pfn = tfc.matchLFN('srmv2', out_lfn)
self.assertEqual(in_pfn, out_pfn)
def trivialFileCatalog(self):
"""
_trivialFileCatalog_
Return an instance of FwkJobRep.TrivialFileCatalog
"""
tfcUrl = self.localStageOut.get('catalog', None)
if tfcUrl == None:
return None
try:
tfcFile = tfcFilename(tfcUrl)
tfcProto = tfcProtocol(tfcUrl)
tfcInstance = readTFC(tfcFile)
tfcInstance.preferredProtocol = tfcProto
except StandardError, ex:
msg = "Unable to load TrivialFileCatalog:\n"
msg += "URL = %s\n" % tfcUrl
raise SiteConfigError, msg
def testSaveTFC(self):
tfc = TrivialFileCatalog()
lfn = "some_lfn"
pfn = "some_pfn"
tfc.addMapping("direct", lfn, pfn, mapping_type = 'lfn-to-pfn')
tfc.addMapping("direct", pfn, lfn, mapping_type = 'pfn-to-lfn')
f = tempfile.NamedTemporaryFile("w+", delete = True) # read / write
tfcStr = str(tfc.getXML())
f.write(tfcStr)
f.flush()
f.seek(0)
tfcInstance = readTFC(f.name)
out_pfn = tfc.matchLFN("direct", lfn)
out_lfn = tfc.matchPFN("direct", pfn)
self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
f.close()
def keys_map(inputDict):
"""
Map function.
"""
return inputDict['key']
return map(keys_map, sites['rows'])
def get_tfc_rules(self, site):
"""
Get the TFC regexp for a given site.
"""
tfc_file = None
try:
self.phedex.getNodeTFC(site)
except Exception, e:
self.logger.exception('PhEDEx exception: %s' % e)
try:
tfc_file = self.phedex.cacheFileName('tfc',
inputdata={'node': site})
except Exception, e:
self.logger.exception('PhEDEx cache exception: %s' % e)
return readTFC(tfc_file)
def terminate(self, parameters=None):
"""
Called when thread is being terminated.
"""
self.pool.close()
self.pool.join()
def keys_map(inputDict):
"""
Map function.
"""
return inputDict["key"]
return map(keys_map, sites["rows"])
def get_tfc_rules(self, site):
"""
Get the TFC regexp for a given site.
"""
tfc_file = None
try:
self.phedex.getNodeTFC(site)
except Exception, e:
self.logger.exception("PhEDEx exception: %s" % e)
try:
tfc_file = self.phedex.cacheFileName("tfc", inputdata={"node": site})
except Exception, e:
self.logger.exception("PhEDEx cache exception: %s" % e)
return readTFC(tfc_file)
def terminate(self, parameters=None):
"""
Called when thread is being terminated.
"""
self.pool.close()
self.pool.join()
def _mock_get_tfc_rules(self, site):
return readTFC(os.path.join(os.path.dirname(AsyncStageOut_t.__file__),\
'fake-tfc.xml'))
def __init__(self):
# FIXME this should really be SE specific
storage_path = '/cvmfs/cms.cern.ch/SITECONF/local/PhEDEx/storage.xml'
self.__catalog = readTFC(storage_path)
self.__dbs = {}