def testTrivialFileCatalog(self):
        """
        Run some simple tests on reading a trivialFileCatalog

        """
        tfcFilename = os.path.join(getTestBase(),
                                   "WMCore_t/Storage_t",
                                   "T1_US_FNAL_TrivialFileCatalog.xml")

        if not os.path.exists(tfcFilename):
            raise Exception("No TrivialFileCatalog found!")


        tfcInstance = readTFC(tfcFilename)

        self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))

        # Look for similarities in each node of the TFC file
        for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
            for x in tfcInstance[mapping]:
                self.assertEqual('path-match-expr' in x, True)
                self.assertEqual('path-match' in x, True)
                self.assertEqual('protocol' in x, True)
                self.assertEqual('result' in x, True)
                self.assertEqual('chain' in x, True)
                self.assertEqual(x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'],
                                 True, 'Could not find protocol %s' % (x['protocol']))
                self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
Esempio n. 2
0
    def testXMLJSON(self):
        """
        Test XML and JSON in the same scope
        """
        site = 'T1_US_FNAL_Buffer'
        httpDict = {
            'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"
        }
        phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
        httpDict = {
            'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"
        }
        phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)

        phedexXML.getNodeTFC(site)
        tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
        tfc_map = {}
        tfc_map[site] = readTFC(tfc_file)
        pfn = tfc_map[site].matchLFN('srmv2',
                                     '/store/user/jblow/dir/test.root')

        self.assertTrue(
            pfn ==
            'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root'
        )

        self.assertTrue(
            phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
Esempio n. 3
0
    def testTrivialFileCatalog(self):
        """
        Run some simple tests on reading a trivialFileCatalog

        """
        tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t",
                                   "T1_US_FNAL_TrivialFileCatalog.xml")

        if not os.path.exists(tfcFilename):
            raise Exception("No TrivialFileCatalog found!")

        tfcInstance = readTFC(tfcFilename)

        self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))

        # Look for similarities in each node of the TFC file
        for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
            for x in tfcInstance[mapping]:
                self.assertEqual('path-match-expr' in x, True)
                self.assertEqual('path-match' in x, True)
                self.assertEqual('protocol' in x, True)
                self.assertEqual('result' in x, True)
                self.assertEqual('chain' in x, True)
                self.assertEqual(
                    x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True,
                    'Could not find protocol %s' % (x['protocol']))
                self.assertEqual(x['chain'], None,
                                 'Invalid chain %s' % (x['chain']))
Esempio n. 4
0
    def get_tfc_rules(self, site):
        """
        Get the TFC regexp for a given site.
        """
        self.phedex.getNodeTFC(site)
        tfc_file = self.phedex.cacheFileName('tfc', inputdata={'node': site})

        return readTFC(tfc_file)
Esempio n. 5
0
def get_tfc_rules(site):
    """
    Get the TFC regexp for a given site.
    """
    phedex = PhEDEx(responseType='xml')
    phedex.getNodeTFC(site)
    tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})

    return readTFC(tfc_file)
def get_tfc_rules(site):
    """
    Get the TFC regexp for a given site.
    """
    phedex = PhEDEx(responseType='xml')
    phedex.getNodeTFC(site)
    tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})

    return readTFC(tfc_file)
Esempio n. 7
0
    def testMultipleRegexMatch(self):
        # Check that an lfn is converted to the right pfn
        in_lfn = '/store/user/fred/data'

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_CH_CERNBOX_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_US_Florida_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_ES_IFCA_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_US_Nebraska_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)
    def testMultipleRegexMatch(self):
        # Check that an lfn is converted to the right pfn
        in_lfn = '/store/user/fred/data'

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_CH_CERNBOX_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_US_Florida_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_ES_IFCA_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)

        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T2_US_Nebraska_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)
        out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
        pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, pfn)
Esempio n. 9
0
 def get_tfc_rules(self, site):
     """
     Get the TFC regexp for a given site.
     """
     self.phedex.getNodeTFC(site)
     try:
         tfc_file = self.phedex.cacheFileName('tfc', inputdata={'node': site})
     except Exception:
         self.logger.exception('A problem occured when getting the TFC regexp: %s')
         return None
     return readTFC(tfc_file)
Esempio n. 10
0
 def get_tfc_rules(self, site):
     """
     Get the TFC regexp for a given site.
     """
     tfc_file = None
     try:
         self.phedex.getNodeTFC(site)
     except Exception as e:
         self.logger.exception('PhEDEx exception: %s' % e)
     try:
         tfc_file = self.phedex.cacheFileName('tfc', inputdata={'node': site})
     except Exception as e:
         self.logger.exception('PhEDEx cache exception: %s' % e)
     return readTFC(tfc_file)
Esempio n. 11
0
 def get_tfc_rules(self, site):
     """
     Get the TFC regexp for a given site.
     """
     tfc_file = None
     try:
         self.phedex.getNodeTFC(site)
     except Exception as e:
         self.logger.exception('PhEDEx exception: %s' % e)
     try:
         tfc_file = self.phedex.cacheFileName('tfc',
                                              inputdata={'node': site})
     except Exception as e:
         self.logger.exception('PhEDEx cache exception: %s' % e)
     return readTFC(tfc_file)
Esempio n. 12
0
    def testDataServiceXML(self):
        # asks for PEM pass phrase ...
        raise nose.SkipTest
        phedex = PhEDEx(responseType='xml')

        site = 'T2_UK_SGrid_Bristol'
        lfn = '/store/users/metson/file'
        protocol = 'srmv2'
        phedex.getNodeTFC(site)

        tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
        tfc = readTFC(tfc_file)

        pfn_dict = phedex.getPFN(site, lfn, protocol)
        phedex_pfn = pfn_dict[(site, lfn)]
        pfn = tfc.matchLFN(protocol, lfn)
        msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn)
        self.assertEqual(phedex_pfn, pfn, msg)
Esempio n. 13
0
    def testDataServiceXML(self):
        # asks for PEM pass phrase ...
        raise nose.SkipTest
        phedex = PhEDEx(responseType='xml')

        site = 'T2_UK_SGrid_Bristol'
        lfn = '/store/users/metson/file'
        protocol = 'srmv2'
        phedex.getNodeTFC(site)

        tfc_file = phedex.cacheFileName('tfc', inputdata={'node': site})
        tfc = readTFC(tfc_file)

        pfn_dict = phedex.getPFN(site, lfn, protocol)
        phedex_pfn = pfn_dict[(site, lfn)]
        pfn = tfc.matchLFN(protocol, lfn)
        msg = 'TFC pfn (%s) did not match PhEDEx pfn (%s)' % (pfn, phedex_pfn)
        self.assertEqual(phedex_pfn, pfn, msg)
Esempio n. 14
0
    def testRoundTrip(self):
        """
        Test PFN, LFN matching upon an existing TFC XML file

        """
        tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t",
                                "T1_US_FNAL_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)

        # Check that an lfn goes to an srmv2 pfn and comes back as the same lfn
        in_lfn = '/store/data/my/data'
        in_pfn = "srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/data/my/data"
        out_pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, in_pfn)
        out_lfn = tfc.matchPFN('srmv2', in_pfn)
        self.assertEqual(in_lfn, out_lfn)
        out_pfn = tfc.matchLFN('srmv2', out_lfn)
        self.assertEqual(in_pfn, out_pfn)
Esempio n. 15
0
    def testXMLJSON(self):
        """
        Test XML and JSON in the same scope
        """
        site = 'T1_US_FNAL_Buffer'
        httpDict = {'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/json/test"}
        phedexJSON = PhEDEx(responseType='json', httpDict=httpDict)
        httpDict = {'endpoint': "https://cmsweb.cern.ch/phedex/datasvc/xml/test"}
        phedexXML = PhEDEx(responseType='xml', httpDict=httpDict)

        phedexXML.getNodeTFC(site)
        tfc_file = phedexXML.cacheFileName('tfc', inputdata={'node': site})
        tfc_map = {}
        tfc_map[site] = readTFC(tfc_file)
        pfn = tfc_map[site].matchLFN('srmv2', '/store/user/jblow/dir/test.root')

        self.assertTrue(pfn == 'srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/user/jblow/dir/test.root')

        self.assertTrue(phedexJSON.getNodeSE('T1_US_FNAL_Buffer') == 'cmssrm.fnal.gov')
Esempio n. 16
0
    def testSaveTFC(self):
        tfc = TrivialFileCatalog()
        lfn = "some_lfn"
        pfn = "some_pfn"
        tfc.addMapping("direct", lfn, pfn, mapping_type='lfn-to-pfn')
        tfc.addMapping("direct", pfn, lfn, mapping_type='pfn-to-lfn')

        f = tempfile.NamedTemporaryFile("w+", delete=True)  # read / write
        tfcStr = str(tfc.getXML())
        f.write(tfcStr)
        f.flush()
        f.seek(0)

        tfcInstance = readTFC(f.name)
        out_pfn = tfc.matchLFN("direct", lfn)
        out_lfn = tfc.matchPFN("direct", pfn)
        self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
        self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
        f.close()
Esempio n. 17
0
    def trivialFileCatalog(self):
        """
        _trivialFileCatalog_

        Return an instance of FwkJobRep.TrivialFileCatalog

        """
        tfcUrl = self.localStageOut.get('catalog', None)
        if tfcUrl == None:
            return None
        try:
            tfcFile = tfcFilename(tfcUrl)
            tfcProto = tfcProtocol(tfcUrl)
            tfcInstance = readTFC(tfcFile)
            tfcInstance.preferredProtocol = tfcProto
        except StandardError, ex:
            msg = "Unable to load TrivialFileCatalog:\n"
            msg += "URL = %s\n" % tfcUrl
            raise SiteConfigError, msg
Esempio n. 18
0
    def testRoundTrip(self):
        """
        Test PFN, LFN matching upon an existing TFC XML file

        """
        tfc_file = os.path.join(getTestBase(),
                                "WMCore_t/Storage_t",
                                "T1_US_FNAL_TrivialFileCatalog.xml")
        tfc = readTFC(tfc_file)

        # Check that an lfn goes to an srmv2 pfn and comes back as the same lfn
        in_lfn = '/store/data/my/data'
        in_pfn = "srm://cmssrm.fnal.gov:8443/srm/managerv2?SFN=/11/store/data/my/data"
        out_pfn = tfc.matchLFN('srmv2', in_lfn)
        self.assertEqual(out_pfn, in_pfn)
        out_lfn = tfc.matchPFN('srmv2', in_pfn)
        self.assertEqual(in_lfn, out_lfn)
        out_pfn = tfc.matchLFN('srmv2', out_lfn)
        self.assertEqual(in_pfn, out_pfn)
Esempio n. 19
0
    def trivialFileCatalog(self):
        """
        _trivialFileCatalog_

        Return an instance of FwkJobRep.TrivialFileCatalog

        """
        tfcUrl = self.localStageOut.get('catalog', None)
        if tfcUrl == None:
            return None
        try:
            tfcFile = tfcFilename(tfcUrl)
            tfcProto = tfcProtocol(tfcUrl)
            tfcInstance = readTFC(tfcFile)
            tfcInstance.preferredProtocol = tfcProto
        except StandardError, ex:
            msg = "Unable to load TrivialFileCatalog:\n"
            msg += "URL = %s\n" % tfcUrl
            raise SiteConfigError, msg
Esempio n. 20
0
    def testSaveTFC(self):
        tfc = TrivialFileCatalog()
        lfn = "some_lfn"
        pfn = "some_pfn"
        tfc.addMapping("direct", lfn, pfn, mapping_type = 'lfn-to-pfn')
        tfc.addMapping("direct", pfn, lfn, mapping_type = 'pfn-to-lfn')

        f = tempfile.NamedTemporaryFile("w+", delete = True) # read / write
        tfcStr = str(tfc.getXML())
        f.write(tfcStr)
        f.flush()
        f.seek(0)

        tfcInstance = readTFC(f.name)
        out_pfn = tfc.matchLFN("direct", lfn)
        out_lfn = tfc.matchPFN("direct", pfn)
        self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
        self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
        f.close()
Esempio n. 21
0
        def keys_map(inputDict):
            """
            Map function.
            """
            return inputDict['key']

        return map(keys_map, sites['rows'])

    def get_tfc_rules(self, site):
        """
        Get the TFC regexp for a given site.
        """
        tfc_file = None
        try:
            self.phedex.getNodeTFC(site)
        except Exception, e:
            self.logger.exception('PhEDEx exception: %s' % e)
        try:
            tfc_file = self.phedex.cacheFileName('tfc',
                                                 inputdata={'node': site})
        except Exception, e:
            self.logger.exception('PhEDEx cache exception: %s' % e)
        return readTFC(tfc_file)

    def terminate(self, parameters=None):
        """
        Called when thread is being terminated.
        """
        self.pool.close()
        self.pool.join()
Esempio n. 22
0
        def keys_map(inputDict):
            """
            Map function.
            """
            return inputDict["key"]

        return map(keys_map, sites["rows"])

    def get_tfc_rules(self, site):
        """
        Get the TFC regexp for a given site.
        """
        tfc_file = None
        try:
            self.phedex.getNodeTFC(site)
        except Exception, e:
            self.logger.exception("PhEDEx exception: %s" % e)
        try:
            tfc_file = self.phedex.cacheFileName("tfc", inputdata={"node": site})
        except Exception, e:
            self.logger.exception("PhEDEx cache exception: %s" % e)
        return readTFC(tfc_file)

    def terminate(self, parameters=None):
        """
        Called when thread is being terminated.
        """
        self.pool.close()
        self.pool.join()
Esempio n. 23
0
 def _mock_get_tfc_rules(self, site):
     return readTFC(os.path.join(os.path.dirname(AsyncStageOut_t.__file__),\
                                  'fake-tfc.xml'))
Esempio n. 24
0
 def __init__(self):
     # FIXME this should really be SE specific
     storage_path = '/cvmfs/cms.cern.ch/SITECONF/local/PhEDEx/storage.xml'
     self.__catalog = readTFC(storage_path)
     self.__dbs = {}