Пример #1
0
 def init(self):
     """Takes care of MongoDB connection"""
     try:
         indexes = [('dataset', DESCENDING), ('site', DESCENDING),
                    ('ts', DESCENDING)]
         for index in indexes:
             create_indexes(self.coll, [index])
         dasmapping   = DASMapping(self.dasconfig)
         service_name = self.config.get('name', 'combined')
         service_api  = self.config.get('api', 'dataset4site_release')
         mapping      = dasmapping.servicemap(service_name)
         self.urls    = mapping[service_api]['services']
         self.expire  = mapping[service_api]['expire']
         services     = self.dasconfig['services']
         if  not self.wthr:
             # Worker thread which update dbs/phedex DB
             self.wthr = start_new_thread('dbs_phedex_worker', worker, \
                  (self.urls, which_dbs, self.uri, \
                  self.dbname, self.collname, self.expire))
         msg = "### DBSPhedexService:init started"
         print(msg)
     except Exception as exc:
         print("### Fail DBSPhedexService:init\n", str(exc))
         self.urls       = None
         self.expire     = 60
         self.wthr       = None
Пример #2
0
 def init(self):
     "Takes care of MongoDB connection since DASMapping requires it"
     try:
         dasmapping  = DASMapping(self.dasconfig)
         mapping     = dasmapping.servicemap(self.service_name)
         self.urls   = mapping[self.service_api]['services']
         self.expire = mapping[self.service_api]['expire']
     except Exception as _exp:
         pass
Пример #3
0
class testDASMapping(unittest.TestCase):
    """
    A test class for the DAS mappingdb class
    """
    def setUp(self):
        """
        set up DAS core module
        """
        debug    = 0
        self.db  = 'test_mapping.db'
        config   = deepcopy(das_readconfig())
        dburi    = config['mongodb']['dburi']
        logger   = PrintManager('TestDASMapping', verbose=debug)
        config['logger']  = logger
        config['verbose'] = debug
        dbname   = 'test_mapping'
        collname = 'db'
        config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
        # add some maps to mapping db
        conn = MongoClient(dburi)
        conn.drop_database(dbname)
        self.coll = conn[dbname][collname]
        self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"},
            {"ui": "Block size", "das": "block.size"}]}, "type": "presentation"}
        self.coll.insert(self.pmap)

        url     = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/'
        dformat = 'JSON'
        system  = 'dbs3'
        expire  = 100
        rec = {'system':system, 'urn': 'acquisitioneras', 'format':dformat,
            'instances': ['prod/global'],
            'url':url, 'expire': expire, 'lookup': 'era',
            'params' : {},
             'das_map': [
                 {"das_key": "era", "rec_key":"era.name", "api_arg":"era"}
                 ],
             'type': 'service'
        }
        self.coll.insert(rec)

        ver_token = verification_token(self.coll.find(**PYMONGO_OPTS))
        rec = {'verification_token':ver_token, 'type':'verification_token'}
        self.coll.insert(rec)

        self.mgr = DASMapping(config)

    def tearDown(self):
        """Invoke after each test"""
        self.mgr.delete_db()

    def test_api(self):
        """test methods for api table"""
        self.mgr.delete_db()

        system  = 'dbs3'
        url     = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
        dformat = 'JSON'
        expire  = 100
        instances = ["prod/global", "prod/phys01"]

        api = 'primarydatasets'
        params = {"primary_ds_name":"*"}
        rec = {'system':system, 'urn':api, 'format':dformat, 'url':url,
            'instances': instances,
            'params': params, 'expire':expire, 'lookup': 'primary_dataset', 'wild_card':'*',
            'das_map' : [dict(das_key='primary_dataset',
                              rec_key='primary_dataset.name',
                              api_arg='primary_dataset')],
            'type': 'service'
        }
        self.mgr.add(rec)
        smap = {api: {'url':url, 'expire':expire, 'keys': ['primary_dataset'],
                'format': dformat, 'wild_card':'*', 'cert':None, 'ckey': None,
                'services': '', 'lookup': 'primary_dataset',
                'params': params }
        }

        rec = {'system':system, 'urn': 'datasetaccesstypes', 'format':dformat,
            'instances': instances,
            'url':url, 'expire': expire, 'lookup': 'status',
            'params' : {'status':'*'},
             'das_map': [
                 {"das_key": "status", "rec_key":"status.name", "api_arg":"status"}
                 ],
             'type': 'service'
        }
        self.mgr.add(rec)


        api = 'datasetaccesstypes'
        daskey = 'status'
        rec_key = 'status.name'
        api_input = 'status'

        res = self.mgr.list_systems()
        self.assertEqual([system], res)

        res = self.mgr.list_apis()
        res.sort()
        self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res)

        res = self.mgr.lookup_keys(system, api, daskey)
        self.assertEqual([rec_key], res)

        value = ''
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        # adding another params which default is None
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        res = self.mgr.api2das(system, api_input)
        self.assertEqual([daskey], res)

        # adding notations
        notations = {'system':system, 'type': 'notation',
            'notations':[
                    {'api_output':'storage_element_name', 'rec_key':'se', 'api':''},
                    {'api_output':'number_of_events', 'rec_key':'nevents', 'api':''},
                        ]
        }
        self.mgr.add(notations)

        res = self.mgr.notation2das(system, 'number_of_events')
        self.assertEqual('nevents', res)

        # API keys
        res = self.mgr.api2daskey(system, api)
        self.assertEqual(['status'], res)

        # build service map
        smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
                'keys': ['status'], 'format':dformat, 'wild_card':'*',
                'services': '', 'lookup': daskey,
                'params': {"status": "*"}
                     }
        })
        res = self.mgr.servicemap(system)
        self.assertEqual(smap, res)

    def test_presentation(self):
        """test presentation method"""
        expect = self.pmap['presentation']['block']
        result = self.mgr.presentation('block')
        self.assertEqual(expect, result)

    def test_notations(self):
        """test notations method"""
        system = "test"
        rec = {'notations': [
        {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
        {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
        {"api_output": "admin.contacts.cms_name", "rec_key":"site.name", "api":""}
        ], "system": system, "type": "notation"}
        self.mgr.add(rec)
        expect = rec['notations']
        result = self.mgr.notations(system)[system]
        self.assertEqual(expect, result)
Пример #4
0
class testDASMapping(unittest.TestCase):
    """
    A test class for the DAS mappingdb class
    """

    def setUp(self):
        """
        set up DAS core module
        """
        debug = 0
        self.db = "test_mapping.db"
        config = deepcopy(das_readconfig())
        dburi = config["mongodb"]["dburi"]
        logger = PrintManager("TestDASMapping", verbose=debug)
        config["logger"] = logger
        config["verbose"] = debug
        dbname = "test_mapping"
        collname = "db"
        config["mappingdb"] = dict(dburi=dburi, dbname=dbname, collname=collname)
        # add some maps to mapping db
        conn = MongoClient(dburi)
        conn.drop_database(dbname)
        self.coll = conn[dbname][collname]
        self.pmap = {
            "presentation": {
                "block": [{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]
            },
            "type": "presentation",
        }
        self.coll.insert(self.pmap)
        ver_token = verification_token(self.coll.find(exhaust=True))
        rec = {"verification_token": ver_token, "type": "verification_token"}
        self.coll.insert(rec)

        self.mgr = DASMapping(config)

    def tearDown(self):
        """Invoke after each test"""
        self.mgr.delete_db()

    def test_api(self):
        """test methods for api table"""
        self.mgr.delete_db()
        self.mgr.init()

        apiversion = "DBS_2_0_8"
        url = "http://a.com"
        dformat = "JSON"
        expire = 100

        api = "listRuns"
        params = {"apiversion": apiversion, "path": "required", "api": api}
        rec = {
            "system": "dbs",
            "urn": api,
            "format": dformat,
            "url": url,
            "params": params,
            "expire": expire,
            "lookup": "run",
            "wild_card": "*",
            "das_map": [dict(das_key="run", rec_key="run.run_number", api_arg="path")],
            "type": "service",
        }
        self.mgr.add(rec)
        smap = {
            api: {
                "url": url,
                "expire": expire,
                "keys": ["run"],
                "format": dformat,
                "wild_card": "*",
                "cert": None,
                "ckey": None,
                "services": "",
                "lookup": "run",
                "params": {"path": "required", "api": api, "apiversion": "DBS_2_0_8"},
            }
        }

        rec = {
            "system": "dbs",
            "urn": "listBlocks",
            "format": dformat,
            "url": url,
            "expire": expire,
            "lookup": "block",
            "params": {
                "apiversion": apiversion,
                "api": "listBlocks",
                "block_name": "*",
                "storage_element_name": "*",
                "user_type": "NORMAL",
            },
            "das_map": [
                {"das_key": "block", "rec_key": "block.name", "api_arg": "block_name"},
                {
                    "das_key": "site",
                    "rec_key": "site.se",
                    "api_arg": "storage_element_name",
                    "pattern": "re.compile('([a-zA-Z0-9]+\.){2}')",
                },
            ],
            "type": "service",
        }
        self.mgr.add(rec)

        system = "dbs"
        api = "listBlocks"
        daskey = "block"
        rec_key = "block.name"
        api_input = "block_name"

        res = self.mgr.list_systems()
        self.assertEqual(["dbs"], res)

        res = self.mgr.list_apis()
        #        self.assertEqual([api], res)
        res.sort()
        self.assertEqual(["listBlocks", "listRuns"], res)

        res = self.mgr.lookup_keys(system, api, daskey)
        self.assertEqual([rec_key], res)

        value = ""
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        # adding another params which default is None
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        res = self.mgr.api2das(system, api_input)
        self.assertEqual([daskey], res)

        # adding notations
        notations = {
            "system": system,
            "type": "notation",
            "notations": [
                {"api_output": "storage_element_name", "rec_key": "se", "api": ""},
                {"api_output": "number_of_events", "rec_key": "nevents", "api": ""},
            ],
        }
        self.mgr.add(notations)

        res = self.mgr.notation2das(system, "number_of_events")
        self.assertEqual("nevents", res)

        # API keys
        res = self.mgr.api2daskey(system, api)
        self.assertEqual(["block", "site"], res)

        # build service map
        smap.update(
            {
                api: {
                    "url": url,
                    "expire": expire,
                    "cert": None,
                    "ckey": None,
                    "keys": ["block", "site"],
                    "format": dformat,
                    "wild_card": "*",
                    "services": "",
                    "lookup": daskey,
                    "params": {
                        "storage_element_name": "*",
                        "api": api,
                        "block_name": "*",
                        "user_type": "NORMAL",
                        "apiversion": "DBS_2_0_8",
                    },
                }
            }
        )
        res = self.mgr.servicemap(system)
        self.assertEqual(smap, res)

    def test_presentation(self):
        """test presentation method"""
        self.mgr.init()
        expect = self.pmap["presentation"]["block"]
        result = self.mgr.presentation("block")
        self.assertEqual(expect, result)

    def test_notations(self):
        """test notations method"""
        self.mgr.init()
        system = "test"
        rec = {
            "notations": [
                {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
                {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
                {"api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": ""},
            ],
            "system": system,
            "type": "notation",
        }
        self.mgr.add(rec)
        expect = rec["notations"]
        result = self.mgr.notations(system)[system]
        self.assertEqual(expect, result)
Пример #5
0
class testDASMapping(unittest.TestCase):
    """
    A test class for the DAS mappingdb class
    """
    def setUp(self):
        """
        set up DAS core module
        """
        debug    = 0
        self.db  = 'test_mapping.db'
        config   = deepcopy(das_readconfig())
        dburi    = config['mongodb']['dburi']
        logger   = PrintManager('TestDASMapping', verbose=debug)
        config['logger']  = logger
        config['verbose'] = debug
        dbname   = 'test_mapping'
        collname = 'db'
        config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname)
        # add some maps to mapping db
        conn = Connection(dburi)
        conn.drop_database(dbname)
        coll = conn[dbname][collname]
        self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"}, 
        {"ui": "Block size", "das": "block.size"}]}}
        coll.insert(self.pmap)

        self.mgr = DASMapping(config)

    def tearDown(self):
        """Invoke after each test"""
        self.mgr.delete_db()

    def test_api(self):                          
        """test methods for api table"""
        self.mgr.delete_db()
        self.mgr.create_db()

        apiversion = 'DBS_2_0_8'
        url     = 'http://a.com'
        dformat = 'JSON'
        expire  = 100

        api = 'listRuns'
        params = { 'apiversion':apiversion, 'path' : 'required', 'api':api}
        rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url,
            'params': params, 'expire':expire, "wild_card": "*",
            'daskeys' : [dict(key='run', map='run.run_number', pattern='')],
            'das2api' : [
                    dict(api_param='path', das_key='dataset', pattern=""),
            ]
        }
        self.mgr.add(rec)
        res = self.mgr.check_dasmap('dbs', api, 'run.bfield')
        self.assertEqual(False, res)
        res = self.mgr.check_dasmap('dbs', api, 'run.run_number')
        self.assertEqual(True, res)
        smap = {api: {'url':url, 'expire':expire, 'keys': ['run'], 
                'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None,
                'params': {'path': 'required', 'api': api, 
                           'apiversion': 'DBS_2_0_8'}
                     }
        }

        rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat,
          'url':url, 'expire': expire,
          'params' : {'apiversion': apiversion, 'api': 'listBlocks',
                      'block_name':'*', 'storage_element_name':'*',
                      'user_type':'NORMAL'},
          'daskeys': [
                 {'key':'block', 'map':'block.name', 'pattern':''},
                 ],
          'das2api': [
                 {'api_param':'storage_element_name', 
                  'das_key':'site', 
                  'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
                 {'api_param':'storage_element_name', 
                  'das_key':'site.se', 
                  'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"},
                 {'api_param':'block_name', 
                  'das_key':'block', 
                  'pattern':""},
                 {'api_param':'block_name', 
                  'das_key':'block.name', 
                  'pattern':""},
                 ]
        } 
        self.mgr.add(rec)


        system = 'dbs'
        api = 'listBlocks'
        daskey = 'block'
        primkey = 'block.name'
        api_input = 'block_name'

        res = self.mgr.list_systems()
        self.assertEqual(['dbs'], res)

        res = self.mgr.list_apis()
#        self.assertEqual([api], res)
        res.sort()
        self.assertEqual(['listBlocks', 'listRuns'], res)

        res = self.mgr.lookup_keys(system, daskey)
        self.assertEqual([primkey], res)

        value = ''
        res = self.mgr.das2api(system, daskey, value)
        self.assertEqual([api_input], res)

        # adding another params which default is None
        res = self.mgr.das2api(system, daskey, value, api)
        self.assertEqual([api_input], res)

        res = self.mgr.api2das(system, api_input)
        self.assertEqual([daskey, primkey], res)

        # adding notations
        notations = {'system':system, 
            'notations':[
                    {'notation':'storage_element_name', 'map':'se', 'api':''},
                    {'notation':'number_of_events', 'map':'nevents', 'api':''},
                        ]
        }
        self.mgr.add(notations)

        res = self.mgr.notation2das(system, 'number_of_events')
        self.assertEqual('nevents', res)

        # API keys
        res = self.mgr.api2daskey(system, api)
        self.assertEqual([daskey], res)

        # build service map
        smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None,
                'keys': ['block'], 'format':dformat, "wild_card": "*",
                'params': {'storage_element_name': '*', 'api':api, 
                           'block_name': '*', 'user_type': 'NORMAL', 
                           'apiversion': 'DBS_2_0_8'}
                     }
        })
        res = self.mgr.servicemap(system)
        self.assertEqual(smap, res)

    def test_presentation(self):                          
        """test presentation method"""
        self.mgr.create_db()
#        rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}}
#        self.mgr.add(rec)
        expect = self.pmap['presentation']['block']
        result = self.mgr.presentation('block')
        self.assertEqual(expect, result)

    def test_notations(self):                          
        """test notations method"""
        self.mgr.create_db()
        system = "test"
        rec = {'notations': [
        {"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""},
        {"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""},
        {"notation": "admin.contacts.cms_name", "map":"site.name", "api":""}
        ], "system": system}
        self.mgr.add(rec)
        expect = rec['notations']
        result = self.mgr.notations(system)[system]
        self.assertEqual(expect, result)
Пример #6
0
class testDASMapping(unittest.TestCase):
    """
    A test class for the DAS mappingdb class
    """
    def setUp(self):
        """
        set up DAS core module
        """
        debug = 0
        self.db = 'test_mapping.db'
        config = deepcopy(das_readconfig())
        dburi = config['mongodb']['dburi']
        logger = PrintManager('TestDASMapping', verbose=debug)
        config['logger'] = logger
        config['verbose'] = debug
        dbname = 'test_mapping'
        collname = 'db'
        config['mappingdb'] = dict(dburi=dburi,
                                   dbname=dbname,
                                   collname=collname)
        # add some maps to mapping db
        conn = MongoClient(dburi)
        conn.drop_database(dbname)
        self.coll = conn[dbname][collname]
        self.pmap = {
            "presentation": {
                "block": [{
                    "ui": "Block name",
                    "das": "block.name"
                }, {
                    "ui": "Block size",
                    "das": "block.size"
                }]
            },
            "type": "presentation"
        }
        self.coll.insert(self.pmap)

        url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/'
        dformat = 'JSON'
        system = 'dbs3'
        expire = 100
        rec = {
            'system':
            system,
            'urn':
            'acquisitioneras',
            'format':
            dformat,
            'instances': ['prod/global'],
            'url':
            url,
            'expire':
            expire,
            'lookup':
            'era',
            'params': {},
            'das_map': [{
                "das_key": "era",
                "rec_key": "era.name",
                "api_arg": "era"
            }],
            'type':
            'service'
        }
        self.coll.insert(rec)

        ver_token = verification_token(self.coll.find(**PYMONGO_OPTS))
        rec = {'verification_token': ver_token, 'type': 'verification_token'}
        self.coll.insert(rec)

        self.mgr = DASMapping(config)

    def tearDown(self):
        """Invoke after each test"""
        self.mgr.delete_db()

    def test_api(self):
        """test methods for api table"""
        self.mgr.delete_db()

        system = 'dbs3'
        url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader'
        dformat = 'JSON'
        expire = 100
        instances = ["prod/global", "prod/phys01"]

        api = 'primarydatasets'
        params = {"primary_ds_name": "*"}
        rec = {
            'system':
            system,
            'urn':
            api,
            'format':
            dformat,
            'url':
            url,
            'instances':
            instances,
            'params':
            params,
            'expire':
            expire,
            'lookup':
            'primary_dataset',
            'wild_card':
            '*',
            'das_map': [
                dict(das_key='primary_dataset',
                     rec_key='primary_dataset.name',
                     api_arg='primary_dataset')
            ],
            'type':
            'service'
        }
        self.mgr.add(rec)
        smap = {
            api: {
                'url': url,
                'expire': expire,
                'keys': ['primary_dataset'],
                'format': dformat,
                'wild_card': '*',
                'cert': None,
                'ckey': None,
                'services': '',
                'lookup': 'primary_dataset',
                'params': params
            }
        }

        rec = {
            'system':
            system,
            'urn':
            'datasetaccesstypes',
            'format':
            dformat,
            'instances':
            instances,
            'url':
            url,
            'expire':
            expire,
            'lookup':
            'status',
            'params': {
                'status': '*'
            },
            'das_map': [{
                "das_key": "status",
                "rec_key": "status.name",
                "api_arg": "status"
            }],
            'type':
            'service'
        }
        self.mgr.add(rec)

        api = 'datasetaccesstypes'
        daskey = 'status'
        rec_key = 'status.name'
        api_input = 'status'

        res = self.mgr.list_systems()
        self.assertEqual([system], res)

        res = self.mgr.list_apis()
        res.sort()
        self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res)

        res = self.mgr.lookup_keys(system, api, daskey)
        self.assertEqual([rec_key], res)

        value = ''
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        # adding another params which default is None
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        res = self.mgr.api2das(system, api_input)
        self.assertEqual([daskey], res)

        # adding notations
        notations = {
            'system':
            system,
            'type':
            'notation',
            'notations': [
                {
                    'api_output': 'storage_element_name',
                    'rec_key': 'se',
                    'api': ''
                },
                {
                    'api_output': 'number_of_events',
                    'rec_key': 'nevents',
                    'api': ''
                },
            ]
        }
        self.mgr.add(notations)

        res = self.mgr.notation2das(system, 'number_of_events')
        self.assertEqual('nevents', res)

        # API keys
        res = self.mgr.api2daskey(system, api)
        self.assertEqual(['status'], res)

        # build service map
        smap.update({
            api: {
                'url': url,
                'expire': expire,
                'cert': None,
                'ckey': None,
                'keys': ['status'],
                'format': dformat,
                'wild_card': '*',
                'services': '',
                'lookup': daskey,
                'params': {
                    "status": "*"
                }
            }
        })
        res = self.mgr.servicemap(system)
        self.assertEqual(smap, res)

    def test_presentation(self):
        """test presentation method"""
        expect = self.pmap['presentation']['block']
        result = self.mgr.presentation('block')
        self.assertEqual(expect, result)

    def test_notations(self):
        """test notations method"""
        system = "test"
        rec = {
            'notations': [{
                "api_output": "site.resource_element.cms_name",
                "rec_key": "site.name",
                "api": ""
            }, {
                "api_output": "site.resource_pledge.cms_name",
                "rec_key": "site.name",
                "api": ""
            }, {
                "api_output": "admin.contacts.cms_name",
                "rec_key": "site.name",
                "api": ""
            }],
            "system":
            system,
            "type":
            "notation"
        }
        self.mgr.add(rec)
        expect = rec['notations']
        result = self.mgr.notations(system)[system]
        self.assertEqual(expect, result)