Esempio n. 1
0
class testDASMapping(unittest.TestCase):
    """
    A test class for the DAS mappingdb class
    """

    def setUp(self):
        """
        set up DAS core module
        """
        debug = 0
        self.db = "test_mapping.db"
        config = deepcopy(das_readconfig())
        dburi = config["mongodb"]["dburi"]
        logger = PrintManager("TestDASMapping", verbose=debug)
        config["logger"] = logger
        config["verbose"] = debug
        dbname = "test_mapping"
        collname = "db"
        config["mappingdb"] = dict(dburi=dburi, dbname=dbname, collname=collname)
        # add some maps to mapping db
        conn = MongoClient(dburi)
        conn.drop_database(dbname)
        self.coll = conn[dbname][collname]
        self.pmap = {
            "presentation": {
                "block": [{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]
            },
            "type": "presentation",
        }
        self.coll.insert(self.pmap)
        ver_token = verification_token(self.coll.find(exhaust=True))
        rec = {"verification_token": ver_token, "type": "verification_token"}
        self.coll.insert(rec)

        self.mgr = DASMapping(config)

    def tearDown(self):
        """Invoke after each test"""
        self.mgr.delete_db()

    def test_api(self):
        """test methods for api table"""
        self.mgr.delete_db()
        self.mgr.init()

        apiversion = "DBS_2_0_8"
        url = "http://a.com"
        dformat = "JSON"
        expire = 100

        api = "listRuns"
        params = {"apiversion": apiversion, "path": "required", "api": api}
        rec = {
            "system": "dbs",
            "urn": api,
            "format": dformat,
            "url": url,
            "params": params,
            "expire": expire,
            "lookup": "run",
            "wild_card": "*",
            "das_map": [dict(das_key="run", rec_key="run.run_number", api_arg="path")],
            "type": "service",
        }
        self.mgr.add(rec)
        smap = {
            api: {
                "url": url,
                "expire": expire,
                "keys": ["run"],
                "format": dformat,
                "wild_card": "*",
                "cert": None,
                "ckey": None,
                "services": "",
                "lookup": "run",
                "params": {"path": "required", "api": api, "apiversion": "DBS_2_0_8"},
            }
        }

        rec = {
            "system": "dbs",
            "urn": "listBlocks",
            "format": dformat,
            "url": url,
            "expire": expire,
            "lookup": "block",
            "params": {
                "apiversion": apiversion,
                "api": "listBlocks",
                "block_name": "*",
                "storage_element_name": "*",
                "user_type": "NORMAL",
            },
            "das_map": [
                {"das_key": "block", "rec_key": "block.name", "api_arg": "block_name"},
                {
                    "das_key": "site",
                    "rec_key": "site.se",
                    "api_arg": "storage_element_name",
                    "pattern": "re.compile('([a-zA-Z0-9]+\.){2}')",
                },
            ],
            "type": "service",
        }
        self.mgr.add(rec)

        system = "dbs"
        api = "listBlocks"
        daskey = "block"
        rec_key = "block.name"
        api_input = "block_name"

        res = self.mgr.list_systems()
        self.assertEqual(["dbs"], res)

        res = self.mgr.list_apis()
        #        self.assertEqual([api], res)
        res.sort()
        self.assertEqual(["listBlocks", "listRuns"], res)

        res = self.mgr.lookup_keys(system, api, daskey)
        self.assertEqual([rec_key], res)

        value = ""
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        # adding another params which default is None
        res = self.mgr.das2api(system, api, rec_key, value)
        self.assertEqual([api_input], res)

        res = self.mgr.api2das(system, api_input)
        self.assertEqual([daskey], res)

        # adding notations
        notations = {
            "system": system,
            "type": "notation",
            "notations": [
                {"api_output": "storage_element_name", "rec_key": "se", "api": ""},
                {"api_output": "number_of_events", "rec_key": "nevents", "api": ""},
            ],
        }
        self.mgr.add(notations)

        res = self.mgr.notation2das(system, "number_of_events")
        self.assertEqual("nevents", res)

        # API keys
        res = self.mgr.api2daskey(system, api)
        self.assertEqual(["block", "site"], res)

        # build service map
        smap.update(
            {
                api: {
                    "url": url,
                    "expire": expire,
                    "cert": None,
                    "ckey": None,
                    "keys": ["block", "site"],
                    "format": dformat,
                    "wild_card": "*",
                    "services": "",
                    "lookup": daskey,
                    "params": {
                        "storage_element_name": "*",
                        "api": api,
                        "block_name": "*",
                        "user_type": "NORMAL",
                        "apiversion": "DBS_2_0_8",
                    },
                }
            }
        )
        res = self.mgr.servicemap(system)
        self.assertEqual(smap, res)

    def test_presentation(self):
        """test presentation method"""
        self.mgr.init()
        expect = self.pmap["presentation"]["block"]
        result = self.mgr.presentation("block")
        self.assertEqual(expect, result)

    def test_notations(self):
        """test notations method"""
        self.mgr.init()
        system = "test"
        rec = {
            "notations": [
                {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""},
                {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""},
                {"api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": ""},
            ],
            "system": system,
            "type": "notation",
        }
        self.mgr.add(rec)
        expect = rec["notations"]
        result = self.mgr.notations(system)[system]
        self.assertEqual(expect, result)
Esempio n. 2
0
class testCMSFakeDataServices(unittest.TestCase):
    """
    A test class for the DAS core module
    """
    def setUp(self):
        """
        set up DAS core module
        """
        debug = 0

        # read DAS config and make fake Mapping DB entry
        collname      = 'test_collection'
        self.dasmerge = 'test_merge'
        self.dascache = 'test_cache'
        self.dasmr    = 'test_mapreduce'
        self.collname = collname
        config        = das_readconfig()
        dburi         = config['mongodb']['dburi']
        self.dburi    = dburi
        logger        = PrintManager('TestCMSFakeDataServices', verbose=debug)
        self.base     = 'http://127.0.0.1:8080' # URL of DASTestDataService
        self.expire   = 100
        config['logger']    = logger
        config['loglevel']  = debug
        config['verbose']   = debug
        config['mappingdb'] = dict(dburi=dburi, dbname='mapping', collname=collname)
        config['analyticsdb'] = dict(dbname='analytics', collname=collname, history=100)
        config['dasdb'] = {'dbname': 'das',
                           'cachecollection': self.dascache,
                           'mrcollection': self.dasmr,
                           'mergecollection': self.dasmerge}
        config['keylearningdb'] = {'collname': collname, 'dbname': 'keylearning'}
        config['parserdb'] = {'collname': collname, 'dbname': 'parser', 
                                'enable': True, 'sizecap': 10000}
        config['services'] = ['dbs', 'phedex', 'sitedb', 'google_maps', 'ip']
        # Do not perform DAS map test, since we overwrite system and urls.
        # This is done to use standard DAS maps, but use local URLs, which
        # cause DAS hash map to be be wrong during a test
        config['map_test'] = False

        # setup DAS mapper
        self.mgr = DASMapping(config)

        # create fresh DB
        self.clear_collections()
        self.mgr.delete_db_collection()
        self.mgr.init()

        # Add fake mapping records
        self.add_service('ip', 'ip.yml')
        self.add_service('google_maps', 'google_maps.yml')
        self.add_service('dbs', 'dbs.yml')
        self.add_service('phedex', 'phedex.yml')
        self.add_service('sitedb', 'sitedb.yml')

        # mongo parser
        self.mongoparser = ql_manager(config)
        config['mongoparser'] = self.mongoparser

        # create DAS handler
        self.das = DASCore(config)

        # start TestDataService
        self.server = Root(config)
        self.server.start()

    def add_service(self, system, ymlfile):
        """
        Add Fake data service mapping records. We provide system name
        which match corresponding name in DASTestDataService and
        associated with this system YML map file.
        """
        fname  = os.path.join(DASPATH, 'services/maps/%s' % ymlfile)
        url    = self.base + '/%s' % system
        for record in read_service_map(fname):
            record['url'] = url
            record['system'] = system
            self.mgr.add(record)
        for record in read_service_map(fname, 'notations'):
            record['system'] = system
            self.mgr.add(record)

    def clear_collections(self):
        """clean-up test collections"""
        conn = Connection(host=self.dburi)
        for dbname in ['mapping', 'analytics', 'das', 'parser', 'keylearning']:
            db = conn[dbname]
            if  dbname != 'das':
                db.drop_collection(self.collname)
            else:
                db.drop_collection(self.dascache)
                db.drop_collection(self.dasmerge)
                db.drop_collection(self.dasmr)
            

    def tearDown(self):
        """Invoke after each test"""
        self.server.stop()
#        self.mgr.delete_db_collection()
#        self.clear_collections()

    def testDBSService(self):
        """test DASCore with test DBS service"""
        query  = "primary_dataset=abc" # invoke query to fill DAS cache
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.call(dquery)
        expect = "ok"
        self.assertEqual(expect, result)

        query  = "primary_dataset=abc" # invoke query to get results from DAS cache
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        result = [r for r in result]
        result = DotDict(result[0]).get('primary_dataset.name')
        expect = 'abc'
        self.assertEqual(expect, result)

    def testPhedexAndSiteDBServices(self):
        """test DASCore with test PhEDEx and SiteDB services"""
        query  = "site=T3_US_Cornell" # invoke query to fill DAS cache
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.call(dquery)
        expect = "ok"
        self.assertEqual(expect, result)

        query  = "site=T3_US_Cornell | grep site.name" # invoke query to get results from DAS cache
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        result = [r for r in result]
        expect = 'T3_US_Cornell'
        self.assertEqual(expect, DotDict(result[0]).get('site.name'))
        expect = ['_id', 'das_id', 'site', 'cache_id', 'das', 'qhash']
        expect.sort()
        rkeys = result[0].keys()
        rkeys.sort()
        self.assertEqual(expect, rkeys)

    def testAggregators(self):
        """test DASCore aggregators via zip service"""
        query  = "zip=1000"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.call(dquery)
        expect = "ok"
        self.assertEqual(expect, result)

        query  = "zip=1000 | count(zip.place.city)"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        result = [r for r in result]
        result = result[0] # take first result
        if  result.has_key('das'):
            del result['das'] # strip off DAS info
        expect = {"function": "count", "result": {"value": 2}, 
                  "key": "zip.place.city", "_id":0}
        self.assertEqual(expect, result)

    def testIPService(self):
        """test DASCore with IP service"""
        query  = "ip=137.138.141.145"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.call(dquery)
        expect = "ok"
        self.assertEqual(expect, result)

        query  = "ip=137.138.141.145 | grep ip.address"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        result = [r for r in result]
        result = DotDict(result[0]).get('ip.address')
        expect = '137.138.141.145'
        self.assertEqual(expect, result)

    def testRecords(self):
        """test records DAS keyword with all services"""
        query  = "ip=137.138.141.145"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.call(dquery)
        expect = "ok"
        self.assertEqual(expect, result)

        query  = "site=T3_US_Cornell"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.call(dquery)
        expect = "ok"
        self.assertEqual(expect, result)

        query  = "records | grep ip.address"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        result = [r for r in result]
        result = DotDict(result[0]).get('ip.address')
        expect = '137.138.141.145'
        self.assertEqual(expect, result)

        query  = "records | grep site.name"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        result = [r for r in result]
        expect = 'T3_US_Cornell'
        self.assertEqual(expect, DotDict(result[0]).get('site.name'))

        query  = "records"
        dquery = DASQuery(query, mongoparser=self.mongoparser)
        result = self.das.get_from_cache(dquery, collection=self.dasmerge)
        res    = []
        for row in result:
            if  row.has_key('das') and row['das'].has_key('empty_record'):
                if  row['das'].get('empty_record'):
                    continue
            if  row.has_key('ip'):
                res.append(DotDict(row).get('ip.address'))
            if  row.has_key('site'):
                for item in row['site']:
                    if  item.has_key('name') and item['name'] not in res:
                        res.append(item['name'])
        res.sort()
        expect = ['137.138.141.145', 'T3_US_Cornell']
        self.assertEqual(expect, res)