class testCMSFakeDataServices(unittest.TestCase): """ A test class for the DAS core module """ def setUp(self): """ set up DAS core module """ debug = 0 # read DAS config and make fake Mapping DB entry collname = 'test_collection' self.dasmerge = 'test_merge' self.dascache = 'test_cache' self.dasmr = 'test_mapreduce' self.collname = collname # config = deepcopy(das_readconfig()) config = das_readconfig() dburi = config['mongodb']['dburi'] self.dburi = dburi logger = PrintManager('TestCMSFakeDataServices', verbose=debug) self.base = 'http://localhost:8080' # URL of DASTestDataService self.expire = 100 config['logger'] = logger config['loglevel'] = debug config['verbose'] = debug config['mappingdb'] = dict(dburi=dburi, dbname='mapping', collname=collname) config['analyticsdb'] = dict(dbname='analytics', collname=collname, history=100) config['dasdb'] = {'dbname': 'das', 'cachecollection': self.dascache, 'mrcollection': self.dasmr, 'mergecollection': self.dasmerge} config['keylearningdb'] = {'collname': collname, 'dbname': 'keylearning'} config['parserdb'] = {'collname': collname, 'dbname': 'parser', 'enable': True, 'sizecap': 10000} config['services'] = ['dbs', 'phedex', 'sitedb', 'google_maps', 'ip'] # mongo parser self.mongoparser = ql_manager(config) config['mongoparser'] = self.mongoparser # setup DAS mapper self.mgr = DASMapping(config) # create fresh DB self.clear_collections() self.mgr.delete_db_collection() self.mgr.create_db() # Add fake mapping records self.add_service('ip', 'ip.yml') self.add_service('google_maps', 'google_maps.yml') self.add_service('dbs', 'dbs.yml') self.add_service('phedex', 'phedex.yml') self.add_service('sitedb', 'sitedb.yml') # create DAS handler self.das = DASCore(config) # start TestDataService self.server = Root(config) self.server.start() def add_service(self, system, ymlfile): """ Add Fake data service mapping records. We provide system name which match corresponding name in DASTestDataService and associated with this system YML map file. """ fname = os.path.join(DASPATH, 'services/maps/%s' % ymlfile) url = self.base + '/%s' % system for record in read_service_map(fname): record['url'] = url record['system'] = system self.mgr.add(record) for record in read_service_map(fname, 'notations'): record['system'] = system self.mgr.add(record) def clear_collections(self): """clean-up test collections""" conn = Connection(host=self.dburi) for dbname in ['mapping', 'analytics', 'das', 'parser', 'keylearning']: db = conn[dbname] if dbname != 'das': db.drop_collection(self.collname) else: db.drop_collection(self.dascache) db.drop_collection(self.dasmerge) db.drop_collection(self.dasmr) def tearDown(self): """Invoke after each test""" self.server.stop() # self.mgr.delete_db_collection() # self.clear_collections() def testDBSService(self): """test DASCore with test DBS service""" query = "primary_dataset=abc" # invoke query to fill DAS cache dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.call(dquery) expect = "ok" self.assertEqual(expect, result) query = "primary_dataset=abc" # invoke query to get results from DAS cache dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) result = [r for r in result] result = DotDict(result[0]).get('primary_dataset.name') expect = 'abc' self.assertEqual(expect, result) def testPhedexAndSiteDBServices(self): """test DASCore with test PhEDEx and SiteDB services""" query = "site=T3_US_Cornell" # invoke query to fill DAS cache dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.call(dquery) expect = "ok" self.assertEqual(expect, result) query = "site=T3_US_Cornell | grep site.name" # invoke query to get results from DAS cache dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) result = [r for r in result] expect = 'T3_US_Cornell' self.assertEqual(expect, DotDict(result[0]).get('site.name')) expect = ['_id', 'das_id', 'site', 'cache_id', 'das', 'qhash'] expect.sort() rkeys = result[0].keys() rkeys.sort() self.assertEqual(expect, rkeys) def testAggregators(self): """test DASCore aggregators via zip service""" query = "zip=1000" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.call(dquery) expect = "ok" self.assertEqual(expect, result) query = "zip=1000 | count(zip.place.city)" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) result = [r for r in result] expect = {"function": "count", "result": {"value": 2}, "key": "zip.place.city", "_id":0} self.assertEqual(expect, result[0]) def testIPService(self): """test DASCore with IP service""" query = "ip=137.138.141.145" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.call(dquery) expect = "ok" self.assertEqual(expect, result) query = "ip=137.138.141.145 | grep ip.address" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) result = [r for r in result] result = DotDict(result[0]).get('ip.address') expect = '137.138.141.145' self.assertEqual(expect, result) def testRecords(self): """test records DAS keyword with all services""" query = "ip=137.138.141.145" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.call(dquery) expect = "ok" self.assertEqual(expect, result) query = "site=T3_US_Cornell" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.call(dquery) expect = "ok" self.assertEqual(expect, result) query = "records | grep ip.address" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) result = [r for r in result] result = DotDict(result[0]).get('ip.address') expect = '137.138.141.145' self.assertEqual(expect, result) query = "records | grep site.name" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) result = [r for r in result] expect = 'T3_US_Cornell' self.assertEqual(expect, DotDict(result[0]).get('site.name')) query = "records" dquery = DASQuery(query, mongoparser=self.mongoparser) result = self.das.get_from_cache(dquery, collection=self.dasmerge) res = [] for row in result: if row.has_key('ip'): res.append(DotDict(row).get('ip.address')) if row.has_key('site'): for item in row['site']: if item.has_key('name') and item['name'] not in res: res.append(item['name']) res.sort() expect = ['137.138.141.145', 'T3_US_Cornell'] self.assertEqual(expect, res)
def main(): "Main function" optmgr = DASOptionParser() (opts, _args) = optmgr.getOpt() dburi = 'mongodb://%s:%s' % (opts.host, opts.port) dasconfig = das_readconfig() dbname, colname = opts.db.split('.') mongodb = dict(dburi=dburi) mappingdb = dict(dbname=dbname, collname=colname) config = dict(verbose=opts.debug, mappingdb=mappingdb, mongodb=mongodb, services=dasconfig['das'].get('services', [])) mgr = DASMapping(config) if opts.listapis: apis = mgr.list_apis(opts.system) print apis sys.exit(0) if opts.listkeys: keys = mgr.daskeys(opts.system) print keys sys.exit(0) if opts.umap: for rec in read_service_map(opts.umap, field='uri'): if opts.debug: print rec spec = {'url':rec['url'], 'urn':rec['urn']} mgr.remove(spec) # remove previous record mgr.add(rec) if opts.nmap: for rec in read_service_map(opts.nmap, field='notations'): if opts.debug: print rec system = rec['system'] spec = {'notations':{'$exists':True}, 'system':system} mgr.remove(spec) # remove previous record mgr.add(rec) if opts.pmap: for rec in read_service_map(opts.pmap, field='presentation'): if opts.debug: print rec spec = {'presentation':{'$exists':True}} mgr.remove(spec) # remove previous record mgr.add(rec) if opts.clean: mgr.delete_db() mgr.create_db() # I need to clear DAS cache/merge since I don't know # a-priory what kind of changes new maps will bring conn = db_connection(dburi) dbname = dasconfig['dasdb']['dbname'] cache = conn[dbname][dasconfig['dasdb']['cachecollection']] cache.remove({}) merge = conn[dbname][dasconfig['dasdb']['mergecollection']] merge.remove({}) if opts.remove: mgr.remove(opts.remove)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = Connection(dburi) conn.drop_database(dbname) coll = conn[dbname][collname] self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]}} coll.insert(self.pmap) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() self.mgr.create_db() apiversion = 'DBS_2_0_8' url = 'http://a.com' dformat = 'JSON' expire = 100 api = 'listRuns' params = { 'apiversion':apiversion, 'path' : 'required', 'api':api} rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url, 'params': params, 'expire':expire, "wild_card": "*", 'daskeys' : [dict(key='run', map='run.run_number', pattern='')], 'das2api' : [ dict(api_param='path', das_key='dataset', pattern=""), ] } self.mgr.add(rec) res = self.mgr.check_dasmap('dbs', api, 'run.bfield') self.assertEqual(False, res) res = self.mgr.check_dasmap('dbs', api, 'run.run_number') self.assertEqual(True, res) smap = {api: {'url':url, 'expire':expire, 'keys': ['run'], 'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None, 'params': {'path': 'required', 'api': api, 'apiversion': 'DBS_2_0_8'} } } rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat, 'url':url, 'expire': expire, 'params' : {'apiversion': apiversion, 'api': 'listBlocks', 'block_name':'*', 'storage_element_name':'*', 'user_type':'NORMAL'}, 'daskeys': [ {'key':'block', 'map':'block.name', 'pattern':''}, ], 'das2api': [ {'api_param':'storage_element_name', 'das_key':'site', 'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"}, {'api_param':'storage_element_name', 'das_key':'site.se', 'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"}, {'api_param':'block_name', 'das_key':'block', 'pattern':""}, {'api_param':'block_name', 'das_key':'block.name', 'pattern':""}, ] } self.mgr.add(rec) system = 'dbs' api = 'listBlocks' daskey = 'block' primkey = 'block.name' api_input = 'block_name' res = self.mgr.list_systems() self.assertEqual(['dbs'], res) res = self.mgr.list_apis() # self.assertEqual([api], res) res.sort() self.assertEqual(['listBlocks', 'listRuns'], res) res = self.mgr.lookup_keys(system, daskey) self.assertEqual([primkey], res) value = '' res = self.mgr.das2api(system, daskey, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, daskey, value, api) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey, primkey], res) # adding notations notations = {'system':system, 'notations':[ {'notation':'storage_element_name', 'map':'se', 'api':''}, {'notation':'number_of_events', 'map':'nevents', 'api':''}, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual([daskey], res) # build service map smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None, 'keys': ['block'], 'format':dformat, "wild_card": "*", 'params': {'storage_element_name': '*', 'api':api, 'block_name': '*', 'user_type': 'NORMAL', 'apiversion': 'DBS_2_0_8'} } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" self.mgr.create_db() # rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}} # self.mgr.add(rec) expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" self.mgr.create_db() system = "test" rec = {'notations': [ {"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""}, {"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""}, {"notation": "admin.contacts.cms_name", "map":"site.name", "api":""} ], "system": system} self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)