class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]}, "type": "presentation"} self.coll.insert(self.pmap) url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/' dformat = 'JSON' system = 'dbs3' expire = 100 rec = {'system':system, 'urn': 'acquisitioneras', 'format':dformat, 'instances': ['prod/global'], 'url':url, 'expire': expire, 'lookup': 'era', 'params' : {}, 'das_map': [ {"das_key": "era", "rec_key":"era.name", "api_arg":"era"} ], 'type': 'service' } self.coll.insert(rec) ver_token = verification_token(self.coll.find(**PYMONGO_OPTS)) rec = {'verification_token':ver_token, 'type':'verification_token'} self.coll.insert(rec) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() system = 'dbs3' url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' dformat = 'JSON' expire = 100 instances = ["prod/global", "prod/phys01"] api = 'primarydatasets' params = {"primary_ds_name":"*"} rec = {'system':system, 'urn':api, 'format':dformat, 'url':url, 'instances': instances, 'params': params, 'expire':expire, 'lookup': 'primary_dataset', 'wild_card':'*', 'das_map' : [dict(das_key='primary_dataset', rec_key='primary_dataset.name', api_arg='primary_dataset')], 'type': 'service' } self.mgr.add(rec) smap = {api: {'url':url, 'expire':expire, 'keys': ['primary_dataset'], 'format': dformat, 'wild_card':'*', 'cert':None, 'ckey': None, 'services': '', 'lookup': 'primary_dataset', 'params': params } } rec = {'system':system, 'urn': 'datasetaccesstypes', 'format':dformat, 'instances': instances, 'url':url, 'expire': expire, 'lookup': 'status', 'params' : {'status':'*'}, 'das_map': [ {"das_key": "status", "rec_key":"status.name", "api_arg":"status"} ], 'type': 'service' } self.mgr.add(rec) api = 'datasetaccesstypes' daskey = 'status' rec_key = 'status.name' api_input = 'status' res = self.mgr.list_systems() self.assertEqual([system], res) res = self.mgr.list_apis() res.sort() self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res) res = self.mgr.lookup_keys(system, api, daskey) self.assertEqual([rec_key], res) value = '' res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey], res) # adding notations notations = {'system':system, 'type': 'notation', 'notations':[ {'api_output':'storage_element_name', 'rec_key':'se', 'api':''}, {'api_output':'number_of_events', 'rec_key':'nevents', 'api':''}, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual(['status'], res) # build service map smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None, 'keys': ['status'], 'format':dformat, 'wild_card':'*', 'services': '', 'lookup': daskey, 'params': {"status": "*"} } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" system = "test" rec = {'notations': [ {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "admin.contacts.cms_name", "rec_key":"site.name", "api":""} ], "system": system, "type": "notation"} self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = "test_mapping.db" config = deepcopy(das_readconfig()) dburi = config["mongodb"]["dburi"] logger = PrintManager("TestDASMapping", verbose=debug) config["logger"] = logger config["verbose"] = debug dbname = "test_mapping" collname = "db" config["mappingdb"] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = { "presentation": { "block": [{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}] }, "type": "presentation", } self.coll.insert(self.pmap) ver_token = verification_token(self.coll.find(exhaust=True)) rec = {"verification_token": ver_token, "type": "verification_token"} self.coll.insert(rec) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() self.mgr.init() apiversion = "DBS_2_0_8" url = "http://a.com" dformat = "JSON" expire = 100 api = "listRuns" params = {"apiversion": apiversion, "path": "required", "api": api} rec = { "system": "dbs", "urn": api, "format": dformat, "url": url, "params": params, "expire": expire, "lookup": "run", "wild_card": "*", "das_map": [dict(das_key="run", rec_key="run.run_number", api_arg="path")], "type": "service", } self.mgr.add(rec) smap = { api: { "url": url, "expire": expire, "keys": ["run"], "format": dformat, "wild_card": "*", "cert": None, "ckey": None, "services": "", "lookup": "run", "params": {"path": "required", "api": api, "apiversion": "DBS_2_0_8"}, } } rec = { "system": "dbs", "urn": "listBlocks", "format": dformat, "url": url, "expire": expire, "lookup": "block", "params": { "apiversion": apiversion, "api": "listBlocks", "block_name": "*", "storage_element_name": "*", "user_type": "NORMAL", }, "das_map": [ {"das_key": "block", "rec_key": "block.name", "api_arg": "block_name"}, { "das_key": "site", "rec_key": "site.se", "api_arg": "storage_element_name", "pattern": "re.compile('([a-zA-Z0-9]+\.){2}')", }, ], "type": "service", } self.mgr.add(rec) system = "dbs" api = "listBlocks" daskey = "block" rec_key = "block.name" api_input = "block_name" res = self.mgr.list_systems() self.assertEqual(["dbs"], res) res = self.mgr.list_apis() # self.assertEqual([api], res) res.sort() self.assertEqual(["listBlocks", "listRuns"], res) res = self.mgr.lookup_keys(system, api, daskey) self.assertEqual([rec_key], res) value = "" res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey], res) # adding notations notations = { "system": system, "type": "notation", "notations": [ {"api_output": "storage_element_name", "rec_key": "se", "api": ""}, {"api_output": "number_of_events", "rec_key": "nevents", "api": ""}, ], } self.mgr.add(notations) res = self.mgr.notation2das(system, "number_of_events") self.assertEqual("nevents", res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual(["block", "site"], res) # build service map smap.update( { api: { "url": url, "expire": expire, "cert": None, "ckey": None, "keys": ["block", "site"], "format": dformat, "wild_card": "*", "services": "", "lookup": daskey, "params": { "storage_element_name": "*", "api": api, "block_name": "*", "user_type": "NORMAL", "apiversion": "DBS_2_0_8", }, } } ) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" self.mgr.init() expect = self.pmap["presentation"]["block"] result = self.mgr.presentation("block") self.assertEqual(expect, result) def test_notations(self): """test notations method""" self.mgr.init() system = "test" rec = { "notations": [ {"api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": ""}, {"api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": ""}, ], "system": system, "type": "notation", } self.mgr.add(rec) expect = rec["notations"] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)
class QLManager(object): """ DAS QL manager. """ def __init__(self, config=None): if not config: config = das_readconfig() self.dasmapping = DASMapping(config) if not self.dasmapping.check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.dasservices = config['services'] self.daskeysmap = self.dasmapping.daskeys() self.operators = list(das_operators()) self.daskeys = list(das_special_keys()) self.verbose = config['verbose'] self.logger = PrintManager('QLManger', self.verbose) for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item) def parse(self, query): """ Parse input query and return query in MongoDB form. """ mongo_query = self.mongo_query(query) self.convert2skeys(mongo_query) return mongo_query def mongo_query(self, query): """ Return mongo query for provided input query """ mongo_query = parse_query(query, self.daskeys, self.dasservices, self.verbose) if set(mongo_query.keys()) & set(['fields', 'spec']) != \ set(['fields', 'spec']): raise Exception('Invalid MongoDB query %s' % mongo_query) if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1: raise Exception(ambiguous_msg(query, mongo_query['spec'].keys())) for key, val in mongo_query['spec'].items(): if isinstance(val, list): raise Exception(ambiguos_val_msg(query, key, val)) return mongo_query def convert2skeys(self, mongo_query): """ Convert DAS input keys into DAS selection keys. """ if not mongo_query['spec']: for key in mongo_query['fields']: for system in self.dasmapping.list_systems(): mapkey = self.dasmapping.find_mapkey(system, key) if mapkey: mongo_query['spec'][mapkey] = '*' return spec = mongo_query['spec'] to_replace = [] for key, val in spec.items(): for system in self.dasmapping.list_systems(): mapkey = self.dasmapping.find_mapkey(system, key, val) if mapkey and mapkey != key and \ key in mongo_query['spec']: to_replace.append((key, mapkey)) continue for key, mapkey in to_replace: if key in mongo_query['spec']: mongo_query['spec'][mapkey] = mongo_query['spec'][key] del mongo_query['spec'][key] def services(self, query): """Find out DAS services to use for provided query""" skeys, cond = decompose(query) if not skeys: skeys = [] if isinstance(skeys, str): skeys = [skeys] slist = [] # look-up services from Mapping DB for key in skeys + [i for i in cond.keys()]: for service, keys in self.daskeysmap.items(): if service not in self.dasservices: continue value = cond.get(key, None) daskeys = self.dasmapping.find_daskey(service, key, value) if set(keys) & set(daskeys) and service not in slist: slist.append(service) # look-up special key condition requested_system = query.get('system', None) if requested_system: if isinstance(requested_system, basestring): requested_system = [requested_system] return list(set(slist) & set(requested_system)) return slist def service_apis_map(self, query): """ Find out which APIs correspond to provided query. Return a map of found services and their apis. """ skeys, cond = decompose(query) if not skeys: skeys = [] if isinstance(skeys, str): skeys = [skeys] adict = {} mapkeys = [ key for key in list(cond.keys()) if key not in das_special_keys() ] services = self.services(query) for srv in services: alist = self.dasmapping.list_apis(srv) for api in alist: daskeys = self.dasmapping.api_info(srv, api)['das_map'] maps = [r['rec_key'] for r in daskeys] if set(mapkeys) & set(maps) == set(mapkeys): if srv in adict: new_list = adict[srv] + [api] adict[srv] = list(set(new_list)) else: adict[srv] = [api] return adict def params(self, query): """ Return dictionary of parameters to be used in DAS Core: selection keys, conditions and services. """ skeys, cond = decompose(query) services = [] for srv in self.services(query): if srv not in services: services.append(srv) return dict(selkeys=skeys, conditions=cond, services=services)
class QLManager(object): """ DAS QL manager. """ def __init__(self, config=None): if not config: config = das_readconfig() self.dasmapping = DASMapping(config) if not self.dasmapping.check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.dasservices = config['services'] self.daskeysmap = self.dasmapping.daskeys() self.operators = list(das_operators()) self.daskeys = list(das_special_keys()) self.verbose = config['verbose'] self.logger = PrintManager('QLManger', self.verbose) for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item) def parse(self, query): """ Parse input query and return query in MongoDB form. """ mongo_query = self.mongo_query(query) self.convert2skeys(mongo_query) return mongo_query def mongo_query(self, query): """ Return mongo query for provided input query """ mongo_query = parse_query(query, self.daskeys, self.dasservices, self.verbose) if set(mongo_query.keys()) & set(['fields', 'spec']) != \ set(['fields', 'spec']): raise Exception('Invalid MongoDB query %s' % mongo_query) if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1: raise Exception(ambiguous_msg(query, mongo_query['spec'].keys())) for key, val in mongo_query['spec'].items(): if isinstance(val, list): raise Exception(ambiguos_val_msg(query, key, val)) return mongo_query def convert2skeys(self, mongo_query): """ Convert DAS input keys into DAS selection keys. """ if not mongo_query['spec']: for key in mongo_query['fields']: for system in self.dasmapping.list_systems(): mapkey = self.dasmapping.find_mapkey(system, key) if mapkey: mongo_query['spec'][mapkey] = '*' return spec = mongo_query['spec'] to_replace = [] for key, val in spec.items(): for system in self.dasmapping.list_systems(): mapkey = self.dasmapping.find_mapkey(system, key, val) if mapkey and mapkey != key and \ key in mongo_query['spec']: to_replace.append((key, mapkey)) continue for key, mapkey in to_replace: if key in mongo_query['spec']: mongo_query['spec'][mapkey] = mongo_query['spec'][key] del mongo_query['spec'][key] def services(self, query): """Find out DAS services to use for provided query""" skeys, cond = decompose(query) if not skeys: skeys = [] if isinstance(skeys, str): skeys = [skeys] slist = [] # look-up services from Mapping DB for key in skeys + [i for i in cond.keys()]: for service, keys in self.daskeysmap.items(): if service not in self.dasservices: continue value = cond.get(key, None) daskeys = self.dasmapping.find_daskey(service, key, value) if set(keys) & set(daskeys) and service not in slist: slist.append(service) # look-up special key condition requested_system = query.get('system', None) if requested_system: if isinstance(requested_system, basestring): requested_system = [requested_system] return list( set(slist) & set(requested_system) ) return slist def service_apis_map(self, query): """ Find out which APIs correspond to provided query. Return a map of found services and their apis. """ skeys, cond = decompose(query) if not skeys: skeys = [] if isinstance(skeys, str): skeys = [skeys] adict = {} mapkeys = [key for key in cond.keys() if key not in das_special_keys()] services = self.services(query) for srv in services: alist = self.dasmapping.list_apis(srv) for api in alist: daskeys = self.dasmapping.api_info(srv, api)['das_map'] maps = [r['rec_key'] for r in daskeys] if set(mapkeys) & set(maps) == set(mapkeys): if srv in adict: new_list = adict[srv] + [api] adict[srv] = list( set(new_list) ) else: adict[srv] = [api] return adict def params(self, query): """ Return dictionary of parameters to be used in DAS Core: selection keys, conditions and services. """ skeys, cond = decompose(query) services = [] for srv in self.services(query): if srv not in services: services.append(srv) return dict(selkeys=skeys, conditions=cond, services=services)
def main(): "Main function" optmgr = DASOptionParser() (opts, _args) = optmgr.getOpt() dburi = 'mongodb://%s:%s' % (opts.host, opts.port) dasconfig = das_readconfig() dbname, colname = opts.db.split('.') mongodb = dict(dburi=dburi) mappingdb = dict(dbname=dbname, collname=colname) config = dict(verbose=opts.debug, mappingdb=mappingdb, mongodb=mongodb, services=dasconfig['das'].get('services', [])) mgr = DASMapping(config) if opts.listapis: apis = mgr.list_apis(opts.system) print apis sys.exit(0) if opts.listkeys: keys = mgr.daskeys(opts.system) print keys sys.exit(0) if opts.umap: for rec in read_service_map(opts.umap, field='uri'): if opts.debug: print rec spec = {'url':rec['url'], 'urn':rec['urn']} mgr.remove(spec) # remove previous record mgr.add(rec) if opts.nmap: for rec in read_service_map(opts.nmap, field='notations'): if opts.debug: print rec system = rec['system'] spec = {'notations':{'$exists':True}, 'system':system} mgr.remove(spec) # remove previous record mgr.add(rec) if opts.pmap: for rec in read_service_map(opts.pmap, field='presentation'): if opts.debug: print rec spec = {'presentation':{'$exists':True}} mgr.remove(spec) # remove previous record mgr.add(rec) if opts.clean: mgr.delete_db() mgr.create_db() # I need to clear DAS cache/merge since I don't know # a-priory what kind of changes new maps will bring conn = db_connection(dburi) dbname = dasconfig['dasdb']['dbname'] cache = conn[dbname][dasconfig['dasdb']['cachecollection']] cache.remove({}) merge = conn[dbname][dasconfig['dasdb']['mergecollection']] merge.remove({}) if opts.remove: mgr.remove(opts.remove)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = Connection(dburi) conn.drop_database(dbname) coll = conn[dbname][collname] self.pmap = {"presentation": {"block":[{"ui": "Block name", "das": "block.name"}, {"ui": "Block size", "das": "block.size"}]}} coll.insert(self.pmap) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() self.mgr.create_db() apiversion = 'DBS_2_0_8' url = 'http://a.com' dformat = 'JSON' expire = 100 api = 'listRuns' params = { 'apiversion':apiversion, 'path' : 'required', 'api':api} rec = {'system' : 'dbs', 'urn':api, 'format':dformat, 'url':url, 'params': params, 'expire':expire, "wild_card": "*", 'daskeys' : [dict(key='run', map='run.run_number', pattern='')], 'das2api' : [ dict(api_param='path', das_key='dataset', pattern=""), ] } self.mgr.add(rec) res = self.mgr.check_dasmap('dbs', api, 'run.bfield') self.assertEqual(False, res) res = self.mgr.check_dasmap('dbs', api, 'run.run_number') self.assertEqual(True, res) smap = {api: {'url':url, 'expire':expire, 'keys': ['run'], 'format': dformat, "wild_card":"*", 'cert':None, 'ckey': None, 'params': {'path': 'required', 'api': api, 'apiversion': 'DBS_2_0_8'} } } rec = {'system':'dbs', 'urn': 'listBlocks', 'format':dformat, 'url':url, 'expire': expire, 'params' : {'apiversion': apiversion, 'api': 'listBlocks', 'block_name':'*', 'storage_element_name':'*', 'user_type':'NORMAL'}, 'daskeys': [ {'key':'block', 'map':'block.name', 'pattern':''}, ], 'das2api': [ {'api_param':'storage_element_name', 'das_key':'site', 'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"}, {'api_param':'storage_element_name', 'das_key':'site.se', 'pattern':"re.compile('([a-zA-Z0-9]+\.){2}')"}, {'api_param':'block_name', 'das_key':'block', 'pattern':""}, {'api_param':'block_name', 'das_key':'block.name', 'pattern':""}, ] } self.mgr.add(rec) system = 'dbs' api = 'listBlocks' daskey = 'block' primkey = 'block.name' api_input = 'block_name' res = self.mgr.list_systems() self.assertEqual(['dbs'], res) res = self.mgr.list_apis() # self.assertEqual([api], res) res.sort() self.assertEqual(['listBlocks', 'listRuns'], res) res = self.mgr.lookup_keys(system, daskey) self.assertEqual([primkey], res) value = '' res = self.mgr.das2api(system, daskey, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, daskey, value, api) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey, primkey], res) # adding notations notations = {'system':system, 'notations':[ {'notation':'storage_element_name', 'map':'se', 'api':''}, {'notation':'number_of_events', 'map':'nevents', 'api':''}, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual([daskey], res) # build service map smap.update({api: {'url':url, 'expire':expire, 'cert':None, 'ckey': None, 'keys': ['block'], 'format':dformat, "wild_card": "*", 'params': {'storage_element_name': '*', 'api':api, 'block_name': '*', 'user_type': 'NORMAL', 'apiversion': 'DBS_2_0_8'} } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" self.mgr.create_db() # rec = {'presentation':{'block':['block.name', 'block.size'], 'size':['size.name']}} # self.mgr.add(rec) expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" self.mgr.create_db() system = "test" rec = {'notations': [ {"notation": "site.resource_element.cms_name", "map": "site.name", "api": ""}, {"notation": "site.resource_pledge.cms_name", "map": "site.name", "api": ""}, {"notation": "admin.contacts.cms_name", "map":"site.name", "api":""} ], "system": system} self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)
class QLManager(object): """ DAS QL manager. """ def __init__(self, config=None): if not config: config = das_readconfig() self.dasmapping = DASMapping(config) if not self.dasmapping.check_maps(): msg = "No DAS maps found in MappingDB" raise Exception(msg) self.analytics = DASAnalytics(config) self.dasservices = config['services'] self.daskeysmap = self.dasmapping.daskeys() self.operators = list(das_operators()) self.daskeys = list(das_special_keys()) self.verbose = config['verbose'] self.logger = PrintManager('QLManger', self.verbose) for val in self.daskeysmap.values(): for item in val: self.daskeys.append(item) parserdir = config['das']['parserdir'] self.parserdir = parserdir self.enabledb = config['parserdb']['enable'] if self.enabledb: self.parserdb = DASParserDB(config) def parse(self, query): """ Parse input query and return query in MongoDB form. Optionally parsed query can be written into analytics DB. """ mongo_query = self.mongo_query(query) self.convert2skeys(mongo_query) return mongo_query def add_to_analytics(self, query, mongo_query): "Add DAS query to analytics DB" self.analytics.add_query(query, mongo_query) def get_ply_query(self, query): """ Get ply object for given query. Since we rely on PLY package and it may fail under the load we use couple of trials. """ ply_query = ply_parse_query(query, self.daskeys, self.dasservices, self.parserdir, self.verbose) return ply_query def mongo_query(self, query): """ Return mongo query for provided input query """ mongo_query = None if self.verbose: ply_output(query, self.daskeys, self.dasservices, self.parserdir, self.verbose) parse_again = True if self.enabledb: status, value = self.parserdb.lookup_query(query) if status == PARSERCACHE_VALID and \ len(last_key_pattern.findall(query)) == 0: mongo_query = value parse_again = False elif status == PARSERCACHE_INVALID: # we unable to find query in parserdb, so will parse again parse_again = True else: ply_query = self.get_ply_query(query) if ply_query: try: mongo_query = ply2mongo(ply_query) parse_again = False except Exception as exc: msg = "Fail in ply2mongo, query=%s, ply_query=%s" \ % (query, ply_query) print msg try: self.parserdb.insert_valid_query(query, mongo_query) except Exception as exc: msg = "Fail to insert into parserdb, exception=%s" \ % str(exc) print_exc(msg, print_traceback=True) if parse_again: try: ply_query = self.get_ply_query(query) mongo_query = ply2mongo(ply_query) except Exception as exc: msg = "Fail to parse query='%s'" % query print_exc(msg, print_traceback=False) raise exc if set(mongo_query.keys()) & set(['fields', 'spec']) != \ set(['fields', 'spec']): raise Exception('Invalid MongoDB query %s' % mongo_query) if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1: raise Exception(ambiguous_msg(query, mongo_query['spec'].keys())) for key, val in mongo_query['spec'].iteritems(): if isinstance(val, list): raise Exception(ambiguos_val_msg(query, key, val)) return mongo_query def convert2skeys(self, mongo_query): """ Convert DAS input keys into DAS selection keys. """ if not mongo_query['spec']: for key in mongo_query['fields']: for system in self.dasmapping.list_systems(): mapkey = self.dasmapping.find_mapkey(system, key) if mapkey: mongo_query['spec'][mapkey] = '*' return spec = mongo_query['spec'] to_replace = [] for key, val in spec.iteritems(): for system in self.dasmapping.list_systems(): mapkey = self.dasmapping.find_mapkey(system, key, val) if mapkey and mapkey != key and \ key in mongo_query['spec']: to_replace.append((key, mapkey)) continue for key, mapkey in to_replace: if key in mongo_query['spec']: mongo_query['spec'][mapkey] = mongo_query['spec'][key] del mongo_query['spec'][key] def services(self, query): """Find out DAS services to use for provided query""" skeys, cond = decompose(query) if not skeys: skeys = [] if isinstance(skeys, str): skeys = [skeys] slist = [] # look-up services from Mapping DB for key in skeys + [i for i in cond.keys()]: for service, keys in self.daskeysmap.iteritems(): if service not in self.dasservices: continue value = cond.get(key, None) daskeys = self.dasmapping.find_daskey(service, key, value) if set(keys) & set(daskeys) and service not in slist: slist.append(service) # look-up special key condition requested_system = query.get('system', None) if requested_system: if isinstance(requested_system, basestring): requested_system = [requested_system] return list( set(slist) & set(requested_system) ) return slist def service_apis_map(self, query): """ Find out which APIs correspond to provided query. Return a map of found services and their apis. """ skeys, cond = decompose(query) if not skeys: skeys = [] if isinstance(skeys, str): skeys = [skeys] adict = {} mapkeys = [key for key in cond.keys() if key not in das_special_keys()] services = self.services(query) for srv in services: alist = self.dasmapping.list_apis(srv) for api in alist: daskeys = self.dasmapping.api_info(srv, api)['das_map'] maps = [r['rec_key'] for r in daskeys] if set(mapkeys) & set(maps) == set(mapkeys): if srv in adict: new_list = adict[srv] + [api] adict[srv] = list( set(new_list) ) else: adict[srv] = [api] return adict def params(self, query): """ Return dictionary of parameters to be used in DAS Core: selection keys, conditions and services. """ skeys, cond = decompose(query) services = [] for srv in self.services(query): if srv not in services: services.append(srv) return dict(selkeys=skeys, conditions=cond, services=services)
class testDASMapping(unittest.TestCase): """ A test class for the DAS mappingdb class """ def setUp(self): """ set up DAS core module """ debug = 0 self.db = 'test_mapping.db' config = deepcopy(das_readconfig()) dburi = config['mongodb']['dburi'] logger = PrintManager('TestDASMapping', verbose=debug) config['logger'] = logger config['verbose'] = debug dbname = 'test_mapping' collname = 'db' config['mappingdb'] = dict(dburi=dburi, dbname=dbname, collname=collname) # add some maps to mapping db conn = MongoClient(dburi) conn.drop_database(dbname) self.coll = conn[dbname][collname] self.pmap = { "presentation": { "block": [{ "ui": "Block name", "das": "block.name" }, { "ui": "Block size", "das": "block.size" }] }, "type": "presentation" } self.coll.insert(self.pmap) url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader/acquisitioneras/' dformat = 'JSON' system = 'dbs3' expire = 100 rec = { 'system': system, 'urn': 'acquisitioneras', 'format': dformat, 'instances': ['prod/global'], 'url': url, 'expire': expire, 'lookup': 'era', 'params': {}, 'das_map': [{ "das_key": "era", "rec_key": "era.name", "api_arg": "era" }], 'type': 'service' } self.coll.insert(rec) ver_token = verification_token(self.coll.find(**PYMONGO_OPTS)) rec = {'verification_token': ver_token, 'type': 'verification_token'} self.coll.insert(rec) self.mgr = DASMapping(config) def tearDown(self): """Invoke after each test""" self.mgr.delete_db() def test_api(self): """test methods for api table""" self.mgr.delete_db() system = 'dbs3' url = 'https://cmsweb.cern.ch/dbs/prod/global/DBSReader' dformat = 'JSON' expire = 100 instances = ["prod/global", "prod/phys01"] api = 'primarydatasets' params = {"primary_ds_name": "*"} rec = { 'system': system, 'urn': api, 'format': dformat, 'url': url, 'instances': instances, 'params': params, 'expire': expire, 'lookup': 'primary_dataset', 'wild_card': '*', 'das_map': [ dict(das_key='primary_dataset', rec_key='primary_dataset.name', api_arg='primary_dataset') ], 'type': 'service' } self.mgr.add(rec) smap = { api: { 'url': url, 'expire': expire, 'keys': ['primary_dataset'], 'format': dformat, 'wild_card': '*', 'cert': None, 'ckey': None, 'services': '', 'lookup': 'primary_dataset', 'params': params } } rec = { 'system': system, 'urn': 'datasetaccesstypes', 'format': dformat, 'instances': instances, 'url': url, 'expire': expire, 'lookup': 'status', 'params': { 'status': '*' }, 'das_map': [{ "das_key": "status", "rec_key": "status.name", "api_arg": "status" }], 'type': 'service' } self.mgr.add(rec) api = 'datasetaccesstypes' daskey = 'status' rec_key = 'status.name' api_input = 'status' res = self.mgr.list_systems() self.assertEqual([system], res) res = self.mgr.list_apis() res.sort() self.assertEqual(['datasetaccesstypes', 'primarydatasets'], res) res = self.mgr.lookup_keys(system, api, daskey) self.assertEqual([rec_key], res) value = '' res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) # adding another params which default is None res = self.mgr.das2api(system, api, rec_key, value) self.assertEqual([api_input], res) res = self.mgr.api2das(system, api_input) self.assertEqual([daskey], res) # adding notations notations = { 'system': system, 'type': 'notation', 'notations': [ { 'api_output': 'storage_element_name', 'rec_key': 'se', 'api': '' }, { 'api_output': 'number_of_events', 'rec_key': 'nevents', 'api': '' }, ] } self.mgr.add(notations) res = self.mgr.notation2das(system, 'number_of_events') self.assertEqual('nevents', res) # API keys res = self.mgr.api2daskey(system, api) self.assertEqual(['status'], res) # build service map smap.update({ api: { 'url': url, 'expire': expire, 'cert': None, 'ckey': None, 'keys': ['status'], 'format': dformat, 'wild_card': '*', 'services': '', 'lookup': daskey, 'params': { "status": "*" } } }) res = self.mgr.servicemap(system) self.assertEqual(smap, res) def test_presentation(self): """test presentation method""" expect = self.pmap['presentation']['block'] result = self.mgr.presentation('block') self.assertEqual(expect, result) def test_notations(self): """test notations method""" system = "test" rec = { 'notations': [{ "api_output": "site.resource_element.cms_name", "rec_key": "site.name", "api": "" }, { "api_output": "site.resource_pledge.cms_name", "rec_key": "site.name", "api": "" }, { "api_output": "admin.contacts.cms_name", "rec_key": "site.name", "api": "" }], "system": system, "type": "notation" } self.mgr.add(rec) expect = rec['notations'] result = self.mgr.notations(system)[system] self.assertEqual(expect, result)