class QLManager(object):
"""
DAS QL manager.
"""
def __init__(self, config=None):
if not config:
config = das_readconfig()
if not config.has_key('dasmapping'):
config['dasmapping'] = DASMapping(config)
if not config.has_key('dasanalytics'):
config['dasanalytics'] = DASAnalytics(config)
if not config['dasmapping'].check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.map = config['dasmapping']
self.analytics = config['dasanalytics']
self.dasservices = config['services']
self.daskeysmap = self.map.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
parserdir = config['das']['parserdir']
self.dasply = DASPLY(parserdir, self.daskeys, self.dasservices,
verbose=self.verbose)
self.enabledb = config['parserdb']['enable']
if self.enabledb:
self.parserdb = DASParserDB(config)
def parse(self, query):
"""
Parse input query and return query in MongoDB form.
Optionally parsed query can be written into analytics DB.
"""
mongo_query = self.mongo_query(query)
self.convert2skeys(mongo_query)
return mongo_query
def add_to_analytics(self, query, mongo_query):
"Add DAS query to analytics DB"
self.analytics.add_query(query, mongo_query)
def mongo_query(self, query):
"""
Return mongo query for provided input query
"""
# NOTE: somehow I need to keep build call just before using
# PLY parser, otherwise it fails to parse.
self.dasply.build()
if self.verbose:
msg = "input query='%s'" % query
self.logger.debug(msg)
self.dasply.test_lexer(query)
if self.enabledb:
status, value = self.parserdb.lookup_query(query)
if status == PARSERCACHE_VALID and \
len(last_key_pattern.findall(query)) == 0:
mongo_query = value
elif status == PARSERCACHE_INVALID:
raise Exception(value)
else:
try:
ply_query = self.dasply.parser.parse(query)
mongo_query = ply2mongo(ply_query)
self.parserdb.insert_valid_query(query, mongo_query)
except Exception as exp:
self.parserdb.insert_invalid_query(query, exp)
print "Input query=%s" % query
raise exp
else:
try:
ply_query = self.dasply.parser.parse(query)
mongo_query = ply2mongo(ply_query)
except Exception as exc:
msg = "Fail to convert input query='%s' into MongoDB format" \
% query
print_exc(msg, print_traceback=False)
raise exc
if set(mongo_query.keys()) & set(['fields', 'spec']) != \
set(['fields', 'spec']):
raise Exception('Invalid MongoDB query %s' % mongo_query)
if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1:
raise Exception(ambiguous_msg(query, mongo_query['spec'].keys()))
for key, val in mongo_query['spec'].iteritems():
if isinstance(val, list):
raise Exception(ambiguos_val_msg(query, key, val))
return mongo_query
def convert2skeys(self, mongo_query):
"""
Convert DAS input keys into DAS selection keys.
"""
if not mongo_query['spec']:
for key in mongo_query['fields']:
for system in self.map.list_systems():
mapkey = self.map.find_mapkey(system, key)
if mapkey:
mongo_query['spec'][mapkey] = '*'
return
spec = mongo_query['spec']
to_replace = []
for key, val in spec.iteritems():
for system in self.map.list_systems():
mapkey = self.map.find_mapkey(system, key, val)
if mapkey and mapkey != key and \
mongo_query['spec'].has_key(key):
to_replace.append((key, mapkey))
continue
for key, mapkey in to_replace:
if mongo_query['spec'].has_key(key):
mongo_query['spec'][mapkey] = mongo_query['spec'][key]
del mongo_query['spec'][key]
def services(self, query):
"""Find out DAS services to use for provided query"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
slist = []
# look-up services from Mapping DB
for key in skeys + [i for i in cond.keys()]:
for service, keys in self.daskeysmap.iteritems():
if service not in self.dasservices:
continue
value = cond.get(key, None)
daskeys = self.map.find_daskey(service, key, value)
if set(keys) & set(daskeys) and service not in slist:
slist.append(service)
# look-up special key condition
requested_system = query.get('system', None)
if requested_system:
if isinstance(requested_system, str):
requested_system = [requested_system]
return list( set(slist) & set(requested_system) )
return slist
def service_apis_map(self, query):
"""
Find out which APIs correspond to provided query.
Return a map of found services and their apis.
"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
adict = {}
mapkeys = [key for key in cond.keys() if key not in das_special_keys()]
services = self.services(query)
for srv in services:
alist = self.map.list_apis(srv)
for api in alist:
daskeys = self.map.api_info(api)['daskeys']
maps = [r['map'] for r in daskeys]
if set(mapkeys) & set(maps) == set(mapkeys):
if adict.has_key(srv):
new_list = adict[srv] + [api]
adict[srv] = list( set(new_list) )
else:
adict[srv] = [api]
return adict
def params(self, query):
"""
Return dictionary of parameters to be used in DAS Core:
selection keys, conditions and services.
"""
skeys, cond = decompose(query)
services = []
for srv in self.services(query):
if srv not in services:
services.append(srv)
return dict(selkeys=skeys, conditions=cond, services=services)
class QLManager(object):
"""
DAS QL manager.
"""
def __init__(self, config=None):
if not config:
config = das_readconfig()
self.dasmapping = DASMapping(config)
if not self.dasmapping.check_maps():
msg = "No DAS maps found in MappingDB"
raise Exception(msg)
self.analytics = DASAnalytics(config)
self.dasservices = config['services']
self.daskeysmap = self.dasmapping.daskeys()
self.operators = list(das_operators())
self.daskeys = list(das_special_keys())
self.verbose = config['verbose']
self.logger = PrintManager('QLManger', self.verbose)
for val in self.daskeysmap.values():
for item in val:
self.daskeys.append(item)
parserdir = config['das']['parserdir']
self.parserdir = parserdir
self.enabledb = config['parserdb']['enable']
if self.enabledb:
self.parserdb = DASParserDB(config)
def parse(self, query):
"""
Parse input query and return query in MongoDB form.
Optionally parsed query can be written into analytics DB.
"""
mongo_query = self.mongo_query(query)
self.convert2skeys(mongo_query)
return mongo_query
def add_to_analytics(self, query, mongo_query):
"Add DAS query to analytics DB"
self.analytics.add_query(query, mongo_query)
def get_ply_query(self, query):
"""
Get ply object for given query. Since we rely on PLY package and it may
fail under the load we use couple of trials.
"""
ply_query = ply_parse_query(query, self.daskeys, self.dasservices,
self.parserdir, self.verbose)
return ply_query
def mongo_query(self, query):
"""
Return mongo query for provided input query
"""
mongo_query = None
if self.verbose:
ply_output(query, self.daskeys, self.dasservices,
self.parserdir, self.verbose)
parse_again = True
if self.enabledb:
status, value = self.parserdb.lookup_query(query)
if status == PARSERCACHE_VALID and \
len(last_key_pattern.findall(query)) == 0:
mongo_query = value
parse_again = False
elif status == PARSERCACHE_INVALID:
# we unable to find query in parserdb, so will parse again
parse_again = True
else:
ply_query = self.get_ply_query(query)
if ply_query:
try:
mongo_query = ply2mongo(ply_query)
parse_again = False
except Exception as exc:
msg = "Fail in ply2mongo, query=%s, ply_query=%s" \
% (query, ply_query)
print msg
try:
self.parserdb.insert_valid_query(query, mongo_query)
except Exception as exc:
msg = "Fail to insert into parserdb, exception=%s" \
% str(exc)
print_exc(msg, print_traceback=True)
if parse_again:
try:
ply_query = self.get_ply_query(query)
mongo_query = ply2mongo(ply_query)
except Exception as exc:
msg = "Fail to parse query='%s'" % query
print_exc(msg, print_traceback=False)
raise exc
if set(mongo_query.keys()) & set(['fields', 'spec']) != \
set(['fields', 'spec']):
raise Exception('Invalid MongoDB query %s' % mongo_query)
if not mongo_query['fields'] and len(mongo_query['spec'].keys()) > 1:
raise Exception(ambiguous_msg(query, mongo_query['spec'].keys()))
for key, val in mongo_query['spec'].iteritems():
if isinstance(val, list):
raise Exception(ambiguos_val_msg(query, key, val))
return mongo_query
def convert2skeys(self, mongo_query):
"""
Convert DAS input keys into DAS selection keys.
"""
if not mongo_query['spec']:
for key in mongo_query['fields']:
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key)
if mapkey:
mongo_query['spec'][mapkey] = '*'
return
spec = mongo_query['spec']
to_replace = []
for key, val in spec.iteritems():
for system in self.dasmapping.list_systems():
mapkey = self.dasmapping.find_mapkey(system, key, val)
if mapkey and mapkey != key and \
key in mongo_query['spec']:
to_replace.append((key, mapkey))
continue
for key, mapkey in to_replace:
if key in mongo_query['spec']:
mongo_query['spec'][mapkey] = mongo_query['spec'][key]
del mongo_query['spec'][key]
def services(self, query):
"""Find out DAS services to use for provided query"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
slist = []
# look-up services from Mapping DB
for key in skeys + [i for i in cond.keys()]:
for service, keys in self.daskeysmap.iteritems():
if service not in self.dasservices:
continue
value = cond.get(key, None)
daskeys = self.dasmapping.find_daskey(service, key, value)
if set(keys) & set(daskeys) and service not in slist:
slist.append(service)
# look-up special key condition
requested_system = query.get('system', None)
if requested_system:
if isinstance(requested_system, basestring):
requested_system = [requested_system]
return list( set(slist) & set(requested_system) )
return slist
def service_apis_map(self, query):
"""
Find out which APIs correspond to provided query.
Return a map of found services and their apis.
"""
skeys, cond = decompose(query)
if not skeys:
skeys = []
if isinstance(skeys, str):
skeys = [skeys]
adict = {}
mapkeys = [key for key in cond.keys() if key not in das_special_keys()]
services = self.services(query)
for srv in services:
alist = self.dasmapping.list_apis(srv)
for api in alist:
daskeys = self.dasmapping.api_info(srv, api)['das_map']
maps = [r['rec_key'] for r in daskeys]
if set(mapkeys) & set(maps) == set(mapkeys):
if srv in adict:
new_list = adict[srv] + [api]
adict[srv] = list( set(new_list) )
else:
adict[srv] = [api]
return adict
def params(self, query):
"""
Return dictionary of parameters to be used in DAS Core:
selection keys, conditions and services.
"""
skeys, cond = decompose(query)
services = []
for srv in self.services(query):
if srv not in services:
services.append(srv)
return dict(selkeys=skeys, conditions=cond, services=services)