def result(self, query, idx=0, limit=None):
"""
Get results either from cache or from explicit call
"""
self.logger.info('input query=%s' % query)
results = []
dasquery = DASQuery(query)
query = dasquery.mongo_query
# check if we have any service which cover the query
# otherwise decompose it into list of queries
service_map = dasquery.service_apis_map()
if not service_map:
msg = 'no APIs found to answer input query, will decompose it'
self.logger.info(msg)
skeys = query['fields']
if not skeys:
skeys = []
for key in skeys:
newquery = DASQuery(dict(fields=[key], spec=query['spec']))
self.call(newquery) # process query
else:
self.call(dasquery) # process query
# lookup provided query in a cache
if not self.noresults:
results = self.get_from_cache(dasquery, idx, limit)
return results
def result(self, query, idx=0, limit=None):
"""
Get results either from cache or from explicit call
"""
self.logger.info('input query=%s' % query)
results = []
dasquery = DASQuery(query)
dasquery.add_to_analytics()
query = dasquery.mongo_query
# check if we have any service which cover the query
# otherwise decompose it into list of queries
service_map = dasquery.service_apis_map()
if not service_map:
msg = 'no APIs found to answer input query, will decompose it'
self.logger.info(msg)
skeys = query['fields']
if not skeys:
skeys = []
for key in skeys:
newquery = DASQuery(dict(fields=[key], spec=query['spec']))
self.call(newquery) # process query
else:
self.call(dasquery) # process query
# lookup provided query in a cache
if not self.noresults:
results = self.get_from_cache(dasquery, idx, limit)
return results
def generate_dasquery(self, uinput, inst, html_error=True):
"""
Check provided input as valid DAS input query.
Returns status and content (either error message or valid DASQuery)
"""
def helper(msg, html_error=None):
"""Helper function which provide error template"""
if not html_error:
return msg
guide = self.templatepage('dbsql_vs_dasql',
operators=', '.join(das_operators()))
page = self.templatepage('das_ambiguous', msg=msg, base=self.base,
guide=guide)
return page
if not uinput:
return 1, helper('No input query')
# Generate DASQuery object, if it fails we catch the exception and
# wrap it for upper layer (web interface)
try:
dasquery = DASQuery(uinput, instance=inst)
except Exception as err:
return 1, helper(das_parser_error(uinput, str(err)), html_error)
fields = dasquery.mongo_query.get('fields', [])
if not fields:
fields = []
spec = dasquery.mongo_query.get('spec', {})
for word in fields+spec.keys():
found = 0
if word in DAS_DB_KEYWORDS:
found = 1
for key in self.daskeys:
if word.find(key) != -1:
found = 1
if not found:
msg = 'Provided input does not contain a valid DAS key'
return 1, helper(msg, html_error)
if isinstance(uinput, dict): # DASQuery w/ {'spec':{'_id:id}}
pass
elif uinput.find('queries') != -1:
pass
elif uinput.find('records') != -1:
pass
else: # normal user DAS query
try:
service_map = dasquery.service_apis_map()
except Exception as exc:
msg = 'Fail to lookup DASQuery service API map'
print msg
print_exc(exc)
return 1, helper(msg, html_error)
if not service_map:
msg = "None of the API's registered in DAS "
msg += "can resolve this query"
return 1, helper(msg, html_error)
return 0, dasquery
def main():
"Main function"
optmgr = DASOptionParser()
opts, _ = optmgr.getOpt()
t0 = time.time()
query = opts.query
if 'instance' not in query:
query += ' instance=cms_dbs_prod_global'
debug = opts.verbose
dascore = DASCore(debug=debug, nores=opts.noresults)
if opts.hash:
dasquery = DASQuery(query)
mongo_query = dasquery.mongo_query
service_map = dasquery.service_apis_map()
str_query = dasquery.storage_query
print "---------------"
print "DAS-QL query :", query
print "DAS query :", dasquery
print "Mongo query :", mongo_query
print "Storage query :", str_query
print "Services :\n"
for srv, val in service_map.items():
print "%s : %s\n" % (srv, ', '.join(val))
sys.exit(0)
sdict = dascore.keys()
if opts.services:
msg = "DAS services:"
print msg
print "-"*len(msg)
keys = sdict.keys()
keys.sort()
for key in keys:
print key
elif opts.service:
msg = "DAS service %s:" % opts.service
print msg
print "-"*len(msg)
keys = sdict[opts.service]
keys.sort()
for key in keys:
print key
elif query:
idx = opts.idx
limit = opts.limit
output = opts.nooutput
plain = opts.plain
if opts.profile:
import cProfile # python profiler
import pstats # profiler statistics
cmd = 'run(dascore,query,idx,limit,output,plain)'
cProfile.runctx(cmd, globals(), locals(), 'profile.dat')
info = pstats.Stats('profile.dat')
info.sort_stats('cumulative')
info.print_stats()
else:
run(dascore, query, idx, limit, output, plain)
elif opts.dasconfig:
print pformat(dascore.dasconfig)
else:
print
print "DAS CLI interface, no actions found,"
print "please use --help for more options."
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
timer = get_das_timer()
print "\nDAS execution time:\n"
if debug:
timelist = []
for _, timerdict in timer.items():
counter = timerdict['counter']
tag = timerdict['tag']
exetime = timerdict['time']
timelist.append((counter, tag, exetime))
timelist.sort()
for _, tag, exetime in timelist:
print "%s %s sec" % (tag, round(exetime, 2))
print "Total %s sec, %s" % (round(time.time()-t0, 2), timestamp)
def main():
"Main function"
optmgr = DASOptionParser()
opts = optmgr.parser.parse_args()
t0 = time.time()
query = opts.query
if 'instance' not in query:
query = ' instance=prod/global ' + query
debug = opts.verbose
dascore = DASCore(debug=debug, nores=opts.noresults)
if opts.hash:
dasquery = DASQuery(query)
mongo_query = dasquery.mongo_query
service_map = dasquery.service_apis_map()
str_query = dasquery.storage_query
print("---------------")
print("DAS-QL query :", query)
print("DAS query :", dasquery)
print("Mongo query :", mongo_query)
print("Storage query :", str_query)
print("Services :\n")
for srv, val in service_map.items():
print("%s : %s\n" % (srv, ', '.join(val)))
sys.exit(0)
sdict = dascore.keys()
if opts.services:
msg = "DAS services:"
print(msg)
print("-"*len(msg))
keys = list(sdict.keys())
keys.sort()
for key in keys:
print(key)
elif opts.service:
msg = "DAS service %s:" % opts.service
print(msg)
print("-"*len(msg))
keys = sdict[opts.service]
keys.sort()
for key in keys:
print(key)
elif opts.jsfile:
kws_js(dascore, query, opts.idx, opts.limit, opts.jsfile, debug)
sys.exit(0)
elif opts.kfile:
keylearning_js(dascore, query, opts.kfile, debug)
sys.exit(0)
elif query:
idx = opts.idx
limit = opts.limit
output = opts.nooutput
plain = opts.plain
qcache = opts.qcache
if opts.profile:
import cProfile # python profiler
import pstats # profiler statistics
cmd = 'run(dascore,query,idx,limit,output,plain)'
cProfile.runctx(cmd, globals(), locals(), 'profile.dat')
info = pstats.Stats('profile.dat')
info.sort_stats('cumulative')
info.print_stats()
else:
run(dascore, query, idx, limit, output, plain)
elif opts.dasconfig:
print(pformat(dascore.dasconfig))
else:
print()
print("DAS CLI interface, no actions found,")
print("please use --help for more options.")
timestamp = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
timer = get_das_timer()
print("\nDAS execution time:\n")
if debug:
timelist = []
for _, timerdict in timer.items():
counter = timerdict['counter']
tag = timerdict['tag']
exetime = timerdict['time']
timelist.append((counter, tag, exetime))
timelist.sort()
for _, tag, exetime in timelist:
print("%s %s sec" % (tag, round(exetime, 2)))
print("Total %s sec, %s" % (round(time.time()-t0, 2), timestamp))
def generate_dasquery(self, uinput, inst, html_mode=True, qcache=0):
"""
Check provided input as valid DAS input query.
Returns status and content (either error message or valid DASQuery)
:param uinput: user's input
:param inst: DBS instance
:param html_mode: whether errors shall be output in html
"""
def error_msg(msg, show_kws=False, tmpl='das_ambiguous', **kwargs):
"""
Helper function which renders an error template, default is
das_ambiguous, but can be overriden via tmpl param.
Template has two versions: html and text for CLI.
The template is passed with msg, base, guide, and **kwargs. """
# TODO: this shall be done by inheriting a parent template
# TODO: no header/footer?
guide = self.templatepage('dbsql_vs_dasql',
operators=', '.join(das_operators()))
# render keyword search loader, if needed
kws = ''
if show_kws:
kws = self.templatepage('kwdsearch_via_ajax',
uinput=uinput,
jsonize=jsonize,
url_extend_params_as_dict=url_extend_params_as_dict,
inst=inst or self.dbs_global,
kws_host=self._get_kws_host())
# render the appropriate template (html vs text mode)
page = self.templatepage(tmpl + ('_txt' if not html_mode else ''),
msg=msg, base=self.base, guide=guide,
kws_enabled=show_kws, kws=kws, **kwargs)
return page
if not uinput:
return 1, error_msg('No input query')
# Generate a DASQuery object, if it fails we catch the exception and
# wrap it for upper layer (web interface)
try:
dasquery = DASQuery(uinput, instance=inst, qcache=qcache)
except WildcardMultipleMatchesException as err:
# TODO: hints could be shown here also, but it makes no sense, as
# they are shown only when no matches are found
if isinstance(err.options.values, list) and err.options.values:
return 1, error_msg(str(err), tmpl='das_wildcard_err',
suggest=err.options.values,
url_extend_params=url_extend_params)
return 1, error_msg(str(err), tmpl='das_wildcard_err',
url_extend_params=url_extend_params)
except WildcardMatchingException as err:
kwds = {'input':uinput, 'instance':inst}
hints = self.hint_datasets(kwds)
page = error_msg(str(err))
for hint in hints:
page += self.templatepage('hint',
url_extend_params=url_extend_params,
hint=hint, base=self.base, dbs=self.dbs_global)
return 1, page
except Exception as err:
# show multiple dataset matches for 1 keyword queries
if hasattr(response, 'dataset_matches_msg'):
return 1, error_msg(response.dataset_matches_msg,
show_kws=self.is_kws_enabled())
# for non Wildcard parsing errors, show the Keyword Search
return 1, error_msg(str(err), show_kws=self.is_kws_enabled())
if dasquery.error:
return 1, error_msg(dasquery.error)
# DAS query validation
if isinstance(uinput, dict): # DASQuery w/ {'spec':{'_id:id}}
pass
elif uinput.find('queries') != -1:
pass
elif uinput.find('records') != -1:
pass
else: # normal user DAS query
try:
service_map = dasquery.service_apis_map()
except Exception as exc:
msg = 'Fail to obtain service API map for this DASQuery'
print(msg)
print_exc(exc)
return 1, error_msg(msg)
if not service_map:
return 1, error_msg('Unable to resolve the query over the '
'available services: %s' % dasquery)
return 0, dasquery
def generate_dasquery(self, uinput, inst, html_mode=True):
"""
Check provided input as valid DAS input query.
Returns status and content (either error message or valid DASQuery)
:param uinput: user's input
:param inst: DBS instance
:param html_mode: whether errors shall be output in html
"""
def error_msg(msg, show_kws=False, tmpl="das_ambiguous", **kwargs):
"""
Helper function which renders an error template, default is
das_ambiguous, but can be overriden via tmpl param.
Template has two versions: html and text for CLI.
The template is passed with msg, base, guide, and **kwargs. """
guide = self.templatepage("dbsql_vs_dasql", operators=", ".join(das_operators()))
# render keyword search loader, if needed
kws = ""
if show_kws:
kws = self.templatepage(
"kwdsearch_via_ajax", uinput=uinput, inst=inst or self.dbs_global, kws_host=self._get_kws_host()
)
# render the appropriate template (html vs text mode)
page = self.templatepage(
tmpl + ("_txt" if not html_mode else ""),
msg=msg,
base=self.base,
guide=guide,
kws_enabled=show_kws,
kws=kws,
**kwargs
)
return page
if not uinput:
return 1, error_msg("No input query")
# Generate a DASQuery object, if it fails we catch the exception and
# wrap it for upper layer (web interface)
try:
dasquery = DASQuery(uinput, instance=inst)
except WildcardMultipleMatchesException as err:
das_parser_error(uinput, str(err).replace("\n", ""))
return 1, error_msg(str(err), tmpl="das_wildcard_err", suggest=err.options.values)
except WildcardMatchingException as err:
das_parser_error(uinput, str(type(err)) + " " + str(err))
return 1, error_msg(str(err))
except Exception as err:
das_parser_error(uinput, str(type(err)) + " " + str(err))
# show multiple dataset matches for 1 keyword queries
if hasattr(response, "dataset_matches_msg"):
return 1, error_msg(response.dataset_matches_msg, show_kws=self.is_kws_enabled())
# for non Wildcard parsing errors, show the Keyword Search
return 1, error_msg(str(err), show_kws=self.is_kws_enabled())
# DAS query validation
if isinstance(uinput, dict): # DASQuery w/ {'spec':{'_id:id}}
pass
elif uinput.find("queries") != -1:
pass
elif uinput.find("records") != -1:
pass
else: # normal user DAS query
try:
service_map = dasquery.service_apis_map()
except Exception as exc:
msg = "Fail to obtain service API map for this DASQuery"
print msg
print_exc(exc)
return 1, error_msg(msg)
if not service_map:
return 1, error_msg("Unable to resolve the query over the " "available services: %s" % dasquery)
return 0, dasquery
