def kws_async(self, **kwargs):
"""
Returns KeywordSearch results for AJAX call (for now as html snippet)
"""
set_no_cache_flags()
uinput = kwargs.get('input', '').strip()
inst = kwargs.get('instance', '').strip()
if not inst:
return 'You must provide DBS instance name'
# it is simplest to run KWS on cherrypy web threads, if more requests
# come than size of thread pool; they'll have to wait. that looks OK...
if not uinput:
return 'The query must be not empty'
if not self.kws:
return 'Query suggestions are unavailable right now...'
timeout = self.dasconfig['keyword_search']['timeout']
show_scores = self.dasconfig['keyword_search'].get(
'show_scores', False)
inst = self._get_dbs_inst(
inst) # validate dbs inst name or use default
return self.kws.handle_search(self,
query=uinput,
dbs_inst=inst,
is_ajax=True,
timeout=timeout,
show_score=show_scores)
def kws_async(self, **kwargs):
"""
Returns KeywordSearch results for AJAX call (for now as html snippet)
"""
set_no_cache_flags()
uinput = kwargs.get('input', '').strip()
inst = kwargs.get('instance', '').strip()
if not inst:
return 'You must provide DBS instance name'
# it is simplest to run KWS on cherrypy web threads, if more requests
# come than size of thread pool; they'll have to wait. that looks OK...
if not uinput:
return 'The query must be not empty'
if not self.kws:
return 'Query suggestions are unavailable right now...'
timeout = self.dasconfig['keyword_search']['timeout']
show_scores = self.dasconfig['keyword_search'].get('show_scores', False)
inst = self._get_dbs_inst(inst) # validate dbs inst name or use default
return self.kws.handle_search(self,
query=uinput,
dbs_inst=inst,
is_ajax=True,
timeout=timeout,
show_score=show_scores)
def check_pid(self, pid):
"""
Check status of given pid. This is a server callback
function for ajaxCheckPid, see js/ajax_utils.js
"""
# do not allow caching
set_no_cache_flags()
img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
page = ''
try:
if self.taskmgr.is_alive(pid):
page = img + " processing PID=%s" % pid
else:
# at this point we don't know if request arrived to this host
# or it was processed. To distinguish the case we'll ask
# request manager for that pid
if self.reqmgr.has_pid(pid):
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
page = 'Request PID=%s is completed' % pid
page += ', please wait for results to load'
else:
# there're no request on this server, re-initiate it
ref = cherrypy.request.headers.get('Referer', None)
if ref:
url = urlparse(ref)
params = dict(parse_qsl(url.query))
return self.request(**params)
else:
msg = 'No referer in cherrypy.request.headers'
msg += '\nHeaders: %s' % cherrypy.request.headers
dasprint(dastimestamp('DAS WEB ERROR '), msg)
except Exception as err:
msg = 'check_pid fails for pid=%s' % pid
dasprint(dastimestamp('DAS WEB ERROR '), msg)
print_exc(err)
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
return self.error(gen_error_msg({'pid':pid}), wrap=False)
return page
def cache(self, **kwargs):
"""
DAS web cache interface. Fire up new process for new requests and
record its pid. The client is in charge to keep track of pid.
The new process uses DAS core call to request the data into cache.
Since query are cached the repeated call with the same query
has no cost to DAS core.
"""
# do not allow caching
set_no_cache_flags()
# if busy return right away
if self.busy():
return self.empty_return(kwargs)
uinput = kwargs.get('input', '').strip()
check_query(uinput)
if not uinput:
head = {'status': 'fail', 'reason': 'No input found',
'args': kwargs, 'ctime': 0, 'input': uinput}
data = []
return self.datastream(dict(head=head, data=data))
self.adjust_input(kwargs)
pid = kwargs.get('pid', '')
inst = kwargs.get('instance', self.dbs_global)
uinput = kwargs.get('input', '')
view = kwargs.get('view', 'list')
qcache = kwargs.get('qcache', 0)
data = []
# textual views need text only error messages...
check, content = self.generate_dasquery(uinput, inst,
html_mode=self._is_web_request(view),
qcache=qcache)
if check:
head = dict(timestamp=time.time())
head.update({'status': 'fail',
'reason': 'Can not interpret the query'+ \
' (while creating DASQuery)',
'ctime': 0})
if not self._is_web_request(view):
head['error_details'] = content
head['reason'] = head['reason'] + '\n\n' + content
return self.datastream(dict(head=head, data=data))
dasquery = content # returned content is valid DAS query
status, error, reason = self.dasmgr.get_status(dasquery)
kwargs.update({'status':status, 'error':error, 'reason':reason})
if not pid:
pid = dasquery.qhash
if status == None and not self.reqmgr.has_pid(pid): # submit new request
uid = cherrypy.request.headers.get('Remote-Addr')
if hasattr(cherrypy.request, 'user'):
uid = cherrypy.request.user.get('dn', None)
_evt, pid = self.taskmgr.spawn(\
self.dasmgr.call, dasquery, uid=uid, pid=dasquery.qhash)
self.reqmgr.add(pid, kwargs)
return pid
if status == 'ok':
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
kwargs['dasquery'] = dasquery
head, data = self.get_data(kwargs)
return self.datastream(dict(head=head, data=data))
kwargs['dasquery'] = dasquery.storage_query
if not self.pid_pat.match(str(pid)) or len(str(pid)) != 32:
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
return self.empty_return(dasquery, 'fail', 'Invalid pid')
elif self.taskmgr.is_alive(pid):
return pid
elif status == None:
# DAS was busy and query expired since status==None
if not self.taskmgr.is_alive(pid) and self.reqmgr.has_pid(pid):
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
return self.empty_return(dasquery, 'fail', 'request expired')
return pid
else: # process is done, get data
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
head, data = self.get_data(kwargs)
return self.datastream(dict(head=head, data=data))
def request(self, **kwargs):
"""
Request data from DAS cache.
"""
# do not allow caching
set_no_cache_flags()
uinput = kwargs.get('input', '').strip()
check_query(uinput)
if not uinput:
kwargs['reason'] = 'No input found'
return self.redirect(**kwargs)
# if busy return right away
if self.busy():
return self.busy_page(uinput)
time0 = time.time()
self.adjust_input(kwargs)
view = kwargs.get('view', 'list')
qcache = kwargs.get('qcache', 0)
if 'instance' in uinput:
form = self.form(uinput=uinput, view=view)
content = 'On DAS web UI please use drop-down menu to specify DBS'
content += ' instance to avoid ambiguity. '
content += 'To proceed please clear your input query.'
return self.page(form + '<div class="box_red">%s</div>' % content)
else:
inst = kwargs.get('instance', self.dbs_global)
uinput = kwargs.get('input', '')
form = self.form(uinput=uinput, instance=inst, view=view)
check, content = self.generate_dasquery(uinput, inst, qcache=qcache)
if check:
if view == 'list' or view == 'table':
return self.page(form + content, ctime=time.time()-time0)
else:
return content
dasquery = content # returned content is valid DAS query
status, error, reason = self.dasmgr.get_status(dasquery)
kwargs.update({'status':status, 'error':error, 'reason':reason})
pid = dasquery.qhash
if status is None: # process new request
kwargs['dasquery'] = dasquery.storage_query
uid = cherrypy.request.headers.get('Remote-Addr')
if hasattr(cherrypy.request, 'user'):
uid = cherrypy.request.user.get('dn', None)
_evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery,
uid=uid, pid=dasquery.qhash)
self.reqmgr.add(pid, kwargs)
elif status == 'ok' or status == 'fail':
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
# check if query can be rewritten via nested PK query
rew_msg = self.q_rewriter and self.q_rewriter.check_fields(dasquery)
if rew_msg:
content = self.templatepage('das_error', msg=rew_msg)
return self.page(form + content, ctime=time.time()-time0)
kwargs['dasquery'] = dasquery
page = self.get_page_content(kwargs, complete_msg=False)
ctime = (time.time()-time0)
if view == 'list' or view == 'table':
return self.page(form + page, ctime=ctime)
return page
if self.taskmgr.is_alive(pid):
page = self.templatepage('das_check_pid', method='check_pid',
uinput=uinput, view=view, urllib=urllib,
base=self.base, pid=pid, interval=self.interval)
elif status == None:
# DAS was busy and query expired since status==None
if not self.taskmgr.is_alive(pid) and self.reqmgr.has_pid(pid):
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
return self.empty_return(dasquery, 'fail', 'request expired')
page = self.templatepage('das_check_pid', method='check_pid',
uinput=uinput, view=view, urllib=urllib,
base=self.base, pid=pid, interval=self.interval)
else:
self.reqmgr.remove(pid)
self.taskmgr.remove(pid)
page = self.get_page_content(kwargs)
ctime = (time.time()-time0)
return self.page(form + page, ctime=ctime)
def request(self, **kwargs):
"""
Request data from DAS cache.
"""
# do not allow caching
set_no_cache_flags()
uinput = kwargs.get("input", "").strip()
if not uinput:
kwargs["reason"] = "No input found"
return self.redirect(**kwargs)
# if busy return right away
if self.busy():
return self.busy_page(uinput)
time0 = time.time()
self.adjust_input(kwargs)
view = kwargs.get("view", "list")
inst = kwargs.get("instance", self.dbs_global)
uinput = kwargs.get("input", "")
form = self.form(uinput=uinput, instance=inst, view=view)
check, content = self.generate_dasquery(uinput, inst)
if check:
if view == "list" or view == "table":
return self.page(form + content, ctime=time.time() - time0)
else:
return content
dasquery = content # returned content is valid DAS query
status, error, reason = self.dasmgr.get_status(dasquery)
kwargs.update({"status": status, "error": error, "reason": reason})
pid = dasquery.qhash
if status == None: # process new request
kwargs["dasquery"] = dasquery.storage_query
addr = cherrypy.request.headers.get("Remote-Addr")
_evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery, uid=addr, pid=dasquery.qhash)
self.reqmgr.add(pid, kwargs)
elif status == "ok" or status == "fail":
self.reqmgr.remove(pid)
# check if query can be rewritten via nested PK query
rew_msg = self.q_rewriter and self.q_rewriter.check_fields(dasquery)
if rew_msg:
content = self.templatepage("das_error", msg=rew_msg)
return self.page(form + content, ctime=time.time() - time0)
kwargs["dasquery"] = dasquery
page = self.get_page_content(kwargs, complete_msg=False)
ctime = time.time() - time0
if view == "list" or view == "table":
return self.page(form + page, ctime=ctime)
return page
if self.taskmgr.is_alive(pid):
page = self.templatepage(
"das_check_pid",
method="check_pid",
uinput=uinput,
view=view,
base=self.base,
pid=pid,
interval=self.interval,
)
else:
self.reqmgr.remove(pid)
page = self.get_page_content(kwargs)
ctime = time.time() - time0
return self.page(form + page, ctime=ctime)
def cache(self, **kwargs):
"""
DAS web cache interface. Fire up new process for new requests and
record its pid. The client is in charge to keep track of pid.
The new process uses DAS core call to request the data into cache.
Since query are cached the repeated call with the same query
has no cost to DAS core.
"""
# do not allow caching
set_no_cache_flags()
# if busy return right away
if self.busy():
nrequests = self.reqmgr.size()
level = nrequests - self.taskmgr.nworkers() - self.queue_limit
reason = "DAS server is busy"
reason += ", #requests=%s, #workers=%s, queue size=%s" % (
self.reqmgr.size(),
self.taskmgr.nworkds(),
self.queue_limit,
)
head = dict(timestamp=time.time())
head.update({"status": "busy", "reason": reason, "ctime": 0})
data = []
return self.datastream(dict(head=head, data=data))
uinput = kwargs.get("input", "").strip()
if not uinput:
head = {"status": "fail", "reason": "No input found", "args": kwargs, "ctime": 0, "input": uinput}
data = []
return self.datastream(dict(head=head, data=data))
self.adjust_input(kwargs)
pid = kwargs.get("pid", "")
inst = kwargs.get("instance", self.dbs_global)
uinput = kwargs.get("input", "")
view = kwargs.get("view", "list")
data = []
# textual views need text only error messages...
check, content = self.generate_dasquery(uinput, inst, html_mode=self._is_web_request(view))
if check:
head = dict(timestamp=time.time())
head.update(
{"status": "fail", "reason": "Can not interpret the query" + " (while creating DASQuery)", "ctime": 0}
)
if not self._is_web_request(view):
head["error_details"] = content
head["reason"] = head["reason"] + "\n\n" + content
return self.datastream(dict(head=head, data=data))
dasquery = content # returned content is valid DAS query
status, error, reason = self.dasmgr.get_status(dasquery)
kwargs.update({"status": status, "error": error, "reason": reason})
if not pid:
pid = dasquery.qhash
if status == None and not self.reqmgr.has_pid(pid): # submit new request
addr = cherrypy.request.headers.get("Remote-Addr")
_evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery, uid=addr, pid=dasquery.qhash)
self.reqmgr.add(pid, kwargs)
return pid
if status == "ok":
self.reqmgr.remove(pid)
kwargs["dasquery"] = dasquery
head, data = self.get_data(kwargs)
return self.datastream(dict(head=head, data=data))
kwargs["dasquery"] = dasquery.storage_query
if not self.pid_pat.match(str(pid)) or len(str(pid)) != 32:
self.reqmgr.remove(pid)
head = {"status": "fail", "reason": "Invalid pid", "args": kwargs, "ctime": 0, "input": uinput}
data = []
return self.datastream(dict(head=head, data=data))
elif self.taskmgr.is_alive(pid):
return pid
else: # process is done, get data
self.reqmgr.remove(pid)
head, data = self.get_data(kwargs)
return self.datastream(dict(head=head, data=data))