Пример #1
0
    def kws_async(self, **kwargs):
        """
        Returns KeywordSearch results for AJAX call (for now as html snippet)
        """
        set_no_cache_flags()

        uinput = kwargs.get('input', '').strip()
        inst = kwargs.get('instance', '').strip()
        if not inst:
            return 'You must provide DBS instance name'

        # it is simplest to run KWS on cherrypy web threads, if more requests
        # come than size of thread pool; they'll have to wait. that looks OK...

        if not uinput:
            return 'The query must be not empty'

        if not self.kws:
            return 'Query suggestions are unavailable right now...'

        timeout = self.dasconfig['keyword_search']['timeout']
        show_scores = self.dasconfig['keyword_search'].get(
            'show_scores', False)
        inst = self._get_dbs_inst(
            inst)  # validate dbs inst name or use default

        return self.kws.handle_search(self,
                                      query=uinput,
                                      dbs_inst=inst,
                                      is_ajax=True,
                                      timeout=timeout,
                                      show_score=show_scores)
Пример #2
0
    def kws_async(self, **kwargs):
        """
        Returns KeywordSearch results for AJAX call (for now as html snippet)
        """
        set_no_cache_flags()

        uinput = kwargs.get('input', '').strip()
        inst = kwargs.get('instance', '').strip()
        if not inst:
            return 'You must provide DBS instance name'

        # it is simplest to run KWS on cherrypy web threads, if more requests
        # come than size of thread pool; they'll have to wait. that looks OK...

        if not uinput:
            return 'The query must be not empty'

        if not self.kws:
            return 'Query suggestions are unavailable right now...'

        timeout = self.dasconfig['keyword_search']['timeout']
        show_scores = self.dasconfig['keyword_search'].get('show_scores', False)
        inst = self._get_dbs_inst(inst)  # validate dbs inst name or use default

        return self.kws.handle_search(self,
                                      query=uinput,
                                      dbs_inst=inst,
                                      is_ajax=True,
                                      timeout=timeout,
                                      show_score=show_scores)
Пример #3
0
    def check_pid(self, pid):
        """
        Check status of given pid. This is a server callback
        function for ajaxCheckPid, see js/ajax_utils.js
        """
        # do not allow caching
        set_no_cache_flags()

        img  = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base
        page = ''
        try:
            if  self.taskmgr.is_alive(pid):
                page = img + " processing PID=%s" % pid
            else:
                # at this point we don't know if request arrived to this host
                # or it was processed. To distinguish the case we'll ask
                # request manager for that pid
                if  self.reqmgr.has_pid(pid):
                    self.reqmgr.remove(pid)
                    self.taskmgr.remove(pid)
                    page  = 'Request PID=%s is completed' % pid
                    page += ', please wait for results to load'
                else:
                    # there're no request on this server, re-initiate it
                    ref = cherrypy.request.headers.get('Referer', None)
                    if  ref:
                        url = urlparse(ref)
                        params = dict(parse_qsl(url.query))
                        return self.request(**params)
                    else:
                        msg  = 'No referer in cherrypy.request.headers'
                        msg += '\nHeaders: %s' % cherrypy.request.headers
                        dasprint(dastimestamp('DAS WEB ERROR '), msg)
        except Exception as err:
            msg = 'check_pid fails for pid=%s' % pid
            dasprint(dastimestamp('DAS WEB ERROR '), msg)
            print_exc(err)
            self.reqmgr.remove(pid)
            self.taskmgr.remove(pid)
            return self.error(gen_error_msg({'pid':pid}), wrap=False)
        return page
Пример #4
0
    def cache(self, **kwargs):
        """
        DAS web cache interface. Fire up new process for new requests and
        record its pid. The client is in charge to keep track of pid.
        The new process uses DAS core call to request the data into cache.
        Since query are cached the repeated call with the same query
        has no cost to DAS core.
        """
        # do not allow caching
        set_no_cache_flags()

        # if busy return right away
        if  self.busy():
            return self.empty_return(kwargs)

        uinput = kwargs.get('input', '').strip()
        check_query(uinput)
        if  not uinput:
            head = {'status': 'fail', 'reason': 'No input found',
                    'args': kwargs, 'ctime': 0, 'input': uinput}
            data = []
            return self.datastream(dict(head=head, data=data))
        self.adjust_input(kwargs)
        pid    = kwargs.get('pid', '')
        inst   = kwargs.get('instance', self.dbs_global)
        uinput = kwargs.get('input', '')
        view   = kwargs.get('view', 'list')
        qcache = kwargs.get('qcache', 0)
        data   = []

        # textual views need text only error messages...
        check, content = self.generate_dasquery(uinput, inst,
                              html_mode=self._is_web_request(view),
                              qcache=qcache)
        if  check:
            head = dict(timestamp=time.time())
            head.update({'status': 'fail',
                         'reason': 'Can not interpret the query'+ \
                                   ' (while creating DASQuery)',
                         'ctime': 0})
            if not self._is_web_request(view):
                head['error_details'] = content
                head['reason'] = head['reason'] + '\n\n' + content
            return self.datastream(dict(head=head, data=data))

        dasquery = content # returned content is valid DAS query
        status, error, reason = self.dasmgr.get_status(dasquery)
        kwargs.update({'status':status, 'error':error, 'reason':reason})
        if  not pid:
            pid = dasquery.qhash
        if  status == None and not self.reqmgr.has_pid(pid): # submit new request
            uid = cherrypy.request.headers.get('Remote-Addr')
            if  hasattr(cherrypy.request, 'user'):
                uid = cherrypy.request.user.get('dn', None)
            _evt, pid = self.taskmgr.spawn(\
                self.dasmgr.call, dasquery, uid=uid, pid=dasquery.qhash)
            self.reqmgr.add(pid, kwargs)
            return pid
        if  status == 'ok':
            self.reqmgr.remove(pid)
            self.taskmgr.remove(pid)
            kwargs['dasquery'] = dasquery
            head, data = self.get_data(kwargs)
            return self.datastream(dict(head=head, data=data))
        kwargs['dasquery'] = dasquery.storage_query
        if  not self.pid_pat.match(str(pid)) or len(str(pid)) != 32:
            self.reqmgr.remove(pid)
            self.taskmgr.remove(pid)
            return self.empty_return(dasquery, 'fail', 'Invalid pid')
        elif self.taskmgr.is_alive(pid):
            return pid
        elif status == None:
            # DAS was busy and query expired since status==None
            if  not self.taskmgr.is_alive(pid) and self.reqmgr.has_pid(pid):
                self.reqmgr.remove(pid)
                self.taskmgr.remove(pid)
                return self.empty_return(dasquery, 'fail', 'request expired')
            return pid
        else: # process is done, get data
            self.reqmgr.remove(pid)
            self.taskmgr.remove(pid)
            head, data = self.get_data(kwargs)
            return self.datastream(dict(head=head, data=data))
Пример #5
0
    def request(self, **kwargs):
        """
        Request data from DAS cache.
        """
        # do not allow caching
        set_no_cache_flags()

        uinput  = kwargs.get('input', '').strip()
        check_query(uinput)
        if  not uinput:
            kwargs['reason'] = 'No input found'
            return self.redirect(**kwargs)

        # if busy return right away
        if  self.busy():
            return self.busy_page(uinput)

        time0   = time.time()
        self.adjust_input(kwargs)
        view    = kwargs.get('view', 'list')
        qcache  = kwargs.get('qcache', 0)
        if  'instance' in uinput:
            form     = self.form(uinput=uinput, view=view)
            content  = 'On DAS web UI please use drop-down menu to specify DBS'
            content += ' instance to avoid ambiguity. '
            content += 'To proceed please clear your input query.'
            return self.page(form + '<div class="box_red">%s</div>' % content)
        else:
            inst = kwargs.get('instance', self.dbs_global)
        uinput  = kwargs.get('input', '')
        form    = self.form(uinput=uinput, instance=inst, view=view)
        check, content = self.generate_dasquery(uinput, inst, qcache=qcache)
        if  check:
            if  view == 'list' or view == 'table':
                return self.page(form + content, ctime=time.time()-time0)
            else:
                return content
        dasquery = content # returned content is valid DAS query
        status, error, reason = self.dasmgr.get_status(dasquery)
        kwargs.update({'status':status, 'error':error, 'reason':reason})
        pid = dasquery.qhash
        if  status is None: # process new request
            kwargs['dasquery'] = dasquery.storage_query
            uid = cherrypy.request.headers.get('Remote-Addr')
            if  hasattr(cherrypy.request, 'user'):
                uid = cherrypy.request.user.get('dn', None)
            _evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery,
                    uid=uid, pid=dasquery.qhash)
            self.reqmgr.add(pid, kwargs)
        elif status == 'ok' or status == 'fail':
            self.reqmgr.remove(pid)
            self.taskmgr.remove(pid)

            # check if query can be rewritten via nested PK query
            rew_msg = self.q_rewriter and self.q_rewriter.check_fields(dasquery)
            if rew_msg:
                content =  self.templatepage('das_error', msg=rew_msg)
                return self.page(form + content, ctime=time.time()-time0)

            kwargs['dasquery'] = dasquery
            page = self.get_page_content(kwargs, complete_msg=False)
            ctime = (time.time()-time0)
            if  view == 'list' or view == 'table':
                return self.page(form + page, ctime=ctime)

            return page
        if  self.taskmgr.is_alive(pid):
            page = self.templatepage('das_check_pid', method='check_pid',
                    uinput=uinput, view=view, urllib=urllib,
                    base=self.base, pid=pid, interval=self.interval)
        elif status == None:
            # DAS was busy and query expired since status==None
            if  not self.taskmgr.is_alive(pid) and self.reqmgr.has_pid(pid):
                self.reqmgr.remove(pid)
                self.taskmgr.remove(pid)
                return self.empty_return(dasquery, 'fail', 'request expired')
            page = self.templatepage('das_check_pid', method='check_pid',
                    uinput=uinput, view=view, urllib=urllib,
                    base=self.base, pid=pid, interval=self.interval)
        else:
            self.reqmgr.remove(pid)
            self.taskmgr.remove(pid)
            page = self.get_page_content(kwargs)
        ctime = (time.time()-time0)
        return self.page(form + page, ctime=ctime)
Пример #6
0
    def request(self, **kwargs):
        """
        Request data from DAS cache.
        """
        # do not allow caching
        set_no_cache_flags()

        uinput = kwargs.get("input", "").strip()
        if not uinput:
            kwargs["reason"] = "No input found"
            return self.redirect(**kwargs)

        # if busy return right away
        if self.busy():
            return self.busy_page(uinput)

        time0 = time.time()
        self.adjust_input(kwargs)
        view = kwargs.get("view", "list")
        inst = kwargs.get("instance", self.dbs_global)
        uinput = kwargs.get("input", "")
        form = self.form(uinput=uinput, instance=inst, view=view)
        check, content = self.generate_dasquery(uinput, inst)
        if check:
            if view == "list" or view == "table":
                return self.page(form + content, ctime=time.time() - time0)
            else:
                return content
        dasquery = content  # returned content is valid DAS query
        status, error, reason = self.dasmgr.get_status(dasquery)
        kwargs.update({"status": status, "error": error, "reason": reason})
        pid = dasquery.qhash
        if status == None:  # process new request
            kwargs["dasquery"] = dasquery.storage_query
            addr = cherrypy.request.headers.get("Remote-Addr")
            _evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery, uid=addr, pid=dasquery.qhash)
            self.reqmgr.add(pid, kwargs)
        elif status == "ok" or status == "fail":
            self.reqmgr.remove(pid)

            # check if query can be rewritten via nested PK query
            rew_msg = self.q_rewriter and self.q_rewriter.check_fields(dasquery)
            if rew_msg:
                content = self.templatepage("das_error", msg=rew_msg)
                return self.page(form + content, ctime=time.time() - time0)

            kwargs["dasquery"] = dasquery
            page = self.get_page_content(kwargs, complete_msg=False)
            ctime = time.time() - time0
            if view == "list" or view == "table":
                return self.page(form + page, ctime=ctime)

            return page
        if self.taskmgr.is_alive(pid):
            page = self.templatepage(
                "das_check_pid",
                method="check_pid",
                uinput=uinput,
                view=view,
                base=self.base,
                pid=pid,
                interval=self.interval,
            )
        else:
            self.reqmgr.remove(pid)
            page = self.get_page_content(kwargs)
        ctime = time.time() - time0
        return self.page(form + page, ctime=ctime)
Пример #7
0
    def cache(self, **kwargs):
        """
        DAS web cache interface. Fire up new process for new requests and
        record its pid. The client is in charge to keep track of pid.
        The new process uses DAS core call to request the data into cache.
        Since query are cached the repeated call with the same query
        has no cost to DAS core.
        """
        # do not allow caching
        set_no_cache_flags()

        # if busy return right away
        if self.busy():
            nrequests = self.reqmgr.size()
            level = nrequests - self.taskmgr.nworkers() - self.queue_limit
            reason = "DAS server is busy"
            reason += ", #requests=%s, #workers=%s, queue size=%s" % (
                self.reqmgr.size(),
                self.taskmgr.nworkds(),
                self.queue_limit,
            )
            head = dict(timestamp=time.time())
            head.update({"status": "busy", "reason": reason, "ctime": 0})
            data = []
            return self.datastream(dict(head=head, data=data))

        uinput = kwargs.get("input", "").strip()
        if not uinput:
            head = {"status": "fail", "reason": "No input found", "args": kwargs, "ctime": 0, "input": uinput}
            data = []
            return self.datastream(dict(head=head, data=data))
        self.adjust_input(kwargs)
        pid = kwargs.get("pid", "")
        inst = kwargs.get("instance", self.dbs_global)
        uinput = kwargs.get("input", "")
        view = kwargs.get("view", "list")
        data = []

        # textual views need text only error messages...
        check, content = self.generate_dasquery(uinput, inst, html_mode=self._is_web_request(view))
        if check:
            head = dict(timestamp=time.time())
            head.update(
                {"status": "fail", "reason": "Can not interpret the query" + " (while creating DASQuery)", "ctime": 0}
            )
            if not self._is_web_request(view):
                head["error_details"] = content
                head["reason"] = head["reason"] + "\n\n" + content
            return self.datastream(dict(head=head, data=data))

        dasquery = content  # returned content is valid DAS query
        status, error, reason = self.dasmgr.get_status(dasquery)
        kwargs.update({"status": status, "error": error, "reason": reason})
        if not pid:
            pid = dasquery.qhash
        if status == None and not self.reqmgr.has_pid(pid):  # submit new request
            addr = cherrypy.request.headers.get("Remote-Addr")
            _evt, pid = self.taskmgr.spawn(self.dasmgr.call, dasquery, uid=addr, pid=dasquery.qhash)
            self.reqmgr.add(pid, kwargs)
            return pid
        if status == "ok":
            self.reqmgr.remove(pid)
            kwargs["dasquery"] = dasquery
            head, data = self.get_data(kwargs)
            return self.datastream(dict(head=head, data=data))
        kwargs["dasquery"] = dasquery.storage_query
        if not self.pid_pat.match(str(pid)) or len(str(pid)) != 32:
            self.reqmgr.remove(pid)
            head = {"status": "fail", "reason": "Invalid pid", "args": kwargs, "ctime": 0, "input": uinput}
            data = []
            return self.datastream(dict(head=head, data=data))
        elif self.taskmgr.is_alive(pid):
            return pid
        else:  # process is done, get data
            self.reqmgr.remove(pid)
            head, data = self.get_data(kwargs)
            return self.datastream(dict(head=head, data=data))