def empty_return(self, dasquery, status='busy', reason=None): "Return header/data when DAS server is busy" if not reason: reason = 'DAS server is busy' reason += ', #requests=%s, #workers=%s, queue size=%s' \ % (self.reqmgr.size(), self.taskmgr.nworkers(), self.queue_limit) head = dict(timestamp=time.time()) head.update({'status': status, 'reason': reason, 'ctime':0}) data = [] dasprint(dastimestamp('DAS INFO '), dasquery, 'server status=%s'%status, reason) return self.datastream(dict(head=head, data=data))
def busy(self): """ Check server load and report busy status if nrequests - nworkers > queue limit """ nrequests = self.reqmgr.size() if (nrequests - self.taskmgr.nworkers()) > self.queue_limit: msg = '#request=%s, queue_limit=%s, #workers=%s' \ % (nrequests, self.taskmgr.nworkers(), self.queue_limit) dasprint(dastimestamp('DAS WEB SERVER IS BUSY '), msg) return True return False
def status(self): """Return list of all current requests in DAS queue""" requests = [r for r in self.reqmgr.items()] page = self.templatepage('das_status', requests=requests, time=time) sdict = self.dasmgr.status() sdict['web'] = self.taskmgr.status() dasprint(dastimestamp('DAS INFO '), "web TaskManager", sdict['web']) for key, val in sdict.items(): dasprint(dastimestamp('DAS INFO '), "%s TaskManager %s" % (key, val)) page += '<h3>Services</h3>' def dump(idict): "Dump input dict" return ', '.join(['<em>%s:</em> %s' % (k, idict[k]) for k in sorted(idict)]) for key, val in sdict.items(): page += '<div>' stats = ', '.join([dump(v) for v in val.values()]) page += '<b>%s</b>: %s' % (key, stats) page += '</div>' return self.page(page)
def check_pid(self, pid): """ Check status of given pid. This is a server callback function for ajaxCheckPid, see js/ajax_utils.js """ # do not allow caching set_no_cache_flags() img = '<img src="%s/images/loading.gif" alt="loading"/>' % self.base page = '' try: if self.taskmgr.is_alive(pid): page = img + " processing PID=%s" % pid else: # at this point we don't know if request arrived to this host # or it was processed. To distinguish the case we'll ask # request manager for that pid if self.reqmgr.has_pid(pid): self.reqmgr.remove(pid) self.taskmgr.remove(pid) page = 'Request PID=%s is completed' % pid page += ', please wait for results to load' else: # there're no request on this server, re-initiate it ref = cherrypy.request.headers.get('Referer', None) if ref: url = urlparse(ref) params = dict(parse_qsl(url.query)) return self.request(**params) else: msg = 'No referer in cherrypy.request.headers' msg += '\nHeaders: %s' % cherrypy.request.headers dasprint(dastimestamp('DAS WEB ERROR '), msg) except Exception as err: msg = 'check_pid fails for pid=%s' % pid dasprint(dastimestamp('DAS WEB ERROR '), msg) print_exc(err) self.reqmgr.remove(pid) self.taskmgr.remove(pid) return self.error(gen_error_msg({'pid':pid}), wrap=False) return page
def get_data(self, kwargs): """ Invoke DAS workflow and get data from the cache. """ head = dict(timestamp=time.time()) head['args'] = kwargs uinput = kwargs.get('input', '') inst = kwargs.get('instance', self.dbs_global) idx = getarg(kwargs, 'idx', 0) limit = getarg(kwargs, 'limit', 0) # do not impose limit coll = kwargs.get('collection', 'merge') status = kwargs.get('status') error = kwargs.get('error') reason = kwargs.get('reason') dasquery = kwargs.get('dasquery', None) time0 = time.time() if dasquery: dasquery = DASQuery(dasquery, instance=inst) if dasquery.error: return self.page(form + dasquery.error, ctime=time.time()-time0) else: check, content = \ self.generate_dasquery(uinput, inst, html_mode=False) if check: head.update({'status': 'fail', 'reason': content, 'ctime': time.time()-time0, 'input': uinput}) data = [] return head, data dasquery = content # returned content is valid DAS query try: nres = self.dasmgr.nresults(dasquery, coll) data = \ self.dasmgr.get_from_cache(dasquery, idx, limit) # check that we got what we expected data = [r for r in data] if nres and not len(data): for retry in range(1, 3, 5): msg = 'retry in %s sec' % retry dasprint(dastimestamp('DAS WARNING '), msg, dasquery) time.sleep(retry) # retry one more time data = \ self.dasmgr.get_from_cache(dasquery, idx, limit) data = [r for r in data] if len(data): break if nres and not len(data): msg = 'fail to get all data for %s, nres=%s, len(data)=%s' \ % (dasquery, nres, len(data)) dasprint(dastimestamp('DAS WARNING '), msg) status = 'fail' reason = 'Fail to retrieve data from DAS cache, please retry' if dasquery.aggregators: # aggregators split DAS record into sub-system and then # apply aggregator functions, therefore we need to correctly # account for nresults. Resolve generator into list and take # its length as nresults value. data = [r for r in data] nres = len(data) if error: # DAS record contains an error status = 'error' head.update({'status':status, 'nresults':nres, 'ctime': time.time()-time0, 'dasquery': dasquery}) except Exception as exc: status = 'fail' reason = str(exc) print_exc(exc) head.update({'status': status, 'ctime': time.time()-time0, 'dasquery': dasquery}) data = [] head.update({'incache':self.dasmgr.incache(dasquery, coll='cache'), 'apilist':self.dasmgr.apilist(dasquery)}) if reason: head.update({'reason': reason}) if status != 'ok': head.update(self.info()) # check if query had dataset input and returned no results # then run hint functions to find dataset in other DBS instances mquery = dasquery.mongo_query empty = False for item in data: if 'dataset.name' in mquery['spec'] and 'dataset' in mquery['fields'] \ and 'result' not in item: if not item['dataset']: empty = True break if empty: # if no results found add dataset from other DBS instances hints = self.hint_datasets(kwargs) for item in data: item.update({'hints': hints}) return head, data