def postagestamp(self, page): ''' Get the postage stamps for a set of query results in the web ''' postage = {} postage['error'] = None postage['images'] = None pagesize = 16 # number of rows (images) in a page pagenum = int(page) # current page number searchvalue = current_session.get('searchvalue', None) if not searchvalue: postage[ 'error'] = 'No query found! Cannot generate images without a query. Go to the Query Page!' return render_template('postage.html', **postage) sort = current_session.get('query_sort', 'cube.mangaid') offset = (pagesize * pagenum) - pagesize q, res = doQuery(searchfilter=searchvalue, release=self._release, sort=sort, limit=10000) plateifus = res.getListOf('plateifu') # if a dap query, grab the unique galaxies if q._isdapquery: plateifus = list(set(plateifus)) # only grab subset if more than 16 galaxies if len(plateifus) > pagesize: plateifus = plateifus[offset:offset + pagesize] # get images imfiles = None try: imfiles = getImagesByList(plateifus, as_url=True, mode='local', release=self._release) except MarvinError as e: postage['error'] = 'Error: could not get images: {0}'.format(e) else: images = buildImageDict(imfiles) # if image grab failed, make placeholders if not imfiles: images = buildImageDict(imfiles, test=True, num=pagesize) # Compute page stats totalpages = int( res.totalcount // pagesize) + int(res.totalcount % pagesize != 0) page = { 'size': pagesize, 'active': int(page), 'total': totalpages, 'count': res.totalcount } postage['page'] = page postage['images'] = images return render_template('postage.html', **postage)
def webtable(self): ''' Do a query for Bootstrap Table interaction in Marvin web ''' form = processRequest(request=request) args = av.manual_parse(self, request, use_params='query') # remove args __tmp__ = args.pop('release', None) __tmp__ = args.pop('searchfilter', None) limit = args.get('limit') offset = args.get('offset') if 'sort' in args: current_session['query_sort'] = args['sort'] # set parameters searchvalue = current_session.get('searchvalue', None) returnparams = current_session.get('returnparams', None) # exit if no searchvalue is found if not searchvalue: output = jsonify({'errmsg': 'No searchvalue found', 'status': -1}) return output # this is to fix the brokeness with sorting on a table column using remote names print('rp', returnparams, args) # do query try: q, res = doQuery(searchfilter=searchvalue, release=self._release, returnparams=returnparams, **args) except Exception as e: errmsg = 'Error generating webtable: {0}'.format(e) output = jsonify({'status': -1, 'errmsg': errmsg}) return output # get subset on a given page try: __results__ = res.getSubset(offset, limit=limit) except Exception as e: errmsg = 'Error getting table page: {0}'.format(e) output = jsonify({'status': -1, 'errmsg': errmsg}) return output else: # get keys cols = res.columns.remote # create output rows = res.getDictOf(format_type='listdict') output = { 'total': res.totalcount, 'rows': rows, 'columns': cols, 'limit': limit, 'offset': offset } output = jsonify(output) return output
def _run_query(searchfilter, **kwargs): ''' Run the query and return the query and results ''' release = kwargs.pop('release', None) kwargs['returnparams'] = kwargs.pop('params', None) kwargs['returntype'] = kwargs.pop('rettype', None) try: q, r = doQuery(searchfilter=searchfilter, release=release, **kwargs) except Exception as e: raise MarvinError('Query failed with {0}: {1}'.format(e.__class__.__name__, e)) else: return q, r
def webtable(self): ''' Do a query for Bootstrap Table interaction in Marvin web ''' form = processRequest(request=request) args = av.manual_parse(self, request, use_params='query') #{'sort': u'cube.mangaid', 'task': None, 'end': None, 'searchfilter': None, #'paramdisplay': None, 'start': None, 'rettype': None, 'limit': 10, 'offset': 30, #'release': u'MPL-4', 'params': None, 'order': u'asc'} # remove args __tmp__ = args.pop('release', None) __tmp__ = args.pop('searchfilter', None) limit = args.get('limit') offset = args.get('offset') # set parameters searchvalue = current_session.get('searchvalue', None) returnparams = current_session.get('returnparams', None) # limit = form.get('limit', 10, type=int) # offset = form.get('offset', None, type=int) # order = form.get('order', None, type=str) # sort = form.get('sort', None, type=str) # search = form.get('search', None, type=str) # exit if no searchvalue is found if not searchvalue: output = jsonify({ 'webtable_error': 'No searchvalue found', 'status': -1 }) return output # do query q, res = doQuery(searchfilter=searchvalue, release=self._release, returnparams=returnparams, **args) # q, res = doQuery(searchfilter=searchvalue, release=self._release, # limit=limit, order=order, sort=sort, returnparams=returnparams) # get subset on a given page __results__ = res.getSubset(offset, limit=limit) # get keys cols = res.mapColumnsToParams() # create output rows = res.getDictOf(format_type='listdict') output = {'total': res.totalcount, 'rows': rows, 'columns': cols} output = jsonify(output) return output
def _getCubes(searchfilter, **kwargs): """Run query locally at Utah.""" release = kwargs.pop('release', None) kwargs['returnparams'] = kwargs.pop('params', None) kwargs['returntype'] = kwargs.pop('rettype', None) try: # q, r = doQuery(searchfilter=searchfilter, returnparams=params, release=release, # mode='local', returntype=rettype, limit=limit, order=order, sort=sort) q, r = doQuery(searchfilter=searchfilter, release=release, **kwargs) except Exception as e: raise MarvinError('Query failed with {0}: {1}'.format( e.__class__.__name__, e)) results = r.results # get the subset keywords start = kwargs.get('start', None) end = kwargs.get('end', None) limit = kwargs.get('limit', None) params = kwargs.get('params', None) # get a subset chunk = None if start: chunk = int(end) - int(start) results = r.getSubset(int(start), limit=chunk) chunk = limit if not chunk else limit runtime = { 'days': q.runtime.days, 'seconds': q.runtime.seconds, 'microseconds': q.runtime.microseconds } output = dict(data=results, query=r.showQuery(), chunk=limit, filter=searchfilter, params=q.params, returnparams=params, runtime=runtime, queryparams_order=q.queryparams_order, count=len(results), totalcount=r.count) return output
def test_success(self, release, mode): q, r = doQuery(searchfilter='nsa.z < 0.1', release=release, mode=mode) assert q is not None assert r is not None
def index(self): # Attempt to retrieve search parameters form = processRequest(request=request) self.search['formparams'] = form self.search['latest_dr'] = self._release.lower() if 'DR' in self._release else config._get_latest_release(dr_only=True).lower() # set the search form and form validation searchform = self.mf.SearchForm(form) searchform.returnparams.choices = [(k.lower(), k) for k in query_params.list_params()] # Add the forms and parameters self.search['paramdata'] = query_params self.search['guideparams'] = [{'id': p.full, 'optgroup': group.name, 'type': 'double' if p.dtype == 'float' else p.dtype, 'validation': {'step': 'any'}} for group in query_params for p in group] self.search['searchform'] = searchform self.search['placeholder'] = getRandomQuery() self.search['returnurl'] = 'https://sdss-marvin.readthedocs.io/en/stable/datamodel/{0}.html#{0}query'.format( current_session['release'].lower().replace('-', '')) # If form parameters then try to do a search if form: self.search.update({'results': None, 'errmsg': None}) args = av.manual_parse(self, request, use_params='search') # get form parameters searchvalue = form['searchbox'] # search filter input returnparams = form.getlist('returnparams', type=str) # dropdown select self.search.update({'returnparams': returnparams}) current_session.update({'searchvalue': searchvalue, 'returnparams': returnparams}) # if main form passes validation then do search if searchform.validate(): # try the query try: q, res = doQuery(search_filter=searchvalue, release=self._release, return_params=returnparams) except (MarvinError, KeyError) as e: self.search['errmsg'] = 'Could not perform query: {0}'.format(refine_error(e)) except NotImplementedError as e: # DAP queries disabled self.search['errmsg'] = 'Could not perform query: {0}'.format(e) else: self.search['filter'] = q.search_filter self.search['count'] = res.totalcount self.search['runtime'] = res.query_time.total_seconds() if res.count > 0: cols = res.columns.remote rows = res.getDictOf(format_type='listdict') output = {'total': res.totalcount, 'rows': rows, 'columns': cols, 'limit': None, 'offset': None} else: output = None self.search['results'] = output self.search['reslen'] = len(res.results) if returnparams: returnparams = [str(r) for r in returnparams] rpstr = 'return_params={0} <br>'.format(returnparams) if returnparams else '' qstr = ', return_params=returnparams' if returnparams else '' self.search['querystring'] = ("<html><samp>from marvin.tools.query import Query<br>\ myfilter='{0}'<br> {1}\ q = Query(search_filter=myfilter{2})<br>\ r = q.run()<br></samp></html>".format(searchvalue, rpstr, qstr)) return render_template('search.html', **self.search)
def index(self): # Attempt to retrieve search parameters form = processRequest(request=request) self.search['formparams'] = form print('search form', form, type(form)) # set the search form and form validation searchform = self.mf.SearchForm(form) q = Query(release=self._release) # allparams = q.get_available_params() bestparams = q.get_best_params() searchform.returnparams.choices = [(k.lower(), k) for k in bestparams] searchform.parambox.validators = [ all_in(bestparams), validators.Optional() ] # Add the forms self.search['searchform'] = searchform self.search['placeholder'] = getRandomQuery() #from flask import abort #abort(500) # If form parameters then try to do a search if form: self.search.update({'results': None, 'errmsg': None}) args = av.manual_parse(self, request, use_params='search') print('search args', args) # get form parameters searchvalue = form['searchbox'] # search filter input returnparams = form.getlist('returnparams', type=str) # dropdown select parambox = form.get('parambox', None, type=str) # from autocomplete if parambox: parms = parambox.split(',') parms = parms if parms[-1].strip() else parms[:-1] parambox = parms if parambox else None # Select the one that is not none returnparams = returnparams if returnparams and not parambox else \ parambox if parambox and not returnparams else \ list(set(returnparams) | set(parambox)) if returnparams and parambox else None print('return params', returnparams) current_session.update({ 'searchvalue': searchvalue, 'returnparams': returnparams }) # if main form passes validation then do search if searchform.validate(): # try the query try: q, res = doQuery(searchfilter=searchvalue, release=self._release, returnparams=returnparams) except MarvinError as e: self.search[ 'errmsg'] = 'Could not perform query: {0}'.format(e) else: self.search['filter'] = q.strfilter self.search['count'] = res.totalcount self.search['runtime'] = res.query_runtime.total_seconds() if res.count > 0: cols = res.mapColumnsToParams() rows = res.getDictOf(format_type='listdict') output = { 'total': res.totalcount, 'rows': rows, 'columns': cols } else: output = None self.search['results'] = output self.search['reslen'] = len(res.results) if returnparams: returnparams = [str(r) for r in returnparams] rpstr = 'returnparams={0} <br>'.format( returnparams) if returnparams else '' qstr = ', returnparams=returnparams' if returnparams else '' self.search['querystring'] = ( "<html><samp>from marvin import \ config<br>from marvin.tools.query import Query<br>config.mode='remote'<br>\ filter='{0}'<br> {1}\ q = Query(searchfilter=filter{2})<br>\ r = q.run()<br></samp></html>".format( searchvalue, rpstr, qstr)) return render_template('search.html', **self.search)