def postagestamp(self, page):
''' Get the postage stamps for a set of query results in the web '''
postage = {}
postage['error'] = None
postage['images'] = None
pagesize = 16 # number of rows (images) in a page
pagenum = int(page) # current page number
searchvalue = current_session.get('searchvalue', None)
if not searchvalue:
postage[
'error'] = 'No query found! Cannot generate images without a query. Go to the Query Page!'
return render_template('postage.html', **postage)
sort = current_session.get('query_sort', 'cube.mangaid')
offset = (pagesize * pagenum) - pagesize
q, res = doQuery(searchfilter=searchvalue,
release=self._release,
sort=sort,
limit=10000)
plateifus = res.getListOf('plateifu')
# if a dap query, grab the unique galaxies
if q._isdapquery:
plateifus = list(set(plateifus))
# only grab subset if more than 16 galaxies
if len(plateifus) > pagesize:
plateifus = plateifus[offset:offset + pagesize]
# get images
imfiles = None
try:
imfiles = getImagesByList(plateifus,
as_url=True,
mode='local',
release=self._release)
except MarvinError as e:
postage['error'] = 'Error: could not get images: {0}'.format(e)
else:
images = buildImageDict(imfiles)
# if image grab failed, make placeholders
if not imfiles:
images = buildImageDict(imfiles, test=True, num=pagesize)
# Compute page stats
totalpages = int(
res.totalcount // pagesize) + int(res.totalcount % pagesize != 0)
page = {
'size': pagesize,
'active': int(page),
'total': totalpages,
'count': res.totalcount
}
postage['page'] = page
postage['images'] = images
return render_template('postage.html', **postage)
def webtable(self):
''' Do a query for Bootstrap Table interaction in Marvin web '''
form = processRequest(request=request)
args = av.manual_parse(self, request, use_params='query')
# remove args
__tmp__ = args.pop('release', None)
__tmp__ = args.pop('searchfilter', None)
limit = args.get('limit')
offset = args.get('offset')
if 'sort' in args:
current_session['query_sort'] = args['sort']
# set parameters
searchvalue = current_session.get('searchvalue', None)
returnparams = current_session.get('returnparams', None)
# exit if no searchvalue is found
if not searchvalue:
output = jsonify({'errmsg': 'No searchvalue found', 'status': -1})
return output
# this is to fix the brokeness with sorting on a table column using remote names
print('rp', returnparams, args)
# do query
try:
q, res = doQuery(searchfilter=searchvalue,
release=self._release,
returnparams=returnparams,
**args)
except Exception as e:
errmsg = 'Error generating webtable: {0}'.format(e)
output = jsonify({'status': -1, 'errmsg': errmsg})
return output
# get subset on a given page
try:
__results__ = res.getSubset(offset, limit=limit)
except Exception as e:
errmsg = 'Error getting table page: {0}'.format(e)
output = jsonify({'status': -1, 'errmsg': errmsg})
return output
else:
# get keys
cols = res.columns.remote
# create output
rows = res.getDictOf(format_type='listdict')
output = {
'total': res.totalcount,
'rows': rows,
'columns': cols,
'limit': limit,
'offset': offset
}
output = jsonify(output)
return output
def _run_query(searchfilter, **kwargs):
''' Run the query and return the query and results '''
release = kwargs.pop('release', None)
kwargs['returnparams'] = kwargs.pop('params', None)
kwargs['returntype'] = kwargs.pop('rettype', None)
try:
q, r = doQuery(searchfilter=searchfilter, release=release, **kwargs)
except Exception as e:
raise MarvinError('Query failed with {0}: {1}'.format(e.__class__.__name__, e))
else:
return q, r
def webtable(self):
''' Do a query for Bootstrap Table interaction in Marvin web '''
form = processRequest(request=request)
args = av.manual_parse(self, request, use_params='query')
#{'sort': u'cube.mangaid', 'task': None, 'end': None, 'searchfilter': None,
#'paramdisplay': None, 'start': None, 'rettype': None, 'limit': 10, 'offset': 30,
#'release': u'MPL-4', 'params': None, 'order': u'asc'}
# remove args
__tmp__ = args.pop('release', None)
__tmp__ = args.pop('searchfilter', None)
limit = args.get('limit')
offset = args.get('offset')
# set parameters
searchvalue = current_session.get('searchvalue', None)
returnparams = current_session.get('returnparams', None)
# limit = form.get('limit', 10, type=int)
# offset = form.get('offset', None, type=int)
# order = form.get('order', None, type=str)
# sort = form.get('sort', None, type=str)
# search = form.get('search', None, type=str)
# exit if no searchvalue is found
if not searchvalue:
output = jsonify({
'webtable_error': 'No searchvalue found',
'status': -1
})
return output
# do query
q, res = doQuery(searchfilter=searchvalue,
release=self._release,
returnparams=returnparams,
**args)
# q, res = doQuery(searchfilter=searchvalue, release=self._release,
# limit=limit, order=order, sort=sort, returnparams=returnparams)
# get subset on a given page
__results__ = res.getSubset(offset, limit=limit)
# get keys
cols = res.mapColumnsToParams()
# create output
rows = res.getDictOf(format_type='listdict')
output = {'total': res.totalcount, 'rows': rows, 'columns': cols}
output = jsonify(output)
return output
def _getCubes(searchfilter, **kwargs):
"""Run query locally at Utah."""
release = kwargs.pop('release', None)
kwargs['returnparams'] = kwargs.pop('params', None)
kwargs['returntype'] = kwargs.pop('rettype', None)
try:
# q, r = doQuery(searchfilter=searchfilter, returnparams=params, release=release,
# mode='local', returntype=rettype, limit=limit, order=order, sort=sort)
q, r = doQuery(searchfilter=searchfilter, release=release, **kwargs)
except Exception as e:
raise MarvinError('Query failed with {0}: {1}'.format(
e.__class__.__name__, e))
results = r.results
# get the subset keywords
start = kwargs.get('start', None)
end = kwargs.get('end', None)
limit = kwargs.get('limit', None)
params = kwargs.get('params', None)
# get a subset
chunk = None
if start:
chunk = int(end) - int(start)
results = r.getSubset(int(start), limit=chunk)
chunk = limit if not chunk else limit
runtime = {
'days': q.runtime.days,
'seconds': q.runtime.seconds,
'microseconds': q.runtime.microseconds
}
output = dict(data=results,
query=r.showQuery(),
chunk=limit,
filter=searchfilter,
params=q.params,
returnparams=params,
runtime=runtime,
queryparams_order=q.queryparams_order,
count=len(results),
totalcount=r.count)
return output
def test_success(self, release, mode):
q, r = doQuery(searchfilter='nsa.z < 0.1', release=release, mode=mode)
assert q is not None
assert r is not None
def index(self):
# Attempt to retrieve search parameters
form = processRequest(request=request)
self.search['formparams'] = form
self.search['latest_dr'] = self._release.lower() if 'DR' in self._release else config._get_latest_release(dr_only=True).lower()
# set the search form and form validation
searchform = self.mf.SearchForm(form)
searchform.returnparams.choices = [(k.lower(), k) for k in query_params.list_params()]
# Add the forms and parameters
self.search['paramdata'] = query_params
self.search['guideparams'] = [{'id': p.full, 'optgroup': group.name, 'type': 'double' if p.dtype == 'float' else p.dtype, 'validation': {'step': 'any'}} for group in query_params for p in group]
self.search['searchform'] = searchform
self.search['placeholder'] = getRandomQuery()
self.search['returnurl'] = 'https://sdss-marvin.readthedocs.io/en/stable/datamodel/{0}.html#{0}query'.format(
current_session['release'].lower().replace('-', ''))
# If form parameters then try to do a search
if form:
self.search.update({'results': None, 'errmsg': None})
args = av.manual_parse(self, request, use_params='search')
# get form parameters
searchvalue = form['searchbox'] # search filter input
returnparams = form.getlist('returnparams', type=str) # dropdown select
self.search.update({'returnparams': returnparams})
current_session.update({'searchvalue': searchvalue, 'returnparams': returnparams})
# if main form passes validation then do search
if searchform.validate():
# try the query
try:
q, res = doQuery(search_filter=searchvalue, release=self._release, return_params=returnparams)
except (MarvinError, KeyError) as e:
self.search['errmsg'] = 'Could not perform query: {0}'.format(refine_error(e))
except NotImplementedError as e: # DAP queries disabled
self.search['errmsg'] = 'Could not perform query: {0}'.format(e)
else:
self.search['filter'] = q.search_filter
self.search['count'] = res.totalcount
self.search['runtime'] = res.query_time.total_seconds()
if res.count > 0:
cols = res.columns.remote
rows = res.getDictOf(format_type='listdict')
output = {'total': res.totalcount, 'rows': rows, 'columns': cols, 'limit': None, 'offset': None}
else:
output = None
self.search['results'] = output
self.search['reslen'] = len(res.results)
if returnparams:
returnparams = [str(r) for r in returnparams]
rpstr = 'return_params={0} <br>'.format(returnparams) if returnparams else ''
qstr = ', return_params=returnparams' if returnparams else ''
self.search['querystring'] = ("<html><samp>from marvin.tools.query import Query<br>\
myfilter='{0}'<br> {1}\
q = Query(search_filter=myfilter{2})<br>\
r = q.run()<br></samp></html>".format(searchvalue, rpstr, qstr))
return render_template('search.html', **self.search)
def index(self):
# Attempt to retrieve search parameters
form = processRequest(request=request)
self.search['formparams'] = form
print('search form', form, type(form))
# set the search form and form validation
searchform = self.mf.SearchForm(form)
q = Query(release=self._release)
# allparams = q.get_available_params()
bestparams = q.get_best_params()
searchform.returnparams.choices = [(k.lower(), k) for k in bestparams]
searchform.parambox.validators = [
all_in(bestparams), validators.Optional()
]
# Add the forms
self.search['searchform'] = searchform
self.search['placeholder'] = getRandomQuery()
#from flask import abort
#abort(500)
# If form parameters then try to do a search
if form:
self.search.update({'results': None, 'errmsg': None})
args = av.manual_parse(self, request, use_params='search')
print('search args', args)
# get form parameters
searchvalue = form['searchbox'] # search filter input
returnparams = form.getlist('returnparams',
type=str) # dropdown select
parambox = form.get('parambox', None,
type=str) # from autocomplete
if parambox:
parms = parambox.split(',')
parms = parms if parms[-1].strip() else parms[:-1]
parambox = parms if parambox else None
# Select the one that is not none
returnparams = returnparams if returnparams and not parambox else \
parambox if parambox and not returnparams else \
list(set(returnparams) | set(parambox)) if returnparams and parambox else None
print('return params', returnparams)
current_session.update({
'searchvalue': searchvalue,
'returnparams': returnparams
})
# if main form passes validation then do search
if searchform.validate():
# try the query
try:
q, res = doQuery(searchfilter=searchvalue,
release=self._release,
returnparams=returnparams)
except MarvinError as e:
self.search[
'errmsg'] = 'Could not perform query: {0}'.format(e)
else:
self.search['filter'] = q.strfilter
self.search['count'] = res.totalcount
self.search['runtime'] = res.query_runtime.total_seconds()
if res.count > 0:
cols = res.mapColumnsToParams()
rows = res.getDictOf(format_type='listdict')
output = {
'total': res.totalcount,
'rows': rows,
'columns': cols
}
else:
output = None
self.search['results'] = output
self.search['reslen'] = len(res.results)
if returnparams:
returnparams = [str(r) for r in returnparams]
rpstr = 'returnparams={0} <br>'.format(
returnparams) if returnparams else ''
qstr = ', returnparams=returnparams' if returnparams else ''
self.search['querystring'] = (
"<html><samp>from marvin import \
config<br>from marvin.tools.query import Query<br>config.mode='remote'<br>\
filter='{0}'<br> {1}\
q = Query(searchfilter=filter{2})<br>\
r = q.run()<br></samp></html>".format(
searchvalue, rpstr, qstr))
return render_template('search.html', **self.search)
