def _query_versions(self, mode='local', release=None):
release = config.release
drpver, dapver = config.lookUpVersions(release=release)
p = 'haflux > 25'
if '4' in release:
name = 'CleanSpaxelProp'
else:
name = 'CleanSpaxelProp{0}'.format(release.split('-')[1])
if release:
q = Query(searchfilter=p, mode=mode, release=release)
else:
q = Query(searchfilter=p, mode=mode)
self.assertEqual(q._drpver, drpver)
self.assertEqual(q._dapver, dapver)
self.assertEqual(q._release, release)
self.assertEqual(q.marvinform._release, release)
self.assertEqual(q.marvinform._param_form_lookup._release, release)
self.assertEqual(
q.marvinform._param_form_lookup['spaxelprop.file'].Meta.model.
__name__, name)
def cleanup(self, args):
''' Clean up idle server-side queries or retrieve the list of them
Do not use!
.. :quickref: Query; Send a cleanup command to the server-side database
:query string release: the release of MaNGA
:form task: ``clean`` or ``getprocs``, the type of task to run
:resjson int status: status of response. 1 if good, -1 if bad.
:resjson string error: error message, null if None
:resjson json inconfig: json of incoming configuration
:resjson json utahconfig: json of outcoming configuration
:resjson string traceback: traceback of an error, null if None
:resjson string data: dictionary of returned data
:json string clean: clean success message
:json list procs: the list of processes currently running on the db
:resheader Content-Type: application/json
:statuscode 200: no error
:statuscode 422: invalid input parameters
**Example request**:
.. sourcecode:: http
GET /marvin2/api/query/cleanup/ HTTP/1.1
Host: api.sdss.org
Accept: application/json, */*
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"status": 1,
"error": null,
"inconfig": {"release": "MPL-5"},
"utahconfig": {"release": "MPL-5", "mode": "local"},
"traceback": null,
"data": 'clean success'
}
'''
task = args.pop('task', None)
if task == 'clean':
q = Query(mode='local')
q._cleanUpQueries()
res = {'status': 1, 'data': 'clean success'}
elif task == 'getprocs':
q = Query(mode='local')
procs = q._getIdleProcesses()
procs = [{k: v for k, v in y.items()} for y in procs]
res = {'status': 1, 'data': procs}
else:
res = {'status': -1, 'data': None, 'error': 'Task is None or not in [clean, getprocs]'}
self.update_results(res)
output = jsonify(self.results)
return output
def test_badreturntype(self, query, objtype, errmsg):
with pytest.raises(AssertionError) as cm:
query = Query(searchfilter=query.searchfilter,
returntype=objtype,
mode=query.mode)
assert cm.type == AssertionError
assert errmsg in str(cm.value)
def test_return_params(self, rps):
base = ['cube.ra', 'cube.dec']
query = Query(search_filter=self.sf,
mode=self.mode,
return_params=base + rps)
params = query._remote_params['returnparams'].split(',')
assert set(rps).issubset(set(params))
def stream(self, args):
''' test query generator stream '''
searchfilter = args.pop('searchfilter', None)
compression = args.pop('compression', config.compression)
mimetype = 'json' if compression == 'json' else 'octet-stream'
release = args.pop('release', None)
args['return_params'] = args.pop('returnparams', None)
args['return_type'] = args.pop('rettype', None)
q = Query(search_filter=searchfilter, release=release, **args)
output = dict(data=None,
chunk=q.limit,
query=q.show(),
filter=searchfilter,
params=q.params,
returnparams=q.return_params,
runtime=None,
queryparams_order=q._query_params_order,
count=None,
totalcount=None)
return Response(stream_with_context(
gen(q.query, compression=compression, params=q.params)),
mimetype='application/{0}'.format(mimetype))
def test_bad_input(self, badinput):
data = 'nsa.z < 0.1' if badinput == 'searchfilter' else ['cube.plate']
kwargs = {badinput: data}
with pytest.raises(TypeError) as cm:
q = Query(**kwargs)
assert "__init__() got an unexpected keyword argument '{0}'".format(
badinput) in str(cm.value)
def test_bad_returnparams(self, rps, errmsg):
with pytest.raises(KeyError) as cm:
query = Query(search_filter=self.sf,
return_params=[rps],
mode=self.mode)
assert cm.type == KeyError
assert errmsg in str(cm.value)
def test_read_best(self):
q = Query(mode='local')
bestkeys = q._read_best_params()
self.assertEqual(type(bestkeys), list)
exp = ['cube.ra', 'cube.dec', 'cube.plate', 'nsa.z', 'spaxelprop.x']
for e in exp:
self.assertIn(e, bestkeys)
def test_dap_query_abs_g_r_remote(self):
# This uses MPL-4 (it will return 4 results if MPL-5)
self._set_remote()
p = 'abs_g_r>-1'
q = Query(searchfilter=p, mode=self.mode)
r = q.run()
self.assertEqual(1389, r.totalcount)
def test_Query_remote_mpl4(self):
self._set_remote()
p = 'nsa.z < 0.12 and ifu.name = 19*'
q = Query(searchfilter=p, mode='remote')
r = q.run()
self.assertEqual([], q.joins)
self.assertEqual(163, r.totalcount) # MPL-4 count
def get_query_params(release, paramdisplay):
q = Query(mode='local', release=release)
if paramdisplay == 'best':
qparams = bestparams
else:
qparams = q.get_available_params('all')
return qparams
def setUp(self):
self._reset_the_config()
self.set_sasurl('local')
self.mode = self.init_mode
config.setMPL('MPL-5')
config.forceDbOn()
self.filter = 'nsa.z < 0.1 and cube.plate==8485'
self.columns = [
'cube.mangaid', 'cube.plate', 'cube.plateifu', 'ifu.name', 'nsa.z'
]
self.remotecols = [u'mangaid', u'plate', u'plateifu', u'name', u'z']
self.coltoparam = OrderedDict([('mangaid', 'cube.mangaid'),
('plate', 'cube.plate'),
('plateifu', 'cube.plateifu'),
('name', 'ifu.name'), ('z', 'nsa.z')])
self.paramtocol = OrderedDict([('cube.mangaid', 'mangaid'),
('cube.plate', 'plate'),
('cube.plateifu', 'plateifu'),
('ifu.name', 'name'), ('nsa.z', 'z')])
self.res = (u'1-209232', 8485, u'8485-1901', u'1901', 0.0407447)
self.resdict = {
'1': (u'1-43148', 8135, u'8135-6101', u'6101', 0.0108501),
'10': (u'1-167079', 8459, u'8459-1901', u'1901', 0.015711),
'11': (u'1-167075', 8459, u'8459-12704', u'12704', 0.0158584),
'21': (u'1-113567', 7815, u'7815-12701', u'12701', 0.0167432),
'31': (u'1-322048', 8552, u'8552-12705', u'12705', 0.0172298),
'36': (u'1-252151', 8335, u'8335-9102', u'9102', 0.0174864),
'41': (u'1-378182', 8134, u'8134-12705', u'12705', 0.0178659),
'46': (u'1-252126', 8335, u'8335-3703', u'3703', 0.0181555)
}
self.q = Query(searchfilter=self.filter, mode=self.mode)
def _fail_query(self, errmsg, tracemsg, *args, **kwargs):
with self.assertRaises(MarvinError) as cm:
q = Query(**kwargs)
r = q.run()
self.assertIn(errmsg, str(cm.exception))
self.assertIsNotNone(config._traceback)
self.assertIsNotNone(bconfig.traceback)
self.assertIn(tracemsg, config._traceback)
def test_Query_only_drpver(self):
p = 'cube.plate==8485 and spaxel.x > 5'
q = Query(searchfilter=p)
r = q.run()
self.assertEqual(self.plate, r.getListOf('cube.plate')[0])
self.assertEqual(self.mangaid, r.getListOf('cube.mangaid')[0])
self.assertIn('drpalias', str(q.query.whereclause))
self.assertNotIn('dapalias', str(q.query.whereclause))
def test_success(self, query, rps):
query = Query(searchfilter=query.searchfilter,
returnparams=rps,
mode=query.mode)
assert 'nsa.z' in query.params
#assert set(rps).issubset(set(query.params))
res = query.run()
assert all([p in res.columns for p in rps]) is True
def test_quality_queries(self, sf, qual, flags, count):
q = Query(search_filter=sf, quality=qual, release='MPL-5')
sql = q.show()
for flag in flags.split(','):
assert flag.strip() in sql
r = q.run()
assert r.results is not None
assert r.totalcount == count
def test_checkonly(self, sf, rps, schema, tables, check):
sf = sf or self.sf
query = Query(search_filter=sf, return_params=rps, mode=self.mode)
isin = query._check_for(query.params,
schema=schema,
tables=tables,
only=True)
assert isin == check
def test_target_queries(self, sf, targ, flags, count):
q = Query(search_filter=sf, targets=targ, release='MPL-4')
sql = q.show()
for flag in flags.split(','):
ff = flag.strip()
assert (ff in sql or ff.replace('%', '%%') in sql)
r = q.run()
assert r.results is not None
assert r.totalcount == count
def test_sort(self, query, sortparam, order):
data = query.expdata['queries']['nsa.z < 0.1']['sorted']
query = Query(search_filter=query.search_filter, mode=query.mode, sort=sortparam, order=order)
res = query.run()
if order == 'asc':
redshift = data['1'][-1]
else:
redshift = data['last'][-1]
assert res.results['z'][0] == redshift
def test_skipdefault(self, query, rps):
query = Query(searchfilter=query.searchfilter,
returnparams=rps,
mode=query.mode)
assert len(query._returnparams) == len(rps)
assert len(query.params) == 6
res = query.run()
assert len(res.returnparams) == len(rps)
assert len(res.columns) == 6
def test_bad_queries(self, expmode, badquery, errmsg):
if expmode is None:
pytest.skip('cannot use queries in local mode without a db')
with pytest.raises(MarvinError) as cm:
query = Query(searchfilter=badquery, mode=expmode)
res = query.run()
assert cm.type == MarvinError
assert errmsg in str(cm.value)
def test_return_params(self, rps):
base = ['cube.ra', 'cube.dec']
query = Query(search_filter=self.sf,
mode=self.mode,
return_params=base + rps)
reals = [
query._marvinform._param_form_lookup.get_real_name(r) for r in rps
]
assert set(reals).issubset(set(query.params))
assert set(reals).issubset(query.return_params)
def test_results_returntype_cube_limit(self):
config.setMPL('MPL-5')
p = 'haflux > 25'
q = Query(searchfilter=p)
r = q.run()
r.convertToTool('cube', limit=5)
self.assertIsNotNone(r.objects)
self.assertEqual(18, r.count)
self.assertEqual(5, len(r.objects))
self.assertEqual(True, isinstance(r.objects[0], Cube))
def fxnr(mode):
if mode == 'remote':
config.forceDbOff()
q = Query(search_filter='nsa.z < 0.1', release='MPL-4', mode=mode)
r = q.run()
yield r
q = None
r = None
if mode == 'remote':
config.forceDbOn()
def test_get_success(self, query, objtype, tool):
if query.mode == 'remote' and config.db is None:
pytest.skip('skipping weird case where nodb, remote mode tried to load a local file')
if config.release == 'MPL-4' and objtype == 'modelcube':
pytest.skip('no modelcubes in mpl-4')
query = Query(search_filter=query.search_filter, return_type=objtype, mode=query.mode, release=query.release)
res = query.run()
assert res.objects is not None
assert len(res.results) == len(res.objects)
assert isinstance(res.objects[0], tool) is True
def test_bad_queries(self, expmode, badquery, errmsg):
if expmode is None:
pytest.skip('cannot use queries in local mode without a db')
set_the_config(config.release)
with pytest.raises((KeyError, MarvinError)) as cm:
query = Query(search_filter=badquery, mode=expmode)
res = query.run()
assert cm.type == KeyError or cm.type == MarvinError
assert errmsg in str(cm.value)
def test_Query_drpver_and_dapver(self):
p = 'cube.plate==8485 and emline_gflux_ha_6564>25'
q = Query(searchfilter=p)
r = q.run()
tmp = r.sort('emline_gflux_ha_6564')
self.assertEqual(self.plate, r.getListOf('cube.plate')[0])
self.assertEqual(self.mangaid, r.getListOf('cube.mangaid')[0])
self.assertEqual(26.112, r.getListOf('emline_gflux_ha_6564')[0])
self.assertGreaterEqual(r.count, 6)
self.assertIn('drpalias', str(q.query.whereclause))
self.assertIn('dapalias', str(q.query.whereclause))
def results(query, request):
searchfilter = request.param if hasattr(
request, 'param') else 'nsa.z < 0.1 and cube.plate==8485'
q = Query(searchfilter=searchfilter,
mode=query.mode,
limit=10,
release=query._release)
r = q.run()
r.expdata = query.expdata
yield r
r = None
def query(request, monkeyauth, release, mode, db):
''' Yields a Query that loops over all modes and db options '''
data = query_data[release]
set_the_config(release)
if mode == 'local' and not db:
pytest.skip('cannot use queries in local mode without a db')
searchfilter = request.param if hasattr(request, 'param') else None
q = Query(searchfilter=searchfilter, mode=mode, release=release)
q.expdata = data
yield q
config.forceDbOn()
q = None
def test_get_list_all(self, results):
q = Query(search_filter=results.search_filter,
mode=results.mode,
limit=1,
release=results.release,
return_params=results.return_params)
r = q.run(start=0, end=1)
assert r.count == 1
mangaids = r.getListOf('mangaid', return_all=True)
assert len(mangaids) == r.totalcount
assert len(mangaids) == results.expdata['queries'][
results.search_filter]['count']
