def stream(self, args): ''' test query generator stream ''' searchfilter = args.pop('searchfilter', None) compression = args.pop('compression', config.compression) mimetype = 'json' if compression == 'json' else 'octet-stream' release = args.pop('release', None) args['return_params'] = args.pop('returnparams', None) args['return_type'] = args.pop('rettype', None) q = Query(search_filter=searchfilter, release=release, **args) output = dict(data=None, chunk=q.limit, query=q.show(), filter=searchfilter, params=q.params, returnparams=q.return_params, runtime=None, queryparams_order=q._query_params_order, count=None, totalcount=None) return Response(stream_with_context( gen(q.query, compression=compression, params=q.params)), mimetype='application/{0}'.format(mimetype))
def get_query_params(release, paramdisplay): q = Query(mode='local', release=release) if paramdisplay == 'best': qparams = bestparams else: qparams = q.get_available_params('all') return qparams
def test_read_best(self): q = Query(mode='local') bestkeys = q._read_best_params() self.assertEqual(type(bestkeys), list) exp = ['cube.ra', 'cube.dec', 'cube.plate', 'nsa.z', 'spaxelprop.x'] for e in exp: self.assertIn(e, bestkeys)
def test_dap_query_abs_g_r_remote(self): # This uses MPL-4 (it will return 4 results if MPL-5) self._set_remote() p = 'abs_g_r>-1' q = Query(searchfilter=p, mode=self.mode) r = q.run() self.assertEqual(1389, r.totalcount)
def _query_versions(self, mode='local', release=None): release = config.release drpver, dapver = config.lookUpVersions(release=release) p = 'haflux > 25' if '4' in release: name = 'CleanSpaxelProp' else: name = 'CleanSpaxelProp{0}'.format(release.split('-')[1]) if release: q = Query(searchfilter=p, mode=mode, release=release) else: q = Query(searchfilter=p, mode=mode) self.assertEqual(q._drpver, drpver) self.assertEqual(q._dapver, dapver) self.assertEqual(q._release, release) self.assertEqual(q.marvinform._release, release) self.assertEqual(q.marvinform._param_form_lookup._release, release) self.assertEqual( q.marvinform._param_form_lookup['spaxelprop.file'].Meta.model. __name__, name)
def test_Query_remote_mpl4(self): self._set_remote() p = 'nsa.z < 0.12 and ifu.name = 19*' q = Query(searchfilter=p, mode='remote') r = q.run() self.assertEqual([], q.joins) self.assertEqual(163, r.totalcount) # MPL-4 count
def setUp(self): self._reset_the_config() self.set_sasurl('local') self.mode = self.init_mode config.setMPL('MPL-5') config.forceDbOn() self.filter = 'nsa.z < 0.1 and cube.plate==8485' self.columns = [ 'cube.mangaid', 'cube.plate', 'cube.plateifu', 'ifu.name', 'nsa.z' ] self.remotecols = [u'mangaid', u'plate', u'plateifu', u'name', u'z'] self.coltoparam = OrderedDict([('mangaid', 'cube.mangaid'), ('plate', 'cube.plate'), ('plateifu', 'cube.plateifu'), ('name', 'ifu.name'), ('z', 'nsa.z')]) self.paramtocol = OrderedDict([('cube.mangaid', 'mangaid'), ('cube.plate', 'plate'), ('cube.plateifu', 'plateifu'), ('ifu.name', 'name'), ('nsa.z', 'z')]) self.res = (u'1-209232', 8485, u'8485-1901', u'1901', 0.0407447) self.resdict = { '1': (u'1-43148', 8135, u'8135-6101', u'6101', 0.0108501), '10': (u'1-167079', 8459, u'8459-1901', u'1901', 0.015711), '11': (u'1-167075', 8459, u'8459-12704', u'12704', 0.0158584), '21': (u'1-113567', 7815, u'7815-12701', u'12701', 0.0167432), '31': (u'1-322048', 8552, u'8552-12705', u'12705', 0.0172298), '36': (u'1-252151', 8335, u'8335-9102', u'9102', 0.0174864), '41': (u'1-378182', 8134, u'8134-12705', u'12705', 0.0178659), '46': (u'1-252126', 8335, u'8335-3703', u'3703', 0.0181555) } self.q = Query(searchfilter=self.filter, mode=self.mode)
def test_success(self, query, rps): query = Query(searchfilter=query.searchfilter, returnparams=rps, mode=query.mode) assert 'nsa.z' in query.params #assert set(rps).issubset(set(query.params)) res = query.run() assert all([p in res.columns for p in rps]) is True
def test_checkonly(self, sf, rps, schema, tables, check): sf = sf or self.sf query = Query(search_filter=sf, return_params=rps, mode=self.mode) isin = query._check_for(query.params, schema=schema, tables=tables, only=True) assert isin == check
def test_quality_queries(self, sf, qual, flags, count): q = Query(search_filter=sf, quality=qual, release='MPL-5') sql = q.show() for flag in flags.split(','): assert flag.strip() in sql r = q.run() assert r.results is not None assert r.totalcount == count
def _fail_query(self, errmsg, tracemsg, *args, **kwargs): with self.assertRaises(MarvinError) as cm: q = Query(**kwargs) r = q.run() self.assertIn(errmsg, str(cm.exception)) self.assertIsNotNone(config._traceback) self.assertIsNotNone(bconfig.traceback) self.assertIn(tracemsg, config._traceback)
def test_Query_only_drpver(self): p = 'cube.plate==8485 and spaxel.x > 5' q = Query(searchfilter=p) r = q.run() self.assertEqual(self.plate, r.getListOf('cube.plate')[0]) self.assertEqual(self.mangaid, r.getListOf('cube.mangaid')[0]) self.assertIn('drpalias', str(q.query.whereclause)) self.assertNotIn('dapalias', str(q.query.whereclause))
def test_bad_queries(self, expmode, badquery, errmsg): if expmode is None: pytest.skip('cannot use queries in local mode without a db') with pytest.raises(MarvinError) as cm: query = Query(searchfilter=badquery, mode=expmode) res = query.run() assert cm.type == MarvinError assert errmsg in str(cm.value)
def test_skipdefault(self, query, rps): query = Query(searchfilter=query.searchfilter, returnparams=rps, mode=query.mode) assert len(query._returnparams) == len(rps) assert len(query.params) == 6 res = query.run() assert len(res.returnparams) == len(rps) assert len(res.columns) == 6
def test_sort(self, query, sortparam, order): data = query.expdata['queries']['nsa.z < 0.1']['sorted'] query = Query(search_filter=query.search_filter, mode=query.mode, sort=sortparam, order=order) res = query.run() if order == 'asc': redshift = data['1'][-1] else: redshift = data['last'][-1] assert res.results['z'][0] == redshift
def test_target_queries(self, sf, targ, flags, count): q = Query(search_filter=sf, targets=targ, release='MPL-4') sql = q.show() for flag in flags.split(','): ff = flag.strip() assert (ff in sql or ff.replace('%', '%%') in sql) r = q.run() assert r.results is not None assert r.totalcount == count
def test_results_returntype_cube_limit(self): config.setMPL('MPL-5') p = 'haflux > 25' q = Query(searchfilter=p) r = q.run() r.convertToTool('cube', limit=5) self.assertIsNotNone(r.objects) self.assertEqual(18, r.count) self.assertEqual(5, len(r.objects)) self.assertEqual(True, isinstance(r.objects[0], Cube))
def fxnr(mode): if mode == 'remote': config.forceDbOff() q = Query(search_filter='nsa.z < 0.1', release='MPL-4', mode=mode) r = q.run() yield r q = None r = None if mode == 'remote': config.forceDbOn()
def results(query, request): searchfilter = request.param if hasattr( request, 'param') else 'nsa.z < 0.1 and cube.plate==8485' q = Query(searchfilter=searchfilter, mode=query.mode, limit=10, release=query._release) r = q.run() r.expdata = query.expdata yield r r = None
def test_bad_queries(self, expmode, badquery, errmsg): if expmode is None: pytest.skip('cannot use queries in local mode without a db') set_the_config(config.release) with pytest.raises((KeyError, MarvinError)) as cm: query = Query(search_filter=badquery, mode=expmode) res = query.run() assert cm.type == KeyError or cm.type == MarvinError assert errmsg in str(cm.value)
def test_get_success(self, query, objtype, tool): if query.mode == 'remote' and config.db is None: pytest.skip('skipping weird case where nodb, remote mode tried to load a local file') if config.release == 'MPL-4' and objtype == 'modelcube': pytest.skip('no modelcubes in mpl-4') query = Query(search_filter=query.search_filter, return_type=objtype, mode=query.mode, release=query.release) res = query.run() assert res.objects is not None assert len(res.results) == len(res.objects) assert isinstance(res.objects[0], tool) is True
def test_Query_drpver_and_dapver(self): p = 'cube.plate==8485 and emline_gflux_ha_6564>25' q = Query(searchfilter=p) r = q.run() tmp = r.sort('emline_gflux_ha_6564') self.assertEqual(self.plate, r.getListOf('cube.plate')[0]) self.assertEqual(self.mangaid, r.getListOf('cube.mangaid')[0]) self.assertEqual(26.112, r.getListOf('emline_gflux_ha_6564')[0]) self.assertGreaterEqual(r.count, 6) self.assertIn('drpalias', str(q.query.whereclause)) self.assertIn('dapalias', str(q.query.whereclause))
def _set_modes(self, expmode=None): p = 'nsa.z < 0.1 and cube.plate == 8485' q = Query(searchfilter=p, mode=self.mode) r = q.run() # this part selects 8485-1901 r.sort('z') r.results = r.results[-4:] r.convertToTool('cube', limit=1) self.assertEqual(expmode, q.mode) self.assertEqual(expmode, r.mode) self.assertEqual(q.mode, r.mode) self.assertEqual(expmode, r.objects[0].mode)
def query(request, monkeyauth, release, mode, db): ''' Yields a Query that loops over all modes and db options ''' data = query_data[release] set_the_config(release) if mode == 'local' and not db: pytest.skip('cannot use queries in local mode without a db') searchfilter = request.param if hasattr(request, 'param') else None q = Query(searchfilter=searchfilter, mode=mode, release=release) q.expdata = data yield q config.forceDbOn() q = None
def test_get_list_all(self, results): q = Query(search_filter=results.search_filter, mode=results.mode, limit=1, release=results.release, return_params=results.return_params) r = q.run(start=0, end=1) assert r.count == 1 mangaids = r.getListOf('mangaid', return_all=True) assert len(mangaids) == r.totalcount assert len(mangaids) == results.expdata['queries'][ results.search_filter]['count']
def getparamslist(self, args): ''' Retrieve a list of all available input parameters into the query .. :quickref: Query; Get a list of all or "best" queryable parameters :query string release: the release of MaNGA :form paramdisplay: ``all`` or ``best``, type of parameters to return :resjson int status: status of response. 1 if good, -1 if bad. :resjson string error: error message, null if None :resjson json inconfig: json of incoming configuration :resjson json utahconfig: json of outcoming configuration :resjson string traceback: traceback of an error, null if None :resjson string data: dictionary of returned data :json list params: the list of queryable parameters :resheader Content-Type: application/json :statuscode 200: no error :statuscode 422: invalid input parameters **Example request**: .. sourcecode:: http GET /marvin2/api/query/getparamslist/ HTTP/1.1 Host: api.sdss.org Accept: application/json, */* **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "status": 1, "error": null, "inconfig": {"release": "MPL-5"}, "utahconfig": {"release": "MPL-5", "mode": "local"}, "traceback": null, "data": ['nsa.z', 'cube.ra', 'cube.dec', ...] } ''' paramdisplay = args.pop('paramdisplay', 'all') q = Query(mode='local', release=args['release']) if paramdisplay == 'all': params = q.get_available_params('all') elif paramdisplay == 'best': params = bestparams self.results['data'] = params self.results['status'] = 1 output = jsonify(self.results) return output
def _pickle_query(self, mode=None, name=None): p = 'nsa.z < 0.1' q = Query(searchfilter=p, mode=mode) path = q.save(name, overwrite=True) self._files_created.append(path) self.assertTrue(os.path.exists(path)) q = None self.assertIsNone(q) q = Query.restore(path) self.assertEqual('nsa.z < 0.1', q.searchfilter) self.assertEqual('remote', q.mode)
def test_badparams(self, query, expmode, rps, errmsg): # set error type based on query mode if expmode == 'remote': error = MarvinError else: error = KeyError with pytest.raises(error) as cm: query = Query(searchfilter=query.searchfilter, returnparams=[rps], mode=query.mode) res = query.run() assert cm.type == error assert errmsg in str(cm.value)
def query(request, allow_dap, monkeyauth, release, mode, db): ''' Yields a Query that loops over all modes and db options ''' data = query_data[release] set_the_config(release) if mode == 'local' and not db: pytest.skip('cannot use queries in local mode without a db') searchfilter = request.param if hasattr(request, 'param') else None q = Query(search_filter=searchfilter, mode=mode, release=release) q.expdata = data if q.mode == 'remote': pytest.xfail('cannot control for DAP spaxel queries on server side; failing all remotes until then') yield q config.forceDbOn() q = None
def results(query, request): searchfilter = request.param if hasattr(request, 'param') else 'nsa.z < 0.1' q = Query(search_filter=searchfilter, mode=query.mode, limit=10, release=query.release) if q.mode == 'remote': pytest.xfail( 'cannot control for DAP spaxel queries on server side; failing all remotes until then' ) r = q.run() r.expdata = query.expdata yield r r = None