def stream(self, *args, **kwargs): """ stream json output http://www.enricozini.org/2011/tips/python-stream-json/ """ cherrypy.response.headers["Content-Type"] = "application/json" uinput = kwargs.get('input', '') chunk_size = int(kwargs.get('chunk_size', 5000)) if chunk_size <= 0 or chunk_size > 500000: chunk_size = 5000 if not uinput: return json.dumps({'error':"empty input"}) manager = cherrypy.engine.qbm mquery = None try: mquery = manager.dbm.explain_query(uinput) except Error: traceback.print_exc() if mquery == None: mquery = manager.qbs.build_query(uinput) if mquery == None: self.log('invalid query', 2) return json.dumps({'error':"invalid query"}) t_list = [ keys.strip() for keys in \ uinput.split('where')[0].split('find')[1].split(',') ] cursor = cherrypy.engine.qbm.dbm.execute(mquery) return pack_stream(cursor, t_list, chunk_size)
def jsonresults(self, *args, **kwargs): """return json results""" cherrypy.response.headers["Content-Type"] = "application/json" uinput = kwargs.get('input', '') if not uinput: return json.dumps({'error':"empty input"}) manager = cherrypy.engine.qbm mquery = None try: mquery = manager.dbm.explain_query(uinput) if mquery == None: mquery = manager.qbs.build_query(uinput) if mquery == None: self.log('invalid query', 2) return json.dumps({'error':"invalid query"}) manager.dbm.set_query_explain(uinput, mquery) except Error: traceback.print_exc() total = self.get_cached_total(mquery) if total > 150000: qline = kwargs.get('input', '') streamapi = "/stream?input=" + qline raise cherrypy.InternalRedirect(streamapi) return self.get_json_data(uinput, mquery)
def pack_stream(cursor, t_list, chunk_size): """ chunk generator """ while True: results = [] if not cursor.closed:# not end rows = cursor.fetchmany(size=chunk_size) if not rows:# no record cursor.close() break for rec in rows:# records loads results.append(dict(izip(t_list, rec))) yield json.dumps(results) + '\n' else: break if not cursor.closed: cursor.close()
def pack_result(cursor, t_list): """ pack result in dictionary and dumps with json """ results = [] rapp = results.append while True: if not cursor.closed: rows = cursor.fetchmany(size=50) if not rows: cursor.close() break for rec in rows: rapp(dict(izip(t_list, rec))) else: break if not cursor.closed: cursor.close() return json.dumps(results)