def stream(self, *args, **kwargs):
"""
stream json output
http://www.enricozini.org/2011/tips/python-stream-json/
"""
cherrypy.response.headers["Content-Type"] = "application/json"
uinput = kwargs.get('input', '')
chunk_size = int(kwargs.get('chunk_size', 5000))
if chunk_size <= 0 or chunk_size > 500000:
chunk_size = 5000
if not uinput:
return json.dumps({'error':"empty input"})
manager = cherrypy.engine.qbm
mquery = None
try:
mquery = manager.dbm.explain_query(uinput)
except Error:
traceback.print_exc()
if mquery == None:
mquery = manager.qbs.build_query(uinput)
if mquery == None:
self.log('invalid query', 2)
return json.dumps({'error':"invalid query"})
t_list = [ keys.strip() for keys in \
uinput.split('where')[0].split('find')[1].split(',') ]
cursor = cherrypy.engine.qbm.dbm.execute(mquery)
return pack_stream(cursor, t_list, chunk_size)
def jsonresults(self, *args, **kwargs):
"""return json results"""
cherrypy.response.headers["Content-Type"] = "application/json"
uinput = kwargs.get('input', '')
if not uinput:
return json.dumps({'error':"empty input"})
manager = cherrypy.engine.qbm
mquery = None
try:
mquery = manager.dbm.explain_query(uinput)
if mquery == None:
mquery = manager.qbs.build_query(uinput)
if mquery == None:
self.log('invalid query', 2)
return json.dumps({'error':"invalid query"})
manager.dbm.set_query_explain(uinput, mquery)
except Error:
traceback.print_exc()
total = self.get_cached_total(mquery)
if total > 150000:
qline = kwargs.get('input', '')
streamapi = "/stream?input=" + qline
raise cherrypy.InternalRedirect(streamapi)
return self.get_json_data(uinput, mquery)
def pack_stream(cursor, t_list, chunk_size):
"""
chunk generator
"""
while True:
results = []
if not cursor.closed:# not end
rows = cursor.fetchmany(size=chunk_size)
if not rows:# no record
cursor.close()
break
for rec in rows:# records loads
results.append(dict(izip(t_list, rec)))
yield json.dumps(results) + '\n'
else:
break
if not cursor.closed:
cursor.close()
def pack_result(cursor, t_list):
"""
pack result in dictionary and dumps with json
"""
results = []
rapp = results.append
while True:
if not cursor.closed:
rows = cursor.fetchmany(size=50)
if not rows:
cursor.close()
break
for rec in rows:
rapp(dict(izip(t_list, rec)))
else:
break
if not cursor.closed:
cursor.close()
return json.dumps(results)
