def query_execute(userquery,options):
global saved_qpresults
query = "select * from earthquake"
if userquery != "":
query = userquery
print >> sys.stderr, "user query: ",query
sdbioptions = {'afl':False}
print >> sys.stderr, "saved_qpresults",saved_qpresults
if saved_qpresults == 0:
saved_qpresults = sdbi.verifyQuery(query,sdbioptions)
#only do this check for new queries
if options['reduce_res_check'] and (saved_qpresults['size'] > sdbi.D3_DATA_THRESHOLD):
return {'reduce_res':True}
if 'reduce_type' in options: # reduction requested
sdbioptions['reduce_res'] = True
srtoptions = {'afl':False}
if 'predicate' in options:
srtoptions['predicate'] = options['predicate']
sdbioptions['reduce_options'] = setup_reduce_type(options['reduce_type'],srtoptions)
else: #return original query
sdbioptions = {'afl':False,'reduce_res':False}
queryresultobj = sdbi.executeQuery(query,sdbioptions)
print >> sys.stderr, "retrieved data from db.\nparsing results..."
sdbioptions={'dimnames':saved_qpresults['dims']}
queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions)
if queryresultobj[1] != 0:
saved_qpresults = queryresultobj[1]
# get the new dim info
queryresultarr['dimnames'] = saved_qpresults['dims'];
queryresultarr['dimbases'] = saved_qpresults['dimbases'];
queryresultarr['dimwidths'] = saved_qpresults['dimwidths'];
print >> sys.stderr, "setting saved_qpresults to 0"
saved_qpresults = 0 # reset so we can use later
return queryresultarr # note that I don't set reduce_res false. JS code will still handle it properly
def process_timeseries():
query_text = request.args['query']
language = request.args['language']
callback = request.args['callback']
db_interface.connect()
query_result = db_interface.executeQuery(str(query_text), str(language),False,False)
if query_result is None: # error processing query
return make_response(json.dumps({
'request': request.json,
'status': 'ERROR',
'message': "error processing query",
}))
data_json = db_interface.getDataInJson(query_result)
db_interface.completeQuery(query_result.queryID)
db_interface.disconnect()
# Make response
json_message = {
'request': request.json,
'status': 'OK',
'data': data_json,
}
message = callback + "(" + json.dumps(json_message) + ")"
response = make_response(message)
return response
def query_execute(userquery,options):
user_id = options['user_id']
resolution = sdbi.D3_DATA_THRESHOLD
if 'resolution' in options and options['resolution'] > 0:
print "got resolution"
resolution = options['resolution']
db = sdbi.scidbOpenConn()
query = userquery
#print "user query: ",query
sdbioptions = {'afl':False,'db':db,'resolution':resolution}
saved_qpresults = None
if 'saved_qpresults' in options:
saved_qpresults = options['saved_qpresults']
#dumb_expert.prefetch(3,options['user_id'])
#tile = ti.getTileByID(2,sbdata.backend_metadata[options['user_id']]['levels'],options['user_id'])
#print "tile: ",tile
#sdbioptions={'dimnames':saved_qpresults['dims']}
#print sdbi.getAllAttrArrFromQueryForJSON(tile[0],sdbioptions)
#fetch_tile(0,1,{'user_id':user_id})
if saved_qpresults is None: # first time
#fetch_first_tile(query,{'user_id':user_id})
saved_qpresults = sdbi.verifyQuery(query,sdbioptions)
#with sbdata.metadata_lock:
# sbdata.backend_metadata[user_id] = {}
# sbdata.backend_metadata[user_id]['orig_query'] = query
# sbdata.backend_metadata[user_id]['saved_qpresults'] = saved_qpresults
# if 'data_threshold' in options:
# sbdata.backend_metadata[user_id]['data_threshold'] = options['data_threshold']
# else: # default
# sbdata.backend_metadata[user_id]['data_threshold'] = sdbi.D3_DATA_THRESHOLD
# sbdata.backend_metadata[user_id]['levels'] = sbdata.default_levels #leave at default levels for now
# #TODO: let # levels vary
#only do this check for new queries
if options['reduce_res_check'] and (saved_qpresults['size'] > resolution):#sdbi.D3_DATA_THRESHOLD):
return {'reduce_res':True,'saved_qpresults':saved_qpresults}
if 'reduce_type' in options: # reduction requested
sdbioptions['reduce_res'] = True
srtoptions = {'afl':False,'saved_qpresults':saved_qpresults}
if 'predicate' in options:
srtoptions['predicate'] = options['predicate']
sdbioptions['reduce_options'] = setup_reduce_type(options['reduce_type'],srtoptions)
else: #return original query
sdbioptions = {'afl':False,'reduce_res':False,'db':db}
queryresultobj = sdbi.executeQuery(query,sdbioptions)
print "queryresultobj:",queryresultobj
print "retrieved data from db.\nparsing results..."
sdbioptions={'dimnames':saved_qpresults['dims']}
queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions)
if queryresultobj[1] != 0:
saved_qpresults = queryresultobj[1]
# get the new dim info
queryresultarr['dimnames'] = saved_qpresults['dims']
queryresultarr['dimbases'] = saved_qpresults['dimbases']
queryresultarr['dimwidths'] = saved_qpresults['dimwidths']
queryresultarr['saved_qpresults'] = saved_qpresults
sdbi.scidbCloseConn(db)
return queryresultarr # note that I don't set reduce_res false. JS code will still handle it properly
def process_query_image():
if request.args['iterative']:
query_text = request.args['query'].replace("@","@"+str(request.args['iteration']))
title = "Iteration " + str(request.args['iteration'])
else:
query_text = str(request.args['query'])
title = query_text
iterative = request.args['iterative']
z_scale = request.args['z_scale']
language = str(request.args['language'])
callback = request.args['callback']
query_lines = query_text.split("\n")
db_interface.connect()
# process all queries except for the last one
for query_line in query_lines[:-1]: # process all lines except the last one
db_interface.executeQuery(str(query_line), str(language))
# process last query
query_result = db_interface.executeQuery(str(query_lines[-1]), str(language))
if query_result is None: # error processing query
return make_response(json.dumps({
'request': request.json,
'status': 'ERROR',
'message': "error processing query",
}))
data_array, array_dim = db_interface.getFirstAttrArrFromQueryInNumPY(query_result)
db_interface.disconnect()
#print >> sys.stdout, response_json
result = SciDB.SciDB_Result(data_array)
# Get the image array
if not iterative:
img_array = result.get_data_array()
elif request.args['iteration'] == 1:
img_array = result.get_data_array()
else:
master = memcache.get(session.pop('previous_data_key', None), None)
img_array = result.get_img_array(master)
# Get z min and max
if z_scale == True:
z_min, z_max = zscale.zscale(result.data_array, nsamples=2000, contrast=0.25)
else:
z_min, z_max = (array_dim[2][0], array_dim[2][1]) #TODO fix this, currently zmin and zmax are unknown
# Make figure
output = scalrr_vis.plot_image(img_array, z_min, z_max)
encoded_image = base64.b64encode(output.getvalue())
# Save session data
memcache[query_text] = result.get_data_array()
session['previous_data_key'] = query_text
# Make response
json_message = {
'request': request.args,
'status': 'OK',
'image': encoded_image,
#"plot": {"title": title, "x_axis": result.x_name, "y_axis": result.y_name, "z_axis": result.z_name},
"dimensions": {
"x":{"min": array_dim[0][0], "max": array_dim[0][1]},
"y":{"min": array_dim[1][0], "max": array_dim[1][1]},
"z":{"min": array_dim[2][0], "max": array_dim[2][1]}
}
}
message = callback + "(" + json.dumps(json_message) + ")"
response = make_response(message)
#print >> sys.stdout, json_message
return response
def process_query_fits():
print "@/process/query/fits/"
query_text = str(request.args['query'])
iterative = request.args['iterative']
iteration = 1
if iterative == "true":
iteration = request.args['iteration']
language = str(request.args['language'])
callback = request.args['callback']
print "iteration:",iteration
#if iterative:
# query_text = request.args['query'].replace("@","@"+str(request.args['iteration']))
query_lines = query_text.split("\n")
try:
db_interface.connect()
except:
message = json.dumps({
'request': request.json,
'status': 'ERROR',
'message': "unable to connect to SciDB",
})
return make_response(callback + "(" + message + ")")
#process the first query
#find the subarray part of the query
queryExecutionTime = 0
query_line_aligned = db_interface.alignQuery(str(query_lines[0]))
if query_line_aligned['done'] == True:
print "first line query:",query_lines[0]
result_one = db_interface.executeQuery(str(query_lines[0]), str(language))
queryExecutionTime += int(result_one.executionTime)
print "time:",queryExecutionTime
# process from the second queriy until the last one
for index in range(0,int(iteration)):
for query_line in query_lines[1:-1]: # process all lines except the last one
print "iterative query:",query_line
result_one = db_interface.executeQuery(str(query_line), str(language))
queryExecutionTime += int(result_one.executionTime)
print "time:",queryExecutionTime
# process last query
print "last query:",query_lines[-1]
query_result = db_interface.executeQuery(str(query_lines[-1]), str(language),True,False)
if query_result is None: # error processing query
message = json.dumps({
'request': request.json,
'status': 'ERROR',
'message': "error processing query",
})
return make_response(callback + "(" + message + ")")
queryExecutionTime += int(query_result.executionTime)
if query_line_aligned['done'] == True:
data_array, array_dim = db_interface.getFirstAttrArrFromQueryInNumPY(query_result,[query_line_aligned['old_row'],query_line_aligned['old_col']],True)
else:
data_array, array_dim = db_interface.getFirstAttrArrFromQueryInNumPY(query_result,[0,0],False)
db_interface.disconnect()
filename = uuid.uuid4().hex + ".fits"
wcs = pywcs.WCS(naxis=2)
try:
sub_arr = query_text.split("subarray")[1].split(",")
wcs.wcs.crpix = [440250-(int(sub_arr[3])-259750), 353595-(int(sub_arr[2])-247750)]
#wcs.wcs.crpix = [440250-(int(sub_arr[3])-260250), 353595-(int(sub_arr[2])-248250)]
except:
sub_arr = query_text.split("between")[1].split(",")
wcs.wcs.crpix = [440250-(int(sub_arr[3])-259750), 353595-(int(sub_arr[2])-247750)]
wcs.wcs.cd = numpy.array([-.0000555555555, 0, 0, .000055555555]).reshape([2,2])
wcs.wcs.crval = [60, 10.8123169635717]
wcs.wcs.ctype = ["RA---STG", "DEC--STG"]
header = wcs.to_header()
try:
sub_arr = query_text.split("subarray")[1].split(",")
header.update("AbsPixX", int(sub_arr[2]))
header.update("AbsPixY", int(sub_arr[3]))
header.update("AbsPixZ", int(sub_arr[1]))
except:
sub_arr = query_text.split("between")[1].split(",")
header.update("AbsPixX", int(sub_arr[2]))
header.update("AbsPixY", int(sub_arr[3]))
header.update("AbsPixZ", int(sub_arr[1]))
header.update("CD1_2", 0)
header.update("CD2_1", 0)
header.update("CD1_1", -0.0000555555555)
header.update("CD2_2", 0.0000555555555)
hdu = pyfits.PrimaryHDU(data_array, header=header)
hdu.writeto(filename)
#encoded_fits = base64.b64encode(output.getvalue())
print "time:",queryExecutionTime
# Make response
json_message = {
'request': request.args,
'status': 'OK',
'time': queryExecutionTime,
'fits_file': filename
}
message = callback + "(" + json.dumps(json_message) + ")"
response = make_response(message)
#print >> sys.stdout, json_message
return response
