def fetch_first_tile2(userquery,options): db = sdbi.scidbOpenConn() global experts query = userquery sdbioptions = {'afl':False,'db':db} saved_qpresults = sdbi.verifyQuery(query,sdbioptions) sdbi.scidbCloseConn(db) print "saved qp results, looking for error" if 'error' in saved_qpresults: print "error found, returning error:",saved_qpresults return saved_qpresults else: print "saved_qpresults:",saved_qpresults user_id = options['user_id'] # setup metadata print "setting up metadata" with sbdata.metadata_lock: sbdata.backend_metadata[user_id] = {} sbdata.backend_metadata[user_id]['orig_query'] = query sbdata.backend_metadata[user_id]['saved_qpresults'] = saved_qpresults if 'data_threshold' in options: sbdata.backend_metadata[user_id]['data_threshold'] = options['data_threshold'] else: # default is whatever is prescribed in scidb server interface code sbdata.backend_metadata[user_id]['data_threshold'] = sdbi.D3_DATA_THRESHOLD sbdata.backend_metadata[user_id]['levels'] = 0 #sbdata.default_levels #leave at default levels for now #TODO: let # levels vary #get tile base_id = [0] * saved_qpresults['numdims'] print "base id:",base_id if 'usenumpy' in options: usenumpy = options['usenumpy'] else: usenumpy = False tile = ti.getTileByIDN(base_id,0,user_id,usenumpy) if 'error' in tile: return tile #save tile info tile_key = str(base_id) with sbdata.user_history_lock: # add tile to history currtime = datetime.now() sbdata.user_history[user_id] = [{'tile_id':base_id,'level':0,'timestamp':currtime}] try: if not os.path.isdir(history_dir): os.makedirs(history_dir) with open(history_dir +"/"+str(user_id)+".txt", "a") as myfile: myfile.write(str(base_id)+'\t0\t'+str(currtime)+"\n") except Exception as e: print e # don't barf if this doesn't work with sbdata.user_tiles_lock:# save tile sbdata.user_tiles[user_id] = {0:{tile_key:tile}} print "added tile to sbdata.user_tiles:"#,sbdata.user_tiles #start prefetching #print "setting up prefetching experts" #experts = range(1) #expert_threads = range(1) #experts[0] = dumb_expert.BasicExpert() #expert_threads[0] = threading.Thread(target=experts[0].prefetch,args=(sbdata.max_prefetched,user_id,)) #expert_threads[0].start() #sdbi.scidbCloseConn(db) return tile
def getTileNoUser(tile_info,orig_query,saved_qpresults,l,levels,k,usenumpy=False,desired_attrs=None): db = sdbi.scidbOpenConn() xbase = 0 ybase = 0 if len(saved_qpresults['dimbases']) > 0: # adjust bases for array if possible xbase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][0]]) ybase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][1]]) xdim = None ydim = None if ('x_label' in tile_info) and ('y_label' in tile_info): xdim = saved_qpresults['indexes'][tile_info['x_label']] ydim = saved_qpresults['indexes'][tile_info['y_label']] if DEBUG_PRINT: print "xdim:",xdim,",ydim:",ydim x = saved_qpresults['dimwidths'][saved_qpresults['dims'][xdim]] y = saved_qpresults['dimwidths'][saved_qpresults['dims'][ydim]] n = saved_qpresults['numdims'] setup_aggr_options = {'afl':False,'saved_qpresults':saved_qpresults} aggr_options = setup_reduce_type('AGGR',setup_aggr_options) aggr_options['db'] = db if tile_info['type'] == "center": queryresultobj = sdbi.getTile(orig_query,tile_info['cx'],tile_info['cy'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options) elif tile_info['type'] == "xy": queryresultobj = sdbi.getTileByIDXY(orig_query,n,xdim,ydim,tile_info['tile_xid'],tile_info['tile_yid'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options) else: queryresultobj = sdbi.getTileByID(orig_query,tile_info['tile_id'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options) total_tiles = queryresultobj[1]['total_tiles'] total_tiles_root = queryresultobj[1]['total_tiles_root'] if DEBUG_PRINT: print "total_tiles_root:",total_tiles_root sdbioptions={'dimnames':saved_qpresults['dims']} if usenumpy: if desired_attrs is None: # only get numpy data for queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0]) else: # only get info for some attrs queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0],desired_attrs) else: queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions) saved_qpresults = queryresultobj[1] # don't need local saved_qpresults anymore, so reuse # get the new dim info queryresultarr['dimnames'] = saved_qpresults['dims'] queryresultarr['dimbases'] = saved_qpresults['dimbases'] queryresultarr['dimwidths'] = saved_qpresults['dimwidths'] queryresultarr['saved_qpresults'] = saved_qpresults queryresultarr['numdims'] = saved_qpresults['numdims'] queryresultarr['indexes'] = saved_qpresults['indexes'] queryresultarr['max_zoom'] = levels queryresultarr['total_tiles'] = total_tiles queryresultarr['total_xtiles'] = saved_qpresults['total_xtiles'] queryresultarr['total_ytiles'] = saved_qpresults['total_ytiles'] queryresultarr['future_xtiles'] = saved_qpresults['future_xtiles'] queryresultarr['future_ytiles'] = saved_qpresults['future_ytiles'] queryresultarr['total_tiles_root'] = total_tiles_root queryresultarr['zoom_diff'] = sbdata.default_diff queryresultarr['total_xtiles_exact'] = saved_qpresults['total_xtiles_exact'] queryresultarr['total_ytiles_exact'] = saved_qpresults['total_ytiles_exact'] queryresultarr['future_xtiles_exact'] = saved_qpresults['future_xtiles_exact'] queryresultarr['future_ytiles_exact'] = saved_qpresults['future_ytiles_exact'] sdbi.scidbCloseConn(db) return queryresultarr
def query_execute(userquery,options): user_id = options['user_id'] resolution = sdbi.D3_DATA_THRESHOLD if 'resolution' in options and options['resolution'] > 0: print "got resolution" resolution = options['resolution'] db = sdbi.scidbOpenConn() query = userquery #print "user query: ",query sdbioptions = {'afl':False,'db':db,'resolution':resolution} saved_qpresults = None if 'saved_qpresults' in options: saved_qpresults = options['saved_qpresults'] #dumb_expert.prefetch(3,options['user_id']) #tile = ti.getTileByID(2,sbdata.backend_metadata[options['user_id']]['levels'],options['user_id']) #print "tile: ",tile #sdbioptions={'dimnames':saved_qpresults['dims']} #print sdbi.getAllAttrArrFromQueryForJSON(tile[0],sdbioptions) #fetch_tile(0,1,{'user_id':user_id}) if saved_qpresults is None: # first time #fetch_first_tile(query,{'user_id':user_id}) saved_qpresults = sdbi.verifyQuery(query,sdbioptions) #with sbdata.metadata_lock: # sbdata.backend_metadata[user_id] = {} # sbdata.backend_metadata[user_id]['orig_query'] = query # sbdata.backend_metadata[user_id]['saved_qpresults'] = saved_qpresults # if 'data_threshold' in options: # sbdata.backend_metadata[user_id]['data_threshold'] = options['data_threshold'] # else: # default # sbdata.backend_metadata[user_id]['data_threshold'] = sdbi.D3_DATA_THRESHOLD # sbdata.backend_metadata[user_id]['levels'] = sbdata.default_levels #leave at default levels for now # #TODO: let # levels vary #only do this check for new queries if options['reduce_res_check'] and (saved_qpresults['size'] > resolution):#sdbi.D3_DATA_THRESHOLD): return {'reduce_res':True,'saved_qpresults':saved_qpresults} if 'reduce_type' in options: # reduction requested sdbioptions['reduce_res'] = True srtoptions = {'afl':False,'saved_qpresults':saved_qpresults} if 'predicate' in options: srtoptions['predicate'] = options['predicate'] sdbioptions['reduce_options'] = setup_reduce_type(options['reduce_type'],srtoptions) else: #return original query sdbioptions = {'afl':False,'reduce_res':False,'db':db} queryresultobj = sdbi.executeQuery(query,sdbioptions) print "queryresultobj:",queryresultobj print "retrieved data from db.\nparsing results..." sdbioptions={'dimnames':saved_qpresults['dims']} queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions) if queryresultobj[1] != 0: saved_qpresults = queryresultobj[1] # get the new dim info queryresultarr['dimnames'] = saved_qpresults['dims'] queryresultarr['dimbases'] = saved_qpresults['dimbases'] queryresultarr['dimwidths'] = saved_qpresults['dimwidths'] queryresultarr['saved_qpresults'] = saved_qpresults sdbi.scidbCloseConn(db) return queryresultarr # note that I don't set reduce_res false. JS code will still handle it properly
def verify_query(query): saved_qpresults = None try: db = sdbi.scidbOpenConn() sdbioptions = {'afl':False,'db':db} saved_qpresults = sdbi.verifyQuery(query,sdbioptions) sdbi.scidbCloseConn(db) except Exception as e: if DEBUG: print "error occured:",e pass return saved_qpresults
def compare_tiles(query,threshold,zoom_diff,x_label,y_label): db = sdbi.scidbOpenConn() sdbioptions = {'afl':False,'db':db} saved_qpresults = sdbi.verifyQuery(query,sdbioptions) sdbi.scidbCloseConn(db) if 'error' in saved_qpresults: print "error found, returning error:",saved_qpresults return saved_qpresults else: if DEBUG_PRINT: print "saved_qpresults:",saved_qpresults if x_label not in saved_qpresults['dims']: print "x_label",x_label,"not found" return if y_label not in saved_qpresults['dims']: print "y_label",y_label,"not found" return n = saved_qpresults['numdims'] root_k = math.ceil(math.pow(threshold,1.0/n)) threshold = root_k ** 2 ## adjust to make it a nice power tsize = saved_qpresults['size'] # get the size of the result levels = 1 if tsize > threshold: # if k happens to be larger than the total results levels = math.ceil(math.log(1.0*tsize/threshold)/(n*math.log(zoom_diff)))+1 # need to account for zoom diff if DEBUG_PRINT: print "levels:",levels tile_metadata = sdbi.get_complete_tile_metadata(root_k,zoom_diff,saved_qpresults) if DEBUG_PRINT: print "tile metadata:",tile_metadata #_id = [0,0] #tile = ti.getTileNoUser(_id,query,saved_qpresults,levels,threshold) #if 'error' in tile: # print 'error found, reeturning error:',tile # return tile for l in range(1,int(levels)): total_tiles_root = int(math.pow(zoom_diff,l)) for xid in range(0,total_tiles_root): for yid in range(0,total_tiles_root): #get tile tile_info = {'type':'xy','tile_xid':xid,'tile_yid':yid,'x_label':x_label,'y_label':y_label} tile = ti.getTileNoUser(tile_info,query,saved_qpresults,l,levels,threshold,USE_NUMPY) if 'error' in tile: print 'error found, reeturning error:',tile return tile #stats = tile['stats'] #if DEBUG_PRINT: print stats if yid+1 < total_tiles_root: for yid2 in range(yid+1,total_tiles_root): #if DEBUG_PRINT: print "comparing tile (lvl:",l,", x:",xid,", y:",yid,") and tile (lvl:",l,", x:",xid,", y:",yid2,")" #check rest of col: xid,yid2 tile_info['tile_xid']=xid tile_info['tile_yid']=yid2 #get other tile newtile = ti.getTileNoUser(tile_info,query,saved_qpresults,l,levels,threshold,USE_NUMPY) if 'error' in newtile: print 'error found, reeturning error:',newtile return newtile #compare tiles if xid+1 < total_tiles_root: for xid2 in range(xid+1,total_tiles_root): for yid2 in range(0,total_tiles_root): #if DEBUG_PRINT: print "comparing tile (lvl:",l,", x:",xid,", y:",yid,") and tile (lvl:",l,", x:",xid2,", y:",yid2,")" #check all tiles w/ > xid: xid2,yid2 tile_info['tile_xid']=xid2 tile_info['tile_yid']=yid2 #get other tile newtile = ti.getTileNoUser(tile_info,query,saved_qpresults,l,levels,threshold,USE_NUMPY) if 'error' in newtile: print 'error found, reeturning error:',newtile return newtile
def getTileHelper(tile_info,l,user_id,usenumpy=False): db = sdbi.scidbOpenConn() with sbdata.metadata_lock: orig_query = sbdata.backend_metadata[user_id]['orig_query'] saved_qpresults = sbdata.backend_metadata[user_id]['saved_qpresults'] xbase = 0 ybase = 0 if len(saved_qpresults['dimbases']) > 0: # adjust bases for array if possible xbase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][0]]) ybase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][1]]) xdim = None ydim = None if ('x_label' in tile_info) and ('y_label' in tile_info): xdim = saved_qpresults['indexes'][tile_info['x_label']] ydim = saved_qpresults['indexes'][tile_info['y_label']] if DEBUG_PRINT: print "xdim:",xdim,",ydim:",ydim x = saved_qpresults['dimwidths'][saved_qpresults['dims'][xdim]] y = saved_qpresults['dimwidths'][saved_qpresults['dims'][ydim]] k = sbdata.backend_metadata[user_id]['data_threshold'] n = saved_qpresults['numdims'] levels = sbdata.backend_metadata[user_id]['levels'] if levels == 0: # need to compute # of levels root_k = math.ceil(math.pow(k,1.0/n)) k = root_k ** 2 ## adjust to make it a nice power if DEBUG_PRINT: print "*************threshold:",k tsize = saved_qpresults['size'] # get the size of the result if tsize <= k: # if k happens to be larger than the total results levels = 1 else: # round up to include the topmost level #levels = math.ceil(math.log(tsize)/math.log(k))+1 levels = math.ceil(math.log(tsize)/(math.log(k)*saved_qpresults['numdims']*math.log(sbdata.default_diff)))+1 # need to account for zoom diff sbdata.backend_metadata[user_id]['levels'] = levels # store this computed value sbdata.backend_metadata[user_id]['data_threshold'] = k setup_aggr_options = {'afl':False,'saved_qpresults':saved_qpresults} aggr_options = setup_reduce_type('AGGR',setup_aggr_options) aggr_options['db'] = db if tile_info['type'] == "center": queryresultobj = sdbi.getTile(orig_query,tile_info['cx'],tile_info['cy'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options) elif tile_info['type'] == "xy": queryresultobj = sdbi.getTileByIDXY(orig_query,n,xdim,ydim,tile_info['tile_xid'],tile_info['tile_yid'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options) else: queryresultobj = sdbi.getTileByID(orig_query,tile_info['tile_id'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options) total_tiles = queryresultobj[1]['total_tiles'] total_tiles_root = queryresultobj[1]['total_tiles_root'] if DEBUG_PRINT: print "total_tiles_root:",total_tiles_root sdbioptions={'dimnames':saved_qpresults['dims']} if usenumpy: queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0]) else: # only get info for some attrs queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions) saved_qpresults = queryresultobj[1] # don't need local saved_qpresults anymore, so reuse # get the new dim info queryresultarr['dimnames'] = saved_qpresults['dims'] queryresultarr['dimbases'] = saved_qpresults['dimbases'] queryresultarr['dimwidths'] = saved_qpresults['dimwidths'] queryresultarr['saved_qpresults'] = saved_qpresults queryresultarr['numdims'] = saved_qpresults['numdims'] queryresultarr['indexes'] = saved_qpresults['indexes'] queryresultarr['max_zoom'] = levels queryresultarr['total_tiles'] = total_tiles queryresultarr['total_xtiles'] = saved_qpresults['total_xtiles'] queryresultarr['total_ytiles'] = saved_qpresults['total_ytiles'] queryresultarr['future_xtiles'] = saved_qpresults['future_xtiles'] queryresultarr['future_ytiles'] = saved_qpresults['future_ytiles'] queryresultarr['total_tiles_root'] = total_tiles_root queryresultarr['zoom_diff'] = sbdata.default_diff queryresultarr['total_xtiles_exact'] = saved_qpresults['total_xtiles_exact'] queryresultarr['total_ytiles_exact'] = saved_qpresults['total_ytiles_exact'] queryresultarr['future_xtiles_exact'] = saved_qpresults['future_xtiles_exact'] queryresultarr['future_ytiles_exact'] = saved_qpresults['future_ytiles_exact'] sdbi.scidbCloseConn(db) return queryresultarr
def getTileHelper3(tile_info,l,user_metadata,usenumpy=False): db = sdbi.scidbOpenConn() tile_id = tile_info['tile_id'] orig_query = user_metadata['original_query'] saved_qpresults = user_metadata['saved_qpresults'] n = saved_qpresults['numdims'] bases= [0]*n widths = [0]*n if n > 0: # adjust bases for array if possible for i in range(n): bases[i] = int(saved_qpresults['dimbases'][saved_qpresults['dims'][i]]) widths[i] = int(saved_qpresults['dimwidths'][saved_qpresults['dims'][i]]) k = user_metadata['threshold'] levels = user_metadata['total_zoom_levels'] if levels == 0: # need to compute # of levels root_k = math.ceil(math.pow(k,1.0/n)) #k = root_k ** 2 k = math.pow(root_k,n) if DEBUG_PRINT: print "root_k:",root_k,"d:",sbdata.default_diff for i in range(n): w_i = widths[i] if DEBUG_PRINT: print "w_i:",w_i l_i = 1 if w_i > root_k: t_i = math.ceil(w_i/root_k) # number of base level tiles if DEBUG_PRINT: print "t_i:",t_i if DEBUG_PRINT: print "log t_i:",math.log(t_i),"log d:",math.log(sbdata.default_diff) l_i = math.ceil(math.log(t_i)/math.log(sbdata.default_diff))+1 if DEBUG_PRINT: print "l_i:",l_i,"levels:",levels if l_i > levels: levels = l_i setup_aggr_options = {'afl':False,'saved_qpresults':saved_qpresults} aggr_options = setup_reduce_type('AGGR',setup_aggr_options) aggr_options['db'] = db queryresultobj = sdbi.getTileByIDN(orig_query,n,tile_info['tile_id'],l,levels-1,sbdata.default_diff,bases,widths,k,aggr_options) if 'error' in queryresultobj: # this is bad, return the error sdbi.scidbCloseConn(db) return queryresultobj #total_tiles = queryresultobj[1]['total_tiles'] #total_tiles_root = queryresultobj[1]['total_tiles_root'] #if DEBUG_PRINT: print "total_tiles_root:",total_tiles_root sdbioptions={'dimnames':saved_qpresults['dims']} if usenumpy: queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0]) data = sdbinp.arrs_to_json(queryresultarr['data']) queryresultarr['data'] = data stats = sdbinp.stats_to_json(queryresultarr['stats']) queryresultarr['stats'] = stats else: queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions) saved_qpresults = queryresultobj[1] # don't need local saved_qpresults anymore, so reuse # get the new dim info queryresultarr['dimnames'] = saved_qpresults['dims'] queryresultarr['dimbases'] = saved_qpresults['dimbases'] queryresultarr['dimwidths'] = saved_qpresults['dimwidths'] queryresultarr['numdims'] = saved_qpresults['numdims'] queryresultarr['indexes'] = saved_qpresults['indexes'] queryresultarr['saved_qpresults'] = saved_qpresults queryresultarr['max_zoom'] = levels queryresultarr['threshold'] = k queryresultarr['zoom_diff'] = sbdata.default_diff queryresultarr['total_tiles'] = saved_qpresults['total_tiles'] queryresultarr['future_tiles'] = saved_qpresults['future_tiles'] queryresultarr['future_tiles_exact'] = saved_qpresults['future_tiles_exact'] if DEBUG_PRINT: print "indexes:",saved_qpresults['indexes'] sdbi.scidbCloseConn(db) return queryresultarr
def teardown_request(exception): """Closes the database again at the end of the request.""" if DEFAULT_DB == SCIDB: sdbi.scidbCloseConn() else: mdbi.mysqlOpenConn()
def dbclose(): """Closes the database again at the end of the request.""" if DEFAULT_DB == SCIDB: sdbi.scidbCloseConn()