Beispiel #1
0
def fetch_first_tile2(userquery,options):
	db = sdbi.scidbOpenConn()
	global experts
	query = userquery
	sdbioptions = {'afl':False,'db':db}
	saved_qpresults = sdbi.verifyQuery(query,sdbioptions)
	sdbi.scidbCloseConn(db)
	print "saved qp results, looking for error"
	if 'error' in saved_qpresults:
		print "error found, returning error:",saved_qpresults
		return saved_qpresults
	else:
		print "saved_qpresults:",saved_qpresults
	user_id = options['user_id']
	# setup metadata
	print "setting up metadata"
	with sbdata.metadata_lock:
		sbdata.backend_metadata[user_id] = {}
		sbdata.backend_metadata[user_id]['orig_query'] = query
		sbdata.backend_metadata[user_id]['saved_qpresults'] = saved_qpresults
		if 'data_threshold' in options:
			sbdata.backend_metadata[user_id]['data_threshold'] = options['data_threshold']
		else: # default is whatever is prescribed in scidb server interface code
			sbdata.backend_metadata[user_id]['data_threshold'] = sdbi.D3_DATA_THRESHOLD
		sbdata.backend_metadata[user_id]['levels'] = 0 #sbdata.default_levels #leave at default levels for now
		#TODO: let # levels vary
	#get tile
	base_id = [0] * saved_qpresults['numdims']
	print "base id:",base_id
	if 'usenumpy' in options:
		usenumpy = options['usenumpy']
	else:
		usenumpy = False
	tile = ti.getTileByIDN(base_id,0,user_id,usenumpy)
	if 'error' in tile:
		return tile
	#save tile info
	tile_key = str(base_id)
	with sbdata.user_history_lock: # add tile to history
		currtime = datetime.now()
		sbdata.user_history[user_id] = [{'tile_id':base_id,'level':0,'timestamp':currtime}]
		try:
			if not os.path.isdir(history_dir):
				os.makedirs(history_dir)
			with open(history_dir +"/"+str(user_id)+".txt", "a") as myfile:
			    myfile.write(str(base_id)+'\t0\t'+str(currtime)+"\n")
		except Exception as e:
			print e # don't barf if this doesn't work
	with sbdata.user_tiles_lock:# save tile
		sbdata.user_tiles[user_id] = {0:{tile_key:tile}}
		print "added tile to sbdata.user_tiles:"#,sbdata.user_tiles
	#start prefetching
	#print "setting up prefetching experts"
	#experts = range(1)
	#expert_threads = range(1)
	#experts[0] = dumb_expert.BasicExpert()
	#expert_threads[0] = threading.Thread(target=experts[0].prefetch,args=(sbdata.max_prefetched,user_id,))
	#expert_threads[0].start()
	#sdbi.scidbCloseConn(db)
	return tile
Beispiel #2
0
def getTileNoUser(tile_info,orig_query,saved_qpresults,l,levels,k,usenumpy=False,desired_attrs=None):
  db = sdbi.scidbOpenConn()
  xbase = 0
  ybase = 0
  if len(saved_qpresults['dimbases']) > 0: # adjust bases for array if possible
    xbase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][0]])
    ybase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][1]])
  xdim = None
  ydim = None
  if ('x_label' in tile_info) and ('y_label' in tile_info):
    xdim = saved_qpresults['indexes'][tile_info['x_label']]
    ydim = saved_qpresults['indexes'][tile_info['y_label']]
  if DEBUG_PRINT: print "xdim:",xdim,",ydim:",ydim
  x = saved_qpresults['dimwidths'][saved_qpresults['dims'][xdim]]
  y = saved_qpresults['dimwidths'][saved_qpresults['dims'][ydim]]
  n = saved_qpresults['numdims']
  setup_aggr_options = {'afl':False,'saved_qpresults':saved_qpresults}
  aggr_options = setup_reduce_type('AGGR',setup_aggr_options)
  aggr_options['db'] = db
  if tile_info['type'] == "center":
    queryresultobj = sdbi.getTile(orig_query,tile_info['cx'],tile_info['cy'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options)
  elif tile_info['type'] == "xy":
    queryresultobj = sdbi.getTileByIDXY(orig_query,n,xdim,ydim,tile_info['tile_xid'],tile_info['tile_yid'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options)
  else:
    queryresultobj = sdbi.getTileByID(orig_query,tile_info['tile_id'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options)
  total_tiles = queryresultobj[1]['total_tiles']
  total_tiles_root = queryresultobj[1]['total_tiles_root']
  if DEBUG_PRINT: print "total_tiles_root:",total_tiles_root
  sdbioptions={'dimnames':saved_qpresults['dims']}
  if usenumpy:
    if desired_attrs is None: # only get numpy data for
      queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0])
    else:  # only get info for some attrs
      queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0],desired_attrs)
  else:
    queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions)
  saved_qpresults = queryresultobj[1] # don't need local saved_qpresults anymore, so reuse
  # get the new dim info
  queryresultarr['dimnames'] = saved_qpresults['dims']
  queryresultarr['dimbases'] = saved_qpresults['dimbases']
  queryresultarr['dimwidths'] = saved_qpresults['dimwidths']
  queryresultarr['saved_qpresults'] = saved_qpresults
  queryresultarr['numdims'] = saved_qpresults['numdims']
  queryresultarr['indexes'] = saved_qpresults['indexes']
  queryresultarr['max_zoom'] = levels
  queryresultarr['total_tiles'] = total_tiles
  queryresultarr['total_xtiles'] = saved_qpresults['total_xtiles']
  queryresultarr['total_ytiles'] = saved_qpresults['total_ytiles']
  queryresultarr['future_xtiles'] = saved_qpresults['future_xtiles']
  queryresultarr['future_ytiles'] = saved_qpresults['future_ytiles']
  queryresultarr['total_tiles_root'] = total_tiles_root
  queryresultarr['zoom_diff'] = sbdata.default_diff
  queryresultarr['total_xtiles_exact'] = saved_qpresults['total_xtiles_exact']
  queryresultarr['total_ytiles_exact'] = saved_qpresults['total_ytiles_exact']
  queryresultarr['future_xtiles_exact'] = saved_qpresults['future_xtiles_exact']
  queryresultarr['future_ytiles_exact'] = saved_qpresults['future_ytiles_exact']
  sdbi.scidbCloseConn(db)
  return queryresultarr
Beispiel #3
0
def query_execute(userquery,options):
	user_id = options['user_id']
	resolution = sdbi.D3_DATA_THRESHOLD
	if 'resolution' in options and options['resolution'] > 0:
		print "got resolution"
        	resolution = options['resolution']
	db = sdbi.scidbOpenConn()
	query = userquery
	#print  "user query: ",query
	sdbioptions = {'afl':False,'db':db,'resolution':resolution}
	saved_qpresults = None
        if 'saved_qpresults' in options:
        	saved_qpresults = options['saved_qpresults']
		#dumb_expert.prefetch(3,options['user_id'])
		#tile = ti.getTileByID(2,sbdata.backend_metadata[options['user_id']]['levels'],options['user_id'])
		#print "tile: ",tile
		#sdbioptions={'dimnames':saved_qpresults['dims']}
		#print sdbi.getAllAttrArrFromQueryForJSON(tile[0],sdbioptions)
		#fetch_tile(0,1,{'user_id':user_id})
	if saved_qpresults is None: # first time
		#fetch_first_tile(query,{'user_id':user_id})
		saved_qpresults = sdbi.verifyQuery(query,sdbioptions)
		#with sbdata.metadata_lock:
		#	sbdata.backend_metadata[user_id] = {}
		#	sbdata.backend_metadata[user_id]['orig_query'] = query
		#	sbdata.backend_metadata[user_id]['saved_qpresults'] = saved_qpresults
		#	if 'data_threshold' in options:
		#		sbdata.backend_metadata[user_id]['data_threshold'] = options['data_threshold']
		#	else: # default
		#		sbdata.backend_metadata[user_id]['data_threshold'] = sdbi.D3_DATA_THRESHOLD
		#	sbdata.backend_metadata[user_id]['levels'] = sbdata.default_levels #leave at default levels for now
		#	#TODO: let # levels vary
		#only do this check for new queries
		if options['reduce_res_check'] and (saved_qpresults['size'] > resolution):#sdbi.D3_DATA_THRESHOLD):
			return {'reduce_res':True,'saved_qpresults':saved_qpresults}
	if 'reduce_type' in options: # reduction requested
		sdbioptions['reduce_res'] = True
		srtoptions = {'afl':False,'saved_qpresults':saved_qpresults}
		if 'predicate' in options:
			srtoptions['predicate'] = options['predicate']
		sdbioptions['reduce_options'] = setup_reduce_type(options['reduce_type'],srtoptions)
	else: #return original query
		sdbioptions = {'afl':False,'reduce_res':False,'db':db}
	queryresultobj = sdbi.executeQuery(query,sdbioptions)
	print "queryresultobj:",queryresultobj
	print  "retrieved data from db.\nparsing results..."
	sdbioptions={'dimnames':saved_qpresults['dims']}
	queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions)
	if queryresultobj[1] != 0:
		saved_qpresults = queryresultobj[1]
	# get the new dim info
	queryresultarr['dimnames'] = saved_qpresults['dims']
	queryresultarr['dimbases'] = saved_qpresults['dimbases']
	queryresultarr['dimwidths'] = saved_qpresults['dimwidths']
	queryresultarr['saved_qpresults'] = saved_qpresults
	sdbi.scidbCloseConn(db)
	return queryresultarr # note that I don't set reduce_res false. JS code will still handle it properly
def verify_query(query):
  saved_qpresults = None
  try:
    db = sdbi.scidbOpenConn()
    sdbioptions = {'afl':False,'db':db}
    saved_qpresults = sdbi.verifyQuery(query,sdbioptions)
    sdbi.scidbCloseConn(db)
  except Exception as e:
    if DEBUG: print "error occured:",e
    pass
  return saved_qpresults
Beispiel #5
0
def compare_tiles(query,threshold,zoom_diff,x_label,y_label):
	db = sdbi.scidbOpenConn()
	sdbioptions = {'afl':False,'db':db}
	saved_qpresults = sdbi.verifyQuery(query,sdbioptions)
	sdbi.scidbCloseConn(db)
	if 'error' in saved_qpresults:
		print "error found, returning error:",saved_qpresults
		return saved_qpresults
	else:
		if DEBUG_PRINT: print "saved_qpresults:",saved_qpresults
	if x_label not in saved_qpresults['dims']:
		print "x_label",x_label,"not found"
		return
	if y_label not in saved_qpresults['dims']:
		print "y_label",y_label,"not found"
		return
	n = saved_qpresults['numdims']
	root_k = math.ceil(math.pow(threshold,1.0/n))
	threshold = root_k ** 2 ## adjust to make it a nice power
	tsize = saved_qpresults['size'] # get the size of the result
	levels = 1
	if tsize > threshold: # if k happens to be larger than the total results
		levels = math.ceil(math.log(1.0*tsize/threshold)/(n*math.log(zoom_diff)))+1 # need to account for zoom diff
	if DEBUG_PRINT: print "levels:",levels

	tile_metadata = sdbi.get_complete_tile_metadata(root_k,zoom_diff,saved_qpresults)
	if DEBUG_PRINT: print "tile metadata:",tile_metadata
	
	#_id = [0,0]
	#tile = ti.getTileNoUser(_id,query,saved_qpresults,levels,threshold)
	#if 'error' in tile:
	#	print 'error found, reeturning error:',tile
	#	return tile
	for l in range(1,int(levels)):
		total_tiles_root = int(math.pow(zoom_diff,l))
		for xid in range(0,total_tiles_root):
			for yid in range(0,total_tiles_root):
				#get tile
				tile_info = {'type':'xy','tile_xid':xid,'tile_yid':yid,'x_label':x_label,'y_label':y_label}
				tile = ti.getTileNoUser(tile_info,query,saved_qpresults,l,levels,threshold,USE_NUMPY)
				if 'error' in tile:
					print 'error found, reeturning error:',tile
					return tile
				#stats = tile['stats']
				#if DEBUG_PRINT: print stats
				if yid+1 < total_tiles_root:
					for yid2 in range(yid+1,total_tiles_root):
						#if DEBUG_PRINT: print "comparing tile (lvl:",l,", x:",xid,", y:",yid,") and tile (lvl:",l,", x:",xid,", y:",yid2,")"
						#check rest of col: xid,yid2
						tile_info['tile_xid']=xid
						tile_info['tile_yid']=yid2
						#get other tile
						newtile = ti.getTileNoUser(tile_info,query,saved_qpresults,l,levels,threshold,USE_NUMPY)
						if 'error' in newtile:
							print 'error found, reeturning error:',newtile
							return newtile
						#compare tiles
				if xid+1 < total_tiles_root:
					for xid2 in range(xid+1,total_tiles_root):
						for yid2 in range(0,total_tiles_root):
							#if DEBUG_PRINT: print "comparing tile (lvl:",l,", x:",xid,", y:",yid,") and tile (lvl:",l,", x:",xid2,", y:",yid2,")"
							#check all tiles w/ > xid: xid2,yid2
							tile_info['tile_xid']=xid2
							tile_info['tile_yid']=yid2
							#get other tile
							newtile = ti.getTileNoUser(tile_info,query,saved_qpresults,l,levels,threshold,USE_NUMPY)
							if 'error' in newtile:
								print 'error found, reeturning error:',newtile
								return newtile
Beispiel #6
0
def getTileHelper(tile_info,l,user_id,usenumpy=False):
  db = sdbi.scidbOpenConn()
  with sbdata.metadata_lock:
    orig_query = sbdata.backend_metadata[user_id]['orig_query']
    saved_qpresults = sbdata.backend_metadata[user_id]['saved_qpresults']
    xbase = 0
    ybase = 0
    if len(saved_qpresults['dimbases']) > 0: # adjust bases for array if possible
      xbase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][0]])
      ybase = int(saved_qpresults['dimbases'][saved_qpresults['dims'][1]])
    xdim = None
    ydim = None
    if ('x_label' in tile_info) and ('y_label' in tile_info):
      xdim = saved_qpresults['indexes'][tile_info['x_label']]
      ydim = saved_qpresults['indexes'][tile_info['y_label']]
    if DEBUG_PRINT: print "xdim:",xdim,",ydim:",ydim
    x = saved_qpresults['dimwidths'][saved_qpresults['dims'][xdim]]
    y = saved_qpresults['dimwidths'][saved_qpresults['dims'][ydim]]
    k = sbdata.backend_metadata[user_id]['data_threshold']
    n = saved_qpresults['numdims']
    levels = sbdata.backend_metadata[user_id]['levels']
    if levels == 0: # need to compute # of levels
      root_k = math.ceil(math.pow(k,1.0/n))
      k = root_k ** 2 ## adjust to make it a nice power
      if DEBUG_PRINT: print "*************threshold:",k
      tsize = saved_qpresults['size'] # get the size of the result
      if tsize <= k: # if k happens to be larger than the total results
        levels = 1
      else: # round up to include the topmost level
        #levels = math.ceil(math.log(tsize)/math.log(k))+1
        levels = math.ceil(math.log(tsize)/(math.log(k)*saved_qpresults['numdims']*math.log(sbdata.default_diff)))+1 # need to account for zoom diff
      sbdata.backend_metadata[user_id]['levels'] = levels # store this computed value
      sbdata.backend_metadata[user_id]['data_threshold'] = k
  setup_aggr_options = {'afl':False,'saved_qpresults':saved_qpresults}
  aggr_options = setup_reduce_type('AGGR',setup_aggr_options)
  aggr_options['db'] = db
  if tile_info['type'] == "center":
    queryresultobj = sdbi.getTile(orig_query,tile_info['cx'],tile_info['cy'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options)
  elif tile_info['type'] == "xy":
    queryresultobj = sdbi.getTileByIDXY(orig_query,n,xdim,ydim,tile_info['tile_xid'],tile_info['tile_yid'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options)
  else:
    queryresultobj = sdbi.getTileByID(orig_query,tile_info['tile_id'],l,levels-1,sbdata.default_diff,x,xbase,y,ybase,k,aggr_options)
  total_tiles = queryresultobj[1]['total_tiles']
  total_tiles_root = queryresultobj[1]['total_tiles_root']
  if DEBUG_PRINT: print "total_tiles_root:",total_tiles_root
  sdbioptions={'dimnames':saved_qpresults['dims']}
  if usenumpy:
    queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0])
  else:  # only get info for some attrs
    queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions)
  saved_qpresults = queryresultobj[1] # don't need local saved_qpresults anymore, so reuse
  # get the new dim info
  queryresultarr['dimnames'] = saved_qpresults['dims']
  queryresultarr['dimbases'] = saved_qpresults['dimbases']
  queryresultarr['dimwidths'] = saved_qpresults['dimwidths']
  queryresultarr['saved_qpresults'] = saved_qpresults
  queryresultarr['numdims'] = saved_qpresults['numdims']
  queryresultarr['indexes'] = saved_qpresults['indexes']
  queryresultarr['max_zoom'] = levels
  queryresultarr['total_tiles'] = total_tiles
  queryresultarr['total_xtiles'] = saved_qpresults['total_xtiles']
  queryresultarr['total_ytiles'] = saved_qpresults['total_ytiles']
  queryresultarr['future_xtiles'] = saved_qpresults['future_xtiles']
  queryresultarr['future_ytiles'] = saved_qpresults['future_ytiles']
  queryresultarr['total_tiles_root'] = total_tiles_root
  queryresultarr['zoom_diff'] = sbdata.default_diff
  queryresultarr['total_xtiles_exact'] = saved_qpresults['total_xtiles_exact']
  queryresultarr['total_ytiles_exact'] = saved_qpresults['total_ytiles_exact']
  queryresultarr['future_xtiles_exact'] = saved_qpresults['future_xtiles_exact']
  queryresultarr['future_ytiles_exact'] = saved_qpresults['future_ytiles_exact']
  sdbi.scidbCloseConn(db)
  return queryresultarr
Beispiel #7
0
def getTileHelper3(tile_info,l,user_metadata,usenumpy=False):
  db = sdbi.scidbOpenConn()
  tile_id = tile_info['tile_id']
  orig_query = user_metadata['original_query']
  saved_qpresults = user_metadata['saved_qpresults']
  n = saved_qpresults['numdims']
  bases= [0]*n
  widths = [0]*n
  if n > 0: # adjust bases for array if possible
    for i in range(n):
      bases[i] = int(saved_qpresults['dimbases'][saved_qpresults['dims'][i]])
      widths[i] = int(saved_qpresults['dimwidths'][saved_qpresults['dims'][i]])
  k = user_metadata['threshold']
  levels = user_metadata['total_zoom_levels']
  if levels == 0: # need to compute # of levels
    root_k = math.ceil(math.pow(k,1.0/n))
    #k = root_k ** 2
    k = math.pow(root_k,n)
    if DEBUG_PRINT: print "root_k:",root_k,"d:",sbdata.default_diff
    for i in range(n):
      w_i = widths[i]
      if DEBUG_PRINT: print "w_i:",w_i
      l_i = 1
      if w_i > root_k:
        t_i = math.ceil(w_i/root_k) # number of base level tiles
        if DEBUG_PRINT: print "t_i:",t_i
        if DEBUG_PRINT: print "log t_i:",math.log(t_i),"log d:",math.log(sbdata.default_diff)
        l_i = math.ceil(math.log(t_i)/math.log(sbdata.default_diff))+1
        if DEBUG_PRINT: print "l_i:",l_i,"levels:",levels
      if l_i > levels:
        levels = l_i
  setup_aggr_options = {'afl':False,'saved_qpresults':saved_qpresults}
  aggr_options = setup_reduce_type('AGGR',setup_aggr_options)
  aggr_options['db'] = db
  queryresultobj = sdbi.getTileByIDN(orig_query,n,tile_info['tile_id'],l,levels-1,sbdata.default_diff,bases,widths,k,aggr_options)
  if 'error' in queryresultobj:
    # this is bad, return the error
    sdbi.scidbCloseConn(db)
    return queryresultobj
  #total_tiles = queryresultobj[1]['total_tiles']
  #total_tiles_root = queryresultobj[1]['total_tiles_root']
  #if DEBUG_PRINT: print "total_tiles_root:",total_tiles_root
  sdbioptions={'dimnames':saved_qpresults['dims']}
  if usenumpy:
    queryresultarr = sdbinp.getAllAttrArrFromQueryForNP(queryresultobj[0])
    data = sdbinp.arrs_to_json(queryresultarr['data'])
    queryresultarr['data'] = data
    stats = sdbinp.stats_to_json(queryresultarr['stats'])
    queryresultarr['stats'] = stats
  else:
    queryresultarr = sdbi.getAllAttrArrFromQueryForJSON(queryresultobj[0],sdbioptions)
  saved_qpresults = queryresultobj[1] # don't need local saved_qpresults anymore, so reuse
  # get the new dim info
  queryresultarr['dimnames'] = saved_qpresults['dims']
  queryresultarr['dimbases'] = saved_qpresults['dimbases']
  queryresultarr['dimwidths'] = saved_qpresults['dimwidths']
  queryresultarr['numdims'] = saved_qpresults['numdims']
  queryresultarr['indexes'] = saved_qpresults['indexes']
  queryresultarr['saved_qpresults'] = saved_qpresults
  queryresultarr['max_zoom'] = levels
  queryresultarr['threshold'] = k
  queryresultarr['zoom_diff'] = sbdata.default_diff
  queryresultarr['total_tiles'] = saved_qpresults['total_tiles']
  queryresultarr['future_tiles'] = saved_qpresults['future_tiles']
  queryresultarr['future_tiles_exact'] = saved_qpresults['future_tiles_exact']
  if DEBUG_PRINT: print "indexes:",saved_qpresults['indexes']
  sdbi.scidbCloseConn(db)
  return queryresultarr
Beispiel #8
0
def teardown_request(exception):
    """Closes the database again at the end of the request."""
    if DEFAULT_DB == SCIDB:
    	sdbi.scidbCloseConn()
    else:
    	mdbi.mysqlOpenConn()
Beispiel #9
0
def dbclose():
    """Closes the database again at the end of the request."""
    if DEFAULT_DB == SCIDB:
    	sdbi.scidbCloseConn()