def exact_match(request, g_id, usr): if usr is None or usr == '' or usr == '/': bbox_ids = Bboxes.objects.filter(group_id=g_id).filter( cur_state__box_done=1).values('bbox_id') else: bbox_ids = Bboxes.objects.filter(group_id=g_id).filter( cur_state__box_done=1).filter( cur_state__user=usr).values('bbox_id') ims = [] for box in bbox_ids: b = Cur_state.objects.filter(bbox_id=box['bbox_id']).get() img, bbox = get_im_bbox(b) im = {} im['img'] = img im['bbox'] = bbox ims.append(im) db = connect_to_db('backup_geocars') cursor = db.cursor() examples = get_images_for_group(int(g_id.encode('ascii', 'ignore')), cursor) db.close() group_name = GroupNames.objects.filter( group_id=g_id).values('group_name').get() return render_to_response( 'streetview/show_exact.html', { 'images': ims, 'examples': examples, 'group_name': group_name, 'group_id': g_id, 'username': usr })
def get_im_bbox(b): db = connect_to_db('bbox_collection_gsv') cursor = db.cursor() img_query = 'select myori_url,oriwidth,oriheight from imagenet_bbox.view_allimage where synsetid=%d and imageid=%d' % ( b.bbox.synsetid, b.bbox.imageid) cursor.execute(img_query) row = cursor.fetchone() bbox_query = 'select pleft, pright, ptop, pbottom, width, height from bbox_answer where bbox_isgood and pleft!=-1 and synsetid=%s and targetsynsetid=%s and imageid=%d and assignid=%d' % ( b.bbox.synsetid, b.bbox.synsetid, b.bbox.imageid, b.bbox.assignid) cursor.execute(bbox_query) bbrow = cursor.fetchone() bbox = {} bbox['bbox_id'] = b.bbox.bbox_id bbox['pleft'] = bbrow[0] bbox['pright'] = bbrow[1] bbox['ptop'] = bbrow[2] bbox['pbottom'] = bbrow[3] bbox['width'] = bbrow[4] bbox['height'] = bbrow[5] img = {} img['url'] = row[0] img['oriwidth'] = row[1] img['oriheight'] = row[2] db.close() return img, bbox
def get_im_bbox(b): db = connect_to_db('bbox_collection_gsv') cursor = db.cursor() img_query='select myori_url,oriwidth,oriheight from imagenet_bbox.view_allimage where synsetid=%d and imageid=%d'%(b.bbox.synsetid, b.bbox.imageid) cursor.execute(img_query) row=cursor.fetchone() bbox_query='select pleft, pright, ptop, pbottom, width, height from bbox_answer where bbox_isgood and pleft!=-1 and synsetid=%s and targetsynsetid=%s and imageid=%d and assignid=%d'%(b.bbox.synsetid,b.bbox.synsetid,b.bbox.imageid,b.bbox.assignid) cursor.execute(bbox_query) bbrow=cursor.fetchone() bbox={} bbox['bbox_id']=b.bbox.bbox_id bbox['pleft']=bbrow[0] bbox['pright']=bbrow[1] bbox['ptop']=bbrow[2] bbox['pbottom']=bbrow[3] bbox['width']=bbrow[4] bbox['height']=bbrow[5] img={} img['url']=row[0] img['oriwidth']=row[1] img['oriheight']=row[2] db.close() return img,bbox
def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue db=connect_to_db('demo') cursor=db.cursor() self.cursor = cursor #self.b=network.QueryBrowser(BROWSER_ID) self.b=network.ThumbBrowser(BROWSER_ID)
def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue db = connect_to_db('demo') cursor = db.cursor() self.cursor = cursor #self.b=network.QueryBrowser(BROWSER_ID) self.b = network.ThumbBrowser(BROWSER_ID)
def __init__(self, queue,out_queue): threading.Thread.__init__(self) self.queue = queue self.out_queue = out_queue db=connect_to_db('demo') cursor=db.cursor() self.cursor = cursor #self.b=network.QueryBrowser(browser_id) self.b=network.ThumbBrowser(browser_id)
def get_submodels(make): if make=='unknown': #Get all submodels from Jon db = connect_to_db('geocars_crawled') cursor = db.cursor() submodel_query='select distinct(submodel) from control_classes order by submodel' else: make_r=make.replace('_',' ') #Get submodel's from Jon's improved table db = connect_to_db('geocars_crawled') cursor = db.cursor() submodel_query='select distinct(submodel) from control_classes where make="%s" order by submodel'%(make_r) cursor.execute(submodel_query) submodels=cursor.fetchall() submodel_list=[] for s in submodels: submodel_list.append(s[0]) return submodel_list
def get_submodels(make): if make == 'unknown': #Get all submodels from Jon db = connect_to_db('geocars_crawled') cursor = db.cursor() submodel_query = 'select distinct(submodel) from control_classes order by submodel' else: make_r = make.replace('_', ' ') #Get submodel's from Jon's improved table db = connect_to_db('geocars_crawled') cursor = db.cursor() submodel_query = 'select distinct(submodel) from control_classes where make="%s" order by submodel' % ( make_r) cursor.execute(submodel_query) submodels = cursor.fetchall() submodel_list = [] for s in submodels: submodel_list.append(s[0]) return submodel_list
def show_ims(request, lat_lng): #exp=True means experimenting to see if images are #being downloaded properly exp = True #exp=False dates_list = [] lat = lat_lng.split('_')[0].encode('ascii', 'ignore') lng = lat_lng.split('_')[1].encode('ascii', 'ignore') db = connect_to_db('geo') cursor = db.cursor() if exp == True: #False: sqls = 'select distinct(im_date) from fixed_timelapse_times where lat=%s and lng=%s and small=0 and corrupt=0 and downloaded=1' % ( lat, lng) cursor.execute(sqls) im_dates = cursor.fetchall() for d in im_dates: dates_list.append(d[0]) else: lld_dict = request.session['lld_dict'] for l in lld_dict[lat_lng]: dates_list.append(l.encode('ascii', 'ignore')) dates_list = list(set(dates_list)) dates_list.sort() dates_list_str = [datetime.strftime(d, "%Y-%b-%d") for d in dates_list] rots = [0, 60, 120, 180, 240, 300] im_list = [] for r in rots: im_rot_list = [] for d, d_str in zip(dates_list, dates_list_str): year = d_str.split('-')[0] month = d_str.split('-')[1] date_str = '%s-%s-01' % (year, month) sqls = 'select corrupt,small,downloaded from geo.fixed_timelapse_times where lat=%s and lng=%s and rot=%s and im_date="%s"' % ( lat, lng, r, datetime.strftime(d, "%Y-%m-%d")) cursor.execute(sqls) res = cursor.fetchone() if int(res[0]) == 0 and int(res[1]) == 0 and int(res[2]) == 1: im_dict = {} im_name = lat_lng_to_path(lat, lng, r, '%s-%s' % (month, year)) print im_name im_dict['im'] = im_name im_dict['date'] = d im_dict['rot'] = r sql_s = 'select x1,y1,x2,y2,desc_val,pscore,group_id from all_cars.city_175_timelapse_detected_cars where lat=%s and lng=%s and rot=%s and im_date="%s"' % ( lat, lng, r, datetime.strftime(d, "%Y-%m-%d")) cursor.execute(sql_s) bboxes_list = cursor.fetchall() bboxes = make_bbox_pred_dict(bboxes_list) im_dict['bboxes'] = bboxes im_rot_list.append(im_dict) im_list.append(im_rot_list) return render_to_response('timelapse/show_ims.html', {'im_list': im_list})
def models_trims(request, make, submodel): request.session['submodel'] = submodel username = request.user.username if make == 'unknown' or submodel == 'unknown': bbox_id = int(request.session.get('bbox', None)['bbox_id']) save_make_submodel(username, make, submodel, bbox_id) return HttpResponseRedirect( "http://imagenet.stanford.edu/streetview/streetview/") else: num_images = 4 make = make.replace('_', ' ') submodel = submodel.replace('_', ' ') db = connect_to_db('geocars_crawled') cursor = db.cursor() #get group names & ids of all groups in submodel group_query = 'select group_name,group_id from control_classes where make="%s" and submodel="%s"' % ( make, submodel) cursor.execute(group_query) group_info = cursor.fetchall() model_dict = {} for g in group_info: #Get positive examples for selected images sql_s = 'select distinct(path),viewpoint from backup_geocars.edmund_examples,backup_geocars.positive_examples where backup_geocars.edmund_examples.group_id=%d and backup_geocars.edmund_examples.group_id=backup_geocars.positive_examples.group_id and path not like "%%flipped%%" order by rand()' % ( g[1]) cursor.execute(sql_s) paths = cursor.fetchall() real_paths = get_images(paths) group_dict = {} model_name, trim_name = get_names(g[0], make, submodel) group_dict['images'] = real_paths group_dict['group_id'] = g[1] group_dict['trim_name'] = trim_name if model_name in model_dict: model_dict[model_name].append(group_dict) else: model_dict[model_name] = [group_dict] models = make_template_dict(model_dict, 'model_name', 'trims') img = request.session.get('image', None) bbox = request.session.get('bbox', None) db.close() return render_to_response( 'streetview/models_trims.html', { 'models': models, 'make': make, 'submodel': submodel, 'image': img, 'bbox': bbox })
def get_image(imageid): synsetid=145622 db = connect_to_db('bbox_collection_gsv') img_query='select myori_url, oriwidth,oriheight from imagenet_bbox.view_allimage where synsetid=%d and imageid=%d'%(synsetid, imageid) cursor = db.cursor() cursor.execute(img_query) im=cursor.fetchone() db.close() im_dict={} im_dict['url']=im[0] im_dict['oriwidth']=im[1] im_dict['oriheight']=im[2] return im_dict
def show_ims(request,lat_lng): #exp=True means experimenting to see if images are #being downloaded properly exp=True #exp=False dates_list=[] lat=lat_lng.split('_')[0].encode('ascii','ignore') lng=lat_lng.split('_')[1].encode('ascii','ignore') db=connect_to_db('geo') cursor=db.cursor() if exp== True:#False: sqls='select distinct(im_date) from fixed_timelapse_times where lat=%s and lng=%s and small=0 and corrupt=0 and downloaded=1'%(lat,lng) cursor.execute(sqls) im_dates=cursor.fetchall() for d in im_dates: dates_list.append(d[0]) else: lld_dict=request.session['lld_dict'] for l in lld_dict[lat_lng]: dates_list.append(l.encode('ascii','ignore')) dates_list=list(set(dates_list)) dates_list.sort() dates_list_str=[datetime.strftime(d,"%Y-%b-%d") for d in dates_list] rots=[0,60,120,180,240,300] im_list=[] for r in rots: im_rot_list=[] for d,d_str in zip(dates_list,dates_list_str): year=d_str.split('-')[0] month=d_str.split('-')[1] date_str='%s-%s-01'%(year,month) sqls='select corrupt,small,downloaded from geo.fixed_timelapse_times where lat=%s and lng=%s and rot=%s and im_date="%s"'%(lat,lng,r,datetime.strftime(d,"%Y-%m-%d")) cursor.execute(sqls) res=cursor.fetchone() if int(res[0])==0 and int(res[1])==0 and int(res[2])==1: im_dict={} im_name=lat_lng_to_path(lat,lng,r,'%s-%s'%(month,year)) print im_name im_dict['im'] =im_name im_dict['date'] =d im_dict['rot'] =r sql_s='select x1,y1,x2,y2,desc_val,pscore,group_id from all_cars.city_175_timelapse_detected_cars where lat=%s and lng=%s and rot=%s and im_date="%s"'%(lat,lng,r,datetime.strftime(d,"%Y-%m-%d")) cursor.execute(sql_s) bboxes_list=cursor.fetchall() bboxes=make_bbox_pred_dict(bboxes_list) im_dict['bboxes']=bboxes im_rot_list.append(im_dict) im_list.append(im_rot_list) return render_to_response('timelapse/show_ims.html',{'im_list':im_list})
def models_trims(request,make,submodel): request.session['submodel']=submodel username=request.user.username if make=='unknown' or submodel=='unknown': bbox_id=int(request.session.get('bbox',None)['bbox_id']) save_make_submodel(username,make,submodel,bbox_id) return HttpResponseRedirect("http://imagenet.stanford.edu/streetview/streetview/") else: num_images=4 make = make.replace('_',' ') submodel = submodel.replace('_',' ') db = connect_to_db('geocars_crawled') cursor = db.cursor() #get group names & ids of all groups in submodel group_query='select group_name,group_id from control_classes where make="%s" and submodel="%s"'%(make,submodel) cursor.execute(group_query) group_info=cursor.fetchall() model_dict={} for g in group_info: #Get positive examples for selected images sql_s='select distinct(path),viewpoint from backup_geocars.edmund_examples,backup_geocars.positive_examples where backup_geocars.edmund_examples.group_id=%d and backup_geocars.edmund_examples.group_id=backup_geocars.positive_examples.group_id and path not like "%%flipped%%" order by rand()'%(g[1]) cursor.execute(sql_s) paths=cursor.fetchall() real_paths=get_images(paths) group_dict={} model_name,trim_name=get_names(g[0],make,submodel) group_dict['images']=real_paths group_dict['group_id']=g[1] group_dict['trim_name']=trim_name if model_name in model_dict: model_dict[model_name].append(group_dict) else: model_dict[model_name]=[group_dict] models=make_template_dict(model_dict,'model_name','trims') img=request.session.get('image',None) bbox=request.session.get('bbox',None) db.close() return render_to_response('streetview/models_trims.html', {'models':models,'make':make,'submodel':submodel,'image':img,'bbox':bbox})
def exact_match(request,g_id,usr): if usr is None or usr=='' or usr=='/': bbox_ids=Bboxes.objects.filter(group_id=g_id).filter(cur_state__box_done=1).values('bbox_id') else: bbox_ids=Bboxes.objects.filter(group_id=g_id).filter(cur_state__box_done=1).filter(cur_state__user=usr).values('bbox_id') ims=[] for box in bbox_ids: b=Cur_state.objects.filter(bbox_id=box['bbox_id']).get() img,bbox=get_im_bbox(b) im ={} im['img']=img im['bbox']=bbox ims.append(im) db = connect_to_db('backup_geocars') cursor = db.cursor() examples=get_images_for_group(int(g_id.encode('ascii','ignore')),cursor) db.close() group_name=GroupNames.objects.filter(group_id=g_id).values('group_name').get() return render_to_response('streetview/show_exact.html',{'images':ims,'examples':examples,'group_name':group_name,'group_id':g_id,'username':usr})
thread_name = self.name while True: task = self.queue.get() insert_queries(task[0], task[1], self.cursor) queue.task_done() def insert_queries(tuples, chunk, cursor): print 'executing chunk %d' % chunk cursor.executemany( """update detected_cars set warped_im_name="%s" where im_name=%s""", tuples) if __name__ == "__main__": db = connect_to_db('all_cars') cursor = db.cursor() sql_s = 'select distinct im_name from detected_cars' print sql_s cursor.execute(sql_s) ims = cursor.fetchall() CHUNKSIZE = 10000 NUMTHREADS = 10 queue = Queue.Queue() for i in range(NUMTHREADS): thread = sqlThread(queue) thread.daemon = True thread.start()
if data is None: return 1 store_in_db(fips, data, vars_list, cursor) return 0 def get_fips(cursor, table, col): print 'Getting FIPS' sql_s = 'select distinct(%s) from demo.%s where %s<>0' % (col, table, col) cursor.execute(sql_s) fips = cursor.fetchall() return fips if __name__ == "__main__": db = connect_to_db('backup_geocars') cursor = db.cursor() fips = get_fips(cursor, 'latlong_fpis', 'fpis') stored_fips = get_fips(cursor, 'census', 'fips') unstored_fips = set(fips) - set(stored_fips) print len(stored_fips), len(unstored_fips), len(fips) queue = Queue.Queue() NUM_THREADS = 100 i = 0 for f in unstored_fips: i += 1 f = str(f[0]) queue.put(f) start = time.time() for i in range(NUM_THREADS):
num_to_try=10 while not connected: try: html=br.DownloadURL(link) connected = True # if line above fails, this is never executed except Exception as e: #catch all exceptions print 'Error in follow_link: %s trying again' %e tried += 1 if tried > num_to_try: print 'cannot download from link: %s' %link return return html if __name__=="__main__": #Load latitude and longitudes db=connect_to_db('') cursor=db.cursor() lat_longs=get_lat_longs(cursor) url_base = "http://maps.googleapis.com/maps/api/geocode/json?latlng=" queue = Queue.Queue() num_threads=100 #Pull zip codes and city names for all lat/longs num_tuples=len(lat_longs) count = 0 for tup in lat_longs: count += 1 print 'adding %s out of %s...'%(count,num_tuples) queue.put(tup) start= time.time()
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db db = connect_to_db('geocars') cursor = db.cursor() price_quantile = open('price_quantile.txt', 'rb').readlines() fine_price_quantile = open('fine_price_quantile.txt', 'rb').readlines() for p in price_quantile: sql_s = 'update car_metadata set price_bracket=%s where price=%s' % ( p.split(',')[1], p.split(',')[0]) cursor.execute(sql_s) print sql_s for p in fine_price_quantile: sql_s = 'update car_metadata set fine_price_bracket=%s where price=%s' % ( p.split(',')[1], p.split(',')[0]) print sql_s cursor.execute(sql_s) db.close()
import sys import os sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db if __name__=="__main__": db_name='all_cars' db = connect_to_db(db_name) cursor = db.cursor() cursor.execute('''show tables from all_cars''') tables=cursor.fetchall() for t in tables: sql_s='alter table %s modify group_id int(4)'%(t[0]) print sql_s cursor.execute(sql_s)
self.db=connect_to_db('all_cars') self.cursor=self.db.cursor() def run(self): thread_name=self.name while True: task = self.queue.get() insert_queries(task[0],task[1],self.cursor) queue.task_done() def insert_queries(tuples,chunk,cursor): print 'executing chunk %d'%chunk cursor.executemany("""update detected_cars set warped_im_name="%s" where im_name=%s""",tuples) if __name__=="__main__": db=connect_to_db('all_cars') cursor=db.cursor() sql_s='select distinct im_name from detected_cars' print sql_s cursor.execute(sql_s) ims=cursor.fetchall() CHUNKSIZE=10000 NUMTHREADS=10 queue=Queue.Queue() for i in range(NUMTHREADS): thread = sqlThread(queue) thread.daemon = True thread.start()
while not connected: try: html = br.DownloadURL(link) connected = True # if line above fails, this is never executed except Exception as e: #catch all exceptions print 'Error in follow_link: %s trying again' % e tried += 1 if tried > num_to_try: print 'cannot download from link: %s' % link return return html if __name__ == "__main__": #Load latitude and longitudes db = connect_to_db('') cursor = db.cursor() lat_longs = get_lat_longs(cursor) url_base = "http://maps.googleapis.com/maps/api/geocode/json?latlng=" queue = Queue.Queue() num_threads = 100 #Pull zip codes and city names for all lat/longs num_tuples = len(lat_longs) count = 0 for tup in lat_longs: count += 1 print 'adding %s out of %s...' % (count, num_tuples) queue.put(tup) start = time.time()
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db db = connect_to_db('demo') cursor = db.cursor() sql_s = 'show tables from demo like "%SF1%"' cursor.execute(sql_s) tables = cursor.fetchall() for t in tables: sql_s = 'drop table %s' % t[0] print sql_s cursor.execute(sql_s)
num_to_try=10 while not connected: try: html=br.DownloadURL(link) connected = True # if line above fails, this is never executed except Exception as e: #catch all exceptions print 'Error in follow_link: %s trying again' %e tried += 1 if tried > num_to_try: print 'cannot download from link: %s' %link return return html if __name__=="__main__": #Load latitude and longitudes db=connect_to_db('backup_geocars') cursor=db.cursor() lat_longs=get_lat_longs(cursor) url_base = "http://data.fcc.gov/api/block/2010/find?" queue = Queue.Queue() out_queue = Queue.Queue() num_threads=1000 #Pull fips codes for all lat/longs num_tuples=len(lat_longs) count = 0 for tup in lat_longs: count += 1 print 'adding %s out of %s...'%(count,num_tuples) queue.put(tup)
def get_lat_lng_list(cityid, exp, request): ll_list = [] if exp == True: #False: db = connect_to_db('geo') zip = 10011 cursor = db.cursor() cityid = 175 sqls = 'select lat,lng,count(distinct(im_date)) from fixed_timelapse_times where cityid=%d and downloaded=1 and small=0 and corrupt=0 group by lat,lng' % ( cityid) #sqls='select distinct t.lat,t.lng, count(im_date) from geo.timelapse_times t, demo.latlong_fpis l where zipcode=%d and l.lat=t.lat and l.lng=t.lng group by lat,lng'%zip cursor.execute(sqls) lat_lng_date = cursor.fetchall() NUM_SAMPLES = 1000 lat_lng_date = random.sample(lat_lng_date, NUM_SAMPLES) lld_dict = {} for l in lat_lng_date: ll_dict = {} #need dict to traverse in view ll_dict['gps'] = '%s_%s' % (str(l[0]), str(l[1])) ll_dict['numdates'] = int(l[2]) ll_list.append(ll_dict) sqls = 'select distinct(im_date) from fixed_timelapse_times where lat=%s and lng=%s' % ( str(l[0]), str(l[1])) cursor.execute(sqls) dates = cursor.fetchone() lld_dict['%s_%s' % (str(l[0]), str(l[1]))] = [str(d) for d in dates] db.close() request.session['lld_dict'] = lld_dict else: #Different file for training/test vs all timelapse #f=open('/imagenetdb/tgebru/scrape/lat_lng_rot_url.txt','rb') #Validation set for 2013 housing data #f=open('/imagenetdb3/tgebru/cvpr2016/housing_data/train_test_split_2013/housing_2013_class_val.txt','rb') #Only loaded images #f=open('/afs/cs.stanford.edu/u/tgebru/cvpr2016/loaded_lat_lng_rot_url.txt', 'rb') NUM_SAMPLES = 1000 #lines=f.readlines() #lines=random.sample(lines,UM_SAMPLES) #f.close() lld_dict = {} #Load lat,lng,dates dict with open( '/afs/cs.stanford.edu/u/tgebru/cvpr2016/ipython_code/kings_lat_lng_date_dict.pickle', 'rb') as f: lat_lng_date_dict = pickle.load(f) ''' for l in lines: #uncomment for all timelapse parts=l.split('\t') lat=parts[0].split('_')[0].strip() lng=parts[0].split('_')[1].strip() date=parts[-1].split('_')[-1][0:-5].strip() #Uncomment for 2013 housing data #lat=l.split('/')[-1].split('_')[0].strip() #lng=l.split('/')[-1].split('_')[1].strip() #date=l.split('/')[-1].split('_')[-1].split(' ')[0][0:-4].strip() #Different file for training/test vs all timelapse lat_lng='%s_%s'%(lat,lng) if lat_lng in lld_dict.keys(): lld_dict[lat_lng].append(date) else: lld_dict[lat_lng]=[date] ''' #Now create a list to send to the view #for k in lld_dict.keys(): keys = random.sample(lat_lng_date_dict.keys(), NUM_SAMPLES) for k in keys: ll_dict = {} ll_dict['gps'] = k dates = list(set(lat_lng_date_dict[k].keys())) numdates = len(dates) ll_dict['numdates'] = numdates lld_dict[k] = dates #Only want to see images with multiple years if numdates > 1: ll_list.append(ll_dict) request.session['lld_dict'] = lld_dict return ll_list
def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue self.db=connect_to_db('all_cars') self.cursor=self.db.cursor()
def get_lat_lng_list(cityid,exp,request): ll_list=[] if exp== True: #False: db = connect_to_db('geo') zip=10011 cursor=db.cursor() cityid=175 sqls='select lat,lng,count(distinct(im_date)) from fixed_timelapse_times where cityid=%d and downloaded=1 and small=0 and corrupt=0 group by lat,lng'%(cityid); #sqls='select distinct t.lat,t.lng, count(im_date) from geo.timelapse_times t, demo.latlong_fpis l where zipcode=%d and l.lat=t.lat and l.lng=t.lng group by lat,lng'%zip cursor.execute(sqls) lat_lng_date=cursor.fetchall() NUM_SAMPLES=1000 lat_lng_date=random.sample(lat_lng_date,NUM_SAMPLES) lld_dict={} for l in lat_lng_date: ll_dict={} #need dict to traverse in view ll_dict['gps']='%s_%s'%(str(l[0]),str(l[1])) ll_dict['numdates']=int(l[2]) ll_list.append(ll_dict) sqls='select distinct(im_date) from fixed_timelapse_times where lat=%s and lng=%s'%(str(l[0]),str(l[1])) cursor.execute(sqls) dates=cursor.fetchone() lld_dict['%s_%s'%(str(l[0]),str(l[1]))]=[str(d) for d in dates] db.close() request.session['lld_dict']=lld_dict else: #Different file for training/test vs all timelapse #f=open('/imagenetdb/tgebru/scrape/lat_lng_rot_url.txt','rb') #Validation set for 2013 housing data #f=open('/imagenetdb3/tgebru/cvpr2016/housing_data/train_test_split_2013/housing_2013_class_val.txt','rb') #Only loaded images #f=open('/afs/cs.stanford.edu/u/tgebru/cvpr2016/loaded_lat_lng_rot_url.txt', 'rb') NUM_SAMPLES=1000 #lines=f.readlines() #lines=random.sample(lines,UM_SAMPLES) #f.close() lld_dict={} #Load lat,lng,dates dict with open('/afs/cs.stanford.edu/u/tgebru/cvpr2016/ipython_code/kings_lat_lng_date_dict.pickle','rb') as f: lat_lng_date_dict=pickle.load(f) ''' for l in lines: #uncomment for all timelapse parts=l.split('\t') lat=parts[0].split('_')[0].strip() lng=parts[0].split('_')[1].strip() date=parts[-1].split('_')[-1][0:-5].strip() #Uncomment for 2013 housing data #lat=l.split('/')[-1].split('_')[0].strip() #lng=l.split('/')[-1].split('_')[1].strip() #date=l.split('/')[-1].split('_')[-1].split(' ')[0][0:-4].strip() #Different file for training/test vs all timelapse lat_lng='%s_%s'%(lat,lng) if lat_lng in lld_dict.keys(): lld_dict[lat_lng].append(date) else: lld_dict[lat_lng]=[date] ''' #Now create a list to send to the view #for k in lld_dict.keys(): keys=random.sample(lat_lng_date_dict.keys(),NUM_SAMPLES) for k in keys: ll_dict={} ll_dict['gps']=k dates=list(set(lat_lng_date_dict[k].keys())) numdates=len(dates) ll_dict['numdates']=numdates lld_dict[k]=dates #Only want to see images with multiple years if numdates > 1: ll_list.append(ll_dict) request.session['lld_dict']=lld_dict return ll_list
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db db=connect_to_db('demo') cursor=db.cursor() sql_s='show tables from demo like "%SF1%"' cursor.execute(sql_s) tables=cursor.fetchall() for t in tables: sql_s='drop table %s'%t[0] print sql_s cursor.execute(sql_s)
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db countries=open('/tmp/makes.csv','rb').readlines() db=connect_to_db('geocars') cursor=db.cursor() for c in countries: parts=c.split(',') sql='select group_id from synsets where make="%s" and group_id is not null'%parts[0].strip() print sql cursor.execute(sql) group_ids=cursor.fetchall() for g in group_ids: print parts[0],g sql='update car_metadata set country="%s",is_foreign=%s where group_id=%s'%(parts[1].strip(),parts[2].strip(),g[0]) print sql cursor.execute(sql) db.close()
def __init__(self, queue): threading.Thread.__init__(self) self.queue = queue self.db = connect_to_db('all_cars') self.cursor = self.db.cursor()
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db db = connect_to_db('boston_cars') cursor = db.cursor() import pickle #Number of All images sql_s = 'select count(distinct(im_name)) from ma_detected_cars' print sql_s cursor.execute(sql_s) num_ims = cursor.fetchall() print 'number of sampled images in MA....' print num_ims #Number of images per zipcode zipcodes_sql = 'select distinct(zipcode),count(distinct(im_name))c from ma_detected_cars m,demo.latlong_fpis l where m.lat=l.lat and m.lng=l.lng group by zipcode order by c' print zipcodes_sql cursor.execute(zipcodes_sql) zipcodes_num_ims = cursor.fetchall() zip_dict = {} for z in zipcodes_num_ims: zip_dict[z[0]] = z[1] print(zipcodes_num_ims) #Number of all ground truth images in the 3 cities gt_sql = 'select distinct(zip_code),sum(veh_tot)s from grid250m_attributes a,grid_quarters_public g where a.g250m_id=g.g250m_id and quarter="2010_q2" and (muni_id=35 or muni_id=281 or muni_id=348) group by zip_code' print gt_sql cursor.execute(gt_sql)
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db import scipy.io import numpy import os if __name__ == "__main__": census_or_acs = sys.argv[1] #'acs' LEVEL = sys.argv[2] #'zipcode' train_or_val = sys.argv[3] db_name = 'all_cars' db = connect_to_db(db_name) cursor = db.cursor() root_dir = '/imagenetdb3/mysql_tmp_dir/car_census' census_var_name_f = '%s_var_names.txt' % census_or_acs census_variables_f = '%s_%s_variables.txt' % (train_or_val, census_or_acs) car_meta_name_f = 'car_meta_names.txt' car_attributes_f = '%s_car_attributes.txt' % (train_or_val) image_names_f = '%s_image_names.txt' % (train_or_val) census_var_file_name = '%s_variables.txt' % census_or_acs census_vars = open(census_var_file_name).readlines() census_variables = [] i = 0 for c in census_vars: census_variables.insert(i, c.split(',')[0].strip()) i += 1
import sys sys.path.append('/imagenetdb/tgebru/') from mysql_utils import connect_to_db db=connect_to_db('boston_cars') cursor=db.cursor() import pickle #Number of All images sql_s='select count(distinct(im_name)) from ma_detected_cars' print sql_s cursor.execute(sql_s) num_ims=cursor.fetchall() print 'number of sampled images in MA....' print num_ims #Number of images per zipcode zipcodes_sql='select distinct(zipcode),count(distinct(im_name))c from ma_detected_cars m,demo.latlong_fpis l where m.lat=l.lat and m.lng=l.lng group by zipcode order by c' print zipcodes_sql cursor.execute(zipcodes_sql) zipcodes_num_ims=cursor.fetchall() zip_dict={} for z in zipcodes_num_ims: zip_dict[z[0]]=z[1] print(zipcodes_num_ims) #Number of all ground truth images in the 3 cities gt_sql='select distinct(zip_code),sum(veh_tot)s from grid250m_attributes a,grid_quarters_public g where a.g250m_id=g.g250m_id and quarter="2010_q2" and (muni_id=35 or muni_id=281 or muni_id=348) group by zip_code' print gt_sql cursor.execute(gt_sql)