def ExtractSift(ID): print ID info = Info.GetVideoInfo(ID) frame_path = info['frame_path'] pano_download_path = info['pano_download_path'] pano_path = info['pano_path'] frame_sift_path = info['frame_sift_path'] pano_sift_path = info['pano_sift_path'] frame_lst_name = frame_path + '/list.txt' pano_lst_name = pano_download_path + '/list.txt' frame_sift_lst_name = pano_path + '/frame_sift_lst.txt' pano_sift_lst_name = pano_path + '/pano_sift_lst.txt' WriteImageList(frame_path, frame_lst_name) WriteImageList(pano_download_path, pano_lst_name) command_1 = 'VisualSFM siftgpu %s' % frame_lst_name command_2 = 'VisualSFM siftgpu %s' % pano_lst_name subprocess.call(command_1, shell=True) subprocess.call(command_2, shell=True) command_1 = 'mv %s/*.sift %s' % (frame_path, frame_sift_path) command_2 = 'mv %s/*.sift %s' % (pano_download_path, pano_sift_path) subprocess.call(command_1, shell=True) subprocess.call(command_2, shell=True) GetSiftList(frame_sift_path, frame_sift_lst_name) GetSiftList(pano_sift_path, pano_sift_lst_name) os.remove(frame_lst_name) os.remove(pano_lst_name)
def DownloadPano(ID): info = Info.GetVideoInfo(ID) print ID circle = PanoProcess.GetCircleBound.GetCircleBound(info['location']) #print circle id_lst = [] start_time = time.time() for loc in circle: #print loc try: panoid = PanoProcess.GetPanoID(loc) except: continue if panoid != None and not panoid in id_lst: id_lst.append(panoid) #a = time.time() PanoProcess.GetPanoByID.GetPanoByID(panoid, info['pano_download_path']) #print time.time() - a pano_lst = [x for x in sorted(os.listdir(info['pano_download_path'])) if x.endswith('.jpg')] f_name = '%s/pano_lst.txt'%info['pano_path'] f = open(f_name, 'w') for img_name in pano_lst: s = img_name + '\n' f.write(s) f.close() end = time.time() print '%s use %f minutes and total %d images'%(ID, (end-start_time)/60, len(id_lst))
def MatchFunM(ID): print ID info = Info.GetVideoInfo(ID) frame_sift_lst = [x for x in sorted(os.listdir(info['frame_sift_path'])) if x.endswith('.sift')] pano_sift_lst = [x for x in sorted(os.listdir(info['pano_sift_path'])) if x.endswith('.sift')] results = np.load('%s/fisher_results.npy'%info['pano_path']) MM = [] for index, name in enumerate(frame_sift_lst): Mi = [] frame_short_name = name.split('.')[0] for i in range(0,results.shape[1]): pano_name = pano_sift_lst[results[index, i]] pano_short_name = pano_name.split('.')[0] kp_pairs = lib_SIFTmatch.flann_match('%s/%s'%(info['frame_sift_path'],frame_short_name), '%s/%s'%(info['pano_sift_path'],pano_short_name)) #print kp_pairs try: (mkp1, mkp2) = zip(*kp_pairs) mkp1_pts = [ (x[0],x[1]) for x in mkp1 ] mkp2_pts = [ (x[0],x[1]) for x in mkp2 ] mkp1_pts = np.float32(mkp1_pts) mkp2_pts = np.float32(mkp2_pts) F, mask = cv2.findFundamentalMat(mkp1_pts,mkp2_pts,cv2.FM_RANSAC,20) q_pts = mkp1_pts[mask.ravel()==1] t_pts = mkp2_pts[mask.ravel()==1] Mi.append(len(q_pts)) except: Mi.append(0) continue MM.append(Mi) np.save('%s/results_fundM'%info['pano_path'],MM)
def GetVideoTime(video_info): ID = video_info[0][0] CutFrame = video_info[0][1] Name = video_info[1] print ID, CutFrame info = Info.GetVideoInfo(ID) f_name = Info.Config.VIDEO_SOURCE_FOLDER + '/%d.mp4' % int(ID) video = mve.VideoFileClip(f_name) end_second = (CutFrame - 1) / video.fps start_second = end_second - 25 frame_lst = [ x for x in sorted(os.listdir(info['frame_path'])) if x.endswith('.jpg') ] if start_second >= 0: f = open('%s/timeline.json' % info['match_path'], 'w') t = start_second inter = 0.2 # 0.25 s d = {} d['time'] = {} for i in frame_lst: #print i d['time'][i] = t t += inter d['time'] = OrderedDict(sorted(d['time'].items())) d['id'] = Name f.write(json.dumps(d, indent=4) + '\n') f.close() return False else: return True
def RunRANSAC(ID): print ID info = Info.GetVideoInfo(ID) subprocess.call('rm %s' % (Info.GetGCPFileName(info)), shell=True) [frame_lst, reconstruct_set, latlon_set] = GetPointData(info) if len(reconstruct_set) == 0 or len(latlon_set) == 0: return #try: [M, model] = RANSAC_2D(reconstruct_set, latlon_set, iteration=1000, tolerance=Info.Config.STEP_TEN_METER) print M #except: # return if not M is None and not model is None: reconstruct = Info.ReadReconstructionData(info)['shots'] f = open(Info.GetGCPFileName(info), 'w') frame_lst = sorted(reconstruct.keys()) for index, frame in enumerate(frame_lst): [x, y, z] = optical_center(reconstruct[frame]) point = np.dot(M, np.transpose(np.array([x, y, 1], dtype=np.float32))) s = '%s\t%f\t%f\n' % (frame, point[0], point[1]) f.write(s) f.close()
def Merge(ID): print ID info = Info.GetVideoInfo(ID) f_name = info['match_path'] + '/' + name f = open(f_name, 'r') data = json.load(f) f.close() key_lst = sorted(data.keys()) pano_lst = [] for key in key_lst: loc = data[key] print key try: panoid = GSV.getIDbyloc(lat=loc[0], lon=loc[1]) except: data.pop(key) continue #print panoid if panoid is None: data.pop(key) continue elif not panoid in pano_lst: pano_lst.append(panoid) try: loc = GSV.getLocationbyID(panoid) data[key] = [float(loc[0]), float(loc[1])] except: data.pop(key) continue else: data.pop(key) f = open(info['match_path'] + '/google_info.json', 'w') f.write(json.dumps(data, indent=4)) f.close()
def CheckState(ID): info = Info.GetVideoInfo(ID) rigid_f = info['deep_match_path'] + '/rigid_distance_result.json' affine_f = info['deep_match_path'] + '/affine_distance_result.json' ourmethod_f = info['deep_match_path'] + '/distance_result.json' func = os.path.isfile if func(rigid_f) and func(affine_f) and func(ourmethod_f): return True return False
def RansacPointExtract(ID): info = Info.GetVideoInfo(ID) f_name = Info.GetTrackFileName(info) f = open(f_name, 'r') obj, graph_feature_data, graph_track_data = dataset.load_tracks_graph(f) ##### #print graph_track_data f.close() track_nodes, image_nodes = nx.algorithms.bipartite.sets(obj) #### buf = Info.ReadReconstructionData(info) image_lst = buf['shots'].keys() ##### reconstruct_data = buf['points'] #### reconstruct_data_key_lst = reconstruct_data.keys() ##### match_data = np.load(Info.GetMatchResultFileName(info)).item() #print match_data.keys() f = open(Info.GetMatchResultPointFileName(info), 'r') pano_data = json.load(f) f.close() #print pano_data.keys() new_pano_data = {} for index, key in enumerate(pano_data): loc_key = pano_data[key].keys()[0] pano_key = match_data[key].keys()[0] tmp = np.reshape(pano_data[key][loc_key], (-1,3)) try: new_pano_data[key][pano_key] = tmp except: new_pano_data[key] = {} new_pano_data[key][pano_key] = tmp recontruct_point = [] pano_point = [] for index, key in enumerate(match_data): #key_tmp = key + '.jpg' key_tmp = key if key_tmp in image_lst: pano_key = match_data[key].keys()[0] for index, image_index in enumerate(match_data[key][pano_key][0]): if image_index in graph_feature_data[key_tmp]: track_id = graph_track_data[key_tmp][str(image_index)] if track_id in reconstruct_data_key_lst: point1 = reconstruct_data[track_id]['coordinates'] point2 = new_pano_data[key_tmp][pano_key][index, :] if point2[np.abs(point2) > 10000].size == 0: recontruct_point.append(point1) pano_point.append(point2) recontruct_point = np.array(recontruct_point, dtype=np.float32) #length = recontruct_point.shape[0] #recontruct_point = np.array([recontruct_point[:,0], recontruct_point[:,1], recontruct_point[:,2], np.ones([length])], dtype=np.float32) pano_point = np.array(pano_point, dtype = np.float32) #pano_point = np.array([pano_point[:,0], pano_point[:,1], pano_point[:,2], np.ones([length])], dtype=np.float32) np.save(Info.GetMatchResultExtractPointFileName(info), [recontruct_point, pano_point]) print recontruct_point.shape print pano_point.shape print '%s extract point finish'%ID
def CheckState(ID): info = Info.GetVideoInfo(ID) deep_f = info['deep_match_path'] + '/distance_result.json' hog_f = info['hog_match_path'] + '/distance_result.json' dsift_f = info['dsift_match_path'] + '/distance_result.json' func = os.path.isfile if func(deep_f) and func(hog_f) and func(dsift_f): return True return False
def GetKnn(ID): print ID info = Info.GetVideoInfo(ID) frame_sift_lst = [ x for x in sorted(os.listdir(info['frame_sift_path'])) if x.endswith('.sift') ] pano_sift_lst = [ x for x in sorted(os.listdir(info['pano_sift_path'])) if x.endswith('.sift') ] #print pano_sift_lst frame_desc = [] pano_desc = [] for one in frame_sift_lst: f_name = info['frame_sift_path'] + '/' + one desc = ReadSift.ReadSift(f_name)[1] if desc.size == 0: desc = np.zeros((0, 128), dtype='uint8') frame_desc.append(desc) for one in pano_sift_lst: f_name = info['pano_sift_path'] + '/' + one desc = ReadSift.ReadSift(f_name)[1] if desc.size == 0: desc = np.zeros((0, 128), dtype='uint8') pano_desc.append(desc) data = np.load(Info.Config.ROOT_PATH + '/gmm_2step.npz') gmm = (data['a'], data['b'], data['c']) mean = data['mean'] pca_transform = data['pca_transform'] image_fvs = [] for image_dec in (frame_desc + pano_desc): image_dec = np.dot(image_dec - mean, pca_transform) fv = ynumpy.fisher(gmm, image_dec, include='mu') image_fvs.append(fv) image_fvs = np.vstack(image_fvs) image_fvs = np.sign(image_fvs) * np.abs(image_fvs)**0.5 norms = np.sqrt(np.sum(image_fvs**2, 1)) image_fvs /= norms.reshape(-1, 1) image_fvs[np.isnan(image_fvs)] = 100 frame_fvs = image_fvs[0:len(frame_sift_lst)] pano_fvs = image_fvs[len(frame_sift_lst):] results, distances = ynumpy.knn(frame_fvs, pano_fvs, nnn=10) #print results #print distances np.save(info['pano_path'] + '/fisher_results', results)
def ExtractFrame(video_info): ID = video_info[0] info = Info.GetVideoInfo(ID) CutFrame = video_info[1] print ID, CutFrame f_name = Info.Config.VIDEO_SOURCE_FOLDER + '/%d.mp4' % int(ID) video = mve.VideoFileClip(f_name) end_second = (CutFrame - 1) / video.fps start_second = end_second - 25 command = 'ffmpeg -i %s -r 5 -qscale:v 1 -ss %f -to %f %s/' % ( f_name, start_second, end_second, info['frame_path']) command += 'image-%5d.jpg' print command subprocess.call(command, shell=True)
def Merge(ID): info = Info.GetVideoInfo(ID) f1_name = Info.Get3DRansacFileName(info) subprocess.call('cp %s %s/ransac_3D_result_old.json' % (f1_name, info['match_path']), shell=True) f2_name = info['match_path'] + '/timeline.json' f1 = open(f1_name, 'r') f2 = open(f2_name, 'r') data = json.load(f1) tim = json.load(f2) data['id'] = tim['id'] data['time'] = tim['time'] f1.close() f2.close() f1 = open(f1_name, 'w') f1.write(json.dumps(data, indent=4)) f1.close()
def Label(ID): info = Info.GetVideoInfo(ID) frame_lst = [ x for x in sorted(os.listdir(info['frame_path'])) if x.endswith ] i = 0 f = open(Info.GetMatchLstFileName(info), 'w') while True: f_name = info['frame_path'] + '/' + frame_lst[i] command = 'eog %s' % f_name subprocess.call(command, shell=True) panoid = raw_input('Pano ID : ') loc = GSV.getLocationbyID(panoid) print loc s = '%s\t%s\t%s\t%s\n' % (frame_lst[i], panoid, loc[0], loc[1]) f.write(s) i += 10 if i >= 125: f.close() return f.close()
def GetResultLst(ID): print ID info = Info.GetVideoInfo(ID) frame_sift_lst = [ x for x in sorted(os.listdir(info['frame_sift_path'])) if x.endswith('.sift') ] pano_sift_lst = [ x for x in sorted(os.listdir(info['pano_sift_path'])) if x.endswith('.sift') ] fisher_result = np.load(Info.GetFisherResultFileName(info)) match_score = np.load(Info.GetMatchFunMFileName(info)) #print match_score f = open(Info.GetMatchLstFileName(info), 'w') for frame_index, frame_sift_name in enumerate(frame_sift_lst): arg_sort_index = np.argsort(match_score[frame_index, :]) highest_index = arg_sort_index[-1] highest_score = match_score[frame_index, highest_index] second_index = arg_sort_index[-2] second_score = match_score[frame_index, second_index] ratio = float(highest_score) / float(second_score) if highest_score >= THRESHOLD: frame_name = frame_sift_name.split('.')[0] + '.jpg' pano_sift_name = pano_sift_lst[fisher_result[frame_index, highest_index]] pano_name = pano_sift_name.split('.')[0] + '.jpg' pano_id = pano_name[5:27] print pano_id loc = GoogleSV.getLocationbyID(pano_id) if loc is None: continue s = '%s\t%s\t%s\t%s\t%d\n' % (frame_name, pano_name, loc[0], loc[1], highest_score) f.write(s) f.close()
def RunRansac(ID): print ID info = Info.GetVideoInfo(ID) data = np.load(Info.GetMatchResultExtractPointFileName(info)) #data2 = data[1] #data[0] = data[0][np.abs(data[1]) <= 3000] #data[1] = data[1][np.abs(data[1]) <= 3000] #data2 = data[1i] try: M, align_data = RANSAC.RANSAC_affine(data[0], data[1], 10000, 5) except: return if M is None: return print M #print np.reshape(M, [12]) try: M = np.reshape(M, [12]).astype(float) except: return gcp = Info.ReadGCPData(info) data = Info.ReadReconstructionData(info) trajectory = {} for shot in data['shots']: [x, y, z] = optical_center(data['shots'][shot]) trajectory[shot] = [x, y, z] data['gcp'] = gcp #print gcp data['transformation'] = list(M) #print type(M[0]) data['trajectory'] = trajectory #print trajectory f_name = Info.Get3DRansacFileName(info) f = open(f_name, 'w') f.write(json.dumps(data, indent=4)) f.close() print '%s ransac finish' % ID
def CopyConfig(ID): info = Info.GetVideoInfo(ID) command = 'cp config.yaml %s' % info['video_path'] subprocess.call(command, shell=True)
def RunExtract(ID): print ID info = Info.GetVideoInfo(ID) RPE.RansacPointExtract(ID)
def Plot(): lst = Info.GetStateList(['match_result'], ['yes']) do_lst = [] for one in lst: if CheckState(one): do_lst.append(one) # Deep dis_data = [] for one in do_lst: info = Info.GetVideoInfo(one) f = GetDeepPath(info) data = LoadFile(f) for key in data: dis_data.append(data[key]) deep_dis_data = np.array(dis_data, dtype=np.float32) print 'Deep Max val : %f'%np.max(deep_dis_data) print 'Deep Min val : %f'%np.min(deep_dis_data) print 'Deep Mean val : %f'%np.mean(deep_dis_data) print 'Deep Standard Deviation : %f'%np.std(deep_dis_data) deep_max = int(np.max(dis_data)) nop = len(deep_dis_data) ##### # Hog dis_data = [] for one in do_lst: info = Info.GetVideoInfo(one) f = GetHogPath(info) data = LoadFile(f) for key in data: dis_data.append(data[key]) hog_dis_data = np.array(dis_data, dtype=np.float32) print 'Hog Max val : %f'%np.max(hog_dis_data) print 'Hog Min val : %f'%np.min(hog_dis_data) print 'Hog Mean val : %f'%np.mean(hog_dis_data) print 'Hog Standard Deviation : %f'%np.std(hog_dis_data) hog_max = int(np.max(dis_data)) ##### # Dsift dis_data = [] for one in do_lst: info = Info.GetVideoInfo(one) f = GetDsiftPath(info) data = LoadFile(f) for key in data: dis_data.append(data[key]) dsift_dis_data = np.array(dis_data, dtype=np.float32) print 'Dsift Max val : %f'%np.max(dsift_dis_data) print 'Dsift Min val : %f'%np.min(dsift_dis_data) print 'Dsift Mean val : %f'%np.mean(dsift_dis_data) print 'Dsift Standard Deviation : %f'%np.std(dsift_dis_data) dsift_max = int(np.max(dis_data)) ##### all_max = np.max([deep_max, hog_max, dsift_max]) top_bound = int(np.ceil(all_max)) x_axis = np.arange(0, top_bound+1) deep_hist = np.zeros([top_bound+1], dtype=np.float32) hog_hist = np.zeros([top_bound+1], dtype=np.float32) dsift_hist = np.zeros([top_bound+1], dtype=np.float32) for i in range(0, top_bound+1): deep_hist[i] = np.sum(deep_dis_data <= i) hog_hist[i] = np.sum(hog_dis_data <= i) dsift_hist[i] = np.sum(dsift_dis_data <= i) x_axis = x_axis[1:] deep_hist = deep_hist[1:] hog_hist = hog_hist[1:] dsift_hist = dsift_hist[1:] plt.subplot('111') plt.plot(x_axis, deep_hist/nop*100, 'r', color='r', linewidth='4', label = 'deep match') plt.plot(x_axis, hog_hist/nop*100, 'r', color='g', linewidth='4', label = 'hog match') plt.plot(x_axis, dsift_hist/nop*100, 'r', color='b', linewidth='4', label = 'dense sift match') plt.xscale('log') #plt.xscale('log') #plt.xlim([1]) plt.xlabel('within distance(m)') plt.ylabel('persent(%)') plt.legend(loc = 'right') plt.savefig('match.jpg') plt.show()
def RunReconstruct(ID): info = Info.GetVideoInfo(ID) command = '%s %s' % (Info.Config.OPENSFM_RECONSTRUCT, info['video_path']) subprocess.call(command, shell=True)
def Plot(): lst = Info.GetStateList(['match_result'], ['yes']) do_lst = [x for x in lst if CheckState(x)] #print len(do_lst) # our our_dis = [] for one in do_lst: info = Info.GetVideoInfo(one) f = GetOurPath(info) data = LoadFile(f) for key in data: our_dis.append(data[key]) number_of_point = len(our_dis) our_dis = np.array(our_dis, dtype=np.float32) print 'Ourmethod Max val : %f' % np.max(our_dis) print 'Ourmethod Min val : %f' % np.min(our_dis) print 'Ourmethod Mean val : %f' % np.mean(our_dis) print 'Ourmethod Standard Deviation : %f' % np.std(our_dis) top_bound = np.ceil(np.max(our_dis)) #print top_bound histogram = np.zeros([int(top_bound) + 1], dtype=np.float32) ceil_data = np.ceil(our_dis) for i in range(0, int(top_bound) + 1): count = np.sum(ceil_data <= i) histogram[i] = count #print histogram our_Y = histogram our_X = np.arange(0, top_bound + 1) # affine affine_dis = [] for one in do_lst: info = Info.GetVideoInfo(one) f = GetAffinePath(info) data = LoadFile(f) for key in data: affine_dis.append(data[key]) affine_dis = np.array(affine_dis, dtype=np.float32) print 'Affine Max val : %f' % np.max(affine_dis) print 'Affine Min val : %f' % np.min(affine_dis) print 'Affine Mean val : %f' % np.mean(affine_dis) print 'Affine Standard Deviation : %f' % np.std(affine_dis) top_bound = np.ceil(np.max(affine_dis)) #print top_bound histogram = np.zeros([int(top_bound) + 1], dtype=np.float32) ceil_data = np.ceil(affine_dis) for i in range(0, int(top_bound) + 1): count = np.sum(ceil_data <= i) histogram[i] = count #print histogram affine_Y = histogram affine_X = np.arange(0, top_bound + 1) # rigid rigid_dis = [] for one in do_lst: info = Info.GetVideoInfo(one) f = GetRigidPath(info) data = LoadFile(f) for key in data: rigid_dis.append(data[key]) rigid_dis = np.array(rigid_dis, dtype=np.float32) print 'Rigid Max val : %f' % np.max(rigid_dis) print 'Rigid Min val : %f' % np.min(rigid_dis) print 'Rigid Mean val : %f' % np.mean(rigid_dis) print 'Rigid Standard Deviation : %f' % np.std(rigid_dis) top_bound = np.ceil(np.max(rigid_dis)) #print top_bound histogram = np.zeros([int(top_bound) + 1], dtype=np.float32) ceil_data = np.ceil(rigid_dis) for i in range(0, int(top_bound) + 1): count = np.sum(ceil_data <= i) histogram[i] = count #print histogram rigid_Y = histogram rigid_X = np.arange(0, top_bound + 1) our_max = np.max(our_Y) affine_max = np.max(affine_Y) rigid_max = np.max(rigid_Y) all_max = np.max([our_max, affine_max, rigid_max]) top_bound = int(np.ceil(all_max)) X_axis = np.arange(0, top_bound + 1) tmp = np.zeros([top_bound + 1], dtype=np.float32) tmp[:] = our_max tmp[0:our_Y.size] = our_Y our_Y = tmp / number_of_point * 100 tmp = np.zeros([top_bound + 1], dtype=np.float32) tmp[:] = affine_max tmp[0:affine_Y.size] = affine_Y affine_Y = tmp / number_of_point * 100 tmp = np.zeros([top_bound + 1], dtype=np.float32) tmp[:] = rigid_max tmp[0:rigid_Y.size] = rigid_Y rigid_Y = tmp / number_of_point * 100 X_axis = X_axis[1:] our_Y = our_Y[1:] affine_Y = affine_Y[1:] rigid_Y = rigid_Y[1:] fig = plt.subplot('111') plt.plot(X_axis, our_Y, 'r', color='r', linewidth=4, label='ourmethod') plt.plot(X_axis, affine_Y, 'r', color='g', linewidth=4, label='affine') plt.plot(X_axis, rigid_Y, 'r', color='b', linewidth=4, label='rigid') plt.xscale('log') #plt.xlim([1, 300]) plt.xlabel('within distance(m)') plt.ylabel('persent(%)') plt.legend(loc='right') plt.savefig('ransac_transform.jpg') plt.show()
def RunMatch(ID): info = Info.GetVideoInfo(ID) command = '%s %s' % (Info.Config.OPENSFM_RUN_MATCH, info['video_path']) subprocess.call(command, shell=True)