def ExtractSift(ID):
print ID
info = Info.GetVideoInfo(ID)
frame_path = info['frame_path']
pano_download_path = info['pano_download_path']
pano_path = info['pano_path']
frame_sift_path = info['frame_sift_path']
pano_sift_path = info['pano_sift_path']
frame_lst_name = frame_path + '/list.txt'
pano_lst_name = pano_download_path + '/list.txt'
frame_sift_lst_name = pano_path + '/frame_sift_lst.txt'
pano_sift_lst_name = pano_path + '/pano_sift_lst.txt'
WriteImageList(frame_path, frame_lst_name)
WriteImageList(pano_download_path, pano_lst_name)
command_1 = 'VisualSFM siftgpu %s' % frame_lst_name
command_2 = 'VisualSFM siftgpu %s' % pano_lst_name
subprocess.call(command_1, shell=True)
subprocess.call(command_2, shell=True)
command_1 = 'mv %s/*.sift %s' % (frame_path, frame_sift_path)
command_2 = 'mv %s/*.sift %s' % (pano_download_path, pano_sift_path)
subprocess.call(command_1, shell=True)
subprocess.call(command_2, shell=True)
GetSiftList(frame_sift_path, frame_sift_lst_name)
GetSiftList(pano_sift_path, pano_sift_lst_name)
os.remove(frame_lst_name)
os.remove(pano_lst_name)
def DownloadPano(ID):
info = Info.GetVideoInfo(ID)
print ID
circle = PanoProcess.GetCircleBound.GetCircleBound(info['location'])
#print circle
id_lst = []
start_time = time.time()
for loc in circle:
#print loc
try:
panoid = PanoProcess.GetPanoID(loc)
except:
continue
if panoid != None and not panoid in id_lst:
id_lst.append(panoid)
#a = time.time()
PanoProcess.GetPanoByID.GetPanoByID(panoid, info['pano_download_path'])
#print time.time() - a
pano_lst = [x for x in sorted(os.listdir(info['pano_download_path'])) if x.endswith('.jpg')]
f_name = '%s/pano_lst.txt'%info['pano_path']
f = open(f_name, 'w')
for img_name in pano_lst:
s = img_name + '\n'
f.write(s)
f.close()
end = time.time()
print '%s use %f minutes and total %d images'%(ID, (end-start_time)/60, len(id_lst))
def MatchFunM(ID):
print ID
info = Info.GetVideoInfo(ID)
frame_sift_lst = [x for x in sorted(os.listdir(info['frame_sift_path'])) if x.endswith('.sift')]
pano_sift_lst = [x for x in sorted(os.listdir(info['pano_sift_path'])) if x.endswith('.sift')]
results = np.load('%s/fisher_results.npy'%info['pano_path'])
MM = []
for index, name in enumerate(frame_sift_lst):
Mi = []
frame_short_name = name.split('.')[0]
for i in range(0,results.shape[1]):
pano_name = pano_sift_lst[results[index, i]]
pano_short_name = pano_name.split('.')[0]
kp_pairs = lib_SIFTmatch.flann_match('%s/%s'%(info['frame_sift_path'],frame_short_name),
'%s/%s'%(info['pano_sift_path'],pano_short_name))
#print kp_pairs
try:
(mkp1, mkp2) = zip(*kp_pairs)
mkp1_pts = [ (x[0],x[1]) for x in mkp1 ]
mkp2_pts = [ (x[0],x[1]) for x in mkp2 ]
mkp1_pts = np.float32(mkp1_pts)
mkp2_pts = np.float32(mkp2_pts)
F, mask = cv2.findFundamentalMat(mkp1_pts,mkp2_pts,cv2.FM_RANSAC,20)
q_pts = mkp1_pts[mask.ravel()==1]
t_pts = mkp2_pts[mask.ravel()==1]
Mi.append(len(q_pts))
except:
Mi.append(0)
continue
MM.append(Mi)
np.save('%s/results_fundM'%info['pano_path'],MM)
def GetVideoTime(video_info):
ID = video_info[0][0]
CutFrame = video_info[0][1]
Name = video_info[1]
print ID, CutFrame
info = Info.GetVideoInfo(ID)
f_name = Info.Config.VIDEO_SOURCE_FOLDER + '/%d.mp4' % int(ID)
video = mve.VideoFileClip(f_name)
end_second = (CutFrame - 1) / video.fps
start_second = end_second - 25
frame_lst = [
x for x in sorted(os.listdir(info['frame_path'])) if x.endswith('.jpg')
]
if start_second >= 0:
f = open('%s/timeline.json' % info['match_path'], 'w')
t = start_second
inter = 0.2 # 0.25 s
d = {}
d['time'] = {}
for i in frame_lst:
#print i
d['time'][i] = t
t += inter
d['time'] = OrderedDict(sorted(d['time'].items()))
d['id'] = Name
f.write(json.dumps(d, indent=4) + '\n')
f.close()
return False
else:
return True
def RunRANSAC(ID):
print ID
info = Info.GetVideoInfo(ID)
subprocess.call('rm %s' % (Info.GetGCPFileName(info)), shell=True)
[frame_lst, reconstruct_set, latlon_set] = GetPointData(info)
if len(reconstruct_set) == 0 or len(latlon_set) == 0:
return
#try:
[M, model] = RANSAC_2D(reconstruct_set,
latlon_set,
iteration=1000,
tolerance=Info.Config.STEP_TEN_METER)
print M
#except:
# return
if not M is None and not model is None:
reconstruct = Info.ReadReconstructionData(info)['shots']
f = open(Info.GetGCPFileName(info), 'w')
frame_lst = sorted(reconstruct.keys())
for index, frame in enumerate(frame_lst):
[x, y, z] = optical_center(reconstruct[frame])
point = np.dot(M,
np.transpose(np.array([x, y, 1], dtype=np.float32)))
s = '%s\t%f\t%f\n' % (frame, point[0], point[1])
f.write(s)
f.close()
def Merge(ID):
print ID
info = Info.GetVideoInfo(ID)
f_name = info['match_path'] + '/' + name
f = open(f_name, 'r')
data = json.load(f)
f.close()
key_lst = sorted(data.keys())
pano_lst = []
for key in key_lst:
loc = data[key]
print key
try:
panoid = GSV.getIDbyloc(lat=loc[0], lon=loc[1])
except:
data.pop(key)
continue
#print panoid
if panoid is None:
data.pop(key)
continue
elif not panoid in pano_lst:
pano_lst.append(panoid)
try:
loc = GSV.getLocationbyID(panoid)
data[key] = [float(loc[0]), float(loc[1])]
except:
data.pop(key)
continue
else:
data.pop(key)
f = open(info['match_path'] + '/google_info.json', 'w')
f.write(json.dumps(data, indent=4))
f.close()
def CheckState(ID):
info = Info.GetVideoInfo(ID)
rigid_f = info['deep_match_path'] + '/rigid_distance_result.json'
affine_f = info['deep_match_path'] + '/affine_distance_result.json'
ourmethod_f = info['deep_match_path'] + '/distance_result.json'
func = os.path.isfile
if func(rigid_f) and func(affine_f) and func(ourmethod_f):
return True
return False
def RansacPointExtract(ID):
info = Info.GetVideoInfo(ID)
f_name = Info.GetTrackFileName(info)
f = open(f_name, 'r')
obj, graph_feature_data, graph_track_data = dataset.load_tracks_graph(f) #####
#print graph_track_data
f.close()
track_nodes, image_nodes = nx.algorithms.bipartite.sets(obj) ####
buf = Info.ReadReconstructionData(info)
image_lst = buf['shots'].keys() #####
reconstruct_data = buf['points'] ####
reconstruct_data_key_lst = reconstruct_data.keys() #####
match_data = np.load(Info.GetMatchResultFileName(info)).item()
#print match_data.keys()
f = open(Info.GetMatchResultPointFileName(info), 'r')
pano_data = json.load(f)
f.close()
#print pano_data.keys()
new_pano_data = {}
for index, key in enumerate(pano_data):
loc_key = pano_data[key].keys()[0]
pano_key = match_data[key].keys()[0]
tmp = np.reshape(pano_data[key][loc_key], (-1,3))
try:
new_pano_data[key][pano_key] = tmp
except:
new_pano_data[key] = {}
new_pano_data[key][pano_key] = tmp
recontruct_point = []
pano_point = []
for index, key in enumerate(match_data):
#key_tmp = key + '.jpg'
key_tmp = key
if key_tmp in image_lst:
pano_key = match_data[key].keys()[0]
for index, image_index in enumerate(match_data[key][pano_key][0]):
if image_index in graph_feature_data[key_tmp]:
track_id = graph_track_data[key_tmp][str(image_index)]
if track_id in reconstruct_data_key_lst:
point1 = reconstruct_data[track_id]['coordinates']
point2 = new_pano_data[key_tmp][pano_key][index, :]
if point2[np.abs(point2) > 10000].size == 0:
recontruct_point.append(point1)
pano_point.append(point2)
recontruct_point = np.array(recontruct_point, dtype=np.float32)
#length = recontruct_point.shape[0]
#recontruct_point = np.array([recontruct_point[:,0], recontruct_point[:,1], recontruct_point[:,2], np.ones([length])], dtype=np.float32)
pano_point = np.array(pano_point, dtype = np.float32)
#pano_point = np.array([pano_point[:,0], pano_point[:,1], pano_point[:,2], np.ones([length])], dtype=np.float32)
np.save(Info.GetMatchResultExtractPointFileName(info), [recontruct_point, pano_point])
print recontruct_point.shape
print pano_point.shape
print '%s extract point finish'%ID
def CheckState(ID):
info = Info.GetVideoInfo(ID)
deep_f = info['deep_match_path'] + '/distance_result.json'
hog_f = info['hog_match_path'] + '/distance_result.json'
dsift_f = info['dsift_match_path'] + '/distance_result.json'
func = os.path.isfile
if func(deep_f) and func(hog_f) and func(dsift_f):
return True
return False
def GetKnn(ID):
print ID
info = Info.GetVideoInfo(ID)
frame_sift_lst = [
x for x in sorted(os.listdir(info['frame_sift_path']))
if x.endswith('.sift')
]
pano_sift_lst = [
x for x in sorted(os.listdir(info['pano_sift_path']))
if x.endswith('.sift')
]
#print pano_sift_lst
frame_desc = []
pano_desc = []
for one in frame_sift_lst:
f_name = info['frame_sift_path'] + '/' + one
desc = ReadSift.ReadSift(f_name)[1]
if desc.size == 0:
desc = np.zeros((0, 128), dtype='uint8')
frame_desc.append(desc)
for one in pano_sift_lst:
f_name = info['pano_sift_path'] + '/' + one
desc = ReadSift.ReadSift(f_name)[1]
if desc.size == 0:
desc = np.zeros((0, 128), dtype='uint8')
pano_desc.append(desc)
data = np.load(Info.Config.ROOT_PATH + '/gmm_2step.npz')
gmm = (data['a'], data['b'], data['c'])
mean = data['mean']
pca_transform = data['pca_transform']
image_fvs = []
for image_dec in (frame_desc + pano_desc):
image_dec = np.dot(image_dec - mean, pca_transform)
fv = ynumpy.fisher(gmm, image_dec, include='mu')
image_fvs.append(fv)
image_fvs = np.vstack(image_fvs)
image_fvs = np.sign(image_fvs) * np.abs(image_fvs)**0.5
norms = np.sqrt(np.sum(image_fvs**2, 1))
image_fvs /= norms.reshape(-1, 1)
image_fvs[np.isnan(image_fvs)] = 100
frame_fvs = image_fvs[0:len(frame_sift_lst)]
pano_fvs = image_fvs[len(frame_sift_lst):]
results, distances = ynumpy.knn(frame_fvs, pano_fvs, nnn=10)
#print results
#print distances
np.save(info['pano_path'] + '/fisher_results', results)
def ExtractFrame(video_info):
ID = video_info[0]
info = Info.GetVideoInfo(ID)
CutFrame = video_info[1]
print ID, CutFrame
f_name = Info.Config.VIDEO_SOURCE_FOLDER + '/%d.mp4' % int(ID)
video = mve.VideoFileClip(f_name)
end_second = (CutFrame - 1) / video.fps
start_second = end_second - 25
command = 'ffmpeg -i %s -r 5 -qscale:v 1 -ss %f -to %f %s/' % (
f_name, start_second, end_second, info['frame_path'])
command += 'image-%5d.jpg'
print command
subprocess.call(command, shell=True)
def Merge(ID):
info = Info.GetVideoInfo(ID)
f1_name = Info.Get3DRansacFileName(info)
subprocess.call('cp %s %s/ransac_3D_result_old.json' %
(f1_name, info['match_path']),
shell=True)
f2_name = info['match_path'] + '/timeline.json'
f1 = open(f1_name, 'r')
f2 = open(f2_name, 'r')
data = json.load(f1)
tim = json.load(f2)
data['id'] = tim['id']
data['time'] = tim['time']
f1.close()
f2.close()
f1 = open(f1_name, 'w')
f1.write(json.dumps(data, indent=4))
f1.close()
def Label(ID):
info = Info.GetVideoInfo(ID)
frame_lst = [
x for x in sorted(os.listdir(info['frame_path'])) if x.endswith
]
i = 0
f = open(Info.GetMatchLstFileName(info), 'w')
while True:
f_name = info['frame_path'] + '/' + frame_lst[i]
command = 'eog %s' % f_name
subprocess.call(command, shell=True)
panoid = raw_input('Pano ID : ')
loc = GSV.getLocationbyID(panoid)
print loc
s = '%s\t%s\t%s\t%s\n' % (frame_lst[i], panoid, loc[0], loc[1])
f.write(s)
i += 10
if i >= 125:
f.close()
return
f.close()
def GetResultLst(ID):
print ID
info = Info.GetVideoInfo(ID)
frame_sift_lst = [
x for x in sorted(os.listdir(info['frame_sift_path']))
if x.endswith('.sift')
]
pano_sift_lst = [
x for x in sorted(os.listdir(info['pano_sift_path']))
if x.endswith('.sift')
]
fisher_result = np.load(Info.GetFisherResultFileName(info))
match_score = np.load(Info.GetMatchFunMFileName(info))
#print match_score
f = open(Info.GetMatchLstFileName(info), 'w')
for frame_index, frame_sift_name in enumerate(frame_sift_lst):
arg_sort_index = np.argsort(match_score[frame_index, :])
highest_index = arg_sort_index[-1]
highest_score = match_score[frame_index, highest_index]
second_index = arg_sort_index[-2]
second_score = match_score[frame_index, second_index]
ratio = float(highest_score) / float(second_score)
if highest_score >= THRESHOLD:
frame_name = frame_sift_name.split('.')[0] + '.jpg'
pano_sift_name = pano_sift_lst[fisher_result[frame_index,
highest_index]]
pano_name = pano_sift_name.split('.')[0] + '.jpg'
pano_id = pano_name[5:27]
print pano_id
loc = GoogleSV.getLocationbyID(pano_id)
if loc is None:
continue
s = '%s\t%s\t%s\t%s\t%d\n' % (frame_name, pano_name, loc[0],
loc[1], highest_score)
f.write(s)
f.close()
def RunRansac(ID):
print ID
info = Info.GetVideoInfo(ID)
data = np.load(Info.GetMatchResultExtractPointFileName(info))
#data2 = data[1]
#data[0] = data[0][np.abs(data[1]) <= 3000]
#data[1] = data[1][np.abs(data[1]) <= 3000]
#data2 = data[1i]
try:
M, align_data = RANSAC.RANSAC_affine(data[0], data[1], 10000, 5)
except:
return
if M is None:
return
print M
#print np.reshape(M, [12])
try:
M = np.reshape(M, [12]).astype(float)
except:
return
gcp = Info.ReadGCPData(info)
data = Info.ReadReconstructionData(info)
trajectory = {}
for shot in data['shots']:
[x, y, z] = optical_center(data['shots'][shot])
trajectory[shot] = [x, y, z]
data['gcp'] = gcp
#print gcp
data['transformation'] = list(M)
#print type(M[0])
data['trajectory'] = trajectory
#print trajectory
f_name = Info.Get3DRansacFileName(info)
f = open(f_name, 'w')
f.write(json.dumps(data, indent=4))
f.close()
print '%s ransac finish' % ID
def CopyConfig(ID):
info = Info.GetVideoInfo(ID)
command = 'cp config.yaml %s' % info['video_path']
subprocess.call(command, shell=True)
def RunExtract(ID):
print ID
info = Info.GetVideoInfo(ID)
RPE.RansacPointExtract(ID)
def Plot():
lst = Info.GetStateList(['match_result'], ['yes'])
do_lst = []
for one in lst:
if CheckState(one):
do_lst.append(one)
# Deep
dis_data = []
for one in do_lst:
info = Info.GetVideoInfo(one)
f = GetDeepPath(info)
data = LoadFile(f)
for key in data:
dis_data.append(data[key])
deep_dis_data = np.array(dis_data, dtype=np.float32)
print 'Deep Max val : %f'%np.max(deep_dis_data)
print 'Deep Min val : %f'%np.min(deep_dis_data)
print 'Deep Mean val : %f'%np.mean(deep_dis_data)
print 'Deep Standard Deviation : %f'%np.std(deep_dis_data)
deep_max = int(np.max(dis_data))
nop = len(deep_dis_data)
#####
# Hog
dis_data = []
for one in do_lst:
info = Info.GetVideoInfo(one)
f = GetHogPath(info)
data = LoadFile(f)
for key in data:
dis_data.append(data[key])
hog_dis_data = np.array(dis_data, dtype=np.float32)
print 'Hog Max val : %f'%np.max(hog_dis_data)
print 'Hog Min val : %f'%np.min(hog_dis_data)
print 'Hog Mean val : %f'%np.mean(hog_dis_data)
print 'Hog Standard Deviation : %f'%np.std(hog_dis_data)
hog_max = int(np.max(dis_data))
#####
# Dsift
dis_data = []
for one in do_lst:
info = Info.GetVideoInfo(one)
f = GetDsiftPath(info)
data = LoadFile(f)
for key in data:
dis_data.append(data[key])
dsift_dis_data = np.array(dis_data, dtype=np.float32)
print 'Dsift Max val : %f'%np.max(dsift_dis_data)
print 'Dsift Min val : %f'%np.min(dsift_dis_data)
print 'Dsift Mean val : %f'%np.mean(dsift_dis_data)
print 'Dsift Standard Deviation : %f'%np.std(dsift_dis_data)
dsift_max = int(np.max(dis_data))
#####
all_max = np.max([deep_max, hog_max, dsift_max])
top_bound = int(np.ceil(all_max))
x_axis = np.arange(0, top_bound+1)
deep_hist = np.zeros([top_bound+1], dtype=np.float32)
hog_hist = np.zeros([top_bound+1], dtype=np.float32)
dsift_hist = np.zeros([top_bound+1], dtype=np.float32)
for i in range(0, top_bound+1):
deep_hist[i] = np.sum(deep_dis_data <= i)
hog_hist[i] = np.sum(hog_dis_data <= i)
dsift_hist[i] = np.sum(dsift_dis_data <= i)
x_axis = x_axis[1:]
deep_hist = deep_hist[1:]
hog_hist = hog_hist[1:]
dsift_hist = dsift_hist[1:]
plt.subplot('111')
plt.plot(x_axis, deep_hist/nop*100, 'r', color='r', linewidth='4', label = 'deep match')
plt.plot(x_axis, hog_hist/nop*100, 'r', color='g', linewidth='4', label = 'hog match')
plt.plot(x_axis, dsift_hist/nop*100, 'r', color='b', linewidth='4', label = 'dense sift match')
plt.xscale('log')
#plt.xscale('log')
#plt.xlim([1])
plt.xlabel('within distance(m)')
plt.ylabel('persent(%)')
plt.legend(loc = 'right')
plt.savefig('match.jpg')
plt.show()
def RunReconstruct(ID):
info = Info.GetVideoInfo(ID)
command = '%s %s' % (Info.Config.OPENSFM_RECONSTRUCT, info['video_path'])
subprocess.call(command, shell=True)
def Plot():
lst = Info.GetStateList(['match_result'], ['yes'])
do_lst = [x for x in lst if CheckState(x)]
#print len(do_lst)
# our
our_dis = []
for one in do_lst:
info = Info.GetVideoInfo(one)
f = GetOurPath(info)
data = LoadFile(f)
for key in data:
our_dis.append(data[key])
number_of_point = len(our_dis)
our_dis = np.array(our_dis, dtype=np.float32)
print 'Ourmethod Max val : %f' % np.max(our_dis)
print 'Ourmethod Min val : %f' % np.min(our_dis)
print 'Ourmethod Mean val : %f' % np.mean(our_dis)
print 'Ourmethod Standard Deviation : %f' % np.std(our_dis)
top_bound = np.ceil(np.max(our_dis))
#print top_bound
histogram = np.zeros([int(top_bound) + 1], dtype=np.float32)
ceil_data = np.ceil(our_dis)
for i in range(0, int(top_bound) + 1):
count = np.sum(ceil_data <= i)
histogram[i] = count
#print histogram
our_Y = histogram
our_X = np.arange(0, top_bound + 1)
# affine
affine_dis = []
for one in do_lst:
info = Info.GetVideoInfo(one)
f = GetAffinePath(info)
data = LoadFile(f)
for key in data:
affine_dis.append(data[key])
affine_dis = np.array(affine_dis, dtype=np.float32)
print 'Affine Max val : %f' % np.max(affine_dis)
print 'Affine Min val : %f' % np.min(affine_dis)
print 'Affine Mean val : %f' % np.mean(affine_dis)
print 'Affine Standard Deviation : %f' % np.std(affine_dis)
top_bound = np.ceil(np.max(affine_dis))
#print top_bound
histogram = np.zeros([int(top_bound) + 1], dtype=np.float32)
ceil_data = np.ceil(affine_dis)
for i in range(0, int(top_bound) + 1):
count = np.sum(ceil_data <= i)
histogram[i] = count
#print histogram
affine_Y = histogram
affine_X = np.arange(0, top_bound + 1)
# rigid
rigid_dis = []
for one in do_lst:
info = Info.GetVideoInfo(one)
f = GetRigidPath(info)
data = LoadFile(f)
for key in data:
rigid_dis.append(data[key])
rigid_dis = np.array(rigid_dis, dtype=np.float32)
print 'Rigid Max val : %f' % np.max(rigid_dis)
print 'Rigid Min val : %f' % np.min(rigid_dis)
print 'Rigid Mean val : %f' % np.mean(rigid_dis)
print 'Rigid Standard Deviation : %f' % np.std(rigid_dis)
top_bound = np.ceil(np.max(rigid_dis))
#print top_bound
histogram = np.zeros([int(top_bound) + 1], dtype=np.float32)
ceil_data = np.ceil(rigid_dis)
for i in range(0, int(top_bound) + 1):
count = np.sum(ceil_data <= i)
histogram[i] = count
#print histogram
rigid_Y = histogram
rigid_X = np.arange(0, top_bound + 1)
our_max = np.max(our_Y)
affine_max = np.max(affine_Y)
rigid_max = np.max(rigid_Y)
all_max = np.max([our_max, affine_max, rigid_max])
top_bound = int(np.ceil(all_max))
X_axis = np.arange(0, top_bound + 1)
tmp = np.zeros([top_bound + 1], dtype=np.float32)
tmp[:] = our_max
tmp[0:our_Y.size] = our_Y
our_Y = tmp / number_of_point * 100
tmp = np.zeros([top_bound + 1], dtype=np.float32)
tmp[:] = affine_max
tmp[0:affine_Y.size] = affine_Y
affine_Y = tmp / number_of_point * 100
tmp = np.zeros([top_bound + 1], dtype=np.float32)
tmp[:] = rigid_max
tmp[0:rigid_Y.size] = rigid_Y
rigid_Y = tmp / number_of_point * 100
X_axis = X_axis[1:]
our_Y = our_Y[1:]
affine_Y = affine_Y[1:]
rigid_Y = rigid_Y[1:]
fig = plt.subplot('111')
plt.plot(X_axis, our_Y, 'r', color='r', linewidth=4, label='ourmethod')
plt.plot(X_axis, affine_Y, 'r', color='g', linewidth=4, label='affine')
plt.plot(X_axis, rigid_Y, 'r', color='b', linewidth=4, label='rigid')
plt.xscale('log')
#plt.xlim([1, 300])
plt.xlabel('within distance(m)')
plt.ylabel('persent(%)')
plt.legend(loc='right')
plt.savefig('ransac_transform.jpg')
plt.show()
def RunMatch(ID):
info = Info.GetVideoInfo(ID)
command = '%s %s' % (Info.Config.OPENSFM_RUN_MATCH, info['video_path'])
subprocess.call(command, shell=True)
