samples=glob("*.zip")
print len(samples), "samples found"
#TL = [-5, -10, -numpy.inf] # is 0.739770037290, 0.772290750017, 0.807449899917
#Alpha = [0.25, 0.5, 0.75] # 0.807809390788, 0.811087250153, 0.772292829366
for alpha in ([0]):
for file_count, file in enumerate(samples):
condition = (file_count != 104)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
print("\t Processing file " + file)
time_tic = time.time()
# Create the object to access the sample
sample = GestureSample(os.path.join(data,file))
#load obser cnn
load_path = os.path.join(cnn_obs_likelihodd_dir,file)
observ_likelihood_cnn = cPickle.load(open(load_path,"rb"))
observ_likelihood_cnn = observ_likelihood_cnn[:-1,:]
#load obser sk --log!
load_path = os.path.join(sk_obs_likelihood_dir,file)
observ_likelihood_file = cPickle.load(open(load_path,"rb"))
observ_likelihood_sk = observ_likelihood_file['log_observ_likelihood']
#print observ_likelihood.shape
##########################
# viterbi path decoding
########################
#log_observ_likelihood = log(observ_likelihood_cnn.T + numpy.finfo(numpy.float32).eps) + observ_likelihood_sk
log_observ_likelihood = alpha * log(observ_likelihood_cnn.T + numpy.finfo(numpy.float32).eps) + (1-alpha) *observ_likelihood_sk
for file_count, file in enumerate(samples):
condition = (file_count > -1)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
save_path = os.path.join(save_dst, file)
if os.path.isfile(save_path):
print "loading exiting file"
data_dic = cPickle.load(open(save_path, 'rb'))
video = data_dic["video"]
Feature_gesture = data_dic["Feature_gesture"]
assert video.shape[0] == Feature_gesture.shape[0]
else:
print("\t Processing file " + file)
# Create the object to access the sample
sample = GestureSample(os.path.join(data, file))
print "finish loading samples"
video, Feature_gesture = sample.get_test_data_wudi_lio(used_joints)
assert video.shape[0] == Feature_gesture.shape[
0] # -*- coding: utf-8 -*-
print "finish preprocessing"
out_file = open(save_path, 'wb')
cPickle.dump({
"video": video,
"Feature_gesture": Feature_gesture
},
out_file,
protocol=cPickle.HIGHEST_PROTOCOL)
out_file.close()
def preprocess(samples, set_label="training"):
for file_count, file in enumerate(sort(samples)):
if (set_label == "training"):
condition = (file_count < 650)
if pc == "wudi":
dest = r"D:\Chalearn2014\Data_processed\train" # dir to destination processed data
elif pc == "wudi_linux":
dest = "/idiap/temp/dwu/chalearn2014_data/Train_processed"
elif pc == "lio":
dest = r"/media/lio/Elements/chalearn/preproc/train"
else:
condition = (file_count >= 650)
if pc == "wudi":
dest = r"D:\Chalearn2014\Data_processed\valid" # dir to destination processed data
elif pc == "wudi_linux":
dest = "/idiap/temp/dwu/chalearn2014_data/Valid_processed"
elif pc == "lio":
dest = r"/media/lio/Elements/chalearn/preproc/valid" # dir to destination processed data
#set == "training" ? (condition = (file_count<650)) : (condition = (file_count>=650))
if condition: #wudi only used first 650 for validation !!! Lio be careful!
print("\t Processing file " + file)
start_time = time.time()
# Create the object to access the sample
sample = GestureSample(os.path.join(data, file))
# ###############################################
# USE Ground Truth information to learn the model
# ###############################################
# Get the list of actions for this frame
gestures = sample.getGestures()
# Iterate for each action in this sample
for gesture in gestures:
skelet, depth, gray, user, c = sample.get_data_wudi(
gesture, vid_res, NEUTRUAL_SEG_LENGTH)
if c:
print 'corrupt'
continue
# preprocess
# skelet_feature: frames * num_features? here gestures because we need netural frames
skelet_feature, Targets, c = proc_skelet_wudi(
sample, used_joints, gesture, STATE_NO,
NEUTRUAL_SEG_LENGTH)
if c:
print 'corrupt'
continue
user_o = user.copy()
user = proc_user(user)
skelet, c = proc_skelet(skelet)
# depth: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1]
user_new, depth, c = proc_depth_wudi(depth, user, user_o,
skelet,
NEUTRUAL_SEG_LENGTH)
if c:
print 'corrupt'
continue
# gray: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1]
gray, c = proc_gray_wudi(gray, user, skelet,
NEUTRUAL_SEG_LENGTH)
if c:
print 'corrupt'
continue
if show_depth:
play_vid_wudi(depth, Targets, wait=1000 / 10, norm=False)
if show_gray:
play_vid_wudi(gray, Targets, wait=1000 / 10, norm=False)
if show_user:
play_vid_wudi(user_new,
Targets,
wait=1000 / 10,
norm=False)
# user_new = user_new.astype("bool")
traj2D, traj3D, ori, pheight, hand, center = skelet
skelet = traj3D, ori, pheight
assert user.dtype == gray.dtype == depth.dtype == traj3D.dtype == ori.dtype == "uint8"
assert gray.shape == depth.shape
if not gray.shape[1] == skelet_feature.shape[
0] == Targets.shape[0]:
print "too early movement or too late,skip one"
continue
# we don't need user info. anyway
video = empty((2, ) + gray.shape, dtype="uint8")
video[0], video[1] = gray, depth
store_preproc_wudi(video, skelet_feature,
Targets.argmax(axis=1), skelet, dest)
end_time = time.time()
print "Processing one batch requires: %d second\n" % (end_time -
start_time)
if condition and file_count == (len(samples) - 1):
dump_last_data(video, skelet_feature, Targets.argmax(axis=1),
skelet, dest)
print 'Process', p_i, 'finished'
print "Processing one batch requires: %d second\n" % (end_time -
start_time)
if condition and file_count == (len(samples) - 1):
dump_last_data(video, skelet_feature, Targets.argmax(axis=1),
skelet, dest)
# we should add the traning data as well
if not condition and file_count == 650 - 1:
dump_last_data(video, skelet_feature, Targets.argmax(axis=1),
skelet, dest)
samples = glob("*.zip")
print len(samples), "samples found"
#TL = [-5, -10, -numpy.inf] # is 0.739770037290, 0.772290750017, 0.807449899917
#Alpha = [0.25, 0.5, 0.75] # 0.807809390788, 0.811087250153, 0.772292829366
for alpha in ([0]):
for file_count, file in enumerate(samples):
condition = (file_count != 104)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
print("\t Processing file " + file)
time_tic = time.time()
# Create the object to access the sample
sample = GestureSample(os.path.join(data, file))
#load obser cnn
load_path = os.path.join(cnn_obs_likelihodd_dir, file)
observ_likelihood_cnn = cPickle.load(open(load_path, "rb"))
observ_likelihood_cnn = observ_likelihood_cnn[:-1, :]
#load obser sk --log!
load_path = os.path.join(sk_obs_likelihood_dir, file)
observ_likelihood_file = cPickle.load(open(load_path, "rb"))
observ_likelihood_sk = observ_likelihood_file[
'log_observ_likelihood']
#print observ_likelihood.shape
##########################
# viterbi path decoding
########################
#log_observ_likelihood = log(observ_likelihood_cnn.T + numpy.finfo(numpy.float32).eps) + observ_likelihood_sk
def predLabel(self, video_file, matrix_file, pred_file, filename):
# time_start = time.time()
print("\t Processing file " + filename)
sample_video = GestureSample(video_file)
observ_likelihood = cPickle.load(open(matrix_file, "rb"))
# print 'fininsh Loadinging obs_likelihodd'
# print observ_likelihood.shape
log_observ_likelihood = log(observ_likelihood.T +
numpy.finfo(numpy.float32).eps)
log_observ_likelihood[-1, 0:5] = 0
log_observ_likelihood[-1, -5:] = 0
#viterbi decoding
[path, predecessor_state_index,
global_score] = viterbi_path_log(log(Prior), log(Transition_matrix),
log_observ_likelihood)
[pred_label, begin_frame, end_frame, Individual_score,
frame_length] = viterbi_colab_states(path,
global_score,
state_no=5,
threshold=-5,
mini_frame=15)
#heuristically we need to add 1 more frame here
begin_frame += 1
end_frame += 5 # because we cut 4 frames as a cuboid so we need add extra 4 frames
#plotting
gesturesList = sample_video.getGestures()
import matplotlib.pyplot as plt
STATE_NO = 5
im = imdisplay(global_score)
plt.plot(range(global_score.shape[-1]),
path,
color='#39FF14',
linewidth=2.0)
plt.xlim((0, global_score.shape[-1]))
plt.ylim((101, 0))
plt.xlabel('Frames')
plt.ylabel('HMM states')
plt.title('Multi_model(DBN+3DCNN)')
# plot ground truth
for gesture in gesturesList:
gestureID, startFrame, endFrame = gesture
frames_count = numpy.array(range(startFrame, endFrame + 1))
pred_label_temp = (
(gestureID - 1) * STATE_NO + 2) * numpy.ones(len(frames_count))
plt.plot(frames_count, pred_label_temp, color='r', linewidth=5.0)
# plot clean path
for i in range(len(begin_frame)):
rames_count = numpy.array(range(begin_frame[i], end_frame[i] + 1))
pred_label_temp = ((pred_label[i] - 1) * STATE_NO +
2) * numpy.ones(len(frames_count))
plt.plot(frames_count,
pred_label_temp,
color='#FFFF33',
linewidth=2.0)
plt.show()
pred = []
for i in range(len(begin_frame)):
pred.append([pred_label[i], begin_frame[i], end_frame[i]])
sample_video.exportPredictions(pred, pred_file)
obs_likelihodd_dir = "/home/fancy/Desktop/sys_project/Test_DBN_state_matrix"
######Test_DBN_state_matrix/Test_CNN_state_matrix/Test_early_fusion_state_matrix_fast_con
outPred = '/home/fancy/Desktop/sys_project/Test_DBN_pred'
######Test_DBN_pred/Test_CNN_pred/Test_early_fusion_pred
os.chdir(obs_likelihodd_dir)
samples = glob("*.zip") # because wudi unzipped all the files already!
#>>>>>>> 23b2a04cbc3637b67926c707408057b995140544
print len(samples), "samples found"
for file_count, file in enumerate(samples):
# print file_count
print("\t Processing file " + file)
time_tic = time.time()
# Create the object to access the sample
sample = GestureSample(os.path.join(data, file))
# print'debug1'
load_path = os.path.join(obs_likelihodd_dir, file)
observ_likelihood = cPickle.load(open(load_path, "rb"))
print 'fininsh Loadinging obs_likelihodd'
#print observ_likelihood.shape
##########################
# viterbi path decoding
########################
log_observ_likelihood = log(observ_likelihood.T +
numpy.finfo(numpy.float32).eps)
log_observ_likelihood[-1, 0:5] = 0
log_observ_likelihood[-1, -5:] = 0
print("\t Viterbi path decoding ")
# do it in log space avoid numeric underflow
condition = (file_count > -1)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
save_path= os.path.join(data, file)
print file
time_start = time()
# we load precomputed feature set or recompute the whole feature set
if os.path.isfile(save_path):
print "loading exiting file"
data_dic = cPickle.load(open(save_path,'rb'))
video = data_dic["video"]
Feature_gesture = data_dic["Feature_gesture"]
assert video.shape[0] == Feature_gesture.shape[0]
else:
print("\t Processing file " + file)
# Create the object to access the sample
sample = GestureSample(os.path.join(data,file))
print "finish loading samples"
video, Feature_gesture = sample.get_test_data_wudi_lio(used_joints)
assert video.shape[0] == Feature_gesture.shape[0]# -*- coding: utf-8 -*-
print "finish preprocessing"
out_file = open(save_path, 'wb')
cPickle.dump({"video":video, "Feature_gesture":Feature_gesture}, out_file, protocol=cPickle.HIGHEST_PROTOCOL)
out_file.close()
print "start computing likelihood"
observ_likelihood = numpy.empty(shape=(video.shape[0],20*STATE_NO+1)) # 20 classed * 5 states + 1 ergodic state
for batchnumber in xrange(video.shape[0]/batch.micro):
video_temp = video[batch.micro*batchnumber:batch.micro*(batchnumber+1),:]
skel_temp = Feature_gesture[batch.micro*batchnumber:batch.micro*(batchnumber+1),:]
def preprocess(samples, set_label="training"):
for file_count, file in enumerate(sort(samples)):
if (set_label == "training"):
condition = (file_count < 650)
if pc == "wudi":
dest = r"D:\Chalearn2014\Data_processed\train" # dir to destination processed data
elif pc == "wudi_linux":
dest = "/idiap/temp/dwu/chalearn2014_data/Train_processed"
elif pc == "lio":
dest = r"/media/lio/Elements/chalearn/preproc/train"
else:
condition = (file_count >= 650)
if pc == "wudi":
dest = r"D:\Chalearn2014\Data_processed\valid" # dir to destination processed data
elif pc == "wudi_linux":
dest = "/idiap/temp/dwu/chalearn2014_data/Valid_processed"
elif pc == "lio":
dest = r"/media/lio/Elements/chalearn/preproc/valid" # dir to destination processed data
#set == "training" ? (condition = (file_count<650)) : (condition = (file_count>=650))
if condition: #wudi only used first 650 for validation !!! Lio be careful!
print("\t Processing file " + file)
start_time = time.time()
# Create the object to access the sample
sample = GestureSample(os.path.join(data, file))
# ###############################################
# USE Ground Truth information to learn the model
# ###############################################
# Get the list of actions for this frame
gestures = sample.getGestures()
# Iterate for each action in this sample
for gesture in gestures:
skelet, depth, gray, user, c = sample.get_data_wudi(gesture, vid_res, NEUTRUAL_SEG_LENGTH)
if c: print 'corrupt'; continue
# preprocess
# skelet_feature: frames * num_features? here gestures because we need netural frames
skelet_feature, Targets, c = proc_skelet_wudi(sample, used_joints, gesture, STATE_NO,
NEUTRUAL_SEG_LENGTH)
if c: print 'corrupt'; continue
user_o = user.copy()
user = proc_user(user)
skelet, c = proc_skelet(skelet)
# depth: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1]
user_new, depth, c = proc_depth_wudi(depth, user, user_o, skelet, NEUTRUAL_SEG_LENGTH)
if c: print 'corrupt'; continue
# gray: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1]
gray, c = proc_gray_wudi(gray, user, skelet, NEUTRUAL_SEG_LENGTH)
if c: print 'corrupt'; continue
if show_depth: play_vid_wudi(depth, Targets, wait=1000 / 10, norm=False)
if show_gray: play_vid_wudi(gray, Targets, wait=1000 / 10, norm=False)
if show_user: play_vid_wudi(user_new, Targets, wait=1000 / 10, norm=False)
# user_new = user_new.astype("bool")
traj2D, traj3D, ori, pheight, hand, center = skelet
skelet = traj3D, ori, pheight
assert user.dtype == gray.dtype == depth.dtype == traj3D.dtype == ori.dtype == "uint8"
assert gray.shape == depth.shape
if not gray.shape[1] == skelet_feature.shape[0] == Targets.shape[0]:
print "too early movement or too late,skip one";
continue
# we don't need user info. anyway
video = empty((2,) + gray.shape, dtype="uint8")
video[0], video[1] = gray, depth
store_preproc_wudi(video, skelet_feature, Targets.argmax(axis=1), skelet, dest)
end_time = time.time()
print "Processing one batch requires: %d second\n"% ( end_time - start_time)
if condition and file_count==(len(samples)-1):
dump_last_data(video,skelet_feature, Targets.argmax(axis=1), skelet, dest)
print 'Process',p_i,'finished'
print "Processing one batch requires: %d second\n"% ( end_time - start_time)
if condition and file_count==(len(samples)-1):
dump_last_data(video,skelet_feature, Targets.argmax(axis=1), skelet, dest)
# we should add the traning data as well
if not condition and file_count == 650-1:
dump_last_data(video,skelet_feature, Targets.argmax(axis=1), skelet, dest)
os.chdir(data)
if pc=="wudi":
samples=glob("*") # because wudi unzipped all the files already!
elif pc=="lio":
samples=glob("*.zip")
print len(samples), "samples found"
for file_count, file in enumerate(samples):
condition = (file_count > 650)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
print("\t Processing file " + file)
# Create the object to access the sample
sample = GestureSample(os.path.join(data,file))
if not load_flag:
video = sample.get_test_data_wudi_lio()
save_path= os.path.join(save_dst, file)
out_file = open(save_path, 'wb')
cPickle.dump(video, out_file, protocol=cPickle.HIGHEST_PROTOCOL)
out_file.close()
else:
save_path= os.path.join(data,file,'test')
video = cPickle.load(open(save_path,"rb"))
print video.shape
print "start computing likelihood"
observ_likelihood = numpy.empty(shape=(video.shape[0],20*5+1)) # 20 classed * 5 states + 1 ergodic state
for batchnumber in xrange(video.shape[0]/batch.micro):
video_temp = video[batch.micro*batchnumber:batch.micro*(batchnumber+1),:]
def preprocess(samples):
for file_count, file in enumerate(sort(samples)):
print "This is the %d th file : " % file_count
##totally 400 training samples, use 360 for training, 40 for validating
if file_count < 360:
if pc == "linux_fancy":
dest = "/home/fancywu/Desktop/GestureRecognition/Train_files/Preprocessed_files/train"
elif pc == "win_fancy":
dest = ""
print "Processing training file ", file
elif file_count >= 360:
if pc == "linux_fancy":
dest = "/home/fancywu/Desktop/GestureRecognition/Train_files/Preprocessed_files/valid"
elif pc == "win_fancy":
dest = ""
print "Processing validating file ", file
start_time = time()
##Create the object to access the sample
sample = GestureSample(os.path.join(data, file))
# print(os.path.join(data, file))
##USE Ground Truth information to learn the model
##Get the list of gesture for this sample
gestures = sample.getGestures()
print "len gestures: ", len(gestures)
# preprocess each gesture
for gesture in gestures:
skelet, depth, gray, user, c = sample.get_data_wudi(
gesture, vid_res, NEUTRUAL_SEG_LENGTH)
if c:
print '1: corrupt'
continue
skelet_feature, Targets, c = proc_skelet_wudi(
sample, used_joints, gesture, HIDDEN_STATE,
NEUTRUAL_SEG_LENGTH)
if c:
print '2: corrupt'
continue
user_o = user.copy()
user = proc_user(user)
skelet, c = proc_skelet(skelet)
if c: print '3: corrupt'
user_new, depth, c = proc_depth_wudi(depth, user, user_o, skelet,
NEUTRUAL_SEG_LENGTH)
if c:
print '4: corrupt'
continue
gray, c = proc_gray_wudi(gray, user, skelet, NEUTRUAL_SEG_LENGTH)
if c:
print '5: corrupt'
continue
if show_depth:
play_vid_wudi(depth, Targets, wait=1000 / 10, norm=False)
if show_gray:
play_vid_wudi(gray, Targets, wait=1000 / 10, norm=False)
if show_user:
play_vid_wudi(user_new, Targets, wait=1000 / 10, norm=False)
traj2D, traj3D, ori, pheight, hand, center = skelet
skelet = traj3D, ori, pheight
assert user.dtype == gray.dtype == depth.dtype == traj3D.dtype == ori.dtype == "uint8"
assert gray.shape == depth.shape
if not gray.shape[1] == skelet_feature.shape[0] == Targets.shape[0]:
print "too early or too late movement, skip one"
continue
##we just use gray and depth videos for training, dont need user
video = empty((2, ) + gray.shape, dtype="uint8")
video[0], video[1] = gray, depth
store_preproc_video_skelet_data(video, skelet_feature,
Targets.argmax(axis=1), skelet,
dest)
print "finished"
print "Processing one batch requires : %d seconds\n" % (time() -
start_time)
if (file_count == len(samples) - 1) or (file_count == 360 - 1):
store_preproc_video_skelet_data(video,
skelet_feature,
Targets.argmax(axis=1),
skelet,
dest,
last_data=True)
print "Processing %d sample requies: %3.3f mins" % (
file_count + 1, (time() - prog_start_time) / 60.)
os.chdir(data)
if pc=="wudi":
samples=glob("*") # because wudi unzipped all the files already!
elif pc=="lio":
samples=glob("*.zip")
print len(samples), "samples found"
for file_count, file in enumerate(samples):
condition = (file_count >= 105)
if condition: #wudi only used first 650 for validation !!! Lio be careful!
print("\t Processing file " + file)
time_tic = time.time()
# Create the object to access the sample
sample = GestureSample(os.path.join(data,file))
#load ober
load_path = os.path.join(obs_likelihodd_dir,file)
observ_likelihood = cPickle.load(open(load_path,"rb"))
#print observ_likelihood.shape
##########################
# viterbi path decoding
########################
log_observ_likelihood = log(observ_likelihood.T + numpy.finfo(numpy.float32).eps)
log_observ_likelihood[-1, 0:5] = 0
log_observ_likelihood[-1, -5:] = 0
print("\t Viterbi path decoding " )
# do it in log space avoid numeric underflow
[path, predecessor_state_index, global_score] = viterbi_path_log(log(Prior), log(Transition_matrix), log_observ_likelihood)
