samples=glob("*.zip") print len(samples), "samples found" #TL = [-5, -10, -numpy.inf] # is 0.739770037290, 0.772290750017, 0.807449899917 #Alpha = [0.25, 0.5, 0.75] # 0.807809390788, 0.811087250153, 0.772292829366 for alpha in ([0]): for file_count, file in enumerate(samples): condition = (file_count != 104) if condition: #wudi only used first 650 for validation !!! Lio be careful! print("\t Processing file " + file) time_tic = time.time() # Create the object to access the sample sample = GestureSample(os.path.join(data,file)) #load obser cnn load_path = os.path.join(cnn_obs_likelihodd_dir,file) observ_likelihood_cnn = cPickle.load(open(load_path,"rb")) observ_likelihood_cnn = observ_likelihood_cnn[:-1,:] #load obser sk --log! load_path = os.path.join(sk_obs_likelihood_dir,file) observ_likelihood_file = cPickle.load(open(load_path,"rb")) observ_likelihood_sk = observ_likelihood_file['log_observ_likelihood'] #print observ_likelihood.shape ########################## # viterbi path decoding ######################## #log_observ_likelihood = log(observ_likelihood_cnn.T + numpy.finfo(numpy.float32).eps) + observ_likelihood_sk log_observ_likelihood = alpha * log(observ_likelihood_cnn.T + numpy.finfo(numpy.float32).eps) + (1-alpha) *observ_likelihood_sk
for file_count, file in enumerate(samples): condition = (file_count > -1) if condition: #wudi only used first 650 for validation !!! Lio be careful! save_path = os.path.join(save_dst, file) if os.path.isfile(save_path): print "loading exiting file" data_dic = cPickle.load(open(save_path, 'rb')) video = data_dic["video"] Feature_gesture = data_dic["Feature_gesture"] assert video.shape[0] == Feature_gesture.shape[0] else: print("\t Processing file " + file) # Create the object to access the sample sample = GestureSample(os.path.join(data, file)) print "finish loading samples" video, Feature_gesture = sample.get_test_data_wudi_lio(used_joints) assert video.shape[0] == Feature_gesture.shape[ 0] # -*- coding: utf-8 -*- print "finish preprocessing" out_file = open(save_path, 'wb') cPickle.dump({ "video": video, "Feature_gesture": Feature_gesture }, out_file, protocol=cPickle.HIGHEST_PROTOCOL) out_file.close()
def preprocess(samples, set_label="training"): for file_count, file in enumerate(sort(samples)): if (set_label == "training"): condition = (file_count < 650) if pc == "wudi": dest = r"D:\Chalearn2014\Data_processed\train" # dir to destination processed data elif pc == "wudi_linux": dest = "/idiap/temp/dwu/chalearn2014_data/Train_processed" elif pc == "lio": dest = r"/media/lio/Elements/chalearn/preproc/train" else: condition = (file_count >= 650) if pc == "wudi": dest = r"D:\Chalearn2014\Data_processed\valid" # dir to destination processed data elif pc == "wudi_linux": dest = "/idiap/temp/dwu/chalearn2014_data/Valid_processed" elif pc == "lio": dest = r"/media/lio/Elements/chalearn/preproc/valid" # dir to destination processed data #set == "training" ? (condition = (file_count<650)) : (condition = (file_count>=650)) if condition: #wudi only used first 650 for validation !!! Lio be careful! print("\t Processing file " + file) start_time = time.time() # Create the object to access the sample sample = GestureSample(os.path.join(data, file)) # ############################################### # USE Ground Truth information to learn the model # ############################################### # Get the list of actions for this frame gestures = sample.getGestures() # Iterate for each action in this sample for gesture in gestures: skelet, depth, gray, user, c = sample.get_data_wudi( gesture, vid_res, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt' continue # preprocess # skelet_feature: frames * num_features? here gestures because we need netural frames skelet_feature, Targets, c = proc_skelet_wudi( sample, used_joints, gesture, STATE_NO, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt' continue user_o = user.copy() user = proc_user(user) skelet, c = proc_skelet(skelet) # depth: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1] user_new, depth, c = proc_depth_wudi(depth, user, user_o, skelet, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt' continue # gray: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1] gray, c = proc_gray_wudi(gray, user, skelet, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt' continue if show_depth: play_vid_wudi(depth, Targets, wait=1000 / 10, norm=False) if show_gray: play_vid_wudi(gray, Targets, wait=1000 / 10, norm=False) if show_user: play_vid_wudi(user_new, Targets, wait=1000 / 10, norm=False) # user_new = user_new.astype("bool") traj2D, traj3D, ori, pheight, hand, center = skelet skelet = traj3D, ori, pheight assert user.dtype == gray.dtype == depth.dtype == traj3D.dtype == ori.dtype == "uint8" assert gray.shape == depth.shape if not gray.shape[1] == skelet_feature.shape[ 0] == Targets.shape[0]: print "too early movement or too late,skip one" continue # we don't need user info. anyway video = empty((2, ) + gray.shape, dtype="uint8") video[0], video[1] = gray, depth store_preproc_wudi(video, skelet_feature, Targets.argmax(axis=1), skelet, dest) end_time = time.time() print "Processing one batch requires: %d second\n" % (end_time - start_time) if condition and file_count == (len(samples) - 1): dump_last_data(video, skelet_feature, Targets.argmax(axis=1), skelet, dest) print 'Process', p_i, 'finished' print "Processing one batch requires: %d second\n" % (end_time - start_time) if condition and file_count == (len(samples) - 1): dump_last_data(video, skelet_feature, Targets.argmax(axis=1), skelet, dest) # we should add the traning data as well if not condition and file_count == 650 - 1: dump_last_data(video, skelet_feature, Targets.argmax(axis=1), skelet, dest)
samples = glob("*.zip") print len(samples), "samples found" #TL = [-5, -10, -numpy.inf] # is 0.739770037290, 0.772290750017, 0.807449899917 #Alpha = [0.25, 0.5, 0.75] # 0.807809390788, 0.811087250153, 0.772292829366 for alpha in ([0]): for file_count, file in enumerate(samples): condition = (file_count != 104) if condition: #wudi only used first 650 for validation !!! Lio be careful! print("\t Processing file " + file) time_tic = time.time() # Create the object to access the sample sample = GestureSample(os.path.join(data, file)) #load obser cnn load_path = os.path.join(cnn_obs_likelihodd_dir, file) observ_likelihood_cnn = cPickle.load(open(load_path, "rb")) observ_likelihood_cnn = observ_likelihood_cnn[:-1, :] #load obser sk --log! load_path = os.path.join(sk_obs_likelihood_dir, file) observ_likelihood_file = cPickle.load(open(load_path, "rb")) observ_likelihood_sk = observ_likelihood_file[ 'log_observ_likelihood'] #print observ_likelihood.shape ########################## # viterbi path decoding ######################## #log_observ_likelihood = log(observ_likelihood_cnn.T + numpy.finfo(numpy.float32).eps) + observ_likelihood_sk
def predLabel(self, video_file, matrix_file, pred_file, filename): # time_start = time.time() print("\t Processing file " + filename) sample_video = GestureSample(video_file) observ_likelihood = cPickle.load(open(matrix_file, "rb")) # print 'fininsh Loadinging obs_likelihodd' # print observ_likelihood.shape log_observ_likelihood = log(observ_likelihood.T + numpy.finfo(numpy.float32).eps) log_observ_likelihood[-1, 0:5] = 0 log_observ_likelihood[-1, -5:] = 0 #viterbi decoding [path, predecessor_state_index, global_score] = viterbi_path_log(log(Prior), log(Transition_matrix), log_observ_likelihood) [pred_label, begin_frame, end_frame, Individual_score, frame_length] = viterbi_colab_states(path, global_score, state_no=5, threshold=-5, mini_frame=15) #heuristically we need to add 1 more frame here begin_frame += 1 end_frame += 5 # because we cut 4 frames as a cuboid so we need add extra 4 frames #plotting gesturesList = sample_video.getGestures() import matplotlib.pyplot as plt STATE_NO = 5 im = imdisplay(global_score) plt.plot(range(global_score.shape[-1]), path, color='#39FF14', linewidth=2.0) plt.xlim((0, global_score.shape[-1])) plt.ylim((101, 0)) plt.xlabel('Frames') plt.ylabel('HMM states') plt.title('Multi_model(DBN+3DCNN)') # plot ground truth for gesture in gesturesList: gestureID, startFrame, endFrame = gesture frames_count = numpy.array(range(startFrame, endFrame + 1)) pred_label_temp = ( (gestureID - 1) * STATE_NO + 2) * numpy.ones(len(frames_count)) plt.plot(frames_count, pred_label_temp, color='r', linewidth=5.0) # plot clean path for i in range(len(begin_frame)): rames_count = numpy.array(range(begin_frame[i], end_frame[i] + 1)) pred_label_temp = ((pred_label[i] - 1) * STATE_NO + 2) * numpy.ones(len(frames_count)) plt.plot(frames_count, pred_label_temp, color='#FFFF33', linewidth=2.0) plt.show() pred = [] for i in range(len(begin_frame)): pred.append([pred_label[i], begin_frame[i], end_frame[i]]) sample_video.exportPredictions(pred, pred_file)
obs_likelihodd_dir = "/home/fancy/Desktop/sys_project/Test_DBN_state_matrix" ######Test_DBN_state_matrix/Test_CNN_state_matrix/Test_early_fusion_state_matrix_fast_con outPred = '/home/fancy/Desktop/sys_project/Test_DBN_pred' ######Test_DBN_pred/Test_CNN_pred/Test_early_fusion_pred os.chdir(obs_likelihodd_dir) samples = glob("*.zip") # because wudi unzipped all the files already! #>>>>>>> 23b2a04cbc3637b67926c707408057b995140544 print len(samples), "samples found" for file_count, file in enumerate(samples): # print file_count print("\t Processing file " + file) time_tic = time.time() # Create the object to access the sample sample = GestureSample(os.path.join(data, file)) # print'debug1' load_path = os.path.join(obs_likelihodd_dir, file) observ_likelihood = cPickle.load(open(load_path, "rb")) print 'fininsh Loadinging obs_likelihodd' #print observ_likelihood.shape ########################## # viterbi path decoding ######################## log_observ_likelihood = log(observ_likelihood.T + numpy.finfo(numpy.float32).eps) log_observ_likelihood[-1, 0:5] = 0 log_observ_likelihood[-1, -5:] = 0 print("\t Viterbi path decoding ") # do it in log space avoid numeric underflow
condition = (file_count > -1) if condition: #wudi only used first 650 for validation !!! Lio be careful! save_path= os.path.join(data, file) print file time_start = time() # we load precomputed feature set or recompute the whole feature set if os.path.isfile(save_path): print "loading exiting file" data_dic = cPickle.load(open(save_path,'rb')) video = data_dic["video"] Feature_gesture = data_dic["Feature_gesture"] assert video.shape[0] == Feature_gesture.shape[0] else: print("\t Processing file " + file) # Create the object to access the sample sample = GestureSample(os.path.join(data,file)) print "finish loading samples" video, Feature_gesture = sample.get_test_data_wudi_lio(used_joints) assert video.shape[0] == Feature_gesture.shape[0]# -*- coding: utf-8 -*- print "finish preprocessing" out_file = open(save_path, 'wb') cPickle.dump({"video":video, "Feature_gesture":Feature_gesture}, out_file, protocol=cPickle.HIGHEST_PROTOCOL) out_file.close() print "start computing likelihood" observ_likelihood = numpy.empty(shape=(video.shape[0],20*STATE_NO+1)) # 20 classed * 5 states + 1 ergodic state for batchnumber in xrange(video.shape[0]/batch.micro): video_temp = video[batch.micro*batchnumber:batch.micro*(batchnumber+1),:] skel_temp = Feature_gesture[batch.micro*batchnumber:batch.micro*(batchnumber+1),:]
def preprocess(samples, set_label="training"): for file_count, file in enumerate(sort(samples)): if (set_label == "training"): condition = (file_count < 650) if pc == "wudi": dest = r"D:\Chalearn2014\Data_processed\train" # dir to destination processed data elif pc == "wudi_linux": dest = "/idiap/temp/dwu/chalearn2014_data/Train_processed" elif pc == "lio": dest = r"/media/lio/Elements/chalearn/preproc/train" else: condition = (file_count >= 650) if pc == "wudi": dest = r"D:\Chalearn2014\Data_processed\valid" # dir to destination processed data elif pc == "wudi_linux": dest = "/idiap/temp/dwu/chalearn2014_data/Valid_processed" elif pc == "lio": dest = r"/media/lio/Elements/chalearn/preproc/valid" # dir to destination processed data #set == "training" ? (condition = (file_count<650)) : (condition = (file_count>=650)) if condition: #wudi only used first 650 for validation !!! Lio be careful! print("\t Processing file " + file) start_time = time.time() # Create the object to access the sample sample = GestureSample(os.path.join(data, file)) # ############################################### # USE Ground Truth information to learn the model # ############################################### # Get the list of actions for this frame gestures = sample.getGestures() # Iterate for each action in this sample for gesture in gestures: skelet, depth, gray, user, c = sample.get_data_wudi(gesture, vid_res, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt'; continue # preprocess # skelet_feature: frames * num_features? here gestures because we need netural frames skelet_feature, Targets, c = proc_skelet_wudi(sample, used_joints, gesture, STATE_NO, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt'; continue user_o = user.copy() user = proc_user(user) skelet, c = proc_skelet(skelet) # depth: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1] user_new, depth, c = proc_depth_wudi(depth, user, user_o, skelet, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt'; continue # gray: 2(h&b) * frames * 5 (stacked frames) * vid_shape_hand[0] *vid_shape_hand[1] gray, c = proc_gray_wudi(gray, user, skelet, NEUTRUAL_SEG_LENGTH) if c: print 'corrupt'; continue if show_depth: play_vid_wudi(depth, Targets, wait=1000 / 10, norm=False) if show_gray: play_vid_wudi(gray, Targets, wait=1000 / 10, norm=False) if show_user: play_vid_wudi(user_new, Targets, wait=1000 / 10, norm=False) # user_new = user_new.astype("bool") traj2D, traj3D, ori, pheight, hand, center = skelet skelet = traj3D, ori, pheight assert user.dtype == gray.dtype == depth.dtype == traj3D.dtype == ori.dtype == "uint8" assert gray.shape == depth.shape if not gray.shape[1] == skelet_feature.shape[0] == Targets.shape[0]: print "too early movement or too late,skip one"; continue # we don't need user info. anyway video = empty((2,) + gray.shape, dtype="uint8") video[0], video[1] = gray, depth store_preproc_wudi(video, skelet_feature, Targets.argmax(axis=1), skelet, dest) end_time = time.time() print "Processing one batch requires: %d second\n"% ( end_time - start_time) if condition and file_count==(len(samples)-1): dump_last_data(video,skelet_feature, Targets.argmax(axis=1), skelet, dest) print 'Process',p_i,'finished' print "Processing one batch requires: %d second\n"% ( end_time - start_time) if condition and file_count==(len(samples)-1): dump_last_data(video,skelet_feature, Targets.argmax(axis=1), skelet, dest) # we should add the traning data as well if not condition and file_count == 650-1: dump_last_data(video,skelet_feature, Targets.argmax(axis=1), skelet, dest)
os.chdir(data) if pc=="wudi": samples=glob("*") # because wudi unzipped all the files already! elif pc=="lio": samples=glob("*.zip") print len(samples), "samples found" for file_count, file in enumerate(samples): condition = (file_count > 650) if condition: #wudi only used first 650 for validation !!! Lio be careful! print("\t Processing file " + file) # Create the object to access the sample sample = GestureSample(os.path.join(data,file)) if not load_flag: video = sample.get_test_data_wudi_lio() save_path= os.path.join(save_dst, file) out_file = open(save_path, 'wb') cPickle.dump(video, out_file, protocol=cPickle.HIGHEST_PROTOCOL) out_file.close() else: save_path= os.path.join(data,file,'test') video = cPickle.load(open(save_path,"rb")) print video.shape print "start computing likelihood" observ_likelihood = numpy.empty(shape=(video.shape[0],20*5+1)) # 20 classed * 5 states + 1 ergodic state for batchnumber in xrange(video.shape[0]/batch.micro): video_temp = video[batch.micro*batchnumber:batch.micro*(batchnumber+1),:]
def preprocess(samples): for file_count, file in enumerate(sort(samples)): print "This is the %d th file : " % file_count ##totally 400 training samples, use 360 for training, 40 for validating if file_count < 360: if pc == "linux_fancy": dest = "/home/fancywu/Desktop/GestureRecognition/Train_files/Preprocessed_files/train" elif pc == "win_fancy": dest = "" print "Processing training file ", file elif file_count >= 360: if pc == "linux_fancy": dest = "/home/fancywu/Desktop/GestureRecognition/Train_files/Preprocessed_files/valid" elif pc == "win_fancy": dest = "" print "Processing validating file ", file start_time = time() ##Create the object to access the sample sample = GestureSample(os.path.join(data, file)) # print(os.path.join(data, file)) ##USE Ground Truth information to learn the model ##Get the list of gesture for this sample gestures = sample.getGestures() print "len gestures: ", len(gestures) # preprocess each gesture for gesture in gestures: skelet, depth, gray, user, c = sample.get_data_wudi( gesture, vid_res, NEUTRUAL_SEG_LENGTH) if c: print '1: corrupt' continue skelet_feature, Targets, c = proc_skelet_wudi( sample, used_joints, gesture, HIDDEN_STATE, NEUTRUAL_SEG_LENGTH) if c: print '2: corrupt' continue user_o = user.copy() user = proc_user(user) skelet, c = proc_skelet(skelet) if c: print '3: corrupt' user_new, depth, c = proc_depth_wudi(depth, user, user_o, skelet, NEUTRUAL_SEG_LENGTH) if c: print '4: corrupt' continue gray, c = proc_gray_wudi(gray, user, skelet, NEUTRUAL_SEG_LENGTH) if c: print '5: corrupt' continue if show_depth: play_vid_wudi(depth, Targets, wait=1000 / 10, norm=False) if show_gray: play_vid_wudi(gray, Targets, wait=1000 / 10, norm=False) if show_user: play_vid_wudi(user_new, Targets, wait=1000 / 10, norm=False) traj2D, traj3D, ori, pheight, hand, center = skelet skelet = traj3D, ori, pheight assert user.dtype == gray.dtype == depth.dtype == traj3D.dtype == ori.dtype == "uint8" assert gray.shape == depth.shape if not gray.shape[1] == skelet_feature.shape[0] == Targets.shape[0]: print "too early or too late movement, skip one" continue ##we just use gray and depth videos for training, dont need user video = empty((2, ) + gray.shape, dtype="uint8") video[0], video[1] = gray, depth store_preproc_video_skelet_data(video, skelet_feature, Targets.argmax(axis=1), skelet, dest) print "finished" print "Processing one batch requires : %d seconds\n" % (time() - start_time) if (file_count == len(samples) - 1) or (file_count == 360 - 1): store_preproc_video_skelet_data(video, skelet_feature, Targets.argmax(axis=1), skelet, dest, last_data=True) print "Processing %d sample requies: %3.3f mins" % ( file_count + 1, (time() - prog_start_time) / 60.)
os.chdir(data) if pc=="wudi": samples=glob("*") # because wudi unzipped all the files already! elif pc=="lio": samples=glob("*.zip") print len(samples), "samples found" for file_count, file in enumerate(samples): condition = (file_count >= 105) if condition: #wudi only used first 650 for validation !!! Lio be careful! print("\t Processing file " + file) time_tic = time.time() # Create the object to access the sample sample = GestureSample(os.path.join(data,file)) #load ober load_path = os.path.join(obs_likelihodd_dir,file) observ_likelihood = cPickle.load(open(load_path,"rb")) #print observ_likelihood.shape ########################## # viterbi path decoding ######################## log_observ_likelihood = log(observ_likelihood.T + numpy.finfo(numpy.float32).eps) log_observ_likelihood[-1, 0:5] = 0 log_observ_likelihood[-1, -5:] = 0 print("\t Viterbi path decoding " ) # do it in log space avoid numeric underflow [path, predecessor_state_index, global_score] = viterbi_path_log(log(Prior), log(Transition_matrix), log_observ_likelihood)