def main_function(): # set the skeleton detector. The two inputs are: operation model and image size Skeleton_Detector = uti_openpose.Skeleton_Detector(OPENPOSE_MODEL, OPENPOSE_IMAGE_SIZE) # track_and_label = Skeleton_Tracker() Images_Loader = uti_commons.Read_Valid_Images( images_folder=SRC_IMAGES_FOLDER, valid_images_txt=SRC_IMAGES_DESCRIPTION_TXT, image_filename_format=IMAGE_FILE_NAME_FORMAT) # Set the images displayer Images_Displayer = uti_images_io.Image_Displayer() # Create the folder for output, if the have not been created os.makedirs(DST_DETECTED_SKELETONS_FOLDER, exist_ok=True) os.makedirs(DST_IMAGES_WITH_DETECTED_SKELETONS, exist_ok=True) # create a list to store the possible invalid image indexs iInvalid_Counter = 0 list_of_invalid = [] iTotal_Number_of_Images = Images_Loader._num_images for iImages_Counter in range(iTotal_Number_of_Images): # Load training images Image, sImage_Info = Images_Loader.read_image() # detect humans Humans = Skeleton_Detector.detect(Image) # display detected skeletons on images Image_DST = Image.copy() Skeleton_Detector.draw(Image_DST, Humans) Images_Displayer.display(Image_DST) # save skeletons coordinates to txt files and the sImage_Info with it at the begining SKELETONS, SCALE_H = Skeleton_Detector.humans_to_skeletons_list(Humans) # SKELETONS_DICT = track_and_label.track(SKELETONS) SKELETONS_DIR = uti_tracker.delete_invalid_skeletons_from_lists( SKELETONS) # add the label infos to this skeleton SKELETONS_DIR.insert(IMAGES_INFO_INDEX, sImage_Info) sFile_Name = SKELETON_FILE_NAME_FORMAT.format(iImages_Counter) uti_commons.save_listlist(DST_DETECTED_SKELETONS_FOLDER + sFile_Name, SKELETONS_DIR) sImage_Name = IMAGE_FILE_NAME_FORMAT.format(iImages_Counter) cv2.imwrite(DST_IMAGES_WITH_DETECTED_SKELETONS + sImage_Name, Image_DST) # the length of this lists should be 2 at this point, if not, then it#s invalid image if len(SKELETONS_DIR) != 2: iInvalid_Counter += 1 list_of_invalid.append(sImage_Name) uti_commons.save_listlist(INVALID_IMAGES_FILE, list_of_invalid) print(f'{iImages_Counter}/{iTotal_Number_of_Images} th image ' f'has {len(SKELETONS_DIR) - 1} people in it') print('Programm End')
def Test_Save_Raw_Skeleton_Data_v1(): '''Try to extract and display the skeleton data and save it in txt files.''' fMax_Framerate = 10 iCamera_Index = 0 Data_Source = uti_images_io.Read_Images_From_Webcam(fMax_Framerate,iCamera_Index) Image_Window = uti_images_io.Image_Displayer() Skeleton_Detector = uti_openpose.Skeleton_Detector("mobilenet_thin", "432x368") # Temporal path and format, use config.yaml file to define it later. SKELETON_X_FOLDER = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Skeletons/Test_Skeleton_X/') SKELETON_Y_FOLDER = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Skeletons/Test_Skeleton_Y/') SKELETON_X_FOLDER_DIR = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Skeletons/Test_Skeleton_X_DIR/') SKELETON_Y_FOLDER_DIR = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Skeletons/Test_Skeleton_Y_DIR/') SKELETON_X_FOLDER_VEL = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Skeletons/Test_Skeleton_X_VEL/') SKELETON_Y_FOLDER_VEL = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Skeletons/Test_Skeleton_Y_VEL/') DST_VIZ_IMGS_FOLDER = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Images/Test_Images/') DST_IMGS_FOLDER = ('/home/zhaj/tf_test/Human_Action_Recognition/Data_Images/Test_Images_ORI/') SKELETON_FILENAME_FORMAT = ('{:05d}.txt') IMG_FILENAME_FORMAT = ('{:05d}.jpg') import itertools for i in itertools.count(): img = Data_Source.Read_Image() if img is None: break print(f"Read {i}th Frame from Video...") Detected_Human = Skeleton_Detector.detect(img) Image_Output = img.copy() Skeleton_Detector.draw(Image_Output, Detected_Human) Image_Window.display(Image_Output) Skeleton_X, Skeleton_Y, Scale_h = Skeleton_Detector.humans_to_skels_list(Detected_Human) if Skeleton_X and Skeleton_Y: #only save no-empty lists txt_filename = SKELETON_FILENAME_FORMAT.format(i) uti_commons.save_listlist(SKELETON_X_FOLDER + txt_filename, Skeleton_X) uti_commons.save_listlist(SKELETON_Y_FOLDER + txt_filename, Skeleton_Y) Skeleton_X_DIR = uti_skeletons_io.Rebuild_Skeletons(Skeleton_X) Skeleton_Y_DIR = uti_skeletons_io.Rebuild_Skeletons(Skeleton_Y) uti_commons.save_listlist(SKELETON_X_FOLDER_DIR + txt_filename, Skeleton_X_DIR) uti_commons.save_listlist(SKELETON_Y_FOLDER_DIR + txt_filename, Skeleton_Y_DIR) print(f"Saved {i}th Skeleton Data from Webcam...") jpg_filename = IMG_FILENAME_FORMAT.format(i) cv2.imwrite( DST_IMGS_FOLDER + jpg_filename, img) cv2.imwrite( DST_VIZ_IMGS_FOLDER + jpg_filename, Image_Output) print(f"Saved {i}th Image with Skeleton Data from Webcam...") print("Program ends")
def main_function(): # initialize the frames counter at -1, so the first incomming frames is 0 iFrames_Counter = -1 # initialize the skeleton detector skeleton_detector = uti_openpose.Skeleton_Detector(OPENPOSE_MODEL, OPENPOSE_IMAGE_SIZE) # load the trained two stream model network = tf.keras.models.load_model(MODEL_PATH) # select the data source # images_loader = uti_images_io.Read_Images_From_Video(VIDEO_PATH_SRC) # images_loader = uti_images_io.Read_Images_From_Webcam(10, 0) images_loader = uti_images_io.Read_Images_From_Folder(IMAGE_PATH) # initialize the skeleton detector Images_Displayer = uti_images_io.Image_Displayer() # initialize the skeleton detector Featurs_Generator = uti_features_extraction.Features_Generator_Multiple( FEATURE_WINDOW_SIZE) # initialize Multiperson Tracker Local_Tracker = uti_tracker.Tracker() # Recorder = uti_images_io.Video_Writer(TEST_OUTPUTS + 'TEST_' + uti_commons.get_time() + '/video', 10) Timestample = uti_commons.get_time() TEST_RESULTS_FOLDER = TEST_OUTPUTS + 'TEST_Webcam' + Timestample + '/scores/' TEST_SKELETONS_FOLDER = TEST_OUTPUTS + 'TEST_Webcam' + Timestample + '/skeletons/' TEST_IMAGES_FOLDER = TEST_OUTPUTS + 'TEST_Webcam' + Timestample + '/images/' if not os.path.exists(TEST_IMAGES_FOLDER): os.makedirs(TEST_IMAGES_FOLDER) ################################################################################################# # Will always be ture, if the webcam is pluged in while images_loader.Image_Captured(): # iterate the frames counter by 1 iFrames_Counter += 1 # grab frames from data source image_src = images_loader.Read_Image() image_display = image_src.copy() # get detected human(s) from openpose humans = skeleton_detector.detect(image_src) # convert human(s) to 2d coordinates in a list(of lists) skeletons_lists_src, scale_h = skeleton_detector.humans_to_skeletons_list( humans) # delete invalid skeletons from lists skeletons_lists = uti_tracker.delete_invalid_skeletons_from_lists( skeletons_lists_src) sText_Name = SKELETON_FILE_NAME_FORMAT.format(iFrames_Counter) sImage_Name = IMAGE_FILE_NAME_FORMAT.format(iFrames_Counter) uti_commons.save_listlist(TEST_SKELETONS_FOLDER + sText_Name, skeletons_lists_src) skeleton_detector.draw(image_display, humans) # sort and track humans in frames skeletons_dict = Local_Tracker.track(skeletons_lists) if len(skeletons_dict) >= 1: # get human ids and skeletons seperatly human_ids, skeletons_tracked_lists = map( list, zip(*skeletons_dict.items())) skeletons_tracked_lists = uti_features_extraction.rebuild_skeleton_joint_order( skeletons_tracked_lists) # uti_images_io.draw_bounding_box_for_multiple_person_on_image(image_display, skeletons_tracked_lists, scale_h) status_list, features_p, features_v = Featurs_Generator.calculate_features_multiple( human_ids, skeletons_tracked_lists) result_dict = predict_action_class(human_ids, status_list, features_p, features_v, network) if len(result_dict) > 0: values_view = result_dict.values() value_iterator = iter(values_view) first_value = next(value_iterator) # result_str = str(result_dict) # np.savetxt(TEST_RESULTS_FOLDER + sText_Name, result_dict) uti_commons.save_result_dict(TEST_RESULTS_FOLDER + sText_Name, result_dict) # only draw all the scores of the first prediction on image uti_images_io.draw_scores_for_one_person_on_image( image_display, first_value) uti_images_io.draw_result_images(image_display, human_ids, skeletons_tracked_lists, result_dict, scale_h, ACTION_CLASSES) cv2.imwrite(TEST_IMAGES_FOLDER + sImage_Name, image_display) Images_Displayer.display(image_display) if cv2.waitKey(1) & 0xFF == ord('q'): break # Remeber to kill the thread, or yyou can't quit this function properly images_loader.Stop() print('Finished')
def train_model_on_batch_v1(network): adam = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08) # original setup from paper SGD = tf.keras.optimizers.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False) RMS = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0) # network.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy']) network.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) network.summary() # network.load_weights(weight_path) # tf.keras.utils.plot_model(network, to_file= FIGURE_PATH + 'Model.png') batch_num = 0 model_save_acc = 0 all_train_accuracy = [] all_train_loss = [] all_tst_accuracy = [] all_tst_loss = [] # load dataset to memory, temp. train_position, train_velocity, train_labels = load_train_datasets(FEATURES_TRAIN) test_position, test_velocity, test_labels = load_test_datasets(FEATURES_TEST) train_data = uti_data_generator.Data_Generator(FEATURES_TRAIN, BATCH_SIZE) train_data_sum = train_data.get_train_data_sum() train_data_index = np.arange(0, train_data_sum) train_data_cursors = train_data.batch_cursors(train_data_sum) index_num = len(train_data_cursors) # test_position = np.expand_dims(test_position, axis=0) # test_velocity = np.expand_dims(test_velocity, axis=0) test_data = uti_data_generator.Data_Generator(FEATURES_TEST, BATCH_SIZE) test_data_sum = test_data.get_test_data_sum() test_data_index = np.arange(0, test_data_sum) test_data_cursors = test_data.batch_cursors(test_data_sum) test_index_num = len(test_data_cursors) for epoch in range(EPOCHS): accuracy_list = [] loss_list = [] test_accuracy_list = [] test_loss_list = [] print(epoch + 1, ' epoch is beginning......') ''' ''' for ind in range(index_num): batch_num += 1 up_data_0, down_data_0, train_labels_0 \ = train_data.generate_batch_data_v1(train_data_index, train_data_cursors[ind], train_position, train_velocity, train_labels) up_data_1, down_data_1, train_labels_1 \ = train_data.generate_batch_data_v1(train_data_index, train_data_cursors[ind], train_position, train_velocity, train_labels) train_loss = network.train_on_batch([up_data_0, up_data_1, down_data_0, down_data_1], train_labels_0) accuracy_list.append(train_loss[1]) loss_list.append(train_loss[0]) if batch_num % 50 == 0: print('the {:03d} batch: loss: {:.3f} accuracy: {:.3%}'.format(batch_num, train_loss[0], train_loss[1])) epoch_accuracy = sum(accuracy_list) / len(accuracy_list) epoch_loss = sum(loss_list) / len(loss_list) all_train_accuracy.append(epoch_accuracy) all_train_loss.append(epoch_loss) print('the {:03d} epoch: mean loss: {:.3f} mean accuracy: {:.3%}'.format(epoch + 1, epoch_loss, epoch_accuracy)) if epoch >= 1: tst_accuracy_list = [] tst_loss_list = [] for num in range(test_index_num): tst_up_0, tst_down_0, tst_labels_0 \ = test_data.get_test_batch(test_data_index, test_data_cursors[num], test_position, test_velocity, test_labels) tst_up_1, tst_down_1, tst_labels_1 \ = test_data.get_test_batch(test_data_index, test_data_cursors[num], test_position, test_velocity, test_labels) tst_loss = network.test_on_batch([tst_up_0, tst_up_1, tst_down_0, tst_down_1], tst_labels_0) # test_result = network.evaluate([tst_up_0, tst_up_1, tst_down_0, tst_down_1], tst_labels_0) tst_loss_list.append(tst_loss[0]) tst_accuracy_list.append(tst_loss[1]) tst_accuracy = sum(tst_accuracy_list) / len(tst_accuracy_list) tst_loss_output = sum(tst_loss_list) / len(tst_loss_list) all_tst_accuracy.append(tst_accuracy) all_tst_loss.append(tst_loss_output) print('The test data accuracy: {:.3%}'.format(tst_accuracy)) if tst_accuracy > model_save_acc: network.save(MODEL_PATH) model_save_acc = tst_accuracy print('Model Saved') uti_commons.save_listlist(TXT_FILE_PATH + 'all_train_loss.txt', all_train_loss) uti_commons.save_listlist(TXT_FILE_PATH + 'all_train_acc.txt', all_train_accuracy) uti_commons.save_listlist(TXT_FILE_PATH + 'all_test_loss.txt', all_tst_loss) uti_commons.save_listlist(TXT_FILE_PATH + 'all_test_acc.txt', all_tst_accuracy) fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8)) fig.suptitle('Training Metrics') axes[0].set_ylabel('Loss', fontsize=14) axes[0].plot(all_train_loss) axes[1].set_ylabel('Accuracy', fontsize=14) axes[1].set_xlabel('Epoch', fontsize=14) axes[1].plot(all_train_accuracy) plt.savefig(FIGURE_PATH + '35frame_train_cc.png') fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8)) fig.suptitle('Test Metrics') axes[0].set_ylabel('Loss', fontsize=14) axes[0].plot(all_tst_loss) axes[1].set_ylabel('Accuracy', fontsize=14) axes[1].set_xlabel('Epoch', fontsize=14) axes[1].plot(all_tst_accuracy) plt.savefig(FIGURE_PATH + '35frame_test_cc.png')