def __init__(self, overall_iters, cfg, mode): """ overall_iters (int): the overall number of iterations of one epoch. cfg (CfgNode): configs. mode (str): `train`, `val`, or `test` mode. """ self.cfg = cfg self.lr = None self.loss = ScalarMeter(cfg.LOG_PERIOD) self.full_ava_test = cfg.AVA.FULL_TEST_ON_VAL self.mode = mode self.iter_timer = Timer() self.data_timer = Timer() self.net_timer = Timer() self.all_preds_train = [] self.all_ori_boxes_train = [] self.all_metadata_train = [] self.all_preds = [] self.all_ori_boxes = [] self.all_metadata = [] self.overall_iters = overall_iters self.categories, self.class_whitelist = read_labelmap( os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.LABEL_MAP_FILE)) gt_filename = os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.GROUNDTRUTH_FILE) self.full_groundtruth = read_csv(gt_filename, self.class_whitelist) self.mini_groundtruth = get_ava_mini_groundtruth(self.full_groundtruth) _, self.video_idx_to_name = ava_helper.load_image_lists( cfg, mode == "train") self.output_dir = cfg.OUTPUT_DIR
def __init__(self, overall_iters, cfg, mode): """ overall_iters (int): the overall number of iterations of one epoch. cfg (CfgNode): configs. mode (str): `train`, `val`, or `test` mode. """ is_custom_dataset = cfg.TRAIN.DATASET == "custom" and cfg.TEST.DATASET == "custom" if is_custom_dataset: logger.info("Creating AVA Meter for custom dataset in %s mode" % mode) else: logger.info("Creating AVA Meter for AVA dataset in %s mode" % mode) self.cfg = cfg self.lr = None self.loss = ScalarMeter(cfg.LOG_PERIOD) self.full_ava_test = cfg.AVA.FULL_TEST_ON_VAL if not is_custom_dataset else cfg.CUSTOM_DATASET.FULL_TEST_ON_VAL self.mode = mode self.iter_timer = Timer() self.all_preds = [] self.all_ori_boxes = [] self.all_metadata = [] self.overall_iters = overall_iters self.excluded_keys = read_exclusions( os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.EXCLUSION_FILE) ) if not is_custom_dataset else read_exclusions( os.path.join(cfg.CUSTOM_DATASET.ANNOTATION_DIR, cfg.CUSTOM_DATASET.EXCLUSION_FILE)) self.categories, self.class_whitelist = read_labelmap( os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.LABEL_MAP_FILE) ) if not is_custom_dataset else read_labelmap( os.path.join(cfg.CUSTOM_DATASET.ANNOTATION_DIR, cfg.CUSTOM_DATASET.LABEL_MAP_FILE)) gt_filename = os.path.join( cfg.AVA.ANNOTATION_DIR, cfg.AVA.GROUNDTRUTH_FILE ) if not is_custom_dataset else os.path.join( cfg.CUSTOM_DATASET.ANNOTATION_DIR, cfg.CUSTOM_DATASET.GROUNDTRUTH_FILE) self.full_groundtruth = read_csv(gt_filename, self.class_whitelist) self.mini_groundtruth = get_ava_mini_groundtruth(self.full_groundtruth) _, self.video_idx_to_name = ava_helper.load_image_lists( cfg, mode == "train" ) if not is_custom_dataset else custom_helper.load_image_lists( cfg, mode == "train")
def __init__(self, cfg, img_height, first_middle_frame_index, frames_per_second, input_detection_queue=None, output_detection_queue=None, output_tracker_queue_visualization=None, output_tracker_queue_action_recognition=None, input_action_recognition_queue=None, output_action_recognition_queue_visualization=None, output_action_recognition_queue_result_export=None): """ Initialize the object :param cfg: our demo config :param img_height: (int) the height of the image :param first_middle_frame_index: (int) the index of the first middle_frame index :param frames_per_second: (float) the fps of the video -> required for determining middle frames :param input_detection_queue: please refer to class MultiProcessDemo :param output_detection_queue: please refer to class MultiProcessDemo :param output_tracker_queue_visualization: please refer to class MultiProcessDemo :param output_tracker_queue_action_recognition: please refer to class MultiProcessDemo :param input_action_recognition_queue: please refer to class MultiProcessDemo :param output_action_recognition_queue_visualization: please refer to class MultiProcessDemo :param output_action_recognition_queue_result_export: please refer to class MultiProcessDemo """ setup_environment() # Setup logging format logging.setup_logging(cfg.OUTPUT_DIR) self.cfg = cfg # The name of the input video self.demo_video_name = Path(self.cfg.DEMO.VIDEO_SOURCE_PATH).stem # Whether we will export an image self.export_video = self.cfg.DEMO.VIDEO_EXPORT_VIDEO_ENABLE if self.export_video: # number of digits for exporting the images (determines how many images can be stored) self.number_of_digits_for_image_export = 10 # The path of the to be created video self.export_video_path = os.path.join( self.cfg.DEMO.OUTPUT_FOLDER, self.demo_video_name + "_annotated.mp4") # Whether we will display an image self.display_video = self.cfg.DEMO.VIDEO_SHOW_VIDEO_ENABLE self.cv2_display_name = "Demo: " + self.demo_video_name # Whether we will display the meta information (Queues Sizes and img idx) self.display_meta_info = cfg.DEMO.VIDEO_SHOW_VIDEO_DEBUGGING_INFO # Used for finding the position of meta info self.img_height = img_height # Used for determining middle_frame_indices (they have the action prediction) self.first_middle_frame_index = first_middle_frame_index self.frames_per_second = frames_per_second # Additional options for displaying the video self.video_display_scaling_factor = cfg.DEMO.VIDEO_DISPLAY_SCALING_FACTOR self.video_action_display_duration_milliseconds = cfg.DEMO.VIDEO_ACTION_DISPLAY_DURATION_MILLISECONDS # The queues containing relevant information self.input_detection_queue = input_detection_queue self.output_detection_queue = output_detection_queue, self.output_tracker_queue_visualization = output_tracker_queue_visualization self.output_tracker_queue_action_recognition = output_tracker_queue_action_recognition, self.input_action_recognition_queue = input_action_recognition_queue self.output_action_recognition_queue_visualization = output_action_recognition_queue_visualization self.output_action_recognition_queue_result_export = output_action_recognition_queue_result_export # The queue sizes as specified in the config files self.queue_size = self.cfg.DEMO.QSIZE_SECONDS * self.cfg.CUSTOM_DATASET.FRAME_RATE # Used for terminating the process successfully self.action_recognition_input_finished = False # The information for displaying actions # Load the categories: self.path_to_label_map_file = os.path.join(cfg.CUSTOM_DATASET.ANNOTATION_DIR, cfg.CUSTOM_DATASET.LABEL_MAP_FILE) \ if not os.path.isfile(cfg.ACTIONRECOGNIZER.LABEL_MAP_FILE) \ else cfg.ACTIONRECOGNIZER.LABEL_MAP_FILE # List of dicts (id, name) self.action_categories, _ = read_labelmap(self.path_to_label_map_file) # A color value for every category self.palette_actions = np.random.randint( 64, 128, (len(self.action_categories), 3)).tolist() # The information required for displaying person_tracking info self.palette_person_ids = (2**11 - 1, 2**15 - 1, 2**20 - 1) # The process for person detection self.display_next_frame_process = mp.Process( target=self.display_and_or_export_next_frame, args=()) # Used to test the correct order of images self.display_img_idx = -1 # The information for action info display self.current_action_output_img_idx = "" self.current_pred_action_category_scores = ""
def __init__(self, cfg, img_height, img_width): """ Initialize the DemoMeter with the relevant paramters :param cfg: :param img_height: (int) the height of the input images :param img_width: (int) the width of input images """ # Set up environment. setup_environment() # Setup logging format logging.setup_logging(cfg.OUTPUT_DIR) self.cfg = cfg # In the case of an AVA-like predictor it is necessary to specify a # cfg.ACTIONRECOGNIZER.LABEL_MAP_FILE, because it comprises all 80 categories. # during the ava challenge only 60 categories were evaluated, an we # want alle categories path_to_label_map_file = os.path.join(cfg.CUSTOM_DATASET.ANNOTATION_DIR, cfg.CUSTOM_DATASET.LABEL_MAP_FILE) \ if not os.path.isfile(cfg.ACTIONRECOGNIZER.LABEL_MAP_FILE) \ else cfg.ACTIONRECOGNIZER.LABEL_MAP_FILE # Export properties datetime_for_filenames = datetime.datetime.now().strftime( "%Y-%m-%d_%H_%M_%S") self.delimiter = "," assert cfg.DEMO.OUTPUT_FOLDER != "", "Please specify cfg.DEMO.OUTPUT_FOLDER to be able to export the output" self.output_dir_path = self.cfg.DEMO.OUTPUT_FOLDER self.file_name_demo_log = datetime_for_filenames + "_" + "demo_log" self.file_name_demo_gt_like_file = datetime_for_filenames + "_" + "demo_gt_format" self.results_gt_like_csv_path = os.path.join( self.output_dir_path, self.file_name_demo_gt_like_file + ".csv") self.results_log_path_prefix = os.path.join(self.output_dir_path, self.file_name_demo_log) self.results_log_csv_path = "" self.results_xes_path = "" # The minimum score for a predicted category to be exported self.min_category_export_score = cfg.DEMO.EXPORT_MIN_CATEGORY_EXPORT_SCORE # Whether a person can do multiple actions at the same time or not # This influences the export options, since only the option with max value is chosen self.multiple_action_possible = cfg.CUSTOM_DATASET.MULTIPLE_ACTION_POSSIBLE # Resolution used for export self.img_height = img_height self.img_width = img_width # List of dict with items "id" and "name" self.categories, _ = read_labelmap(path_to_label_map_file) # Replace delimiter out of category_name to guarantee good csv export for idx in range(0, len(self.categories)): self.categories[idx]["name"] = self.categories[idx][ "name"].replace(self.delimiter, "") # The list-variables we use to store the demo prediction results. # They will be used to export the information into csv or xes self.res_person_tracking_outputs = [] self.res_pred_action_category_scores = [] self.res_all_metadata = [] self.res_case_ids = [] # This df is used to assign correct case_concept_name and concept_instance values self.case_and_instance_df = self.create_empty_case_and_instance_df() # Used to indicate that the activity instance has not yet completed self.video_second_not_complete = -1 # Used as test for correct functionality of lifecycle_transition self.not_closed_list = []