def main(): set_determinstic_mode(seed=1538574472) args = _parse_args() trainer = Trainer(args.config, args.device) train_data = breakfast.get_mstcn_data('train') test_data = breakfast.get_mstcn_data('test') trainer.train(train_data, test_data)
def main(): set_determinstic_mode() args = _parse_args() if args.model == 'mstcn': from scripts.action_segmentation.train_mstcn import Trainer elif args.model == 'coarse-inputs': from scripts.action_segmentation.train_coarse_inputs import Trainer elif args.model == 'coarse-inputs-boundary-true': from scripts.action_segmentation.train_coarse_inputs_boundary_true import Trainer else: raise ValueError('no such model') submission_dir = os.path.join(SUBMISSION_DIR, args.model, args.config) if not os.path.exists(submission_dir): os.makedirs(submission_dir) else: raise ValueError( submission_dir + ' exists, please delete if you want a new submission with this name' ) trainer = Trainer(args.config, args.device) if 'boundary' in args.model: # this is for boundary aware models i3d_feats, timestamps = get_boundary_aware_submission_data() frame_level_predictions = trainer.predict(i3d_feats, timestamps) else: submission_segments, _, _ = breakfast.get_mstcn_data(split='test') frame_level_predictions = trainer.predict(submission_segments) return get_cls_results(frame_level_predictions, submission_dir)
def get_mstcn_action_reg_data(split): i3d_files, labels, logits = get_mstcn_data(split=split) reg_files = [] segment_windows = [] reg_labels = [] reg_logits = [] for j, label in enumerate(labels): feat_file = i3d_files[j] prev_label = label[0] prev_idx = 0 label_len = len(label) for curr_idx in range(1, label_len): curr_label = label[curr_idx] if curr_label != prev_label: if prev_label != 'SIL': segment_windows.append([prev_idx, curr_idx]) # start idx, end_idx reg_files.append(feat_file) reg_labels.append(prev_label) reg_logits.append(logits[j][prev_idx]) prev_label = curr_label prev_idx = curr_idx # add the final one if prev_label != 'SIL': segment_windows.append([prev_idx, label_len]) # start idx, end_idx reg_files.append(feat_file) reg_labels.append(prev_label) reg_logits.append(logits[j][prev_idx]) assert len(reg_files) == len(segment_windows) == len(reg_labels) == len( reg_logits) return reg_files, segment_windows, reg_labels, reg_logits
def main(): set_determinstic_mode() args = _parse_args() submission_dir = os.path.join(SUBMISSION_DIR, 'zhangcan', args.config) submission_feats, _, _ = breakfast.get_mstcn_data(split='test') frame_prediction_dir = submission_dir video_names = [ os.path.split(feat_file)[-1] for feat_file in submission_feats ] video_names = [feat_file.split('.')[0] for feat_file in video_names] mapping_dict = breakfast.read_mapping_file() frame_level_predictions = [] for i, video_name in enumerate(video_names): frame_prediction_file = os.path.join(frame_prediction_dir, video_name) with open(frame_prediction_file, 'r') as f: frame_predictions = f.readlines()[1] frame_predictions = frame_predictions.strip().split(' ') frame_predictions = [ mapping_dict[prediction] for prediction in frame_predictions ] frame_level_predictions.append(frame_predictions) with open(breakfast.SUBMISSION_LABEL_FILE, 'r') as f: submission_timestamps = f.readlines() submission_timestamps = [ line.strip().split(' ') for line in submission_timestamps ] submission_timestamps = [ np.array(timestamps).astype(int) for timestamps in submission_timestamps ] n_segments = 0 submission_str = 'Id,Category\n' for i, video_name in enumerate(video_names): video_timestamps = submission_timestamps[i] n_timestamps = len(video_timestamps) video_frame_predictions = frame_level_predictions[i] for j in range(n_timestamps - 1): start = video_timestamps[j] end = video_timestamps[j + 1] segment_frame_predictions = video_frame_predictions[start:end] counts = np.bincount(segment_frame_predictions) segment_prediction = np.argmax(counts).item() submission_str += '{0},{1}\n'.format(n_segments, segment_prediction) n_segments += 1 submission_file = os.path.join(submission_dir, 'submission.csv') with open(submission_file, 'w') as f: f.write(submission_str) get_submission_accuracy(submission_file)
def get_boundary_aware_submission_data(): with open(breakfast.SUBMISSION_LABEL_FILE, 'r') as f: submission_timestamps = f.readlines() submission_timestamps = [ line.strip().split(' ') for line in submission_timestamps ] submission_timestamps = [ np.array(timestamps).astype(int) for timestamps in submission_timestamps ] i3d_files, _, _ = breakfast.get_mstcn_data(split='test') assert len(submission_timestamps) == len(i3d_files) return i3d_files, submission_timestamps
def get_cls_results(segment_predictions, submission_dir, postprocess='midpoint'): submission_feats, _, _ = breakfast.get_mstcn_data(split='test') video_names = [ os.path.split(feat_file)[-1] for feat_file in submission_feats ] with open(breakfast.SUBMISSION_LABEL_FILE, 'r') as f: submission_timestamps = f.readlines() submission_timestamps = [ line.strip().split(' ') for line in submission_timestamps ] submission_timestamps = [ np.array(timestamps).astype(int) for timestamps in submission_timestamps ] n_segments = 0 submission_str = 'Id,Category\n' segment_idx = 0 for i, video_name in enumerate(video_names): video_timestamps = submission_timestamps[i] n_timestamps = len(video_timestamps) for j in range(n_timestamps - 1): start = video_timestamps[j] end = video_timestamps[j + 1] vid_len = end - start assert vid_len > 0 segment_prediction = segment_predictions[segment_idx] segment_idx += 1 submission_str += '{0},{1}\n'.format(n_segments, segment_prediction) n_segments += 1 assert n_segments == len(segment_predictions) submission_file = os.path.join(submission_dir, 'submission.csv') with open(submission_file, 'w') as f: f.write(submission_str) return get_submission_accuracy(submission_file)
def get_cls_results(frame_level_predictions, submission_dir, postprocess='midpoint'): submission_feats, _, _ = breakfast.get_mstcn_data(split='test') frame_prediction_dir = os.path.join(submission_dir, 'frame-level-predictions') if not os.path.exists(frame_prediction_dir): os.makedirs(frame_prediction_dir) video_names = [ os.path.split(feat_file)[-1] for feat_file in submission_feats ] video_names = [feat_file.split('.')[0] for feat_file in video_names] for i, video_name in enumerate(video_names): frame_prediction_file = os.path.join(frame_prediction_dir, video_name + '.txt') frame_predictions = np.array(frame_level_predictions[i]) with open(frame_prediction_file, 'w') as f: frame_predictions = ' '.join(frame_predictions.astype(str)) f.write(frame_predictions + '\n') with open(breakfast.SUBMISSION_LABEL_FILE, 'r') as f: submission_timestamps = f.readlines() submission_timestamps = [ line.strip().split(' ') for line in submission_timestamps ] submission_timestamps = [ np.array(timestamps).astype(int) for timestamps in submission_timestamps ] print('INFO: creating segment unaware predictions') n_segments = 0 submission_str = 'Id,Category\n' for i, video_name in enumerate(tqdm(video_names)): video_timestamps = submission_timestamps[i] n_gt_segments = len(video_timestamps) - 1 video_frame_predictions = frame_level_predictions[i] segments = [] prediction_len = 1 prev_prediction = video_frame_predictions[i] for j in range(1, len(video_frame_predictions)): curr_prediction = video_frame_predictions[j] if curr_prediction != prev_prediction: if 0 < prev_prediction < 48: segments.append([prev_prediction, prediction_len]) prev_prediction = curr_prediction prediction_len = 1 else: prediction_len += 1 is_selected = np.zeros(shape=len(segments)) while len(segments) < n_gt_segments: segments.append(segments[-1]) is_selected[:] = 1 segment_lengths = np.array(segments)[:, 1] sorted_segment_lengths = np.flip( np.sort(np.unique(np.array(segments)[:, 1]))) idx = 0 while np.sum(is_selected).item() < n_gt_segments: max_length = sorted_segment_lengths[idx] is_selected = segment_lengths >= max_length idx += 1 selected_segments = list() for ii in range(len(segments)): if is_selected[ii]: selected_segments.append(segments[ii]) if len(selected_segments) > n_gt_segments: selected_segments = selected_segments[:n_gt_segments] for segment in selected_segments: segment_prediction = segment[0] submission_str += '{0},{1}\n'.format(n_segments, segment_prediction) n_segments += 1 submission_file = os.path.join(submission_dir, 'submission.csv') with open(submission_file, 'w') as f: f.write(submission_str) return get_submission_accuracy(submission_file)