def main(out_dir=None, data_dir=None, labels_dir=None): out_dir = os.path.expanduser(out_dir) data_dir = os.path.expanduser(data_dir) labels_dir = os.path.expanduser(labels_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) filenames = [ utils.stripExtension(fn) for fn in glob.glob(os.path.join(labels_dir, '*.csv')) ] metadata = utils.loadMetadata(data_dir) metadata['seq_id'] = metadata.index metadata = metadata.set_index( 'dir_name', drop=False).loc[filenames].set_index('seq_id') seq_ids = np.sort(metadata.index.to_numpy()) logger.info(f"Loaded {len(seq_ids)} sequences from {labels_dir}") vocab = [] for i, seq_id in enumerate(seq_ids): seq_id_str = f"seq={seq_id}" seq_dir_name = metadata['dir_name'].loc[seq_id] labels_fn = os.path.join(labels_dir, f'{seq_dir_name}.csv') event_labels = utils.loadVariable(f'{seq_id_str}_labels', data_dir) assembly_actions = pd.read_csv(labels_fn) label_seq = parseActions(assembly_actions, event_labels.shape[0], vocab) utils.saveVariable(label_seq, f'{seq_id_str}_label-seq', out_data_dir) plotLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.png'), label_seq) writeLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.csv'), label_seq, vocab) utils.saveMetadata(metadata, out_data_dir) utils.saveVariable(vocab, 'vocab', out_data_dir)
def main(out_dir=None, data_dir=None, annotation_dir=None, frames_dir=None, col_format='standard', win_params={}, slowfast_csv_params={}, label_types=('event', 'action', 'part')): out_dir = os.path.expanduser(out_dir) data_dir = os.path.expanduser(data_dir) annotation_dir = os.path.expanduser(annotation_dir) frames_dir = os.path.expanduser(frames_dir) annotation_dir = os.path.join(annotation_dir, 'action_annotations') logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_labels_dir = os.path.join(out_dir, 'labels') if not os.path.exists(out_labels_dir): os.makedirs(out_labels_dir) data_dirs = { name: os.path.join(out_dir, f"{name}-dataset") for name in label_types } for name, dir_ in data_dirs.items(): if not os.path.exists(dir_): os.makedirs(dir_) event_vocab_df, part_vocab, action_vocab = load_vocabs( os.path.join(data_dir, 'ANU_ikea_dataset', 'indexing_files', 'atomic_action_list.txt')) event_vocab_df.to_csv(os.path.join(out_labels_dir, 'event-vocab.csv')) event_vocab = event_vocab_df.index.tolist() vocabs = {'event': event_vocab, 'action': action_vocab, 'part': part_vocab} vocabs = {label_name: vocabs[label_name] for label_name in label_types} for name, vocab in vocabs.items(): utils.saveVariable(vocab, 'vocab', data_dirs[name]) label_fn = os.path.join(annotation_dir, 'gt_segments.json') seq_ids, event_labels, metadata = load_action_labels( label_fn, event_vocab_df) utils.saveMetadata(metadata, out_labels_dir) for name, dir_ in data_dirs.items(): utils.saveMetadata(metadata, dir_) logger.info(f"Loaded {len(seq_ids)} sequences from {label_fn}") part_names = [name for name in part_vocab if name != ''] col_names = [f"{name}_active" for name in part_names] integerizers = { label_name: {name: i for i, name in enumerate(label_vocab)} for label_name, label_vocab in vocabs.items() } all_slowfast_labels_seg = collections.defaultdict(list) all_slowfast_labels_win = collections.defaultdict(list) counts = np.zeros((len(action_vocab), len(part_vocab)), dtype=int) for i, seq_id in enumerate(seq_ids): seq_id_str = f"seq={seq_id}" seq_dir_name = metadata['dir_name'].loc[seq_id] event_segs = event_labels[i] if not event_segs.any(axis=None): logger.warning(f"No event labels for sequence {seq_id}") continue event_data = make_event_data( event_segs, sorted(glob.glob(os.path.join(frames_dir, seq_dir_name, '*.jpg'))), integerizers['event'], integerizers['action'], integerizers['part'], event_vocab.index('NA'), action_vocab.index('NA'), False) event_wins = make_window_clips(event_data, vocabs['event'], vocabs['action'], **win_params) event_data.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_data.csv"), index=False) event_segs.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_segs.csv"), index=False) filenames = event_data['fn'].to_list() label_indices = {} for name in label_types: if name == 'part': label_indices[name] = event_data[col_names].to_numpy() seg_labels_slowfast = make_slowfast_labels( event_segs[['start', 'end']], getActivePart(event_segs[col_names], part_names), event_data['fn'], integerizers[name], col_format=col_format) win_labels_slowfast = make_slowfast_labels( event_wins[['start', 'end']], getActivePart(event_wins[col_names], part_names), event_data['fn'], integerizers[name], col_format=col_format) else: label_indices[name] = event_data[name].to_numpy() seg_labels_slowfast = make_slowfast_labels( event_segs[['start', 'end']], event_segs[name], event_data['fn'], integerizers[name], col_format=col_format) win_labels_slowfast = make_slowfast_labels( event_wins[['start', 'end']], event_wins[name], event_data['fn'], integerizers[name], col_format=col_format) utils.saveVariable(filenames, f'{seq_id_str}_frame-fns', data_dirs[name]) utils.saveVariable(label_indices[name], f'{seq_id_str}_labels', data_dirs[name]) seg_labels_slowfast.to_csv( os.path.join(data_dirs[name], f'{seq_id_str}_slowfast-labels.csv'), **slowfast_csv_params) win_labels_slowfast.to_csv( os.path.join(data_dirs[name], f'{seq_id_str}_slowfast-labels.csv'), **slowfast_csv_params) all_slowfast_labels_seg[name].append(seg_labels_slowfast) all_slowfast_labels_win[name].append(win_labels_slowfast) plot_event_labels(os.path.join(fig_dir, f"{seq_id_str}.png"), label_indices['event'], label_indices['action'], label_indices['part'], event_vocab, action_vocab, part_names) for part_activity_row, action_index in zip(label_indices['part'], label_indices['action']): for i, is_active in enumerate(part_activity_row): part_index = integerizers['part'][part_names[i]] counts[action_index, part_index] += int(is_active) for name, labels in all_slowfast_labels_seg.items(): pd.concat(labels, axis=0).to_csv( os.path.join(data_dirs[name], 'slowfast-labels_seg.csv'), **slowfast_csv_params) for name, labels in all_slowfast_labels_win.items(): pd.concat(labels, axis=0).to_csv( os.path.join(data_dirs[name], 'slowfast-labels_win.csv'), **slowfast_csv_params) utils.saveVariable(counts, 'action-part-counts', out_labels_dir) plt.matshow(counts) plt.xticks(ticks=range(len(part_vocab)), labels=part_vocab, rotation='vertical') plt.yticks(ticks=range(len(action_vocab)), labels=action_vocab) plt.savefig(os.path.join(fig_dir, 'action-part-coocurrence.png'), bbox_inches='tight')
def saveVariable(var, var_name, to_dir=out_data_dir): utils.saveVariable(var, var_name, to_dir)
def main( out_dir=None, data_dir=None, use_vid_ids_from=None, output_data=None, magnitude_centering=None, resting_from_gt=None, remove_before_first_touch=None, include_signals=None, fig_type=None): data_dir = os.path.expanduser(data_dir) out_dir = os.path.expanduser(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) logger.info(f"Reading from: {data_dir}") logger.info(f"Writing to: {out_dir}") fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) def loadAll(seq_ids, var_name, from_dir=data_dir, prefix='trial='): all_data = tuple( utils.loadVariable(f"{prefix}{seq_id}_{var_name}", from_dir) for seq_id in seq_ids ) return all_data def saveVariable(var, var_name, to_dir=out_data_dir): utils.saveVariable(var, var_name, to_dir) if fig_type is None: fig_type = 'multi' # Load data if use_vid_ids_from is None: trial_ids = utils.getUniqueIds(data_dir, prefix='trial=', to_array=True) else: use_vid_ids_from = os.path.expanduser(use_vid_ids_from) trial_ids = utils.getUniqueIds(use_vid_ids_from, prefix='trial-', to_array=True) accel_seqs = loadAll(trial_ids, 'accel-samples.pkl') gyro_seqs = loadAll(trial_ids, 'gyro-samples.pkl') action_seqs = loadAll(trial_ids, 'action-seq.pkl') rgb_timestamp_seqs = loadAll(trial_ids, 'rgb-frame-timestamp-seq.pkl') def validate_imu(seqs): def is_valid(d): return not any(np.isnan(x).any() for x in d.values()) return np.array([is_valid(d) for d in seqs]) imu_is_valid = validate_imu(accel_seqs) & validate_imu(gyro_seqs) logger.info( f"Ignoring {(~imu_is_valid).sum()} IMU sequences with NaN-valued samples " f"(of {len(imu_is_valid)} total)" ) def chooseValid(seq): return tuple(x for x, is_valid in zip(seq, imu_is_valid) if is_valid) trial_ids = np.array(list(chooseValid(trial_ids))) accel_seqs = chooseValid(accel_seqs) gyro_seqs = chooseValid(gyro_seqs) action_seqs = chooseValid(action_seqs) rgb_timestamp_seqs = chooseValid(rgb_timestamp_seqs) vocab = [] metadata = utils.loadMetadata(data_dir, rows=trial_ids) utils.saveMetadata(metadata, out_data_dir) utils.saveVariable(vocab, 'vocab', out_data_dir) def norm(x): norm = np.linalg.norm(imu.getImuSamples(x), axis=1)[:, None] return norm accel_mag_seqs = tuple(map(lambda x: dictToArray(x, transform=norm), accel_seqs)) gyro_mag_seqs = tuple(map(lambda x: dictToArray(x, transform=norm), gyro_seqs)) imu_timestamp_seqs = utils.batchProcess(makeTimestamps, accel_seqs, gyro_seqs) if remove_before_first_touch: before_first_touch_seqs = utils.batchProcess( beforeFirstTouch, action_seqs, rgb_timestamp_seqs, imu_timestamp_seqs ) num_ignored = sum(b is None for b in before_first_touch_seqs) logger.info( f"Ignoring {num_ignored} sequences without first-touch annotations " f"(of {len(before_first_touch_seqs)} total)" ) trials_missing_first_touch = [ i for b, i in zip(before_first_touch_seqs, trial_ids) if b is None ] logger.info(f"Trials without first touch: {trials_missing_first_touch}") def clip(signal, bool_array): return signal[~bool_array, ...] accel_mag_seqs = tuple( clip(signal, b) for signal, b in zip(accel_mag_seqs, before_first_touch_seqs) if b is not None ) gyro_mag_seqs = tuple( clip(signal, b) for signal, b in zip(gyro_mag_seqs, before_first_touch_seqs) if b is not None ) imu_timestamp_seqs = tuple( clip(signal, b) for signal, b in zip(imu_timestamp_seqs, before_first_touch_seqs) if b is not None ) trial_ids = tuple( x for x, b in zip(trial_ids, before_first_touch_seqs) if b is not None ) action_seqs = tuple( x for x, b in zip(action_seqs, before_first_touch_seqs) if b is not None ) rgb_timestamp_seqs = tuple( x for x, b in zip(rgb_timestamp_seqs, before_first_touch_seqs) if b is not None ) assembly_seqs = utils.batchProcess( parseActions, action_seqs, rgb_timestamp_seqs, imu_timestamp_seqs ) if output_data == 'components': accel_feat_seqs = accel_mag_seqs gyro_feat_seqs = gyro_mag_seqs unique_components = {frozenset(): 0} imu_label_seqs = zip( *tuple( labels.componentLabels(*args, unique_components) for args in zip(action_seqs, rgb_timestamp_seqs, imu_timestamp_seqs) ) ) saveVariable(unique_components, 'unique_components') elif output_data == 'pairwise components': imu_label_seqs = utils.batchProcess( labels.pairwiseComponentLabels, assembly_seqs, static_kwargs={'lower_tri_only': True, 'include_action_labels': False} ) accel_feat_seqs = tuple(map(imu.pairwiseFeats, accel_mag_seqs)) gyro_feat_seqs = tuple(map(imu.pairwiseFeats, gyro_mag_seqs)) else: raise AssertionError() signals = {'accel': accel_feat_seqs, 'gyro': gyro_feat_seqs} if include_signals is None: include_signals = tuple(signals.keys()) signals = tuple(signals[key] for key in include_signals) imu_feature_seqs = tuple(np.stack(x, axis=-1).squeeze(axis=-1) for x in zip(*signals)) video_seqs = tuple(zip(imu_feature_seqs, imu_label_seqs, trial_ids)) imu.plot_prediction_eg(video_seqs, fig_dir, fig_type=fig_type, output_data=output_data) video_seqs = tuple( zip(assembly_seqs, imu_feature_seqs, imu_timestamp_seqs, imu_label_seqs, trial_ids) ) for assembly_seq, feature_seq, timestamp_seq, label_seq, trial_id in video_seqs: id_string = f"trial={trial_id}" saveVariable(assembly_seq, f'{id_string}_assembly-seq') saveVariable(feature_seq, f'{id_string}_feature-seq') saveVariable(timestamp_seq, f'{id_string}_timestamp-seq') saveVariable(label_seq, f'{id_string}_label-seq')
def main(out_dir=None, data_dir=None, event_scores_dir=None, action_scores_dir=None, part_scores_dir=None, as_atomic_events=False, only_fold=None, plot_io=None, prefix='seq=', results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) event_scores_dir = os.path.expanduser(event_scores_dir) action_scores_dir = os.path.expanduser(action_scores_dir) part_scores_dir = os.path.expanduser(part_scores_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) loader = DataLoader( { 'event': os.path.join(data_dir, 'event-dataset'), 'action': os.path.join(data_dir, 'action-dataset'), 'part': os.path.join(data_dir, 'part-dataset') }, { 'event': event_scores_dir, 'action': action_scores_dir, 'part': part_scores_dir }, prefix=prefix) logger.info( f"Loaded ids for {len(loader.seq_ids)} sequences from {data_dir}") all_metrics = collections.defaultdict(list) if as_atomic_events: map_df = pd.read_csv(os.path.join(data_dir, 'labels', 'event-vocab.csv'), index_col=False, keep_default_na=False) else: map_df = pd.DataFrame( [[f"{action}({part})", action] + [part == n for n in loader.vocabs['part'] if n] for action in loader.vocabs['action'] for part in loader.vocabs['part']], columns=['event', 'action'] + [f"{n}_active" for n in loader.vocabs['part'] if n]) event_vocab = map_df['event'].to_list() utils.saveVariable(event_vocab, 'vocab', out_data_dir) # Define cross-validation folds cv_folds = utils.makeDataSplits(len(loader.seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) for cv_index, cv_fold in enumerate(cv_folds): if only_fold is not None and cv_index != only_fold: continue train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) action_part_counts = sum( count_action_part_bigrams(loader.vocabs['action'], loader.vocabs['part'], la, lp) for la, lp in loader._getLabels(train_indices, label_name=['action', 'part'], load_from_data=True)) model = AttributeClassifier(action_part_counts) plot_action_part_bigrams( loader.vocabs['action'], loader.vocabs['part'], (model.action_part_counts > 0).astype(int), os.path.join(fig_dir, f'cvfold={cv_index}_action-part-coocurrence.png')) for i in test_indices: seq_id = loader.seq_ids[i] logger.info(f" Processing sequence {seq_id}...") action_score_seq, part_score_seq, true_action_seq, true_part_seq = loader[ seq_id] event_score_seq = model.forward(action_score_seq, part_score_seq) pred_action_seq, pred_part_seq = model.predict(event_score_seq) true_event_seq = ap_to_event(true_action_seq, true_part_seq, map_df, loader.vocabs) pred_event_seq = ap_to_event(pred_action_seq, pred_part_seq, map_df, loader.vocabs) metric_dict = {} metric_dict = eval_metrics(pred_event_seq, true_event_seq, name_prefix='Event ', append_to=metric_dict) metric_dict = eval_metrics(pred_action_seq, true_action_seq, name_prefix='Action ', append_to=metric_dict) metric_dict = eval_metrics(pred_part_seq, true_part_seq, name_prefix='Part ', append_to=metric_dict) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") all_metrics[name].append(value) seq_id_str = f"seq={seq_id}" utils.saveVariable(event_score_seq, f'{seq_id_str}_score-seq', out_data_dir) utils.saveVariable(true_event_seq, f'{seq_id_str}_true-label-seq', out_data_dir) utils.saveVariable(pred_event_seq, f'{seq_id_str}_pred-label-seq', out_data_dir) utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) if plot_io: utils.plot_array( event_score_seq.reshape(event_score_seq.shape[0], -1).T, (true_event_seq, pred_event_seq), ('true', 'pred'), fn=os.path.join(fig_dir, f"seq={seq_id:03d}_event.png")) utils.plot_array(action_score_seq.T, (true_action_seq, pred_action_seq), ('true', 'pred'), fn=os.path.join( fig_dir, f"seq={seq_id:03d}_action.png")) utils.plot_array(part_score_seq.T, (true_part_seq, pred_part_seq), ('true', 'pred'), fn=os.path.join(fig_dir, f"seq={seq_id:03d}_part.png"))
def main(out_dir=None, data_dir=None, segs_dir=None, scores_dir=None, vocab_dir=None, label_type='edges', gpu_dev_id=None, start_from=None, stop_at=None, num_disp_imgs=None, results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) segs_dir = os.path.expanduser(segs_dir) scores_dir = os.path.expanduser(scores_dir) vocab_dir = os.path.expanduser(vocab_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir_images = os.path.join(fig_dir, 'model-io_images') if not os.path.exists(io_dir_images): os.makedirs(io_dir_images) io_dir_plots = os.path.join(fig_dir, 'model-io_plots') if not os.path.exists(io_dir_plots): os.makedirs(io_dir_plots) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(scores_dir, prefix='trial=', suffix='score-seq.*', to_array=True) logger.info( f"Loaded scores for {len(seq_ids)} sequences from {scores_dir}") link_vocab = {} joint_vocab = {} joint_type_vocab = {} vocab, parts_vocab, part_labels = load_vocab(link_vocab, joint_vocab, joint_type_vocab, vocab_dir) pred_vocab = [] # FIXME if label_type == 'assembly': logger.info("Converting assemblies -> edges") state_pred_seqs = tuple( utils.loadVariable(f"trial={seq_id}_pred-label-seq", scores_dir) for seq_id in seq_ids) state_true_seqs = tuple( utils.loadVariable(f"trial={seq_id}_true-label-seq", scores_dir) for seq_id in seq_ids) edge_pred_seqs = tuple(part_labels[seq] for seq in state_pred_seqs) edge_true_seqs = tuple(part_labels[seq] for seq in state_true_seqs) elif label_type == 'edge': logger.info("Converting edges -> assemblies (will take a few minutes)") edge_pred_seqs = tuple( utils.loadVariable(f"trial={seq_id}_pred-label-seq", scores_dir) for seq_id in seq_ids) edge_true_seqs = tuple( utils.loadVariable(f"trial={seq_id}_true-label-seq", scores_dir) for seq_id in seq_ids) state_pred_seqs = tuple( edges_to_assemblies(seq, pred_vocab, parts_vocab, part_labels) for seq in edge_pred_seqs) state_true_seqs = tuple( edges_to_assemblies(seq, vocab, parts_vocab, part_labels) for seq in edge_true_seqs) device = torchutils.selectDevice(gpu_dev_id) dataset = sim2real.LabeledConnectionDataset( utils.loadVariable('parts-vocab', vocab_dir), utils.loadVariable('part-labels', vocab_dir), utils.loadVariable('vocab', vocab_dir), device=device) all_metrics = collections.defaultdict(list) # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) for cv_index, cv_fold in enumerate(cv_folds): train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) train_states = np.hstack( tuple(state_true_seqs[i] for i in (train_indices))) train_edges = part_labels[train_states] # state_train_vocab = np.unique(train_states) # edge_train_vocab = part_labels[state_train_vocab] train_freq_bigram, train_freq_unigram = edge_joint_freqs(train_edges) # state_probs = utils.makeHistogram(len(vocab), train_states, normalize=True) test_states = np.hstack( tuple(state_true_seqs[i] for i in (test_indices))) test_edges = part_labels[test_states] # state_test_vocab = np.unique(test_states) # edge_test_vocab = part_labels[state_test_vocab] test_freq_bigram, test_freq_unigram = edge_joint_freqs(test_edges) f, axes = plt.subplots(1, 2) axes[0].matshow(train_freq_bigram) axes[0].set_title('Train') axes[1].matshow(test_freq_bigram) axes[1].set_title('Test') plt.tight_layout() plt.savefig( os.path.join(fig_dir, f"edge-freqs-bigram_cvfold={cv_index}.png")) f, axis = plt.subplots(1) axis.stem(train_freq_unigram, label='Train', linefmt='C0-', markerfmt='C0o') axis.stem(test_freq_unigram, label='Test', linefmt='C1--', markerfmt='C1o') plt.legend() plt.tight_layout() plt.savefig( os.path.join(fig_dir, f"edge-freqs-unigram_cvfold={cv_index}.png")) for i in test_indices: seq_id = seq_ids[i] logger.info(f" Processing sequence {seq_id}...") trial_prefix = f"trial={seq_id}" # I include the '.' to differentiate between 'rgb-frame-seq' and # 'rgb-frame-seq-before-first-touch' # rgb_seq = utils.loadVariable(f"{trial_prefix}_rgb-frame-seq.", data_dir) # seg_seq = utils.loadVariable(f"{trial_prefix}_seg-labels-seq", segs_dir) score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", scores_dir) # if score_seq.shape[0] != rgb_seq.shape[0]: # err_str = f"scores shape {score_seq.shape} != data shape {rgb_seq.shape}" # raise AssertionError(err_str) edge_pred_seq = edge_pred_seqs[i] edge_true_seq = edge_true_seqs[i] state_pred_seq = state_pred_seqs[i] state_true_seq = state_true_seqs[i] num_types = np.unique(state_pred_seq).shape[0] num_samples = state_pred_seq.shape[0] num_total = len(pred_vocab) logger.info( f" {num_types} assemblies predicted ({num_total} total); " f"{num_samples} samples") # edge_freq_bigram, edge_freq_unigram = edge_joint_freqs(edge_true_seq) # dist_shift = np.linalg.norm(train_freq_unigram - edge_freq_unigram) metric_dict = { # 'State OOV rate': oov_rate_state(state_true_seq, state_train_vocab), # 'Edge OOV rate': oov_rate_edges(edge_true_seq, edge_train_vocab), # 'State avg prob, true': state_probs[state_true_seq].mean(), # 'State avg prob, pred': state_probs[state_pred_seq].mean(), # 'Edge distribution shift': dist_shift } metric_dict = eval_edge_metrics(edge_pred_seq, edge_true_seq, append_to=metric_dict) metric_dict = eval_state_metrics(state_pred_seq, state_true_seq, append_to=metric_dict) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") all_metrics[name].append(value) utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) if num_disp_imgs is not None: pred_images = tuple( render(dataset, vocab[seg_label]) for seg_label in utils.computeSegments(state_pred_seq)[0]) imageprocessing.displayImages( *pred_images, file_path=os.path.join( io_dir_images, f"seq={seq_id:03d}_pred-assemblies.png"), num_rows=None, num_cols=5) true_images = tuple( render(dataset, vocab[seg_label]) for seg_label in utils.computeSegments(state_true_seq)[0]) imageprocessing.displayImages( *true_images, file_path=os.path.join( io_dir_images, f"seq={seq_id:03d}_true-assemblies.png"), num_rows=None, num_cols=5) utils.plot_array(score_seq.T, (edge_true_seq.T, edge_pred_seq.T), ('true', 'pred'), fn=os.path.join(io_dir_plots, f"seq={seq_id:03d}.png"))
def save_vocabs(self, out_dir): if not os.path.exists(out_dir): os.makedirs(out_dir) for name, vocab in self.vocabs.items(): utils.saveVariable(vocab, name.replace('_', '-'), out_dir)
def main(out_dir=None, data_dir=None, assembly_data_dir=None, scores_dir=None, event_attr_fn=None, connection_attr_fn=None, assembly_attr_fn=None, only_fold=None, plot_io=None, prefix='seq=', stop_after=None, background_action='', model_params={}, cv_params={}, stride=None, results_file=None, sweep_param_name=None): data_dir = os.path.expanduser(data_dir) assembly_data_dir = os.path.expanduser(assembly_data_dir) scores_dir = os.path.expanduser(scores_dir) event_attr_fn = os.path.expanduser(event_attr_fn) connection_attr_fn = os.path.expanduser(connection_attr_fn) assembly_attr_fn = os.path.expanduser(assembly_attr_fn) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) misc_dir = os.path.join(out_dir, 'misc') if not os.path.exists(misc_dir): os.makedirs(misc_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(data_dir, prefix=prefix, suffix='labels.*', to_array=True) dataset = utils.FeaturelessCvDataset(seq_ids, data_dir, prefix=prefix, label_fn_format='labels') logger.info( f"Loaded scores for {len(seq_ids)} sequences from {scores_dir}") # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) # Load event, connection attributes assembly_vocab = tuple( tuple(sorted(tuple(sorted(joint)) for joint in a)) for a in utils.loadVariable('vocab', assembly_data_dir)) (*probs, assembly_transition_probs), vocabs = loadPartInfo( event_attr_fn, connection_attr_fn, assembly_attr_fn, background_action=background_action, assembly_vocab=assembly_vocab) event_assembly_scores = event_to_assembly_scores(*probs, vocabs) assembly_transition_scores = np.log(assembly_transition_probs) viz_transition_probs(os.path.join(fig_dir, 'action-transitions'), np.exp(event_assembly_scores), vocabs['event_vocab']) write_transition_probs(os.path.join(misc_dir, 'action-transitions'), np.exp(event_assembly_scores), vocabs['event_vocab'], vocabs['assembly_vocab']) for cv_index, cv_fold in enumerate(cv_folds): if only_fold is not None and cv_index != only_fold: continue train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) train_data, val_data, test_data = dataset.getFold(cv_fold) cv_str = f'cvfold={cv_index}' class_priors, event_dur_probs = count_priors(train_data[0], len(dataset.vocab), stride=stride, approx_upto=0.95, support_only=True) event_dur_scores = np.log(event_dur_probs) event_dur_scores = np.zeros_like(event_dur_scores) scores = (event_dur_scores, event_assembly_scores, assembly_transition_scores) model = decode.AssemblyActionRecognizer(scores, vocabs, model_params) viz_priors(os.path.join(fig_dir, f'{cv_str}_priors'), class_priors, event_dur_probs) model.write_fsts(os.path.join(misc_dir, f'{cv_str}_fsts')) model.save_vocabs(os.path.join(out_data_dir, f'{cv_str}_model-vocabs')) for i, (_, seq_id) in enumerate(zip(*test_data)): if stop_after is not None and i >= stop_after: break trial_prefix = f"{prefix}{seq_id}" if model_params['return_label'] == 'input': true_seq = utils.loadVariable(f"{trial_prefix}_true-label-seq", scores_dir) elif model_params['return_label'] == 'output': try: true_seq = utils.loadVariable(f"{trial_prefix}_label-seq", assembly_data_dir) true_seq = true_seq[::stride] except AssertionError: # logger.info(f' Skipping sequence {seq_id}: {e}') continue logger.info(f" Processing sequence {seq_id}...") event_score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", scores_dir) if event_score_seq.shape[0] != true_seq.shape[0]: err_str = (f'Event scores shape {event_score_seq.shape} ' f'!= labels shape {true_seq.shape}') raise AssertionError(err_str) # FIXME: the serialized variables are probs, not log-probs # event_score_seq = suppress_nonmax(event_score_seq) # event_score_seq = np.log(event_score_seq) decode_score_seq = model.forward(event_score_seq) pred_seq = model.predict(decode_score_seq) metric_dict = eval_metrics(pred_seq, true_seq) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) utils.saveVariable(decode_score_seq, f'{trial_prefix}_score-seq', out_data_dir) utils.saveVariable(pred_seq, f'{trial_prefix}_pred-label-seq', out_data_dir) utils.saveVariable(true_seq, f'{trial_prefix}_true-label-seq', out_data_dir) if plot_io: utils.plot_array(event_score_seq.T, (pred_seq.T, true_seq.T), ('pred', 'true'), fn=os.path.join(fig_dir, f"seq={seq_id:03d}.png")) write_labels( os.path.join(misc_dir, f"seq={seq_id:03d}_pred-seq.txt"), pred_seq, model.output_vocab.as_raw()) write_labels( os.path.join(misc_dir, f"seq={seq_id:03d}_true-seq.txt"), true_seq, model.output_vocab.as_raw())
def main(out_dir=None, data_dir=None, model_name=None, predict_mode='classify', gpu_dev_id=None, batch_size=None, learning_rate=None, independent_signals=None, active_only=None, output_dim_from_vocab=False, prefix='trial=', feature_fn_format='feature-seq.pkl', label_fn_format='label_seq.pkl', dataset_params={}, model_params={}, cv_params={}, train_params={}, viz_params={}, metric_names=['Loss', 'Accuracy', 'Precision', 'Recall', 'F1'], plot_predictions=None, results_file=None, sweep_param_name=None): data_dir = os.path.expanduser(data_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) def saveVariable(var, var_name, to_dir=out_data_dir): return utils.saveVariable(var, var_name, to_dir) # Load data device = torchutils.selectDevice(gpu_dev_id) trial_ids = utils.getUniqueIds(data_dir, prefix=prefix, suffix=feature_fn_format, to_array=True) dataset = utils.CvDataset( trial_ids, data_dir, prefix=prefix, feature_fn_format=feature_fn_format, label_fn_format=label_fn_format, ) utils.saveMetadata(dataset.metadata, out_data_dir) utils.saveVariable(dataset.vocab, 'vocab', out_data_dir) # Define cross-validation folds cv_folds = utils.makeDataSplits(len(trial_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) if predict_mode == 'binary multiclass': # criterion = torchutils.BootstrappedCriterion( # 0.25, base_criterion=torch.nn.functional.binary_cross_entropy_with_logits, # ) criterion = torch.nn.BCEWithLogitsLoss() labels_dtype = torch.float elif predict_mode == 'multiclass': criterion = torch.nn.CrossEntropyLoss() labels_dtype = torch.long elif predict_mode == 'classify': criterion = torch.nn.CrossEntropyLoss() labels_dtype = torch.long else: raise AssertionError() def make_dataset(feats, labels, ids, shuffle=True): dataset = torchutils.SequenceDataset(feats, labels, device=device, labels_dtype=labels_dtype, seq_ids=ids, **dataset_params) loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) return dataset, loader for cv_index, cv_fold in enumerate(cv_folds): train_data, val_data, test_data = dataset.getFold(cv_fold) if independent_signals: train_data = splitSeqs(*train_data, active_only=active_only) val_data = splitSeqs(*val_data, active_only=active_only) test_data = splitSeqs(*test_data, active_only=False) train_set, train_loader = make_dataset(*train_data, shuffle=True) test_set, test_loader = make_dataset(*test_data, shuffle=False) val_set, val_loader = make_dataset(*val_data, shuffle=True) logger.info( f'CV fold {cv_index + 1} / {len(cv_folds)}: {len(dataset.trial_ids)} total ' f'({len(train_set)} train, {len(val_set)} val, {len(test_set)} test)' ) logger.info(f'{train_set.num_label_types} unique labels in train set; ' f'vocab size is {len(dataset.vocab)}') input_dim = train_set.num_obsv_dims output_dim = train_set.num_label_types if output_dim_from_vocab: output_dim = len(dataset.vocab) if model_name == 'linear': model = torchutils.LinearClassifier( input_dim, output_dim, **model_params).to(device=device) elif model_name == 'conv': model = ConvClassifier(input_dim, output_dim, **model_params).to(device=device) elif model_name == 'TCN': if predict_mode == 'multiclass': num_multiclass = train_set[0][1].shape[-1] output_dim = max([ train_set.num_label_types, test_set.num_label_types, val_set.num_label_types ]) else: num_multiclass = None model = TcnClassifier(input_dim, output_dim, num_multiclass=num_multiclass, **model_params).to(device=device) elif model_name == 'LSTM': if predict_mode == 'multiclass': num_multiclass = train_set[0][1].shape[-1] output_dim = max([ train_set.num_label_types, test_set.num_label_types, val_set.num_label_types ]) else: num_multiclass = None model = LstmClassifier(input_dim, output_dim, num_multiclass=num_multiclass, **model_params).to(device=device) else: raise AssertionError() optimizer_ft = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=1, gamma=1.00) train_epoch_log = collections.defaultdict(list) val_epoch_log = collections.defaultdict(list) metric_dict = {name: metrics.makeMetric(name) for name in metric_names} model, last_model_wts = torchutils.trainModel( model, criterion, optimizer_ft, lr_scheduler, train_loader, val_loader, device=device, metrics=metric_dict, train_epoch_log=train_epoch_log, val_epoch_log=val_epoch_log, **train_params) # Test model metric_dict = {name: metrics.makeMetric(name) for name in metric_names} test_io_history = torchutils.predictSamples( model.to(device=device), test_loader, criterion=criterion, device=device, metrics=metric_dict, data_labeled=True, update_model=False, seq_as_batch=train_params['seq_as_batch'], return_io_history=True) if independent_signals: test_io_history = tuple(joinSeqs(test_io_history)) logger.info('[TST] ' + ' '.join(str(m) for m in metric_dict.values())) utils.writeResults(results_file, {k: v.value for k, v in metric_dict.items()}, sweep_param_name, model_params) if plot_predictions: io_fig_dir = os.path.join(fig_dir, 'model-io') if not os.path.exists(io_fig_dir): os.makedirs(io_fig_dir) label_names = ('gt', 'pred') preds, scores, inputs, gt_labels, ids = zip(*test_io_history) for batch in test_io_history: batch = tuple( x.cpu().numpy() if isinstance(x, torch.Tensor) else x for x in batch) for preds, _, inputs, gt_labels, seq_id in zip(*batch): fn = os.path.join(io_fig_dir, f"{prefix}{seq_id}_model-io.png") utils.plot_array(inputs, (gt_labels.T, preds.T), label_names, fn=fn) for batch in test_io_history: batch = tuple(x.cpu().numpy() if isinstance(x, torch.Tensor) else x for x in batch) for pred_seq, score_seq, feat_seq, label_seq, trial_id in zip( *batch): saveVariable(pred_seq, f'{prefix}{trial_id}_pred-label-seq') saveVariable(score_seq, f'{prefix}{trial_id}_score-seq') saveVariable(label_seq, f'{prefix}{trial_id}_true-label-seq') saveVariable(model, f'cvfold={cv_index}_{model_name}-best') train_fig_dir = os.path.join(fig_dir, 'train-plots') if not os.path.exists(train_fig_dir): os.makedirs(train_fig_dir) if train_epoch_log: torchutils.plotEpochLog(train_epoch_log, subfig_size=(10, 2.5), title='Training performance', fn=os.path.join( train_fig_dir, f'cvfold={cv_index}_train-plot.png')) if val_epoch_log: torchutils.plotEpochLog(val_epoch_log, subfig_size=(10, 2.5), title='Heldout performance', fn=os.path.join( train_fig_dir, f'cvfold={cv_index}_val-plot.png'))
def main(out_dir=None, data_dir=None, scores_dir=None, frames_dir=None, vocab_from_scores_dir=None, only_fold=None, plot_io=None, prefix='seq=', results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) scores_dir = os.path.expanduser(scores_dir) frames_dir = os.path.expanduser(frames_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir_images = os.path.join(fig_dir, 'model-io_images') if not os.path.exists(io_dir_images): os.makedirs(io_dir_images) io_dir_plots = os.path.join(fig_dir, 'model-io_plots') if not os.path.exists(io_dir_plots): os.makedirs(io_dir_plots) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(scores_dir, prefix=prefix, suffix='pred-label-seq.*', to_array=True) logger.info( f"Loaded scores for {len(seq_ids)} sequences from {scores_dir}") if vocab_from_scores_dir: vocab = utils.loadVariable('vocab', scores_dir) else: vocab = utils.loadVariable('vocab', data_dir) all_metrics = collections.defaultdict(list) # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) all_pred_seqs = [] all_true_seqs = [] for cv_index, cv_fold in enumerate(cv_folds): if only_fold is not None and cv_index != only_fold: continue train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) for i in test_indices: seq_id = seq_ids[i] logger.info(f" Processing sequence {seq_id}...") trial_prefix = f"{prefix}{seq_id}" score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", scores_dir) pred_seq = utils.loadVariable(f"{trial_prefix}_pred-label-seq", scores_dir) true_seq = utils.loadVariable(f"{trial_prefix}_true-label-seq", scores_dir) metric_dict = eval_metrics(pred_seq, true_seq) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") all_metrics[name].append(value) utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) all_pred_seqs.append(pred_seq) all_true_seqs.append(true_seq) if plot_io: utils.plot_array(score_seq.T, (true_seq, pred_seq), ('true', 'pred'), fn=os.path.join(io_dir_plots, f"seq={seq_id:03d}.png")) if False: confusions = metrics.confusionMatrix(all_pred_seqs, all_true_seqs, len(vocab)) utils.saveVariable(confusions, "confusions", out_data_dir) per_class_acc, class_counts = metrics.perClassAcc(confusions, return_counts=True) class_preds = confusions.sum(axis=1) logger.info(f"MACRO ACC: {np.nanmean(per_class_acc) * 100:.2f}%") metrics.plotConfusions(os.path.join(fig_dir, 'confusions.png'), confusions, vocab) metrics.plotPerClassAcc(os.path.join(fig_dir, 'per-class-results.png'), vocab, per_class_acc, class_preds, class_counts)
def main(out_dir=None, data_dir=None, actions_dir=None, parts_dir=None, events_dir=None, edges_dir=None, prefix='seq=', feature_fn_format='score-seq', label_fn_format='true-label-seq', stop_after=None, only_fold=None, plot_io=None, dataset_params={}, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) actions_dir = os.path.expanduser(actions_dir) parts_dir = os.path.expanduser(parts_dir) events_dir = os.path.expanduser(events_dir) edges_dir = os.path.expanduser(edges_dir) out_dir = os.path.expanduser(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) # vocab = utils.loadVariable( # 'assembly-action-vocab', # os.path.join(data_dir, 'event-dataset') # ) # vocab = [BlockAssembly()] + list(abs(x) for x in vocab) dataset = FusionDataset( actions_dir, parts_dir, events_dir, edges_dir, prefix=prefix, **dataset_params, # vocab=vocab, ) utils.saveMetadata(dataset.metadata, out_data_dir) utils.saveVariable(dataset.vocab, 'vocab', out_data_dir) seq_ids = dataset.trial_ids logger.info(f"Loaded scores for {len(seq_ids)} sequences from {data_dir}") # Define cross-validation folds # cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) # utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) for i, seq_id in enumerate(seq_ids): try: labels = dataset.loadTargets(seq_id) features = dataset.loadInputs(seq_id) except AssertionError as e: logger.warning(f'Skipping sequence {seq_id}: {e}') continue logger.info(f"Processing sequence {seq_id}") if labels.shape[0] != features.shape[0]: message = f'Label shape {labels.shape} != feature shape {features.shape}' raise AssertionError(message) seq_prefix = f"seq={seq_id}" utils.saveVariable(features, f'{seq_prefix}_feature-seq', out_data_dir) utils.saveVariable(labels, f'{seq_prefix}_label-seq', out_data_dir) if plot_io: fn = os.path.join(fig_dir, f'{seq_prefix}.png') utils.plot_array(features.T, (labels.T, ), ('gt', ), fn=fn)
def main( out_dir=None, data_dir=None, results_file=None, cv_file=None, take_log=False, col_format=None, win_params={}, slowfast_csv_params={}): out_dir = os.path.expanduser(out_dir) data_dir = os.path.expanduser(data_dir) results_file = os.path.expanduser(results_file) cv_file = os.path.expanduser(cv_file) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) vocab = utils.loadVariable('vocab', data_dir) metadata = utils.loadMetadata(data_dir) slowfast_labels = pd.read_csv( cv_file, keep_default_na=False, index_col=0, **slowfast_csv_params ) seg_ids = slowfast_labels.index.to_numpy() vid_names = slowfast_labels['video_name'].unique().tolist() metadata['seq_id'] = metadata.index vid_ids = metadata.set_index('dir_name').loc[vid_names].set_index('seq_id').index metadata = metadata.drop('seq_id', axis=1) with open(results_file, 'rb') as file_: model_probs, gt_labels = pickle.load(file_) model_probs = model_probs.numpy() gt_labels = gt_labels.numpy() if len(model_probs) != len(seg_ids): err_str = f"{len(model_probs)} segment scores != {slowfast_labels.shape[0]} CSV rows" raise AssertionError(err_str) logger.info(f"Loaded {len(seg_ids)} segments, {len(vid_ids)} videos") for vid_id, vid_name in zip(vid_ids, vid_names): matches_video = (slowfast_labels['video_name'] == vid_name).to_numpy() win_labels = gt_labels[matches_video] win_probs = model_probs[matches_video, :] if win_labels.shape == win_probs.shape: win_preds = (win_probs > 0.5).astype(int) else: win_preds = win_probs.argmax(axis=1) if take_log: win_probs = np.log(win_probs) seq_id_str = f"seq={vid_id}" utils.saveVariable(win_probs, f'{seq_id_str}_score-seq', out_data_dir) utils.saveVariable(win_labels, f'{seq_id_str}_true-label-seq', out_data_dir) utils.saveVariable(win_preds, f'{seq_id_str}_pred-label-seq', out_data_dir) utils.plot_array( win_probs.T, (win_labels.T, win_preds.T), ('true', 'pred'), tick_names=vocab, fn=os.path.join(fig_dir, f"{seq_id_str}.png"), subplot_width=12, subplot_height=5 ) utils.saveVariable(vocab, 'vocab', out_data_dir) utils.saveMetadata(metadata, out_data_dir)
def main(out_dir=None, data_dir=None, scores_dirs={}, vocab_from_scores_dir=None, only_fold=None, plot_io=None, prefix='seq=', results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) scores_dirs = { name: os.path.expanduser(dir_) for name, dir_ in scores_dirs.items() } out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if len(scores_dirs) != 2: err_str = ( f"scores_dirs has {len(scores_dirs)} entries, but this script " "compare exactly 2") raise NotImplementedError(err_str) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(data_dir, prefix=prefix, suffix='labels.*', to_array=True) logger.info(f"Loaded scores for {len(seq_ids)} sequences from {data_dir}") # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) vocabs = {} confusions = {} accs = {} counts = {} for expt_name, scores_dir in scores_dirs.items(): if vocab_from_scores_dir: vocabs[expt_name] = utils.loadVariable('vocab', scores_dir) else: vocabs[expt_name] = utils.loadVariable('vocab', data_dir) confusions[expt_name] = utils.loadVariable("confusions", scores_dir) per_class_accs, class_counts = metrics.perClassAcc( confusions[expt_name], return_counts=True) accs[expt_name] = per_class_accs counts[expt_name] = class_counts vocab = utils.reduce_all_equal(tuple(vocabs.values())) class_counts = utils.reduce_all_equal(tuple(counts.values())) first_name, second_name = scores_dirs.keys() confusions_diff = confusions[first_name] - confusions[second_name] acc_diff = accs[first_name] - accs[second_name] metrics.plotConfusions( os.path.join(fig_dir, f'confusions_{first_name}-minus-{second_name}.png'), confusions_diff, vocab) metrics.plotPerClassAcc( os.path.join(fig_dir, f'accs_{first_name}-minus-{second_name}.png'), vocab, acc_diff, confusions_diff.sum(axis=1), class_counts)
def main( out_dir=None, modalities=['rgb', 'imu'], gpu_dev_id=None, plot_io=None, rgb_data_dir=None, rgb_attributes_dir=None, imu_data_dir=None, imu_attributes_dir=None): out_dir = os.path.expanduser(out_dir) rgb_data_dir = os.path.expanduser(rgb_data_dir) rgb_attributes_dir = os.path.expanduser(rgb_attributes_dir) imu_data_dir = os.path.expanduser(imu_data_dir) imu_attributes_dir = os.path.expanduser(imu_attributes_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) # Load data if modalities == ['rgb']: trial_ids = utils.getUniqueIds(rgb_data_dir, prefix='trial=', to_array=True) logger.info(f"Processing {len(trial_ids)} videos") else: rgb_trial_ids = utils.getUniqueIds(rgb_data_dir, prefix='trial=', to_array=True) imu_trial_ids = utils.getUniqueIds(imu_data_dir, prefix='trial=', to_array=True) trial_ids = np.array(sorted(set(rgb_trial_ids.tolist()) & set(imu_trial_ids.tolist()))) logger.info( f"Processing {len(trial_ids)} videos common to " f"RGB ({len(rgb_trial_ids)} total) and IMU ({len(imu_trial_ids)} total)" ) device = torchutils.selectDevice(gpu_dev_id) dataset = FusionDataset( trial_ids, rgb_attributes_dir, rgb_data_dir, imu_attributes_dir, imu_data_dir, device=device, modalities=modalities, ) utils.saveMetadata(dataset.metadata, out_data_dir) utils.saveVariable(dataset.vocab, 'vocab', out_data_dir) for i, trial_id in enumerate(trial_ids): logger.info(f"Processing sequence {trial_id}...") true_label_seq = dataset.loadTargets(trial_id) attribute_feats = dataset.loadInputs(trial_id) # (Process the samples here if we need to) attribute_feats = attribute_feats.cpu().numpy() true_label_seq = true_label_seq.cpu().numpy() trial_prefix = f"trial={trial_id}" utils.saveVariable(attribute_feats, f'{trial_prefix}_feature-seq', out_data_dir) utils.saveVariable(true_label_seq, f'{trial_prefix}_label-seq', out_data_dir) if plot_io: fn = os.path.join(fig_dir, f'{trial_prefix}.png') utils.plot_array( attribute_feats.T, (true_label_seq,), ('gt',), fn=fn )
def main(out_dir=None, metadata_file=None, corpus_name=None, default_annotator=None, metadata_criteria={}, win_params={}, slowfast_csv_params={}, label_types=('event', 'action', 'part')): out_dir = os.path.expanduser(out_dir) metadata_file = os.path.expanduser(metadata_file) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_labels_dir = os.path.join(out_dir, 'labels') if not os.path.exists(out_labels_dir): os.makedirs(out_labels_dir) data_dirs = { name: os.path.join(out_dir, f"{name}-dataset") for name in label_types } for name, dir_ in data_dirs.items(): if not os.path.exists(dir_): os.makedirs(dir_) assembly_data_dir = os.path.join(out_dir, 'assembly-dataset') if not os.path.exists(assembly_data_dir): os.makedirs(assembly_data_dir) (seq_ids, event_labels, assembly_seqs, frame_fn_seqs, frame_fn_idx_seqs, vocabs, assembly_action_vocab, metadata) = load_all_labels(corpus_name, default_annotator, metadata_file, metadata_criteria, start_video_from_first_touch=True, subsample_period=None, use_coarse_actions=True) assembly_vocab = [] assembly_idx_seqs = tuple( tuple(labels.gen_eq_classes(state_seq, assembly_vocab)) for state_seq in assembly_seqs) utils.saveVariable(assembly_vocab, 'vocab', assembly_data_dir) utils.saveVariable(assembly_action_vocab, 'assembly-action-vocab', data_dirs['event']) vocabs = {label_name: vocabs[label_name] for label_name in label_types} for name, vocab in vocabs.items(): utils.saveVariable(vocab, 'vocab', data_dirs[name]) utils.saveMetadata(metadata, out_labels_dir) for name, dir_ in data_dirs.items(): utils.saveMetadata(metadata, dir_) assembly_attrs = labels.blockConnectionsSeq(assembly_vocab) utils.saveVariable(assembly_attrs, 'assembly-attrs', assembly_data_dir) plt.matshow(assembly_attrs.T) plt.savefig(os.path.join(fig_dir, 'assembly-attrs.png')) logger.info( f"Loaded {len(seq_ids)} sequence labels from {corpus_name} dataset") part_names = [name for name in vocabs['part'] if name != ''] col_names = [f"{name}_active" for name in part_names] integerizers = { label_name: {name: i for i, name in enumerate(label_vocab)} for label_name, label_vocab in vocabs.items() } all_slowfast_labels_seg = collections.defaultdict(list) all_slowfast_labels_win = collections.defaultdict(list) counts = np.zeros((len(vocabs['action']), len(vocabs['part'])), dtype=int) for i, seq_id in enumerate(seq_ids): logger.info(f"Processing sequence {i + 1} / {len(seq_ids)}") seq_id_str = f"seq={seq_id}" event_segs = event_labels[i] frame_fns = frame_fn_seqs[i] frame_fn_idxs = frame_fn_idx_seqs[i] assembly_seq = assembly_seqs[i] assembly_label_seq = make_assembly_labels(assembly_seq, assembly_idx_seqs[i], **win_params) # video_dir = os.path.dirname(frame_fns[0]).split('/')[-1] video_dir = f"{seq_id}" event_data = make_event_data(event_segs, frame_fns, frame_fn_idxs, integerizers['event'], integerizers['action'], integerizers['part'], vocabs['event'].index(''), vocabs['action'].index(''), False) # Redefining event segments from the sequence catches background segments # that are not annotated in the source labels event_segs = make_clips(event_data, vocabs['event'], vocabs['action'], clip_type='segment') event_wins = make_clips(event_data, vocabs['event'], vocabs['action'], clip_type='window', **win_params) for name in ('event', 'action'): event_segs[f'{name}_id'] = [ integerizers[name][n] for n in event_segs[name] ] event_wins[f'{name}_id'] = [ integerizers[name][n] for n in event_wins[name] ] event_data.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_data.csv"), index=False) event_segs.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_segs.csv"), index=False) event_wins.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_wins.csv"), index=False) utils.saveVariable(assembly_label_seq, f'seq={seq_id}_label-seq', assembly_data_dir) filenames = event_data['fn'].to_list() label_indices = {} bound_keys = ['start', 'end'] for name in label_types: if name == 'part': label_indices[name] = event_data[col_names].to_numpy() label_keys = col_names else: label_indices[name] = event_data[name].to_numpy() label_keys = [f'{name}_id'] seg_labels_slowfast = make_slowfast_labels( event_segs[bound_keys], event_segs[label_keys], [video_dir for _ in range(event_segs.shape[0])]) win_labels_slowfast = make_slowfast_labels( event_wins[bound_keys], event_wins[label_keys], [video_dir for _ in range(event_wins.shape[0])]) utils.saveVariable(filenames, f'{seq_id_str}_frame-fns', data_dirs[name]) utils.saveVariable(label_indices[name], f'{seq_id_str}_labels', data_dirs[name]) seg_labels_slowfast.to_csv(os.path.join( data_dirs[name], f'{seq_id_str}_slowfast-labels.csv'), index=False, **slowfast_csv_params) win_labels_slowfast.to_csv(os.path.join( data_dirs[name], f'{seq_id_str}_slowfast-labels.csv'), index=False, **slowfast_csv_params) all_slowfast_labels_seg[name].append(seg_labels_slowfast) all_slowfast_labels_win[name].append(win_labels_slowfast) plot_event_labels(os.path.join(fig_dir, f"{seq_id_str}.png"), label_indices['event'], label_indices['action'], label_indices['part'], vocabs['event'], vocabs['action'], part_names) plot_assembly_labels( os.path.join(fig_dir, f"{seq_id_str}_assembly.png"), assembly_label_seq, label_indices['event'], vocabs['event']) for part_activity_row, action_index in zip(label_indices['part'], label_indices['action']): for i, is_active in enumerate(part_activity_row): part_index = integerizers['part'][part_names[i]] counts[action_index, part_index] += int(is_active) for name, sf_labels in all_slowfast_labels_seg.items(): pd.concat(sf_labels, axis=0).to_csv( os.path.join(data_dirs[name], 'slowfast-labels_seg.csv'), **slowfast_csv_params) for name, sf_labels in all_slowfast_labels_win.items(): pd.concat(sf_labels, axis=0).to_csv( os.path.join(data_dirs[name], 'slowfast-labels_win.csv'), **slowfast_csv_params) utils.saveVariable(counts, 'action-part-counts', out_labels_dir) plt.matshow(counts) plt.xticks(ticks=range(len(vocabs['part'])), labels=vocabs['part'], rotation='vertical') plt.yticks(ticks=range(len(vocabs['action'])), labels=vocabs['action']) plt.savefig(os.path.join(fig_dir, 'action-part-coocurrence.png'), bbox_inches='tight')
def main( out_dir=None, annotation_dir=None, frames_dir=None, win_params={}, slowfast_csv_params={}, label_types=('event', 'action', 'part')): out_dir = os.path.expanduser(out_dir) annotation_dir = os.path.expanduser(annotation_dir) frames_dir = os.path.expanduser(frames_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_labels_dir = os.path.join(out_dir, 'labels') if not os.path.exists(out_labels_dir): os.makedirs(out_labels_dir) data_dirs = {name: os.path.join(out_dir, f"{name}-dataset") for name in label_types} for name, dir_ in data_dirs.items(): if not os.path.exists(dir_): os.makedirs(dir_) seq_ids, event_labels, vocabs, metadata = load_all_labels(annotation_dir) vocabs = {label_name: vocabs[label_name] for label_name in label_types} for name, vocab in vocabs.items(): utils.saveVariable(vocab, 'vocab', data_dirs[name]) utils.saveMetadata(metadata, out_labels_dir) for name, dir_ in data_dirs.items(): utils.saveMetadata(metadata, dir_) logger.info(f"Loaded {len(seq_ids)} sequence labels from {annotation_dir}") part_names = [name for name in vocabs['part'] if name != ''] col_names = [f"{name}_active" for name in part_names] integerizers = { label_name: {name: i for i, name in enumerate(label_vocab)} for label_name, label_vocab in vocabs.items() } all_slowfast_labels_seg = collections.defaultdict(list) all_slowfast_labels_win = collections.defaultdict(list) counts = np.zeros((len(vocabs['action']), len(vocabs['part'])), dtype=int) for i, seq_id in enumerate(seq_ids): logger.info(f"Processing sequence {i + 1} / {len(seq_ids)}") seq_id_str = f"seq={seq_id}" event_segs = event_labels[i] # Ignore 'check booklet' events because they don't have an impact on construction event_segs = event_segs.loc[event_segs['event'] != 'check_booklet'] event_data = make_event_data( event_segs, sorted(glob.glob(os.path.join(frames_dir, f'{seq_id}', '*.jpg'))), integerizers['event'], integerizers['action'], integerizers['part'], vocabs['event'].index(''), vocabs['action'].index(''), False ) # Redefining event segments from the sequence catches background segments # that are not annotated in the source labels event_segs = make_clips( event_data, vocabs['event'], vocabs['action'], clip_type='segment' ) event_wins = make_clips( event_data, vocabs['event'], vocabs['action'], clip_type='window', **win_params ) for name in ('event', 'action'): event_segs[f'{name}_id'] = [integerizers[name][n] for n in event_segs[name]] event_wins[f'{name}_id'] = [integerizers[name][n] for n in event_wins[name]] event_data.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_data.csv"), index=False) event_segs.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_segs.csv"), index=False) event_wins.to_csv(os.path.join(out_labels_dir, f"{seq_id_str}_wins.csv"), index=False) filenames = event_data['fn'].to_list() label_indices = {} bound_keys = ['start', 'end'] for name in label_types: if name == 'part': label_indices[name] = event_data[col_names].to_numpy() label_keys = col_names else: label_indices[name] = event_data[name].to_numpy() label_keys = [f'{name}_id'] seg_labels_slowfast = make_slowfast_labels( event_segs[bound_keys], event_segs[label_keys], event_data['fn'] ) win_labels_slowfast = make_slowfast_labels( event_wins[bound_keys], event_wins[label_keys], event_data['fn'] ) utils.saveVariable(filenames, f'{seq_id_str}_frame-fns', data_dirs[name]) utils.saveVariable(label_indices[name], f'{seq_id_str}_labels', data_dirs[name]) seg_labels_slowfast.to_csv( os.path.join(data_dirs[name], f'{seq_id_str}_slowfast-labels.csv'), index=False, **slowfast_csv_params ) win_labels_slowfast.to_csv( os.path.join(data_dirs[name], f'{seq_id_str}_slowfast-labels.csv'), index=False, **slowfast_csv_params ) all_slowfast_labels_seg[name].append(seg_labels_slowfast) all_slowfast_labels_win[name].append(win_labels_slowfast) plot_event_labels( os.path.join(fig_dir, f"{seq_id_str}.png"), label_indices['event'], label_indices['action'], label_indices['part'], vocabs['event'], vocabs['action'], part_names ) for part_activity_row, action_index in zip(label_indices['part'], label_indices['action']): for i, is_active in enumerate(part_activity_row): part_index = integerizers['part'][part_names[i]] counts[action_index, part_index] += int(is_active) for name, labels in all_slowfast_labels_seg.items(): pd.concat(labels, axis=0).to_csv( os.path.join(data_dirs[name], 'slowfast-labels_seg.csv'), **slowfast_csv_params ) for name, labels in all_slowfast_labels_win.items(): pd.concat(labels, axis=0).to_csv( os.path.join(data_dirs[name], 'slowfast-labels_win.csv'), **slowfast_csv_params ) utils.saveVariable(counts, 'action-part-counts', out_labels_dir) plt.matshow(counts) plt.xticks(ticks=range(len(vocabs['part'])), labels=vocabs['part'], rotation='vertical') plt.yticks(ticks=range(len(vocabs['action'])), labels=vocabs['action']) plt.savefig(os.path.join(fig_dir, 'action-part-coocurrence.png'), bbox_inches='tight')