def loadInputs(self, seq_id): if self.modalities == ['rgb']: return self.loadInputsRgb(seq_id) trial_prefix = f"trial={seq_id}" rgb_attribute_seq = torch.tensor( utils.loadVariable(f"{trial_prefix}_score-seq", self.rgb_attributes_dir), dtype=torch.float, device=self.device ) rgb_timestamp_seq = utils.loadVariable( f"{trial_prefix}_rgb-frame-timestamp-seq", from_dir=self.rgb_data_dir ) imu_attribute_seq = torch.tensor( utils.loadVariable(f"{trial_prefix}_score-seq", self.imu_attributes_dir), dtype=torch.float, device=self.device ) imu_timestamp_seq = utils.loadVariable(f"{trial_prefix}_timestamp-seq", self.imu_data_dir) rgb_attribute_seq, imu_attribute_seq = resample( rgb_attribute_seq, rgb_timestamp_seq, imu_attribute_seq, imu_timestamp_seq ) attribute_feats = { 'rgb': make_attribute_features(rgb_attribute_seq), 'imu': make_attribute_features(imu_attribute_seq) } attribute_feats = torch.cat( tuple(attribute_feats[name] for name in self.modalities), dim=1 ) return attribute_feats
def loadInputs(self, seq_id, prefix=None, stride=None): if prefix is None: prefix = self.prefix actions_seq = utils.loadVariable( f"{prefix}{seq_id}_{self.feature_fn_format}", self.actions_dir) actions_seq = scipy.special.softmax(actions_seq, axis=1) events_seq = utils.loadVariable( f"{prefix}{seq_id}_{self.feature_fn_format}", self.events_dir) events_seq = scipy.special.softmax(events_seq, axis=1) parts_seq = utils.loadVariable( f"{prefix}{seq_id}_{self.feature_fn_format}", self.parts_dir) parts_seq = scipy.special.expit(parts_seq) prefix = 'trial=' edges_seq = utils.loadVariable( f"{prefix}{seq_id}_{self.feature_fn_format}", self.edges_dir) edges_seq = scipy.special.softmax(edges_seq, axis=1) edges_seq = np.reshape(edges_seq, (edges_seq.shape[0], -1)) edges_seq = edges_seq[::5] attribute_feats = { 'actions': make_attribute_features(actions_seq), 'parts': make_attribute_features(parts_seq), 'events': make_attribute_features(events_seq), 'edges': make_attribute_features(edges_seq) } attribute_feats = np.concatenate(tuple(attribute_feats[name] for name in self.modalities), axis=1) return attribute_feats
def loadPartInfo(assembly_data_dir, event_attr_fn, connection_attr_fn, background_action=''): assembly_vocab = utils.loadVariable('vocab', assembly_data_dir) assembly_attrs = utils.loadVariable('assembly-attrs', assembly_data_dir) # joint_vocab = tuple(tuple(sorted(joint)) for joint in data['joint_vocab']) # part_vocab = tuple(data['part_vocab']) joint_vocab = None # FIXME part_vocab = None # FIXME part_categories = {} final_assembly_attrs = assembly_attrs # FIXME connection_probs, action_vocab, connection_vocab = connection_attrs_to_probs( pd.read_csv(connection_attr_fn, index_col=False, keep_default_na=False), # normalize=True ) with open(event_attr_fn, 'rt') as file_: event_probs, event_vocab = event_attrs_to_probs( json.load(file_), part_categories, action_vocab, joint_vocab, background_action=background_action, # normalize=True ) assembly_probs, assembly_vocab = assembly_attrs_to_probs( assembly_attrs, joint_vocab, connection_vocab, # normalize=True ) num_assemblies = len(assembly_attrs) transition_probs = np.zeros((num_assemblies + 1, num_assemblies + 1), dtype=float) transition_probs[0, :-1] = prior_probs(assembly_attrs) transition_probs[1:, -1] = final_probs(assembly_attrs, final_assembly_attrs) transition_probs[1:, :-1] = assembly_transition_probs( assembly_attrs, allow_self_transitions=True ) probs = (event_probs, connection_probs, assembly_probs, transition_probs) vocabs = { 'event_vocab': event_vocab, 'part_vocab': part_vocab, 'action_vocab': action_vocab, 'joint_vocab': joint_vocab, 'connection_vocab': connection_vocab, 'assembly_vocab': assembly_vocab } return probs, vocabs
def loadTargets(self, seq_id): trial_prefix = f"trial={seq_id}" true_label_seq = torch.tensor( utils.loadVariable(f'{trial_prefix}_true-label-seq', self.rgb_attributes_dir), dtype=torch.long, device=self.device ) return true_label_seq
def loadAssemblies(self, seq_id, var_name, vocab, prefix='trial='): assembly_seq = utils.loadVariable(f"{prefix}{seq_id}_{var_name}", self.data_dir) labels = np.zeros(assembly_seq[-1].end_idx, dtype=int) for assembly in assembly_seq: i = utils.getIndex(assembly, vocab) labels[assembly.start_idx:assembly.end_idx] = i return labels
def loadTargets(self, seq_id, prefix=None): if prefix is None: prefix = self.prefix if self.labels == 'edges': prefix = 'trial=' true_label_seq = utils.loadVariable( f'{prefix}{seq_id}_{self.label_fn_format}', self.data_dirs[self.labels]) true_label_seq = true_label_seq[::5] else: true_label_seq = utils.loadVariable( f'{prefix}{seq_id}_{self.label_fn_format}', self.data_dirs[self.labels]) # true_label_seq = self.part_labels[true_label_seq] return true_label_seq
def loadTargets(self, seq_id): trial_prefix = f"trial={seq_id}" assembly_seq = utils.loadVariable(f'{trial_prefix}_assembly-seq', self.rgb_data_dir) true_label_seq = torch.tensor(make_labels(assembly_seq, self.vocab), dtype=torch.long, device=self.device) return true_label_seq
def __init__( self, trial_ids, rgb_attributes_dir, rgb_data_dir, imu_attributes_dir, imu_data_dir, device=None, modalities=None): self.trial_ids = trial_ids self.metadata = utils.loadMetadata(rgb_data_dir, rows=trial_ids) self.vocab = utils.loadVariable('vocab', rgb_attributes_dir) self.rgb_attributes_dir = rgb_attributes_dir self.rgb_data_dir = rgb_data_dir self.imu_attributes_dir = imu_attributes_dir self.imu_data_dir = imu_data_dir self.device = device self.modalities = modalities
def loadInputsRgb(self, seq_id): trial_prefix = f"trial={seq_id}" rgb_attribute_seq = torch.tensor( utils.loadVariable(f"{trial_prefix}_score-seq", self.rgb_attributes_dir), dtype=torch.float, device=self.device ) attribute_feats = {'rgb': make_attribute_features(rgb_attribute_seq)} attribute_feats = torch.cat( tuple(attribute_feats[name] for name in self.modalities), dim=1 ) return attribute_feats
def __init__(self, data_dirs, scores_dirs, prefix='seq='): self.prefix = prefix self.data_dirs = data_dirs self.scores_dirs = scores_dirs self.seq_ids = utils.getUniqueIds(self.data_dirs['event'], prefix=prefix, suffix='labels.*', to_array=True) self.vocabs = { name: utils.loadVariable('vocab', dir_) for name, dir_ in self.data_dirs.items() }
def _getScores(self, index, label_name='event'): if isinstance(index, collections.abc.Iterable): return tuple( self._getScores(i, label_name=label_name) for i in index) if isinstance(label_name, collections.abc.Iterable) and not isinstance( label_name, str): return tuple( self._getScores(index, label_name=ln) for ln in label_name) trial_prefix = f"{self.prefix}{index}" score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", self.scores_dirs[label_name]) return score_seq
def load_vocab(link_vocab, joint_vocab, joint_type_vocab, vocab_dir): assembly_vocab = utils.loadVariable('vocab', vocab_dir) # FIXME: Convert keys from vertex pairs to edge indices edge_vocab = utils.loadVariable('parts-vocab', vocab_dir) edge_labels = utils.loadVariable('part-labels', vocab_dir) assembly_vocab = tuple( lib_assembly.Assembly.from_blockassembly( a, link_vocab=link_vocab, joint_vocab=joint_vocab, joint_type_vocab=joint_type_vocab) for a in assembly_vocab) edge_vocab = { edge_key: tuple( lib_assembly.Assembly.from_blockassembly( a, link_vocab=link_vocab, joint_vocab=joint_vocab, joint_type_vocab=joint_type_vocab) for a in assemblies) for edge_key, assemblies in edge_vocab.items() } return assembly_vocab, edge_vocab, edge_labels
def main(out_dir=None, data_dir=None, labels_dir=None): out_dir = os.path.expanduser(out_dir) data_dir = os.path.expanduser(data_dir) labels_dir = os.path.expanduser(labels_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) filenames = [ utils.stripExtension(fn) for fn in glob.glob(os.path.join(labels_dir, '*.csv')) ] metadata = utils.loadMetadata(data_dir) metadata['seq_id'] = metadata.index metadata = metadata.set_index( 'dir_name', drop=False).loc[filenames].set_index('seq_id') seq_ids = np.sort(metadata.index.to_numpy()) logger.info(f"Loaded {len(seq_ids)} sequences from {labels_dir}") vocab = [] for i, seq_id in enumerate(seq_ids): seq_id_str = f"seq={seq_id}" seq_dir_name = metadata['dir_name'].loc[seq_id] labels_fn = os.path.join(labels_dir, f'{seq_dir_name}.csv') event_labels = utils.loadVariable(f'{seq_id_str}_labels', data_dir) assembly_actions = pd.read_csv(labels_fn) label_seq = parseActions(assembly_actions, event_labels.shape[0], vocab) utils.saveVariable(label_seq, f'{seq_id_str}_label-seq', out_data_dir) plotLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.png'), label_seq) writeLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.csv'), label_seq, vocab) utils.saveMetadata(metadata, out_data_dir) utils.saveVariable(vocab, 'vocab', out_data_dir)
def __init__(self, actions_dir, parts_dir, events_dir, edges_dir, vocab=None, modalities=('actions', 'parts', 'edges'), labels='edges', prefix='seq=', feature_fn_format='score-seq', label_fn_format='true-label-seq'): self.modalities = modalities self.labels = labels self.actions_dir = actions_dir self.parts_dir = parts_dir self.events_dir = events_dir self.edges_dir = edges_dir self.data_dirs = { 'actions': self.actions_dir, 'parts': self.parts_dir, 'events': self.events_dir, 'edges': self.edges_dir } labels_dir = self.data_dirs[self.labels] self.trial_ids = utils.getUniqueIds( labels_dir, prefix='trial=' if self.labels == 'edges' else prefix, suffix=f'{label_fn_format}.*', to_array=True) if vocab is None: vocab = utils.loadVariable('vocab', labels_dir) self.vocab = vocab self.metadata = utils.loadMetadata(labels_dir, rows=self.trial_ids) self.prefix = prefix self.feature_fn_format = feature_fn_format self.label_fn_format = label_fn_format
def _getLabels(self, index, label_name='event', load_from_data=False): if isinstance(index, collections.abc.Iterable): return tuple( self._getLabels( i, label_name=label_name, load_from_data=load_from_data) for i in index) if isinstance(label_name, collections.abc.Iterable) and not isinstance( label_name, str): return tuple( self._getLabels( index, label_name=ln, load_from_data=load_from_data) for ln in label_name) trial_prefix = f"{self.prefix}{index}" if load_from_data: dir_ = self.data_dirs[label_name] fn = f"{trial_prefix}_labels" else: dir_ = self.scores_dirs[label_name] fn = f"{trial_prefix}_true-label-seq" return utils.loadVariable(fn, dir_)
def main(out_dir=None, data_dir=None, scores_dir=None, frames_dir=None, vocab_from_scores_dir=None, only_fold=None, plot_io=None, prefix='seq=', results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) scores_dir = os.path.expanduser(scores_dir) frames_dir = os.path.expanduser(frames_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir_images = os.path.join(fig_dir, 'model-io_images') if not os.path.exists(io_dir_images): os.makedirs(io_dir_images) io_dir_plots = os.path.join(fig_dir, 'model-io_plots') if not os.path.exists(io_dir_plots): os.makedirs(io_dir_plots) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(scores_dir, prefix=prefix, suffix='pred-label-seq.*', to_array=True) logger.info( f"Loaded scores for {len(seq_ids)} sequences from {scores_dir}") if vocab_from_scores_dir: vocab = utils.loadVariable('vocab', scores_dir) else: vocab = utils.loadVariable('vocab', data_dir) all_metrics = collections.defaultdict(list) # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) all_pred_seqs = [] all_true_seqs = [] for cv_index, cv_fold in enumerate(cv_folds): if only_fold is not None and cv_index != only_fold: continue train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) for i in test_indices: seq_id = seq_ids[i] logger.info(f" Processing sequence {seq_id}...") trial_prefix = f"{prefix}{seq_id}" score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", scores_dir) pred_seq = utils.loadVariable(f"{trial_prefix}_pred-label-seq", scores_dir) true_seq = utils.loadVariable(f"{trial_prefix}_true-label-seq", scores_dir) metric_dict = eval_metrics(pred_seq, true_seq) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") all_metrics[name].append(value) utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) all_pred_seqs.append(pred_seq) all_true_seqs.append(true_seq) if plot_io: utils.plot_array(score_seq.T, (true_seq, pred_seq), ('true', 'pred'), fn=os.path.join(io_dir_plots, f"seq={seq_id:03d}.png")) if False: confusions = metrics.confusionMatrix(all_pred_seqs, all_true_seqs, len(vocab)) utils.saveVariable(confusions, "confusions", out_data_dir) per_class_acc, class_counts = metrics.perClassAcc(confusions, return_counts=True) class_preds = confusions.sum(axis=1) logger.info(f"MACRO ACC: {np.nanmean(per_class_acc) * 100:.2f}%") metrics.plotConfusions(os.path.join(fig_dir, 'confusions.png'), confusions, vocab) metrics.plotPerClassAcc(os.path.join(fig_dir, 'per-class-results.png'), vocab, per_class_acc, class_preds, class_counts)
def main(out_dir=None, data_dir=None, segs_dir=None, scores_dir=None, vocab_dir=None, label_type='edges', gpu_dev_id=None, start_from=None, stop_at=None, num_disp_imgs=None, results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) segs_dir = os.path.expanduser(segs_dir) scores_dir = os.path.expanduser(scores_dir) vocab_dir = os.path.expanduser(vocab_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir_images = os.path.join(fig_dir, 'model-io_images') if not os.path.exists(io_dir_images): os.makedirs(io_dir_images) io_dir_plots = os.path.join(fig_dir, 'model-io_plots') if not os.path.exists(io_dir_plots): os.makedirs(io_dir_plots) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(scores_dir, prefix='trial=', suffix='score-seq.*', to_array=True) logger.info( f"Loaded scores for {len(seq_ids)} sequences from {scores_dir}") link_vocab = {} joint_vocab = {} joint_type_vocab = {} vocab, parts_vocab, part_labels = load_vocab(link_vocab, joint_vocab, joint_type_vocab, vocab_dir) pred_vocab = [] # FIXME if label_type == 'assembly': logger.info("Converting assemblies -> edges") state_pred_seqs = tuple( utils.loadVariable(f"trial={seq_id}_pred-label-seq", scores_dir) for seq_id in seq_ids) state_true_seqs = tuple( utils.loadVariable(f"trial={seq_id}_true-label-seq", scores_dir) for seq_id in seq_ids) edge_pred_seqs = tuple(part_labels[seq] for seq in state_pred_seqs) edge_true_seqs = tuple(part_labels[seq] for seq in state_true_seqs) elif label_type == 'edge': logger.info("Converting edges -> assemblies (will take a few minutes)") edge_pred_seqs = tuple( utils.loadVariable(f"trial={seq_id}_pred-label-seq", scores_dir) for seq_id in seq_ids) edge_true_seqs = tuple( utils.loadVariable(f"trial={seq_id}_true-label-seq", scores_dir) for seq_id in seq_ids) state_pred_seqs = tuple( edges_to_assemblies(seq, pred_vocab, parts_vocab, part_labels) for seq in edge_pred_seqs) state_true_seqs = tuple( edges_to_assemblies(seq, vocab, parts_vocab, part_labels) for seq in edge_true_seqs) device = torchutils.selectDevice(gpu_dev_id) dataset = sim2real.LabeledConnectionDataset( utils.loadVariable('parts-vocab', vocab_dir), utils.loadVariable('part-labels', vocab_dir), utils.loadVariable('vocab', vocab_dir), device=device) all_metrics = collections.defaultdict(list) # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) for cv_index, cv_fold in enumerate(cv_folds): train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) train_states = np.hstack( tuple(state_true_seqs[i] for i in (train_indices))) train_edges = part_labels[train_states] # state_train_vocab = np.unique(train_states) # edge_train_vocab = part_labels[state_train_vocab] train_freq_bigram, train_freq_unigram = edge_joint_freqs(train_edges) # state_probs = utils.makeHistogram(len(vocab), train_states, normalize=True) test_states = np.hstack( tuple(state_true_seqs[i] for i in (test_indices))) test_edges = part_labels[test_states] # state_test_vocab = np.unique(test_states) # edge_test_vocab = part_labels[state_test_vocab] test_freq_bigram, test_freq_unigram = edge_joint_freqs(test_edges) f, axes = plt.subplots(1, 2) axes[0].matshow(train_freq_bigram) axes[0].set_title('Train') axes[1].matshow(test_freq_bigram) axes[1].set_title('Test') plt.tight_layout() plt.savefig( os.path.join(fig_dir, f"edge-freqs-bigram_cvfold={cv_index}.png")) f, axis = plt.subplots(1) axis.stem(train_freq_unigram, label='Train', linefmt='C0-', markerfmt='C0o') axis.stem(test_freq_unigram, label='Test', linefmt='C1--', markerfmt='C1o') plt.legend() plt.tight_layout() plt.savefig( os.path.join(fig_dir, f"edge-freqs-unigram_cvfold={cv_index}.png")) for i in test_indices: seq_id = seq_ids[i] logger.info(f" Processing sequence {seq_id}...") trial_prefix = f"trial={seq_id}" # I include the '.' to differentiate between 'rgb-frame-seq' and # 'rgb-frame-seq-before-first-touch' # rgb_seq = utils.loadVariable(f"{trial_prefix}_rgb-frame-seq.", data_dir) # seg_seq = utils.loadVariable(f"{trial_prefix}_seg-labels-seq", segs_dir) score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", scores_dir) # if score_seq.shape[0] != rgb_seq.shape[0]: # err_str = f"scores shape {score_seq.shape} != data shape {rgb_seq.shape}" # raise AssertionError(err_str) edge_pred_seq = edge_pred_seqs[i] edge_true_seq = edge_true_seqs[i] state_pred_seq = state_pred_seqs[i] state_true_seq = state_true_seqs[i] num_types = np.unique(state_pred_seq).shape[0] num_samples = state_pred_seq.shape[0] num_total = len(pred_vocab) logger.info( f" {num_types} assemblies predicted ({num_total} total); " f"{num_samples} samples") # edge_freq_bigram, edge_freq_unigram = edge_joint_freqs(edge_true_seq) # dist_shift = np.linalg.norm(train_freq_unigram - edge_freq_unigram) metric_dict = { # 'State OOV rate': oov_rate_state(state_true_seq, state_train_vocab), # 'Edge OOV rate': oov_rate_edges(edge_true_seq, edge_train_vocab), # 'State avg prob, true': state_probs[state_true_seq].mean(), # 'State avg prob, pred': state_probs[state_pred_seq].mean(), # 'Edge distribution shift': dist_shift } metric_dict = eval_edge_metrics(edge_pred_seq, edge_true_seq, append_to=metric_dict) metric_dict = eval_state_metrics(state_pred_seq, state_true_seq, append_to=metric_dict) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") all_metrics[name].append(value) utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) if num_disp_imgs is not None: pred_images = tuple( render(dataset, vocab[seg_label]) for seg_label in utils.computeSegments(state_pred_seq)[0]) imageprocessing.displayImages( *pred_images, file_path=os.path.join( io_dir_images, f"seq={seq_id:03d}_pred-assemblies.png"), num_rows=None, num_cols=5) true_images = tuple( render(dataset, vocab[seg_label]) for seg_label in utils.computeSegments(state_true_seq)[0]) imageprocessing.displayImages( *true_images, file_path=os.path.join( io_dir_images, f"seq={seq_id:03d}_true-assemblies.png"), num_rows=None, num_cols=5) utils.plot_array(score_seq.T, (edge_true_seq.T, edge_pred_seq.T), ('true', 'pred'), fn=os.path.join(io_dir_plots, f"seq={seq_id:03d}.png"))
def loadVariable(var_name, from_dir=scores_dir): var = utils.loadVariable(var_name, from_dir) return var
def main(out_dir=None, rgb_data_dir=None, rgb_attributes_dir=None, rgb_vocab_dir=None, imu_data_dir=None, imu_attributes_dir=None, modalities=['rgb', 'imu'], gpu_dev_id=None, plot_predictions=None, results_file=None, sweep_param_name=None, model_params={}, cv_params={}, train_params={}, viz_params={}): out_dir = os.path.expanduser(out_dir) rgb_data_dir = os.path.expanduser(rgb_data_dir) rgb_attributes_dir = os.path.expanduser(rgb_attributes_dir) rgb_vocab_dir = os.path.expanduser(rgb_vocab_dir) imu_data_dir = os.path.expanduser(imu_data_dir) imu_attributes_dir = os.path.expanduser(imu_attributes_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) def saveVariable(var, var_name, to_dir=out_data_dir): utils.saveVariable(var, var_name, to_dir) # Load data if modalities == ['rgb']: trial_ids = utils.getUniqueIds(rgb_data_dir, prefix='trial=', to_array=True) logger.info(f"Processing {len(trial_ids)} videos") else: rgb_trial_ids = utils.getUniqueIds(rgb_data_dir, prefix='trial=', to_array=True) imu_trial_ids = utils.getUniqueIds(imu_data_dir, prefix='trial=', to_array=True) trial_ids = np.array( sorted(set(rgb_trial_ids.tolist()) & set(imu_trial_ids.tolist()))) logger.info( f"Processing {len(trial_ids)} videos common to " f"RGB ({len(rgb_trial_ids)} total) and IMU ({len(imu_trial_ids)} total)" ) device = torchutils.selectDevice(gpu_dev_id) dataset = FusionDataset(trial_ids, rgb_attributes_dir, rgb_data_dir, imu_attributes_dir, imu_data_dir, device=device, modalities=modalities) utils.saveMetadata(dataset.metadata, out_data_dir) saveVariable(dataset.vocab, 'vocab') # parts_vocab = loadVariable('parts-vocab') edge_labels = { 'rgb': utils.loadVariable('part-labels', rgb_vocab_dir), 'imu': np.stack([ labels.inSameComponent(a, lower_tri_only=True) for a in dataset.vocab ]) } # edge_labels = revise_edge_labels(edge_labels, input_seqs) attribute_labels = tuple(edge_labels[name] for name in modalities) logger.info('Making transition probs...') transition_probs = make_transition_scores(dataset.vocab) saveVariable(transition_probs, 'transition-probs') model = AttributeModel(*attribute_labels, device=device) if plot_predictions: figsize = (12, 3) fig, axis = plt.subplots(1, figsize=figsize) axis.imshow(edge_labels['rgb'].T, interpolation='none', aspect='auto') plt.savefig(os.path.join(fig_dir, "edge-labels.png")) plt.close() for i, trial_id in enumerate(trial_ids): logger.info(f"Processing sequence {trial_id}...") trial_prefix = f"trial={trial_id}" true_label_seq = dataset.loadTargets(trial_id) attribute_feats = dataset.loadInputs(trial_id) score_seq = model(attribute_feats) pred_label_seq = model.predict(score_seq) attribute_feats = attribute_feats.cpu().numpy() score_seq = score_seq.cpu().numpy() true_label_seq = true_label_seq.cpu().numpy() pred_label_seq = pred_label_seq.cpu().numpy() saveVariable(score_seq.T, f'{trial_prefix}_score-seq') saveVariable(true_label_seq.T, f'{trial_prefix}_label-seq') if plot_predictions: fn = os.path.join(fig_dir, f'{trial_prefix}.png') utils.plot_array(attribute_feats.T, (true_label_seq, pred_label_seq, score_seq), ('gt', 'pred', 'scores'), fn=fn) metric_dict = eval_metrics(pred_label_seq, true_label_seq) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") utils.writeResults(results_file, metric_dict, sweep_param_name, model_params)
def main(out_dir=None, data_dir=None, scores_dirs={}, vocab_from_scores_dir=None, only_fold=None, plot_io=None, prefix='seq=', results_file=None, sweep_param_name=None, model_params={}, cv_params={}): data_dir = os.path.expanduser(data_dir) scores_dirs = { name: os.path.expanduser(dir_) for name, dir_ in scores_dirs.items() } out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if len(scores_dirs) != 2: err_str = ( f"scores_dirs has {len(scores_dirs)} entries, but this script " "compare exactly 2") raise NotImplementedError(err_str) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(data_dir, prefix=prefix, suffix='labels.*', to_array=True) logger.info(f"Loaded scores for {len(seq_ids)} sequences from {data_dir}") # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) vocabs = {} confusions = {} accs = {} counts = {} for expt_name, scores_dir in scores_dirs.items(): if vocab_from_scores_dir: vocabs[expt_name] = utils.loadVariable('vocab', scores_dir) else: vocabs[expt_name] = utils.loadVariable('vocab', data_dir) confusions[expt_name] = utils.loadVariable("confusions", scores_dir) per_class_accs, class_counts = metrics.perClassAcc( confusions[expt_name], return_counts=True) accs[expt_name] = per_class_accs counts[expt_name] = class_counts vocab = utils.reduce_all_equal(tuple(vocabs.values())) class_counts = utils.reduce_all_equal(tuple(counts.values())) first_name, second_name = scores_dirs.keys() confusions_diff = confusions[first_name] - confusions[second_name] acc_diff = accs[first_name] - accs[second_name] metrics.plotConfusions( os.path.join(fig_dir, f'confusions_{first_name}-minus-{second_name}.png'), confusions_diff, vocab) metrics.plotPerClassAcc( os.path.join(fig_dir, f'accs_{first_name}-minus-{second_name}.png'), vocab, acc_diff, confusions_diff.sum(axis=1), class_counts)
def main(out_dir=None, data_dir=None, assembly_data_dir=None, scores_dir=None, event_attr_fn=None, connection_attr_fn=None, assembly_attr_fn=None, only_fold=None, plot_io=None, prefix='seq=', stop_after=None, background_action='', model_params={}, cv_params={}, stride=None, results_file=None, sweep_param_name=None): data_dir = os.path.expanduser(data_dir) assembly_data_dir = os.path.expanduser(assembly_data_dir) scores_dir = os.path.expanduser(scores_dir) event_attr_fn = os.path.expanduser(event_attr_fn) connection_attr_fn = os.path.expanduser(connection_attr_fn) assembly_attr_fn = os.path.expanduser(assembly_attr_fn) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) misc_dir = os.path.join(out_dir, 'misc') if not os.path.exists(misc_dir): os.makedirs(misc_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) seq_ids = utils.getUniqueIds(data_dir, prefix=prefix, suffix='labels.*', to_array=True) dataset = utils.FeaturelessCvDataset(seq_ids, data_dir, prefix=prefix, label_fn_format='labels') logger.info( f"Loaded scores for {len(seq_ids)} sequences from {scores_dir}") # Define cross-validation folds cv_folds = utils.makeDataSplits(len(seq_ids), **cv_params) utils.saveVariable(cv_folds, 'cv-folds', out_data_dir) # Load event, connection attributes assembly_vocab = tuple( tuple(sorted(tuple(sorted(joint)) for joint in a)) for a in utils.loadVariable('vocab', assembly_data_dir)) (*probs, assembly_transition_probs), vocabs = loadPartInfo( event_attr_fn, connection_attr_fn, assembly_attr_fn, background_action=background_action, assembly_vocab=assembly_vocab) event_assembly_scores = event_to_assembly_scores(*probs, vocabs) assembly_transition_scores = np.log(assembly_transition_probs) viz_transition_probs(os.path.join(fig_dir, 'action-transitions'), np.exp(event_assembly_scores), vocabs['event_vocab']) write_transition_probs(os.path.join(misc_dir, 'action-transitions'), np.exp(event_assembly_scores), vocabs['event_vocab'], vocabs['assembly_vocab']) for cv_index, cv_fold in enumerate(cv_folds): if only_fold is not None and cv_index != only_fold: continue train_indices, val_indices, test_indices = cv_fold logger.info( f"CV FOLD {cv_index + 1} / {len(cv_folds)}: " f"{len(train_indices)} train, {len(val_indices)} val, {len(test_indices)} test" ) train_data, val_data, test_data = dataset.getFold(cv_fold) cv_str = f'cvfold={cv_index}' class_priors, event_dur_probs = count_priors(train_data[0], len(dataset.vocab), stride=stride, approx_upto=0.95, support_only=True) event_dur_scores = np.log(event_dur_probs) event_dur_scores = np.zeros_like(event_dur_scores) scores = (event_dur_scores, event_assembly_scores, assembly_transition_scores) model = decode.AssemblyActionRecognizer(scores, vocabs, model_params) viz_priors(os.path.join(fig_dir, f'{cv_str}_priors'), class_priors, event_dur_probs) model.write_fsts(os.path.join(misc_dir, f'{cv_str}_fsts')) model.save_vocabs(os.path.join(out_data_dir, f'{cv_str}_model-vocabs')) for i, (_, seq_id) in enumerate(zip(*test_data)): if stop_after is not None and i >= stop_after: break trial_prefix = f"{prefix}{seq_id}" if model_params['return_label'] == 'input': true_seq = utils.loadVariable(f"{trial_prefix}_true-label-seq", scores_dir) elif model_params['return_label'] == 'output': try: true_seq = utils.loadVariable(f"{trial_prefix}_label-seq", assembly_data_dir) true_seq = true_seq[::stride] except AssertionError: # logger.info(f' Skipping sequence {seq_id}: {e}') continue logger.info(f" Processing sequence {seq_id}...") event_score_seq = utils.loadVariable(f"{trial_prefix}_score-seq", scores_dir) if event_score_seq.shape[0] != true_seq.shape[0]: err_str = (f'Event scores shape {event_score_seq.shape} ' f'!= labels shape {true_seq.shape}') raise AssertionError(err_str) # FIXME: the serialized variables are probs, not log-probs # event_score_seq = suppress_nonmax(event_score_seq) # event_score_seq = np.log(event_score_seq) decode_score_seq = model.forward(event_score_seq) pred_seq = model.predict(decode_score_seq) metric_dict = eval_metrics(pred_seq, true_seq) for name, value in metric_dict.items(): logger.info(f" {name}: {value * 100:.2f}%") utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) utils.saveVariable(decode_score_seq, f'{trial_prefix}_score-seq', out_data_dir) utils.saveVariable(pred_seq, f'{trial_prefix}_pred-label-seq', out_data_dir) utils.saveVariable(true_seq, f'{trial_prefix}_true-label-seq', out_data_dir) if plot_io: utils.plot_array(event_score_seq.T, (pred_seq.T, true_seq.T), ('pred', 'true'), fn=os.path.join(fig_dir, f"seq={seq_id:03d}.png")) write_labels( os.path.join(misc_dir, f"seq={seq_id:03d}_pred-seq.txt"), pred_seq, model.output_vocab.as_raw()) write_labels( os.path.join(misc_dir, f"seq={seq_id:03d}_true-seq.txt"), true_seq, model.output_vocab.as_raw())
def main(out_dir=None, data_dir=None, model_name=None, gpu_dev_id=None, batch_size=None, learning_rate=None, model_params={}, cv_params={}, train_params={}, viz_params={}, load_masks_params={}, kornia_tfs={}, only_edge=None, num_disp_imgs=None, results_file=None, sweep_param_name=None): data_dir = os.path.expanduser(data_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir = os.path.join(fig_dir, 'model-io') if not os.path.exists(io_dir): os.makedirs(io_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) def saveVariable(var, var_name, to_dir=out_data_dir): return utils.saveVariable(var, var_name, to_dir) trial_ids = utils.getUniqueIds(data_dir, prefix='trial=', to_array=True) vocab = [BlockAssembly() ] + [make_single_block_state(i) for i in range(len(defn.blocks))] for seq_id in trial_ids: assembly_seq = utils.loadVariable(f"trial={seq_id}_assembly-seq", data_dir) for assembly in assembly_seq: utils.getIndex(assembly, vocab) parts_vocab, part_labels = labels_lib.make_parts_vocab( vocab, lower_tri_only=True, append_to_vocab=True) if only_edge is not None: part_labels = part_labels[:, only_edge:only_edge + 1] logger.info( f"Loaded {len(trial_ids)} sequences; {len(vocab)} unique assemblies") saveVariable(vocab, 'vocab') saveVariable(parts_vocab, 'parts-vocab') saveVariable(part_labels, 'part-labels') device = torchutils.selectDevice(gpu_dev_id) if model_name == 'AAE': Dataset = sim2real.DenoisingDataset elif model_name == 'Resnet': Dataset = sim2real.RenderDataset elif model_name == 'Connections': Dataset = sim2real.ConnectionDataset elif model_name == 'Labeled Connections': Dataset = sim2real.LabeledConnectionDataset occlusion_masks = loadMasks(**load_masks_params) if occlusion_masks is not None: logger.info(f"Loaded {occlusion_masks.shape[0]} occlusion masks") def make_data(shuffle=True): dataset = Dataset( parts_vocab, part_labels, vocab, device=device, occlusion_masks=occlusion_masks, kornia_tfs=kornia_tfs, ) data_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle) return dataset, data_loader for cv_index, cv_splits in enumerate(range(1)): cv_str = f"cvfold={cv_index}" train_set, train_loader = make_data(shuffle=True) test_set, test_loader = make_data(shuffle=False) val_set, val_loader = make_data(shuffle=True) if model_name == 'AAE': model = sim2real.AugmentedAutoEncoder(train_set.data_shape, train_set.num_classes) criterion = torchutils.BootstrappedCriterion( 0.25, base_criterion=torch.nn.functional.mse_loss, ) metric_names = ('Reciprocal Loss', ) elif model_name == 'Resnet': model = sim2real.ImageClassifier(train_set.num_classes, **model_params) criterion = torch.nn.CrossEntropyLoss() metric_names = ('Loss', 'Accuracy') elif model_name == 'Connections': model = sim2real.ConnectionClassifier(train_set.label_shape[0], **model_params) criterion = torch.nn.BCEWithLogitsLoss() metric_names = ('Loss', 'Accuracy', 'Precision', 'Recall', 'F1') elif model_name == 'Labeled Connections': out_dim = int(part_labels.max()) + 1 num_vertices = len(defn.blocks) edges = np.column_stack(np.tril_indices(num_vertices, k=-1)) if only_edge is not None: edges = edges[only_edge:only_edge + 1] model = sim2real.LabeledConnectionClassifier( out_dim, num_vertices, edges, **model_params) if only_edge is not None: logger.info(f"Class freqs: {train_set.class_freqs}") # criterion = torch.nn.CrossEntropyLoss(weight=1 / train_set.class_freqs[:, 0]) criterion = torch.nn.CrossEntropyLoss() else: criterion = torch.nn.CrossEntropyLoss() # criterion = torchutils.BootstrappedCriterion( # 0.25, base_criterion=torch.nn.functional.cross_entropy, # ) metric_names = ('Loss', 'Accuracy', 'Precision', 'Recall', 'F1') model = model.to(device=device) optimizer_ft = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=1, gamma=1.00) train_epoch_log = collections.defaultdict(list) val_epoch_log = collections.defaultdict(list) metric_dict = {name: metrics.makeMetric(name) for name in metric_names} model, last_model_wts = torchutils.trainModel( model, criterion, optimizer_ft, lr_scheduler, train_loader, val_loader, device=device, metrics=metric_dict, train_epoch_log=train_epoch_log, val_epoch_log=val_epoch_log, **train_params) # Test model metric_dict = {name: metrics.makeMetric(name) for name in metric_names} test_io_batches = torchutils.predictSamples( model.to(device=device), test_loader, criterion=criterion, device=device, metrics=metric_dict, data_labeled=True, update_model=False, seq_as_batch=train_params['seq_as_batch'], return_io_history=True) metric_str = ' '.join(str(m) for m in metric_dict.values()) logger.info('[TST] ' + metric_str) utils.writeResults(results_file, metric_dict, sweep_param_name, model_params) for pred_seq, score_seq, feat_seq, label_seq, trial_id in test_io_batches: trial_str = f"trial={trial_id}" saveVariable(pred_seq.cpu().numpy(), f'{trial_str}_pred-label-seq') saveVariable(score_seq.cpu().numpy(), f'{trial_str}_score-seq') saveVariable(label_seq.cpu().numpy(), f'{trial_str}_true-label-seq') saveVariable(model, f'{cv_str}_model-best') if train_epoch_log: torchutils.plotEpochLog(train_epoch_log, subfig_size=(10, 2.5), title='Training performance', fn=os.path.join( fig_dir, f'{cv_str}_train-plot.png')) if val_epoch_log: torchutils.plotEpochLog(val_epoch_log, subfig_size=(10, 2.5), title='Heldout performance', fn=os.path.join(fig_dir, f'{cv_str}_val-plot.png')) if num_disp_imgs is not None: model.plotBatches(test_io_batches, io_dir, dataset=test_set)
def main( out_dir=None, data_dir=None, results_file=None, cv_file=None, take_log=False, col_format=None, win_params={}, slowfast_csv_params={}): out_dir = os.path.expanduser(out_dir) data_dir = os.path.expanduser(data_dir) results_file = os.path.expanduser(results_file) cv_file = os.path.expanduser(cv_file) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) vocab = utils.loadVariable('vocab', data_dir) metadata = utils.loadMetadata(data_dir) slowfast_labels = pd.read_csv( cv_file, keep_default_na=False, index_col=0, **slowfast_csv_params ) seg_ids = slowfast_labels.index.to_numpy() vid_names = slowfast_labels['video_name'].unique().tolist() metadata['seq_id'] = metadata.index vid_ids = metadata.set_index('dir_name').loc[vid_names].set_index('seq_id').index metadata = metadata.drop('seq_id', axis=1) with open(results_file, 'rb') as file_: model_probs, gt_labels = pickle.load(file_) model_probs = model_probs.numpy() gt_labels = gt_labels.numpy() if len(model_probs) != len(seg_ids): err_str = f"{len(model_probs)} segment scores != {slowfast_labels.shape[0]} CSV rows" raise AssertionError(err_str) logger.info(f"Loaded {len(seg_ids)} segments, {len(vid_ids)} videos") for vid_id, vid_name in zip(vid_ids, vid_names): matches_video = (slowfast_labels['video_name'] == vid_name).to_numpy() win_labels = gt_labels[matches_video] win_probs = model_probs[matches_video, :] if win_labels.shape == win_probs.shape: win_preds = (win_probs > 0.5).astype(int) else: win_preds = win_probs.argmax(axis=1) if take_log: win_probs = np.log(win_probs) seq_id_str = f"seq={vid_id}" utils.saveVariable(win_probs, f'{seq_id_str}_score-seq', out_data_dir) utils.saveVariable(win_labels, f'{seq_id_str}_true-label-seq', out_data_dir) utils.saveVariable(win_preds, f'{seq_id_str}_pred-label-seq', out_data_dir) utils.plot_array( win_probs.T, (win_labels.T, win_preds.T), ('true', 'pred'), tick_names=vocab, fn=os.path.join(fig_dir, f"{seq_id_str}.png"), subplot_width=12, subplot_height=5 ) utils.saveVariable(vocab, 'vocab', out_data_dir) utils.saveMetadata(metadata, out_data_dir)
def loadData(self, seq_id): rgb_frames = utils.loadVariable(f"trial={seq_id}_rgb-frame-seq.", self.data_dir) seg_frames = utils.loadVariable(f"trial={seq_id}_seg-labels-seq", self.seg_dir).astype(int) return rgb_frames, seg_frames
def main(out_dir=None, scores_dir=None, preprocessed_data_dir=None, keyframe_model_name=None, subsample_period=None, window_size=None, corpus_name=None, default_annotator=None, cv_scheme=None, max_trials_per_fold=None, model_name=None, numeric_backend=None, gpu_dev_id=None, visualize=False, model_config={}, camera_params_config={}): out_dir = os.path.expanduser(out_dir) scores_dir = os.path.expanduser(scores_dir) preprocessed_data_dir = os.path.expanduser(preprocessed_data_dir) m.set_backend('numpy') def loadFromWorkingDir(var_name): return joblib.load(os.path.join(scores_dir, f"{var_name}.pkl")) def saveToWorkingDir(var, var_name): joblib.dump(var, os.path.join(out_dir, f"{var_name}.pkl")) # Load camera parameters from external file and add them to model config kwargs model_config['init_kwargs'].update( render.loadCameraParams(**camera_params_config, as_dict=True)) trial_ids = joblib.load( os.path.join(preprocessed_data_dir, 'trial_ids.pkl')) corpus = duplocorpus.DuploCorpus(corpus_name) assembly_seqs = tuple( labels.parseLabelSeq( corpus.readLabels(trial_id, default_annotator)[0]) for trial_id in trial_ids) logger.info(f"Selecting keyframes...") keyframe_idx_seqs = [] rgb_keyframe_seqs = [] depth_keyframe_seqs = [] seg_keyframe_seqs = [] background_keyframe_seqs = [] assembly_keyframe_seqs = [] for seq_idx, trial_id in enumerate(trial_ids): trial_str = f"trial-{trial_id}" rgb_frame_seq = loadFromWorkingDir(f'{trial_str}_rgb-frame-seq') depth_frame_seq = loadFromWorkingDir(f'{trial_str}_depth-frame-seq') segment_seq = loadFromWorkingDir(f'{trial_str}_segment-seq') frame_scores = loadFromWorkingDir(f'{trial_str}_frame-scores') background_plane_seq = loadFromWorkingDir( f'{trial_str}_background-plane-seq') assembly_seq = assembly_seqs[seq_idx] # FIXME: Get the real frame index numbers instead of approximating assembly_seq[-1].end_idx = len(rgb_frame_seq) * subsample_period keyframe_idxs = videoprocessing.selectSegmentKeyframes( frame_scores, score_thresh=0, prepend_first=True) selectKeyframes = functools.partial(utils.select, keyframe_idxs) rgb_keyframe_seq = selectKeyframes(rgb_frame_seq) depth_keyframe_seq = selectKeyframes(depth_frame_seq) seg_keyframe_seq = selectKeyframes(segment_seq) background_keyframe_seq = selectKeyframes(background_plane_seq) # FIXME: Get the real frame index numbers instead of approximating keyframe_idxs_orig = keyframe_idxs * subsample_period assembly_keyframe_seq = labels.resampleStateSeq( keyframe_idxs_orig, assembly_seq) # Store all keyframe sequences in memory keyframe_idx_seqs.append(keyframe_idxs) rgb_keyframe_seqs.append(rgb_keyframe_seq) depth_keyframe_seqs.append(depth_keyframe_seq) seg_keyframe_seqs.append(seg_keyframe_seq) background_keyframe_seqs.append(background_keyframe_seq) assembly_keyframe_seqs.append(assembly_keyframe_seq) # Split into train and test sets if cv_scheme == 'leave one out': num_seqs = len(trial_ids) cv_folds = [] for i in range(num_seqs): test_fold = (i, ) train_fold = tuple(range(0, i)) + tuple(range(i + 1, num_seqs)) cv_folds.append((train_fold, test_fold)) elif cv_scheme == 'train on child': child_corpus = duplocorpus.DuploCorpus('child') child_trial_ids = utils.loadVariable('trial_ids', 'preprocess-all-data', 'child') child_assembly_seqs = [ labels.parseLabelSeq( child_corpus.readLabels(trial_id, 'Cathryn')[0]) for trial_id in child_trial_ids ] num_easy = len(assembly_keyframe_seqs) num_child = len(child_assembly_seqs) cv_folds = [(tuple(range(num_easy, num_easy + num_child)), tuple(range(num_easy)))] assembly_keyframe_seqs = assembly_keyframe_seqs + child_assembly_seqs rgb_keyframe_seqs = tuple( tuple( imageprocessing.saturateImage(rgb_image, background_mask=segment_image == 0) for rgb_image, segment_image in zip(rgb_frame_seq, seg_frame_seq)) for rgb_frame_seq, seg_frame_seq in zip(rgb_keyframe_seqs, seg_keyframe_seqs)) depth_keyframe_seqs = tuple( tuple(depth_image.astype(float) for depth_image in depth_frame_seq) for depth_frame_seq in depth_keyframe_seqs) device = torchutils.selectDevice(gpu_dev_id) m.set_backend('torch') m.set_default_device(device) assembly_keyframe_seqs = tuple( tuple(a.to(device=device, in_place=False) for a in seq) for seq in assembly_keyframe_seqs) assembly_seqs = tuple( tuple(a.to(device=device, in_place=False) for a in seq) for seq in assembly_seqs) rgb_keyframe_seqs = tuple( tuple(m.np.array(frame, dtype=torch.float) for frame in rgb_frame_seq) for rgb_frame_seq in rgb_keyframe_seqs) depth_keyframe_seqs = tuple( tuple( m.np.array(frame, dtype=torch.float) for frame in depth_frame_seq) for depth_frame_seq in depth_keyframe_seqs) seg_keyframe_seqs = tuple( tuple(m.np.array(frame, dtype=torch.int) for frame in seg_frame_seq) for seg_frame_seq in seg_keyframe_seqs) num_cv_folds = len(cv_folds) saveToWorkingDir(cv_folds, f'cv-folds') for fold_index, (train_idxs, test_idxs) in enumerate(cv_folds): logger.info(f"CV FOLD {fold_index + 1} / {num_cv_folds}") # Initialize and train model utils.validateCvFold(train_idxs, test_idxs) selectTrain = functools.partial(utils.select, train_idxs) train_assembly_seqs = selectTrain(assembly_keyframe_seqs) model = getattr(models, model_name)(**model_config['init_kwargs']) logger.info( f" Training {model_name} on {len(train_idxs)} sequences...") model.fit(train_assembly_seqs, **model_config['fit_kwargs']) logger.info( f' Model trained on {model.num_states} unique assembly states') # saveToWorkingDir(model, f'model-fold{fold_index}') # Decode on the test set selectTest = functools.partial(utils.select, test_idxs) test_trial_ids = selectTest(trial_ids) test_rgb_keyframe_seqs = selectTest(rgb_keyframe_seqs) test_depth_keyframe_seqs = selectTest(depth_keyframe_seqs) test_seg_keyframe_seqs = selectTest(seg_keyframe_seqs) test_background_keyframe_seqs = selectTest(background_keyframe_seqs) test_assembly_keyframe_seqs = selectTest(assembly_keyframe_seqs) test_assembly_seqs = selectTest(assembly_seqs) logger.info(f" Testing model on {len(test_idxs)} sequences...") for i, trial_id in enumerate(test_trial_ids): if max_trials_per_fold is not None and i >= max_trials_per_fold: break rgb_frame_seq = test_rgb_keyframe_seqs[i] depth_frame_seq = test_depth_keyframe_seqs[i] seg_frame_seq = test_seg_keyframe_seqs[i] background_plane_seq = test_background_keyframe_seqs[i] true_assembly_seq = test_assembly_keyframe_seqs[i] true_assembly_seq_orig = test_assembly_seqs[i] rgb_background_seq, depth_background_seq = utils.batchProcess( model.renderPlane, background_plane_seq, unzip=True) logger.info(f' Decoding video {trial_id}...') start_time = time.process_time() out = model.predictSeq(rgb_frame_seq, depth_frame_seq, seg_frame_seq, rgb_background_seq, depth_background_seq, **model_config['decode_kwargs']) pred_assembly_seq, pred_idx_seq, max_log_probs, log_likelihoods, poses_seq = out end_time = time.process_time() logger.info(utils.makeProcessTimeStr(end_time - start_time)) num_correct, num_total = metrics.numberCorrect( true_assembly_seq, pred_assembly_seq) logger.info(f' ACCURACY: {num_correct} / {num_total}') num_correct, num_total = metrics.numberCorrect( true_assembly_seq, pred_assembly_seq, ignore_empty_true=True) logger.info(f' RECALL: {num_correct} / {num_total}') num_correct, num_total = metrics.numberCorrect( true_assembly_seq, pred_assembly_seq, ignore_empty_pred=True) logger.info(f' PRECISION: {num_correct} / {num_total}') # Save intermediate results logger.info(f"Saving output...") saveToWorkingDir(segment_seq, f'segment_seq-{trial_id}') saveToWorkingDir(true_assembly_seq_orig, f'true_state_seq_orig-{trial_id}') saveToWorkingDir(true_assembly_seq, f'true_state_seq-{trial_id}') saveToWorkingDir(pred_assembly_seq, f'pred_state_seq-{trial_id}') saveToWorkingDir(poses_seq, f'poses_seq-{trial_id}') saveToWorkingDir(background_plane_seq, f'background_plane_seq-{trial_id}') saveToWorkingDir(max_log_probs, f'max_log_probs-{trial_id}') saveToWorkingDir(log_likelihoods, f'log_likelihoods-{trial_id}') # Save figures if visualize: rgb_rendered_seq, depth_rendered_seq, label_rendered_seq = utils.batchProcess( model.renderScene, pred_assembly_seq, poses_seq, rgb_background_seq, depth_background_seq, unzip=True, static_kwargs={'as_numpy': True}) if utils.in_ipython_console(): file_path = None else: trial_str = f"trial-{trial_id}" file_path = os.path.join(out_dir, f'{trial_str}_best-frames.png') rgb_frame_seq = tuple(img.cpu().numpy() for img in rgb_frame_seq) imageprocessing.displayImages(*rgb_frame_seq, *rgb_rendered_seq, num_rows=2, file_path=file_path)
def main(out_dir=None, data_dir=None, prefix='trial=', model_name=None, gpu_dev_id=None, batch_size=None, learning_rate=None, file_fn_format=None, label_fn_format=None, start_from=None, stop_at=None, model_params={}, cv_params={}, train_params={}, viz_params={}, num_disp_imgs=None, viz_templates=None, results_file=None, sweep_param_name=None): data_dir = os.path.expanduser(data_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir = os.path.join(fig_dir, 'model-io') if not os.path.exists(io_dir): os.makedirs(io_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) def saveVariable(var, var_name, to_dir=out_data_dir): utils.saveVariable(var, var_name, to_dir) # Load data trial_ids = utils.getUniqueIds(data_dir, prefix=prefix, to_array=True) vocab = utils.loadVariable('vocab', data_dir) saveVariable(vocab, 'vocab') # Define cross-validation folds data_loader = utils.CvDataset(trial_ids, data_dir, vocab=vocab, prefix=prefix, feature_fn_format=file_fn_format, label_fn_format=label_fn_format) cv_folds = utils.makeDataSplits(len(data_loader.trial_ids), **cv_params) device = torchutils.selectDevice(gpu_dev_id) labels_dtype = torch.long criterion = torch.nn.CrossEntropyLoss() metric_names = ('Loss', 'Accuracy') def make_dataset(fns, labels, ids, batch_mode='sample', shuffle=True): dataset = VideoDataset(fns, labels, device=device, labels_dtype=labels_dtype, seq_ids=ids, batch_size=batch_size, batch_mode=batch_mode) loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=shuffle) return dataset, loader for cv_index, cv_fold in enumerate(cv_folds): if start_from is not None and cv_index < start_from: continue if stop_at is not None and cv_index > stop_at: break train_data, val_data, test_data = data_loader.getFold(cv_fold) train_set, train_loader = make_dataset(*train_data, batch_mode='flatten', shuffle=True) test_set, test_loader = make_dataset(*test_data, batch_mode='flatten', shuffle=False) val_set, val_loader = make_dataset(*val_data, batch_mode='flatten', shuffle=True) logger.info( f'CV fold {cv_index + 1} / {len(cv_folds)}: {len(data_loader.trial_ids)} total ' f'({len(train_set)} train, {len(val_set)} val, {len(test_set)} test)' ) model = ImageClassifier(len(vocab), **model_params) optimizer_ft = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=1, gamma=1.00) train_epoch_log = collections.defaultdict(list) val_epoch_log = collections.defaultdict(list) metric_dict = {name: metrics.makeMetric(name) for name in metric_names} model, last_model_wts = torchutils.trainModel( model, criterion, optimizer_ft, lr_scheduler, train_loader, val_loader, device=device, metrics=metric_dict, train_epoch_log=train_epoch_log, val_epoch_log=val_epoch_log, **train_params) # Test model metric_dict = {name: metrics.makeMetric(name) for name in metric_names} test_io_history = torchutils.predictSamples( model.to(device=device), test_loader, criterion=criterion, device=device, metrics=metric_dict, data_labeled=True, update_model=False, seq_as_batch=train_params['seq_as_batch'], return_io_history=True) metric_str = ' '.join(str(m) for m in metric_dict.values()) logger.info('[TST] ' + metric_str) utils.writeResults(results_file, {name: m.value for name, m in metric_dict.items()}, sweep_param_name, model_params) for pred_seq, score_seq, feat_seq, label_seq, batch_id in test_io_history: prefix = f'cvfold={cv_index}_batch={batch_id}' saveVariable(pred_seq.cpu().numpy(), f'{prefix}_pred-label-seq') saveVariable(score_seq.cpu().numpy(), f'{prefix}_score-seq') saveVariable(label_seq.cpu().numpy(), f'{prefix}_true-label-seq') saveVariable(test_set.unflatten, f'cvfold={cv_index}_test-set-unflatten') saveVariable(model, f'cvfold={cv_index}_{model_name}-best') if train_epoch_log: torchutils.plotEpochLog(train_epoch_log, subfig_size=(10, 2.5), title='Training performance', fn=os.path.join( fig_dir, f'cvfold={cv_index}_train-plot.png')) if val_epoch_log: torchutils.plotEpochLog(val_epoch_log, subfig_size=(10, 2.5), title='Heldout performance', fn=os.path.join( fig_dir, f'cvfold={cv_index}_val-plot.png'))
def loadAll(seq_ids, var_name, from_dir=data_dir, prefix='trial='): all_data = tuple( utils.loadVariable(f"{prefix}{seq_id}_{var_name}", from_dir) for seq_id in seq_ids ) return all_data
def main(out_dir=None, data_dir=None, segs_dir=None, pretrained_model_dir=None, model_name=None, gpu_dev_id=None, batch_size=None, learning_rate=None, start_from=None, stop_at=None, model_params={}, cv_params={}, train_params={}, viz_params={}, num_disp_imgs=None, viz_templates=None, results_file=None, sweep_param_name=None): data_dir = os.path.expanduser(data_dir) segs_dir = os.path.expanduser(segs_dir) pretrained_model_dir = os.path.expanduser(pretrained_model_dir) out_dir = os.path.expanduser(out_dir) if not os.path.exists(out_dir): os.makedirs(out_dir) logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt')) if results_file is None: results_file = os.path.join(out_dir, 'results.csv') else: results_file = os.path.expanduser(results_file) fig_dir = os.path.join(out_dir, 'figures') if not os.path.exists(fig_dir): os.makedirs(fig_dir) io_dir = os.path.join(fig_dir, 'model-io') if not os.path.exists(io_dir): os.makedirs(io_dir) out_data_dir = os.path.join(out_dir, 'data') if not os.path.exists(out_data_dir): os.makedirs(out_data_dir) def saveVariable(var, var_name, to_dir=out_data_dir): utils.saveVariable(var, var_name, to_dir) # Load data trial_ids = utils.getUniqueIds(data_dir, prefix='trial=', to_array=True) vocab = utils.loadVariable('vocab', pretrained_model_dir) parts_vocab = utils.loadVariable('parts-vocab', pretrained_model_dir) edge_labels = utils.loadVariable('part-labels', pretrained_model_dir) saveVariable(vocab, 'vocab') saveVariable(parts_vocab, 'parts-vocab') saveVariable(edge_labels, 'part-labels') # Define cross-validation folds data_loader = VideoLoader(trial_ids, data_dir, segs_dir, vocab=vocab, label_fn_format='assembly-seq') cv_folds = utils.makeDataSplits(len(data_loader.trial_ids), **cv_params) Dataset = sim2real.BlocksConnectionDataset device = torchutils.selectDevice(gpu_dev_id) label_dtype = torch.long labels_dtype = torch.long # FIXME criterion = torch.nn.CrossEntropyLoss() def make_dataset(labels, ids, batch_mode='sample', shuffle=True): dataset = Dataset(vocab, edge_labels, label_dtype, data_loader.loadData, labels, device=device, labels_dtype=labels_dtype, seq_ids=ids, batch_size=batch_size, batch_mode=batch_mode) loader = torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=shuffle) return dataset, loader for cv_index, cv_fold in enumerate(cv_folds): if start_from is not None and cv_index < start_from: continue if stop_at is not None and cv_index > stop_at: break train_data, val_data, test_data = data_loader.getFold(cv_fold) train_set, train_loader = make_dataset(*train_data, batch_mode='sample', shuffle=True) test_set, test_loader = make_dataset(*test_data, batch_mode='flatten', shuffle=False) val_set, val_loader = make_dataset(*val_data, batch_mode='sample', shuffle=True) logger.info( f'CV fold {cv_index + 1} / {len(cv_folds)}: {len(data_loader.trial_ids)} total ' f'({len(train_set)} train, {len(val_set)} val, {len(test_set)} test)' ) logger.info( f"Class freqs (train): {np.squeeze(train_set.class_freqs)}") logger.info(f"Class freqs (val): {np.squeeze(val_set.class_freqs)}") logger.info(f"Class freqs (test): {np.squeeze(test_set.class_freqs)}") if model_name == 'template': model = sim2real.AssemblyClassifier(vocab, **model_params) elif model_name == 'pretrained': pretrained_model = utils.loadVariable("cvfold=0_model-best", pretrained_model_dir) model = sim2real.SceneClassifier(pretrained_model, **model_params) metric_names = ('Loss', 'Accuracy', 'Precision', 'Recall', 'F1') criterion = torch.nn.CrossEntropyLoss() # criterion = torchutils.BootstrappedCriterion( # 0.25, base_criterion=torch.nn.functional.cross_entropy, # ) else: raise AssertionError() optimizer_ft = torch.optim.Adam(model.parameters(), lr=learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=0, amsgrad=False) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer_ft, step_size=1, gamma=1.00) train_epoch_log = collections.defaultdict(list) val_epoch_log = collections.defaultdict(list) metric_dict = {name: metrics.makeMetric(name) for name in metric_names} model, last_model_wts = torchutils.trainModel( model, criterion, optimizer_ft, lr_scheduler, train_loader, val_loader, device=device, metrics=metric_dict, train_epoch_log=train_epoch_log, val_epoch_log=val_epoch_log, **train_params) # Test model metric_dict = {name: metrics.makeMetric(name) for name in metric_names} test_io_history = torchutils.predictSamples( model.to(device=device), test_loader, criterion=criterion, device=device, metrics=metric_dict, data_labeled=True, update_model=False, seq_as_batch=train_params['seq_as_batch'], return_io_history=True) metric_str = ' '.join(str(m) for m in metric_dict.values()) logger.info('[TST] ' + metric_str) utils.writeResults(results_file, {name: m.value for name, m in metric_dict.items()}, sweep_param_name, model_params) for pred_seq, score_seq, feat_seq, label_seq, batch_id in test_io_history: prefix = f'cvfold={cv_index}_batch={batch_id}' saveVariable(pred_seq.cpu().numpy(), f'{prefix}_pred-label-seq') saveVariable(score_seq.cpu().numpy(), f'{prefix}_score-seq') saveVariable(label_seq.cpu().numpy(), f'{prefix}_true-label-seq') saveVariable(test_set.unflatten, f'cvfold={cv_index}_test-set-unflatten') saveVariable(model, f'cvfold={cv_index}_{model_name}-best') if train_epoch_log: torchutils.plotEpochLog(train_epoch_log, subfig_size=(10, 2.5), title='Training performance', fn=os.path.join( fig_dir, f'cvfold={cv_index}_train-plot.png')) if val_epoch_log: torchutils.plotEpochLog(val_epoch_log, subfig_size=(10, 2.5), title='Heldout performance', fn=os.path.join( fig_dir, f'cvfold={cv_index}_val-plot.png')) if model_name == 'pretrained' and num_disp_imgs is not None: cvfold_dir = os.path.join(io_dir, f'cvfold={cv_index}') if not os.path.exists(cvfold_dir): os.makedirs(cvfold_dir) model.plotBatches(test_io_history, cvfold_dir, images_per_fig=num_disp_imgs, dataset=test_set) if model_name == 'template' and num_disp_imgs is not None: io_dir = os.path.join(fig_dir, 'model-io') if not os.path.exists(io_dir): os.makedirs(io_dir) plot_topk(model, test_io_history, num_disp_imgs, os.path.join(io_dir, f"cvfold={cv_index}.png")) if viz_templates: sim2real.viz_model_params(model, templates_dir=None)