def grouped_pca(grouped_features, n_components: int, pca_models_by_group=None):
# grouped_features: Dict[group_name: str, Dict[vid_name: str, np array]]
if pca_models_by_group is not None:
assert set(grouped_features.keys()) == set(pca_models_by_group.keys())
else:
pca_models_by_group = {}
for group_name, vid_dict in grouped_features.items():
# rows should be data points, so all groups should have the same number of cols
assert all_equal(v.shape[1] for v in vid_dict.values())
X_l = []
for vid, features in vid_dict.items():
X_l.append(features)
X = np.vstack(X_l)
pca = PCA(n_components=min(n_components, X.shape[1]))
pca.fit(X)
logger.debug("group {}: {} instances".format(group_name, len(X_l)))
logger.debug("group {}: pca explained {} of the variance".format(
group_name, pca.explained_variance_ratio_.sum()))
pca_models_by_group[group_name] = pca
transformed = {
group_name: {
vid_name: pca_models_by_group[group_name].transform(x)
for vid_name, x in vid_dict.items()
}
for group_name, vid_dict in grouped_features.items()
}
return transformed, pca_models_by_group
def merge_grouped(grouped_features):
# grouped_features: Dict[group_name: str, Dict[vid_name: str, np array]]
merged = {}
# should have the same vid_name s for each group_name
assert all_equal(group_dict.keys()
for group_dict in grouped_features.values())
for vid_name in next(iter(grouped_features.values())):
values = [
t[1][vid_name]
for t in sorted(grouped_features.items(), key=lambda t: t[0])
]
merged[vid_name] = np.hstack(values)
return merged
def forward(self, features, lengths, valid_classes_per_instance=None):
# batch_size x max_len x seq_hidden_size
encoded = self.encoder(features, lengths, output_padding_value=0)
# batch_size x max_len x num_classes
logits = self.proj(encoded)
if valid_classes_per_instance is not None:
assert all_equal(set(vc.detach().cpu().numpy()) for vc in
valid_classes_per_instance), "must have same valid_classes for all instances in the batch"
valid_classes = valid_classes_per_instance[0]
mask = torch.full_like(logits, -float("inf"))
mask[:,:,valid_classes] = 0
logits = logits + mask
return logits
def predict(self, test_data: Datasplit):
self.model.eval()
predictions = {}
loader = make_data_loader(self.args, test_data, batch_by_task=False, shuffle=False, batch_size=1)
for batch in loader:
features = batch['features']
lengths = batch['lengths']
task_indices = batch['task_indices']
if self.args.cuda:
features = features.cuda()
lengths = lengths.cuda()
task_indices = [indx.cuda() for indx in task_indices]
videos = batch['video_name']
assert all_equal(videos)
video = next(iter(videos))
# batch_size x length x num_classes
with torch.no_grad():
logits = self.model(features, lengths, valid_classes_per_instance=task_indices)
preds = logits.max(dim=-1)[1]
preds = preds.squeeze(0)
assert preds.ndim == 1
predictions[video] = preds.detach().cpu().numpy()
return predictions
def viterbi(self, features, lengths, valid_classes_per_instance, add_eos=True, use_mean_z=False,
additional_allowed_ends_per_instance=None, constraints=None, predict_single=False, return_elp=False):
if valid_classes_per_instance is not None:
assert all_equal(set(vc.detach().cpu().numpy()) for vc in
valid_classes_per_instance), "must have same valid_classes for all instances in the batch"
valid_classes = valid_classes_per_instance[0]
C = len(valid_classes)
else:
valid_classes = None
C = self.n_classes
scores, log_det, elp = self.score_features(features, lengths, valid_classes, add_eos=add_eos, use_mean_z=use_mean_z,
additional_allowed_ends_per_instance=additional_allowed_ends_per_instance,
constraints=constraints, return_elp=True)
if add_eos:
eos_lengths = lengths + 1
else:
eos_lengths = lengths
dist = SemiMarkovCRF(scores, lengths=eos_lengths)
pred_spans, extra = dist.struct.from_parts(dist.argmax)
# convert to class labels
# pred_spans_trim = self.model.trim(pred_spans, lengths, check_eos=add_eos)
pred_spans_unmap = pred_spans.detach().cpu()
if valid_classes is not None:
mapping = {index: cls.item() for index, cls in enumerate(valid_classes)}
assert len(mapping.values()) == len(mapping), "valid_classes must be unique"
assert -1 not in mapping.values()
mapping[-1] = -1
mapping[C] = self.n_classes # map EOS
# unmap
pred_spans_unmap.apply_(lambda x: mapping[x])
if return_elp:
return pred_spans_unmap, elp
else:
return pred_spans_unmap
class BreakfastCorpus(Corpus):
BACKGROUND_LABELS = ["SIL"]
TASKS = [
'coffee', 'cereals', 'tea', 'milk', 'juice', 'sandwich',
'scrambledegg', 'friedegg', 'salat', 'pancake'
]
DATASPLITS = {
's1': ["P{:02d}".format(d) for d in range(3, 16)],
's2': ["P{:02d}".format(d) for d in range(16, 29)],
's3': ["P{:02d}".format(d) for d in range(29, 42)],
's4': ["P{:02d}".format(d) for d in range(42, 55)],
}
assert all_equal(len(v) for v in DATASPLITS.values())
def __init__(self,
mapping_file,
feature_root,
label_root,
task_specific_steps=False):
self._mapping_file = mapping_file
self._feature_root = feature_root
self._label_root = label_root
self._task_specific_steps = task_specific_steps
assert not task_specific_steps
self.annotate_background_with_previous = False
super(BreakfastCorpus,
self).__init__(background_labels=self.BACKGROUND_LABELS)
def _get_components_for_label(self, label):
return label.split('_')
def _load_mapping(self):
with open(self._mapping_file, 'r') as f:
for line in f:
index, label = line.strip().split()
index = int(index)
_index = self._index(label)
if label in self._background_labels:
assert index in self._background_indices
if index in self._background_indices:
assert label in self._background_labels
assert _index == index
def get_datasplit(self,
remove_background,
task_filter=None,
splits=None,
full=True,
subsample=1,
feature_downscale=1.0,
feature_permutation_seed=None):
return BreakfastDatasplit(
self,
remove_background,
task_filter=task_filter,
splits=splits,
full=full,
subsample=subsample,
feature_downscale=feature_downscale,
feature_permutation_seed=feature_permutation_seed)
def predict(self, test_data):
self.model.eval()
self.model.flatten_parameters()
predictions = {}
loader = make_data_loader(self.args, test_data, shuffle=False, batch_by_task=True, batch_size=self.args.batch_size)
# print('{} videos in prediction data'.format(len(loader.dataset)))
# for batch in tqdm.tqdm(loader, ncols=80):
for batch in loader:
features = batch['features']
task_indices = batch['task_indices']
lengths = batch['lengths']
# add a batch dimension
# lengths = torch.LongTensor([features.size(0)]).unsqueeze(0)
# features = features.unsqueeze(0)
# task_indices = task_indices.unsqueeze(0)
videos = batch['video_name']
tasks = batch['task_name']
assert len(set(tasks)) == 1
task = next(iter(tasks))
if 'test' in self.args.sm_constrain_with_narration:
assert all_equal(tasks)
constraints_expanded = self.expand_constraints(
test_data, task, task_indices[0], 1 - batch['constraints']
)
constraints_expanded *= self.args.sm_constrain_narration_weight
else:
constraints_expanded = None
if self.args.cuda:
features = features.cuda()
task_indices = [ti.cuda() for ti in task_indices]
lengths = lengths.cuda()
if constraints_expanded is not None:
constraints_expanded = constraints_expanded.cuda()
addl_allowed_ends = self.make_additional_allowed_ends(tasks, lengths)
def predict(constraints):
# TODO: figure out under which eval conditions use_mean_z should be False
pred_spans, elp = self.model.viterbi(features, lengths, task_indices, add_eos=True, use_mean_z=True,
additional_allowed_ends_per_instance=addl_allowed_ends,
constraints=constraints, return_elp=True)
pred_labels = semimarkov_utils.spans_to_labels(pred_spans)
# if self.args.sm_predict_single:
# # pred_spans: batch_size x T
# pred_labels_single = torch.zeros_like(pred_labels)
# for i in pred_labels.size(0):
# for lab in torch.unique(pred_labels[i,:lengths[i]]):
# #emission_scores: b x N x C
# pred_labels
# pass
# if self.args.sm_constrain_transitions:
# all_pred_span_indices = [
# [ix for ix, count in this_rle_spans]
# for this_rle_spans in semimarkov_utils.rle_spans(pred_spans, lengths)
# ]
# for i, indices in enumerate(all_pred_span_indices):
# remove_cons_dups = [ix for ix, group in itertools.groupby(indices)
# if not ix in test_data.corpus._background_indices]
# non_bg_indices = [
# ix for ix in test_data.corpus.indices_by_task(task)
# if ix not in test_data.corpus._background_indices
# ]
# if len(remove_cons_dups) != len(non_bg_indices) and lengths[i].item() != len(remove_cons_dups):
# print("deduped: {}, indices: {}, length {}".format(
# remove_cons_dups, non_bg_indices, lengths[i].item()
# ))
# # assert lengths[i].item() < len(non_bg_indices)
pred_labels_trim_s = self.model.trim(pred_labels, lengths, check_eos=True)
return pred_labels_trim_s
pred_labels_trim_s = predict(constraints_expanded)
# assert len(pred_labels_trim_s) == 1, "batch size should be 1"
for ix, (video, pred_labels_trim) in enumerate(zip(videos, pred_labels_trim_s)):
preds = pred_labels_trim.numpy()
predictions[video] = preds
# if constraints_expanded is not None:
# this_cons = batch['constraints'][ix]
# if this_cons.sum() > 0:
# step_indices = test_data.get_ordered_indices_no_background()[task]
# for t, label in enumerate(preds):
# if label in step_indices:
# label_ix = step_indices.index(label)
# assert batch['constraints'][ix,t,label_ix] == 1
assert self.model.n_classes not in predictions[video], "predictions should not contain EOS: {}".format(
predictions[video])
return predictions
def fit(self, train_data: Datasplit, use_labels: bool, callback_fn=None):
self.model.train()
self.model.flatten_parameters()
if use_labels:
assert not self.args.sm_constrain_transitions
initialize = True
if use_labels and self.args.sm_supervised_method in ['closed-form', 'closed-then-gradient']:
self.fit_supervised(train_data)
if self.args.sm_supervised_method == 'closed-then-gradient':
initialize = False
callback_fn(-1, {})
else:
return
if self.args.sm_init_non_projection_parameters_from:
initialize = False
if callback_fn:
callback_fn(-1, {})
optimizer, scheduler = make_optimizer(self.args, self.model.parameters())
big_loader = make_data_loader(self.args, train_data, batch_by_task=False, shuffle=True, batch_size=100)
samp = next(iter(big_loader))
big_features = samp['features']
big_lengths = samp['lengths']
if self.args.cuda:
big_features = big_features.cuda()
big_lengths = big_lengths.cuda()
if initialize:
self.model.initialize_gaussian(big_features, big_lengths)
loader = make_data_loader(self.args, train_data, batch_by_task=True, shuffle=True, batch_size=self.args.batch_size)
# print('{} videos in training data'.format(len(loader.dataset)))
# all_features = [sample['features'] for batch in loader for sample in batch]
# if self.args.cuda:
# all_features = [feats.cuda() for feats in all_features]
C = self.n_classes
K = self.args.sm_max_span_length
for epoch in range(self.args.epochs):
start_time = time.time()
# call here since we may set eval in callback_fn
self.model.train()
losses = []
multi_batch_losses = []
nlls = []
kls = []
log_dets = []
num_frames = 0
num_videos = 0
train_nll = 0
train_kl = 0
train_log_det = 0
# for batch_ix, batch in enumerate(tqdm.tqdm(loader, ncols=80)):
for batch_ix, batch in enumerate(loader):
if self.args.train_limit and batch_ix >= self.args.train_limit:
break
# if self.args.cuda:
# features = features.cuda()
# task_indices = task_indices.cuda()
# gt_single = gt_single.cuda()
tasks = batch['task_name']
videos = batch['video_name']
features = batch['features']
task_indices = batch['task_indices']
lengths = batch['lengths']
if 'train' in self.args.sm_constrain_with_narration:
assert all_equal(tasks)
constraints_expanded = self.expand_constraints(
train_data, tasks[0], task_indices[0], 1 - batch['constraints']
)
constraints_expanded *= self.args.sm_constrain_narration_weight
else:
constraints_expanded = None
num_frames += lengths.sum().item()
num_videos += len(lengths)
# assert len( task_indices) == self.n_classes, "remove_background and multi-task fit() not implemented"
if self.args.cuda:
features = features.cuda()
lengths = lengths.cuda()
if constraints_expanded is not None:
constraints_expanded = constraints_expanded.cuda()
if use_labels:
labels = batch['gt_single']
if self.args.cuda:
labels = labels.cuda()
spans = semimarkov_utils.labels_to_spans(labels, max_k=K)
use_mean_z = True
else:
spans = None
use_mean_z = False
addl_allowed_ends = self.make_additional_allowed_ends(tasks, lengths)
ll, log_det = self.model.log_likelihood(features,
lengths,
valid_classes_per_instance=task_indices,
spans=spans,
add_eos=True,
use_mean_z=use_mean_z,
additional_allowed_ends_per_instance=addl_allowed_ends,
constraints=constraints_expanded)
nll = -ll
kl = self.model.kl.mean()
if use_labels:
this_loss = nll - log_det
else:
this_loss = nll - log_det + kl
multi_batch_losses.append(this_loss)
nlls.append(nll.item())
kls.append(kl.item())
log_dets.append(log_det.item())
train_nll += (nll.item() * len(videos))
train_kl += (kl.item() * len(videos))
train_log_det += (log_det.item() * len(videos))
losses.append(this_loss.item())
if len(multi_batch_losses) >= self.args.batch_accumulation:
loss = sum(multi_batch_losses) / len(multi_batch_losses)
loss.backward()
multi_batch_losses = []
if self.args.print_every and (batch_ix % self.args.print_every == 0):
param_norm = sum([p.norm()**2 for p in self.model.parameters()
if p.requires_grad]).item()**0.5
gparam_norm = sum([p.grad.norm()**2 for p in self.model.parameters()
if p.requires_grad and p.grad is not None]).item()**0.5
log_str = 'Epoch: %02d, Batch: %03d/%03d, |Param|: %.6f, |GParam|: %.2f, lr: %.2E, ' + \
'loss: %.4f, recon: %.4f, kl: %.4f, log_det: %.4f, recon_bound: %.2f, Throughput: %.2f vid / sec'
print(log_str %
(epoch, batch_ix, len(loader), param_norm, gparam_norm,
optimizer.param_groups[0]["lr"],
(train_nll + train_kl + train_log_det) / num_videos, # loss
train_nll / num_frames, # recon
train_kl / num_frames, # kl
train_log_det / num_videos, # log_det
(train_nll + train_kl) / num_frames, # recon_bound
num_videos / (time.time() - start_time))) # Throughput
if self.args.max_grad_norm is not None:
torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.args.max_grad_norm)
optimizer.step()
self.model.zero_grad()
train_loss = np.mean(losses)
if scheduler is not None:
scheduler.step(train_loss)
callback_fn(epoch, {'train_loss': train_loss,
'train_nll_frame_avg': train_nll / num_frames,
'train_kl_vid_avg': train_kl / num_videos,
'train_recon_bound': (train_nll + train_kl) / num_frames})
def log_likelihood(self, features, lengths, valid_classes_per_instance, spans=None, add_eos=True, use_mean_z=False,
additional_allowed_ends_per_instance=None, constraints=None):
if valid_classes_per_instance is not None:
assert all_equal(set(vc.detach().cpu().numpy()) for vc in
valid_classes_per_instance), "must have same valid_classes for all instances in the batch"
valid_classes = valid_classes_per_instance[0]
C = len(valid_classes)
else:
valid_classes = None
C = self.n_classes
scores, log_det = self.score_features(features, lengths, valid_classes, add_eos=add_eos, use_mean_z=use_mean_z,
additional_allowed_ends_per_instance=additional_allowed_ends_per_instance,
constraints=constraints)
K = scores.size(2)
assert K <= self.max_k or (self.max_k == 1 and K == 2)
if add_eos:
eos_lengths = lengths + 1
eos_spans = self.add_eos(spans, lengths) if spans is not None else spans
eos_C = C + 1
else:
eos_lengths = lengths
eos_spans = spans
eos_C = C
dist = SemiMarkovCRF(scores, lengths=eos_lengths)
if eos_spans is not None:
eos_spans_mapped = eos_spans.detach().cpu().clone()
if valid_classes is not None:
# unmap
mapping = {cls.item(): index for index, cls in enumerate(valid_classes)}
assert len(mapping) == len(valid_classes), "valid_classes must be unique"
assert -1 not in mapping
mapping[-1] = -1
mapping[self.n_classes] = C # map EOS
if 0 not in mapping:
# TODO: hack, 0 sometimes will signify padding
mapping[0] = 0
eos_spans_mapped.apply_(lambda x: mapping[x])
# features = features[:,:this_N,:]
# spans = spans[:,:this_N]
parts = SemiMarkovCRF.struct.to_parts(eos_spans_mapped, (eos_C, K),
lengths=eos_lengths).type_as(scores)
if self.args.sm_train_discriminatively:
# this maximizes p(y | x)
log_likelihood = dist.log_prob(parts).mean()
else:
# this maximizes p(x, y)
d = parts.dim()
batch_dims = range(d - len(dist.event_shape))
log_likelihood = dist.struct().score(
dist.log_potentials,
parts.type_as(dist.log_potentials),
batch_dims=batch_dims,
).mean()
else:
log_likelihood = dist.partition.mean()
return log_likelihood, log_det.mean()
