class ICarl(ContinualModel):
NAME = 'icarl'
COMPATIBILITY = ['class-il', 'domain-il', 'task-il']
def __init__(self, backbone, loss, args, transform):
super(ICarl, self).__init__(backbone, loss, args, transform)
self.dataset = get_dataset(args)
# Instantiate buffers
self.buffer = Buffer(self.args.buffer_size, self.device)
if type(self.dataset.N_CLASSES_PER_TASK) == list:
nc = int(np.sum(self.dataset.N_CLASSES_PER_TASK))
else:
nc = self.dataset.N_CLASSES_PER_TASK * self.dataset.N_TASKS
self.eye = torch.eye(nc).to(self.device)
self.class_means = None
self.old_net = None
self.current_task = 0
def forward(self, x):
if self.class_means is None:
with torch.no_grad():
self.compute_class_means()
feats = self.net.features(x)
feats = feats.unsqueeze(1)
pred = (self.class_means.unsqueeze(0) - feats).pow(2).sum(2)
return -pred
def observe(self, inputs, labels, not_aug_inputs, logits=None):
labels = labels.long()
if not hasattr(self, 'classes_so_far'):
self.register_buffer('classes_so_far', labels.unique().to('cpu'))
else:
self.register_buffer(
'classes_so_far',
torch.cat((self.classes_so_far, labels.to('cpu'))).unique())
self.class_means = None
if self.current_task > 0:
with torch.no_grad():
logits = torch.sigmoid(self.old_net(inputs))
self.opt.zero_grad()
loss = self.get_loss(inputs, labels, self.current_task, logits)
loss.backward()
self.opt.step()
return loss.item()
@staticmethod
def binary_cross_entropy(pred, y):
return -(pred.log() * y + (1 - y) * (1 - pred).log()).mean()
def get_loss(self, inputs: torch.Tensor, labels: torch.Tensor,
task_idx: int, logits: torch.Tensor) -> torch.Tensor:
"""
Computes the loss tensor.
:param inputs: the images to be fed to the network
:param labels: the ground-truth labels
:param task_idx: the task index
:return: the differentiable loss value
"""
if type(self.dataset.N_CLASSES_PER_TASK) == list:
pc = int(np.sum(self.dataset.N_CLASSES_PER_TASK[:task_idx]))
ac = int(np.sum(self.dataset.N_CLASSES_PER_TASK[:task_idx + 1]))
else:
pc = task_idx * self.dataset.N_CLASSES_PER_TASK
ac = (task_idx + 1) * self.dataset.N_CLASSES_PER_TASK
outputs = self.net(inputs)[:, :ac]
if task_idx == 0:
# Compute loss on the current task
targets = self.eye[labels][:, :ac]
loss = F.binary_cross_entropy_with_logits(outputs, targets)
assert loss >= 0
else:
targets = self.eye[labels][:, pc:ac]
comb_targets = torch.cat((logits[:, :pc], targets), dim=1)
loss = F.binary_cross_entropy_with_logits(outputs, comb_targets)
assert loss >= 0
if self.args.wd_reg:
loss += self.args.wd_reg * torch.sum(self.net.get_params()**2)
return loss
def begin_task(self, dataset):
denorm = (lambda x: x) if dataset.get_denormalization_transform()\
is None else dataset.get_denormalization_transform()
if self.current_task > 0:
if self.args.dataset != 'seq-core50':
dataset.train_loader.dataset.targets = np.concatenate([
dataset.train_loader.dataset.targets,
self.buffer.labels.cpu().numpy()
[:self.buffer.num_seen_examples]
])
if type(dataset.train_loader.dataset.data) == torch.Tensor:
dataset.train_loader.dataset.data = torch.cat([
dataset.train_loader.dataset.data,
torch.stack([
denorm(self.buffer.examples[i].type(
torch.uint8).cpu())
for i in range(self.buffer.num_seen_examples)
]).squeeze(1)
])
else:
dataset.train_loader.dataset.data = np.concatenate([
dataset.train_loader.dataset.data,
torch.stack([
(denorm(self.buffer.examples[i] * 255).type(
torch.uint8).cpu())
for i in range(self.buffer.num_seen_examples)
]).numpy().swapaxes(1, 3)
])
else:
print(
torch.stack([(denorm(self.buffer.examples[i]).cpu())
for i in range(self.buffer.num_seen_examples)
]).numpy().shape)
dataset.train_loader.dataset.add_more_data(
more_targets=self.buffer.labels.cpu().numpy()
[:self.buffer.num_seen_examples],
more_data=torch.stack([
(denorm(self.buffer.examples[i]).cpu())
for i in range(self.buffer.num_seen_examples)
]).numpy().swapaxes(1, 3))
def end_task(self, dataset) -> None:
self.old_net = deepcopy(self.net.eval())
self.net.train()
with torch.no_grad():
self.fill_buffer(self.buffer, dataset, self.current_task)
self.current_task += 1
self.class_means = None
def compute_class_means(self) -> None:
"""
Computes a vector representing mean features for each class.
"""
# This function caches class means
transform = self.dataset.get_normalization_transform()
class_means = []
examples, labels, _ = self.buffer.get_all_data(transform)
for _y in self.classes_so_far:
x_buf = torch.stack([
examples[i] for i in range(0, len(examples))
if labels[i].cpu() == _y
]).to(self.device)
class_means.append(self.net.features(x_buf).mean(0))
self.class_means = torch.stack(class_means)
def fill_buffer(self, mem_buffer: Buffer, dataset, t_idx: int) -> None:
"""
Adds examples from the current task to the memory buffer
by means of the herding strategy.
:param mem_buffer: the memory buffer
:param dataset: the dataset from which take the examples
:param t_idx: the task index
"""
mode = self.net.training
self.net.eval()
samples_per_class = mem_buffer.buffer_size // len(self.classes_so_far)
if t_idx > 0:
# 1) First, subsample prior classes
buf_x, buf_y, buf_l = self.buffer.get_all_data()
mem_buffer.empty()
for _y in buf_y.unique():
idx = (buf_y == _y)
_y_x, _y_y, _y_l = buf_x[idx], buf_y[idx], buf_l[idx]
mem_buffer.add_data(examples=_y_x[:samples_per_class],
labels=_y_y[:samples_per_class],
logits=_y_l[:samples_per_class])
# 2) Then, fill with current tasks
loader = dataset.not_aug_dataloader(self.args, self.args.batch_size)
# 2.1 Extract all features
a_x, a_y, a_f, a_l = [], [], [], []
for x, y, not_norm_x in loader:
x, y, not_norm_x = (a.to(self.device) for a in [x, y, not_norm_x])
a_x.append(not_norm_x.to('cpu'))
a_y.append(y.to('cpu'))
feats = self.net.features(x)
a_f.append(feats.cpu())
a_l.append(torch.sigmoid(self.net.classifier(feats)).cpu())
a_x, a_y, a_f, a_l = torch.cat(a_x), torch.cat(a_y), torch.cat(
a_f), torch.cat(a_l)
# 2.2 Compute class means
for _y in a_y.unique():
idx = (a_y == _y)
_x, _y, _l = a_x[idx], a_y[idx], a_l[idx]
feats = a_f[idx]
mean_feat = feats.mean(0, keepdim=True)
running_sum = torch.zeros_like(mean_feat)
i = 0
while i < samples_per_class and i < feats.shape[0]:
cost = (mean_feat - (feats + running_sum) / (i + 1)).norm(2, 1)
idx_min = cost.argmin().item()
mem_buffer.add_data(
examples=_x[idx_min:idx_min + 1].to(self.device),
labels=_y[idx_min:idx_min + 1].to(self.device),
logits=_l[idx_min:idx_min + 1].to(self.device))
running_sum += feats[idx_min:idx_min + 1]
feats[idx_min] = feats[idx_min] + 1e6
i += 1
assert len(mem_buffer.examples) <= mem_buffer.buffer_size
self.net.train(mode)
class Fdr(ContinualModel):
NAME = 'fdr'
COMPATIBILITY = ['class-il', 'domain-il', 'task-il', 'general-continual']
def __init__(self, backbone, loss, args, transform):
super(Fdr, self).__init__(backbone, loss, args, transform)
self.buffer = Buffer(self.args.buffer_size, self.device)
self.current_task = 0
self.i = 0
self.soft = torch.nn.Softmax(dim=1)
self.logsoft = torch.nn.LogSoftmax(dim=1)
def end_task(self, dataset):
self.current_task += 1
examples_per_task = self.args.buffer_size // self.current_task
if self.current_task > 1:
buf_x, buf_log, buf_tl = self.buffer.get_all_data()
self.buffer.empty()
for ttl in buf_tl.unique():
idx = (buf_tl == ttl)
ex, log, tasklab = buf_x[idx], buf_log[idx], buf_tl[idx]
first = min(ex.shape[0], examples_per_task)
self.buffer.add_data(examples=ex[:first],
logits=log[:first],
task_labels=tasklab[:first])
counter = 0
with torch.no_grad():
for i, data in enumerate(dataset.train_loader):
inputs, labels, not_aug_inputs = data
inputs = inputs.to(self.device)
not_aug_inputs = not_aug_inputs.to(self.device)
outputs = self.net(inputs)
if examples_per_task - counter < 0:
break
self.buffer.add_data(
examples=not_aug_inputs[:(examples_per_task - counter)],
logits=outputs.data[:(examples_per_task - counter)],
task_labels=(torch.ones(self.args.batch_size) *
(self.current_task - 1))[:(examples_per_task -
counter)])
counter += self.args.batch_size
def observe(self, inputs, labels, not_aug_inputs):
self.i += 1
self.opt.zero_grad()
outputs = self.net(inputs)
loss = self.loss(outputs, labels)
loss.backward()
self.opt.step()
if not self.buffer.is_empty():
self.opt.zero_grad()
buf_inputs, buf_logits, _ = self.buffer.get_data(
self.args.minibatch_size, transform=self.transform)
buf_outputs = self.net(buf_inputs)
loss = torch.norm(
self.soft(buf_outputs) - self.soft(buf_logits), 2, 1).mean()
assert not torch.isnan(loss)
loss.backward()
self.opt.step()
return loss.item()