def __init__(self, main_trainer: "LLLTrainer", task_idx:int):
self.task_idx = task_idx
self.main_trainer = main_trainer
self.device_name = main_trainer.device_name
self.config = main_trainer.config
self.init_models()
self.init_writer()
self.criterion = nn.CrossEntropyLoss()
self.optim = self.construct_optimizer()
self.attrs = self.model.attrs if hasattr(self.model, 'attrs') else None
self.task_ds_train, self.task_ds_test = main_trainer.data_splits[task_idx]
self.output_mask = construct_output_mask(main_trainer.class_splits[task_idx], self.config.lll_setup.num_classes)
self.classes = self.main_trainer.class_splits[self.task_idx]
self.learned_classes = np.unique(flatten(self.main_trainer.class_splits[:self.task_idx])).tolist()
self.learned_classes_mask = construct_output_mask(self.learned_classes, self.config.data.num_classes)
self.seen_classes = np.unique(flatten(self.main_trainer.class_splits[:self.task_idx + 1])).tolist()
self.seen_classes_mask = construct_output_mask(self.seen_classes, self.config.data.num_classes)
self.init_dataloaders()
self.init_episodic_memory()
self.test_acc_batch_history = []
self.after_iter_done_callbacks = []
self.num_iters_done = 0
self.num_epochs_done = 0
self._after_init_hook()
def compute_generalized_forgetting_measure(logits_history: List[List[float]],
targets: List[int],
class_splits: List[List[int]],
task_idx: int = -1) -> float:
"""
Computes Generalized Forgetting Measure for task {task_idx}
GFM — is a forgetting measure where we do not use task identities
:param logits_history: matrix of size [NUM_TASKS x NUM_TASKS]
:param task_idx: for which task we should compute this measure
:return: generalized forgetting measure
"""
if task_idx == 0:
return 0. # We cannot forget anything if we have not learned anything
elif task_idx == -1:
task_idx = len(logits_history)
logits_history = np.array(logits_history)
targets = np.array(targets)
seen_classes = set(flatten(class_splits[:task_idx]))
test_seen_idx = [i for i, y in enumerate(targets) if y in seen_classes]
# Removing unseen samples
targets = targets[test_seen_idx]
logits_history = [ls[test_seen_idx] for ls in logits_history]
guessed_history = [(ls.argmax(axis=1) == targets) for ls in logits_history]
return np.mean([g.mean() for g in guessed_history])
def compute_seen_classes_acc_history(
logits_history: List[List[List[float]]],
targets: List[int],
class_splits: List[List[int]],
restrict_space: bool = True) -> List[float]:
"""
Computes zero-shot history on all the remaining tasks before starting each task
:param logits_history: history of model logits, evaluated AFTER each task,
i.e. matrix of size [NUM_TASKS x DATASET_SIZE x NUM_CLASSES]
:param targets: targets for the objects of size [DATASET_SIZE]
:param class_splits: list of classes for each task of size [NUM_TASKS x NUM_CLASSES_PER_TASK]
:param restrict_space: should we restrict prediction space to specified classes or not
:return: zero-shot accuracies of size [NUM_TASKS]
"""
seen_classes = [
np.unique(flatten(class_splits[:i + 1]))
for i in range(len(class_splits))
]
accs = [
compute_acc_for_classes(l, targets, cs, restrict_space)
for l, cs in zip(logits_history, seen_classes)
]
return accs
def compute_joined_ausuc_history(logits_history: List[List[List[float]]],
targets: List[int],
class_splits: List[List[int]]) -> List[float]:
"""
Computes AUSUC history on all the remaining tasks before starting each task
:param logits_history: history of model logits, evaluated BEFORE each task,
i.e. matrix of size [NUM_TASKS x DATASET_SIZE x NUM_CLASSES]
:param targets: targets for the objects of size [DATASET_SIZE]
:param class_splits: list of classes for each task of size [NUM_TASKS x NUM_CLASSES_PER_TASK]
:return: AUSUC scores of size [NUM_TASKS]
"""
num_classes = len(logits_history[0][0])
seen_classes = [
np.unique(flatten(class_splits[:i])) for i in range(len(class_splits))
]
seen_classes_masks = [
construct_output_mask(cs, num_classes) for cs in seen_classes
]
ausuc_scores = [
compute_ausuc(l, targets, m)[0]
for l, m in zip(logits_history, seen_classes_masks)
]
return ausuc_scores
def update_episodic_memory(self):
"""
Adds examples from own data to episodic memory
:param:
- task_idx — task index
- num_samples_per_class — max number of samples of each class to add
"""
num_samples_per_class = self.config.hp.num_mem_samples_per_class
unique_labels = set(
[y for _, y in self.load_dataset(self.task_ds_train)])
groups = [[(x, y) for (x, y) in self.load_dataset(self.task_ds_train)
if y == label]
for label in unique_labels] # Slow but concise
num_samples_to_add = [
min(len(g), num_samples_per_class) for g in groups
]
task_memory = [
random.sample(g, n) for g, n in zip(groups, num_samples_to_add)
]
task_memory = flatten(task_memory)
task_mask = construct_output_mask(
self.main_trainer.class_splits[self.task_idx],
self.config.lll_setup.num_classes)
task_mask = task_mask.reshape(1, -1).repeat(len(task_memory), axis=0)
assert len(task_memory) <= num_samples_per_class * len(groups)
assert len(task_mask) <= num_samples_per_class * len(groups)
self.episodic_memory.extend(task_memory)
self.episodic_memory_output_mask.extend([m for m in task_mask])
def reduce_episodic_memory(self, num_samples_per_class: int):
class_memories = [[(x, y) for x, y in self.episodic_memory if y == c]
for c in self.learned_classes]
class_memories_reduced = [
mem[:num_samples_per_class] for mem in class_memories
]
self.episodic_memory = flatten(class_memories_reduced)
def _after_init_hook(self):
seen_classes = np.unique(flatten(self.main_trainer.class_splits[:self.task_idx + 1]))
self.task_ds_train = [ds_train for ds_train, ds_test in self.main_trainer.data_splits[:self.task_idx+1]]
self.task_ds_train = [(x, y) for ds in self.task_ds_train for (x, y) in ds]
self.joint_output_mask = construct_output_mask(seen_classes, self.config.lll_setup.num_classes)
self.original_train_dataloader = self.train_dataloader
self.train_dataloader = DataLoader(self.task_ds_train, batch_size=self.config.hp.batch_size,
collate_fn=lambda b: list(zip(*b)), shuffle=True)
def init_dataloaders(self):
self.ds_train, self.ds_test, self.class_attributes = load_data(
self.config.data, self.config.hp.get('img_target_shape'))
self.class_splits = split_classes_for_tasks(self.config.lll_setup,
self.config.random_seed)
classes_used = set(flatten(self.class_splits))
if len(classes_used) < self.config.data.num_classes:
self.ds_train = self.ds_train.filter_out_classes(classes_used)
self.ds_test = self.ds_test.filter_out_classes(classes_used)
self.data_splits = get_train_test_data_splits(self.class_splits,
self.ds_train,
self.ds_test)
for task_idx, task_classes in enumerate(self.class_splits):
print(f'[Task {task_idx}]:', task_classes)
def compute_ausuc_matrix(logits_history: np.ndarray, targets: List[int],
class_splits: List[List[int]]) -> np.ndarray:
"""
Computes pairwise AUSUC scores between tasks given logits history
:param logits_history: history of model logits, evaluated BEFORE each task,
i.e. matrix of size [NUM_TASKS x DATASET_SIZE x NUM_CLASSES]
:param targets: targets for the objects of size [DATASET_SIZE]
:param class_splits: list of classes for each task of size [NUM_TASKS x NUM_CLASSES_PER_TASK]
:return: AUCSUC value and a matrix of pairwise AUSUCS
"""
num_tasks = len(logits_history)
ausuc_matrix = []
for task_from in range(num_tasks):
ausucs = []
for task_to in range(num_tasks):
classes = set(
flatten([class_splits[task_to], class_splits[task_from]]))
curr_logits = [
l for l, t in zip(logits_history[task_from], targets)
if t in classes
]
curr_targets = [t for t in targets if t in classes]
classes = list(classes)
curr_targets = remap_targets(curr_targets, classes)
seen_classes_mask = np.array(
[c in class_splits[task_from] for c in classes]).astype(bool)
ausuc, _ = compute_ausuc(
np.array(curr_logits)[:, classes], curr_targets,
seen_classes_mask)
ausucs.append(ausuc)
ausuc_matrix.append(ausucs)
return np.array(ausuc_matrix)