def distill_loss_fct(config, X, Y_logits, T_soft_logits, data): assert np.all(np.equal(X.shape[:2], T_soft_logits.shape[:2])) # Note, targets and predictions might have different head sizes if a # growing softmax is used. assert np.all(np.equal(Y_logits.shape[:2], T_soft_logits.shape[:2])) # Disillation temperature. T = 2. n_digs = config.ssmnist_seq_len # Note, smnist samples have the end-of-sequence bit as last # timestep, the rest is padded. Since there are `n_digs` digits per # sample, we consider the last end-of-sequence digit to determine the # unpadded sequence length. eod_features = X[:, :, 3].cpu().numpy() seq_lengths = np.argsort(eod_features, axis=0)[-n_digs:].max(axis=0) inds = seq_lengths - 1 inds[inds < 0] = 0 # Only compute loss for last timestep. Y_logits = Y_logits[inds, np.arange(inds.size), :] T_soft_logits = T_soft_logits[inds, np.arange(inds.size), :] target_mapping = None if config.all_task_softmax: target_mapping = list(range(T_soft_logits.shape[1])) return Classifier.knowledge_distillation_loss( Y_logits, T_soft_logits, target_mapping=target_mapping, device=Y_logits.device, T=T)
def distill_loss_fct(config, X, Y_logits, T_soft_logits, data, in_seq_lens=None): if in_seq_lens is None: raise NotImplementedError( 'This distillation loss is currently ' + 'only implemented if sequence lengths are provided, as they ' + 'can\'t be inferred easily.') # Note, input and output sequence lengths are identical for the PoS # dataset. assert np.all(np.equal(X.shape[:2], T_soft_logits.shape[:2])) # Note, targets and predictions might have different head sizes if a # growing softmax is used. assert np.all(np.equal(Y_logits.shape[:2], T_soft_logits.shape[:2])) # Disillation temperature. T = 2. target_mapping = None if config.all_task_softmax: target_mapping = list(range(T_soft_logits.shape[2])) dloss = 0 total_num_ts = 0 for bid in range(X.shape[1]): sl = int(in_seq_lens[bid]) total_num_ts += sl Y_logits_i = Y_logits[:sl, bid, :] T_soft_logits_i = T_soft_logits[:sl, bid, :] dloss += Classifier.knowledge_distillation_loss( Y_logits_i, T_soft_logits_i, target_mapping=target_mapping, device=Y_logits.device, T=T) * sl return dloss / total_num_ts
def distill_loss_fct(config, X, Y_logits, T_soft_logits, data): # Note, targets and predictions might have different head sizes if a # growing softmax is used. assert np.all(np.equal(Y_logits.shape[:2], T_soft_logits.shape[:2])) # Only compute loss for last timestep. Y_logits = Y_logits[-1, :, :] T_soft_logits = T_soft_logits[-1, :, :] target_mapping = None if config.all_task_softmax: target_mapping = list(range(T_soft_logits.shape[1])) return Classifier.knowledge_distillation_loss( Y_logits, T_soft_logits, target_mapping=target_mapping, device=Y_logits.device, T=2.)
def distill_loss_fct(config, X, Y_logits, T_soft_logits, data): assert np.all(np.equal(X.shape[:2], T_soft_logits.shape[:2])) # Note, targets and predictions might have different head sizes if a # growing softmax is used. assert np.all(np.equal(Y_logits.shape[:2], T_soft_logits.shape[:2])) # Disillation temperature. T = 2. if config.distill_across_time: # Emd-of-digit feature. # Note, the softmax would be an easy way to obtain per-timestep # probabilities, that a digit has ended. But since this feature # vector `X[:, :, 3]` is ideally a 1-hot encoding, it shouldn't # be squashed via a softmax, that would blur out the probabilities. #ts_weights = F.softmax(X[:, :, 3], dim=0).detach() ts_weights = X[:, :, 3].clone() ts_weights[ts_weights < 0] = 0 # Avoid division by zero in case all elements of `X[:, :, 3]` are # negative. ts_weights /= ts_weights.sum() + 1e-5 ts_weights = ts_weights.detach() # For distillation, we use a tempered softmax. T_soft = F.softmax(T_soft_logits / T, dim=2) if config.all_task_softmax and Y_logits.shape[2] != T_soft.shape[2]: # Pad new classes to soft targets. T_soft = F.pad(T_soft, (0, data.num_classes), mode='constant', value=0) assert Y_logits.shape[2] == T_soft.shape[2] # Distillation loss. loss = -(T_soft * F.log_softmax(Y_logits / T, dim=2)).sum(dim=2) * \ T**2 loss *= ts_weights # Sum across time (note, weights sum to 1) and mean across batch # dimension. return loss.sum(dim=0).mean() else: # Note, smnist samples have the end-of-sequence bit as last # timestep, the rest is padded. seq_lengths = X[:, :, 3].argmax(dim=0) inds = seq_lengths.cpu().numpy() - 1 inds[inds < 0] = 0 # Only compute loss for last timestep. Y_logits = Y_logits[inds, np.arange(inds.size), :] T_soft_logits = T_soft_logits[inds, np.arange(inds.size), :] target_mapping = None if config.all_task_softmax: target_mapping = list(range(T_soft_logits.shape[1])) return Classifier.knowledge_distillation_loss( Y_logits, T_soft_logits, target_mapping=target_mapping, device=Y_logits.device, T=T)
def get_fake_data_loss(dhandlers_rp, net, dec, d_hnet, device, config, writer, t, i, net_copy): """ Sample fake data from generator for tasks up to t and compute a loss compared to predictions of a checkpointed network. We must take caution when considering the different learning scenarios and methods and training stages, see detailed comments in the code. In general, we build a batch of replayed data from all previous tasks. Since we do not know the labels of the replayed data, we consider the output of the checkpointed network as ground thruth i.e. we must compute a loss between two logits.See :class:`mnets.classifier_interface.Classifier` for a detailed describtion of the different loss functions. Args: (....): See docstring of function :func:`train_tasks`. t: Task id. i: Current training iteration. net_copy: Copy/checkpoint of the classifier network before learning task ``t``. Returns: The loss between predictions and predictions of a checkpointed network or replayed data. """ all_Y_hat_ls = [] all_targets = [] # we have to choose from which embeddings (multiple?!) to sample from if config.class_incremental or config.single_class_replay: # if we trained every class with a different generator emb_num = t * config.out_dim else: # here samples from the whole task come from one generator emb_num = t # we have to choose from which embeddings to sample from if config.fake_data_full_range: ran = range(0, emb_num) bs_per_task = int(np.ceil(config.batch_size / emb_num)) else: random_t = np.random.randint(0, emb_num) ran = range(random_t, random_t + 1) bs_per_task = config.batch_size for re in ran: # exchange replay data with real data to compute upper bounds if config.upper_bound: real_batch = dhandlers_rp[re].next_train_batch(bs_per_task) X_fake = dhandlers_rp[re].input_to_torch_tensor(real_batch[0], device, mode='train') else: # get fake data if config.replay_method == 'gan': X_fake = sample_gan(dec, d_hnet, config, re, device, bs=bs_per_task) else: X_fake = sample_vae(dec, d_hnet, config, re, device, bs=bs_per_task) # save some fake data to the writer if i % 100 == 0: if X_fake.shape[0] >= 15: fig_fake = _plotImages(X_fake, config, bs_per_task) writer.add_figure('train_class_' + str(re) + '_fake', fig_fake, global_step=i) # compute soft targets with copied network target_logits = net_copy.forward(X_fake).detach() Y_hat_ls = net.forward(X_fake.detach()) ############### # BUILD TARGETS ############### od = config.out_dim if config.class_incremental or config.training_task_infer: # This is a bit complicated: If we train class/task incrementally # we skip thraining the classifier on the first task. # So when starting to train the classifier on task 2, we have to # build a hard target for this first output neuron trained by # replay data. A soft target (on an untrained output) would not # make sense. # output head over all output neurons already available task_out = [0, (t + 1) * od] # create target with zero everywhere except from the current re zeros = torch.zeros(target_logits[:, 0:(t + 1) * od].shape).to(device) if config.hard_targets or (t == 1 and re == 0): zeros[:, re] = 1 else: zeros[:, 0:t * od] = target_logits[:, 0:t * od] targets = zeros Y_hat_ls = Y_hat_ls[:, task_out[0]:task_out[1]] elif config.cl_scenario == 1 or config.cl_scenario == 2: if config.cl_scenario == 1: # take the task specific output neuron task_out = [re * od, re * od + od] else: # always all output neurons, only one head is used task_out = [0, od] Y_hat_ls = Y_hat_ls[:, task_out[0]:task_out[1]] target_logits = target_logits[:, task_out[0]:task_out[1]] # build hard targets i.e. one hots if this option is chosen if config.hard_targets: soft_targets = torch.sigmoid(target_logits) zeros = torch.zeros(Y_hat_ls.shape).to(device) _, argmax = torch.max(soft_targets, 1) targets = zeros.scatter_(1, argmax.view(-1, 1), 1) else: # loss expects logits targets = target_logits else: # take all neurons used up until now # output head over all output neurons already available task_out = [0, (t + 1) * od] # create target with zero everywhere except from the current re zeros = torch.zeros(target_logits[:, 0:(t + 1) * od].shape).to(device) # sigmoid over the output head(s) from all previous task soft_targets = torch.sigmoid(target_logits[:, 0:t * od]) # compute one hots if config.hard_targets: _, argmax = torch.max(soft_targets, 1) zeros.scatter_(1, argmax.view(-1, 1), 1) else: # loss expects logits zeros[:, 0:t * od] = target_logits[:, 0:t * od] targets = zeros # choose the correct output size for the actual Y_hat_ls = Y_hat_ls[:, task_out[0]:task_out[1]] # add to list all_targets.append(targets) all_Y_hat_ls.append(Y_hat_ls) # cat to one tensor all_targets = torch.cat(all_targets) Y_hat_ls = torch.cat(all_Y_hat_ls) if i % 200 == 0: classifier_accuracy = Classifier.accuracy(Y_hat_ls, all_targets) * 100.0 msg = 'Training step {}: Classifier Accuracy: {:.3f} ' + \ '(on current FAKE DATA training batch).' print(msg.format(i, classifier_accuracy)) # dependent on the target softness, the loss function is chosen if config.hard_targets or (config.class_incremental and t == 1): return Classifier.logit_cross_entropy_loss(Y_hat_ls, all_targets) else: return Classifier.knowledge_distillation_loss(Y_hat_ls, all_targets)
def get_fake_data_loss(dhandlers_rp, net, dec, d_hnet, device, config, writer, t, i, net_copy): """ Sample fake data from generator for tasks up to t and compute a loss compared to predictions of a checkpointed network. We must take caution when considering the different learning scenarios and methods and training stages, see detailed comments in the code. In general, we build a batch of replayed data from all previous tasks. Since we do not know the labels of the replayed data, we consider the output of the checkpointed network as ground thruth i.e. we must compute a loss between two logits.See :class:`mnets.classifier_interface.Classifier` for a detailed describtion of the different loss functions. Args: (....): See docstring of function :func:`train_tasks`. t: Task id. i: Current training iteration. net_copy: Copy/checkpoint of the classifier network before learning task ``t``. Returns: The loss between predictions and predictions of a checkpointed network or replayed data. """ all_Y_hat_ls = [] all_targets = [] # we have to choose from which embeddings (multiple?!) to sample from if config.class_incremental or config.single_class_replay: # if we trained every class with a different generator emb_num = t * config.out_dim else: # here samples from the whole task come from one generator emb_num = t # we have to choose from which embeddings to sample from if config.fake_data_full_range: ran = range(0, emb_num) bs_per_task = int(np.ceil(config.batch_size / emb_num)) else: random_t = np.random.randint(0, emb_num) ran = range(random_t, random_t + 1) bs_per_task = config.batch_size # print('config.upper_bound: ',config.upper_bound) # print('config.num_embeddings: ',config.num_embeddings) for re in ran: # exchange replay data with real data to compute upper bounds if config.upper_bound: real_batch = dhandlers_rp[re].next_train_batch(bs_per_task) #15 X_fake = dhandlers_rp[re].input_to_torch_tensor( real_batch[0], device, mode='train') #each batch 128 else: # get fake data if config.replay_method == 'gan': X_fake = sample_gan(dec, d_hnet, config, re, device, bs=bs_per_task) else: X_fake = sample_vae(dec, d_hnet, config, re, device, bs=bs_per_task) # print('X_fake: ',X_fake.size()) # save some fake data to the writer # if i % 100 == 0: # if X_fake.shape[0] >= 15: # fig_fake = _plotImages(X_fake, config, bs_per_task) # writer.add_figure('train_class_' + str(re) + '_fake', # fig_fake, global_step=i) # compute soft targets with copied network target_logits = net_copy.forward(X_fake).detach() Y_hat_ls = net.forward(X_fake.detach()) ############### # BUILD TARGETS ############### if config.cl_scenario == 1: # take the task specific output neuron task_out = [sum(config.dims[:re]), sum(config.dims[:re + 1])] Y_hat_ls = Y_hat_ls[:, task_out[0]:task_out[1]] target_logits = target_logits[:, task_out[0]:task_out[1]] # build hard targets i.e. one hots if this option is chosen if config.hard_targets: soft_targets = torch.sigmoid(target_logits) zeros = torch.zeros(Y_hat_ls.shape).to(device) _, argmax = torch.max(soft_targets, 1) targets = zeros.scatter_(1, argmax.view(-1, 1), 1) else: # loss expects logits targets = target_logits # add to list all_targets.append(targets) all_Y_hat_ls.append(Y_hat_ls) # cat to one tensor # all_targets = torch.cat(all_targets) # Y_hat_ls = torch.cat(all_Y_hat_ls) all_targets = all_targets Y_hat_ls = all_Y_hat_ls if i % 200 == 0: classifier_accuracy = Classifier.accuracy(Y_hat_ls, all_targets) * 100.0 msg = 'Training step {}: Classifier Accuracy: {:.3f} ' + \ '(on current FAKE DATA training batch).' print(msg.format(i, classifier_accuracy)) # dependent on the target softness, the loss function is chosen if config.hard_targets or (config.class_incremental and t == 1): return Classifier.logit_cross_entropy_loss(Y_hat_ls, all_targets) else: return Classifier.knowledge_distillation_loss(Y_hat_ls, all_targets)