def vectorized_negatives_loss(image_outputs, audio_outputs, negatives_output, nframes, margin, symfun): """ Computes the triplet margin ranking loss for each anchor image/caption pair using the specific negative, in a vectorized way """ # I = image_outputs.view(image_outputs.size(0), embedding_dim) # A = audio_outputs.view(audio_outputs.size(0), embedding_dim) # num_negatives = len(negatives_output) num_units = image_outputs.size(1) output_loss = [] n = image_outputs.size(0) loss = torch.zeros(1, requires_grad=True).type(image_outputs.data.type()) first = 0 last = num_units # first = (j*num_units)//num_negatives # last = ((j+1)*num_units)//num_negatives for i in range(n): nF = nframes[i] anchorsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i][first:last], audio_outputs[i][first:last, :, 0:nF], symfun)) Aimpsim = utils.matchmap_sim( utils.compute_matchmap(negatives_output[i][first:last], audio_outputs[i][first:last, :, 0:nF], symfun)) I2A_simdif = margin + Aimpsim - anchorsim output_loss.append(I2A_simdif) return output_loss
def combined_random_sampled_margin_rank_loss(image_outputs, audio_outputs, negatives_output, nframes, margin, symfun): """ Computes the triplet margin ranking loss for each anchor image/caption pair using both a random negative from the positive images batch, and also the specific negative for the image. The returned loss for each sample is the highest of the two """ # I = image_outputs.view(image_outputs.size(0), embedding_dim) # A = audio_outputs.view(audio_outputs.size(0), embedding_dim) n = image_outputs.size(0) loss = torch.zeros(1, requires_grad=True).type(image_outputs.data.type()) for i in range(n): I_imp_ind = i A_imp_ind = i while I_imp_ind == i: I_imp_ind = np.random.randint(0, n) while A_imp_ind == i: A_imp_ind = np.random.randint(0, n) nF = nframes[i] nFimp = nframes[A_imp_ind] anchorsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[i][:, :, 0:nF], symfun)) Iimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[I_imp_ind], audio_outputs[i][:, :, 0:nF], symfun)) Aimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[A_imp_ind][:, :, 0:nFimp], symfun)) anchorsim_neg = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[i][:, :, 0:nF], symfun)) Aimpsim_neg = utils.matchmap_sim( utils.compute_matchmap(negatives_output[i], audio_outputs[i][:, :, 0:nF], symfun)) I2A_simdif_neg = margin + Aimpsim_neg - anchorsim_neg A2I_simdif = margin + Iimpsim - anchorsim if (A2I_simdif.data > 0).all(): loss = loss + A2I_simdif I2A_simdif = margin + Aimpsim - anchorsim if (I2A_simdif.data > 0).all(): loss = loss + torch.max(I2A_simdif, I2A_simdif_neg) loss = loss / n return loss
def sampled_margin_rank_loss(image_outputs, audio_outputs, nframes, margin, symfun): """ Computes the triplet margin ranking loss for each anchor image/caption pair The impostor image/caption is randomly sampled from the minibatch """ # I = image_outputs.view(image_outputs.size(0), embedding_dim) # A = audio_outputs.view(audio_outputs.size(0), embedding_dim) n = image_outputs.size(0) loss = torch.zeros(1, requires_grad=True).type(image_outputs.data.type()) for i in range(n): I_imp_ind = i A_imp_ind = i while I_imp_ind == i: I_imp_ind = np.random.randint(0, n) while A_imp_ind == i: A_imp_ind = np.random.randint(0, n) nF = nframes[i] nFimp = nframes[A_imp_ind] anchorsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[i][:, :, 0:nF], symfun)) Iimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[I_imp_ind], audio_outputs[i][:, :, 0:nF], symfun)) Aimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[A_imp_ind][:, :, 0:nFimp], symfun)) A2I_simdif = margin + Iimpsim - anchorsim if (A2I_simdif.data > 0).all(): loss = loss + A2I_simdif I2A_simdif = margin + Aimpsim - anchorsim if (I2A_simdif.data > 0).all(): loss = loss + I2A_simdif loss = loss / n return loss
def hard_negative_loss(image_outputs, audio_outputs, nframes, margin, symfun): """ Computes the triplet margin ranking loss for each anchor image/caption pair using the hardes sample from the positive images batch """ # I = image_outputs.view(image_outputs.size(0), embedding_dim) # A = audio_outputs.view(audio_outputs.size(0), embedding_dim) n = image_outputs.size(0) with torch.no_grad(): N = image_outputs.size(0) similarity_loss = torch.zeros(N, N, requires_grad=False).type( image_outputs.data.type()) D = image_outputs.size(1) H = image_outputs.size(2) W = image_outputs.size(3) T = audio_outputs.size(3) image_outputs_hard = image_outputs.detach() audio_outputs_hard = audio_outputs.detach() image_outputs_hard = image_outputs_hard.view(N, 1, D, H, W).expand( N, N, D, H, W).contiguous().view(-1, D, H, W) audio_outputs_hard = audio_outputs_hard.view(1, N, D, 1, T).expand( N, N, D, 1, T).contiguous().view(-1, D, 1, T) match_hard = utils.compute_matchmap_vectorized( image_outputs_hard, audio_outputs_hard).view(N, N, H, W, T) match_hard, _ = match_hard.max(3) match_hard, _ = match_hard.max(2) for i in range(N): similarity_loss[:, i] = match_hard[:, i, 0:nframes[i]].mean(1) loss = torch.zeros(1, requires_grad=True).type(image_outputs.data.type()) for i in range(n): _, rank_image = torch.topk(similarity_loss[:, i], 3) _, rank_audio = torch.topk(similarity_loss[i, :], 3) I_imp_ind = rank_image[0] A_imp_ind = rank_audio[0] if I_imp_ind == i: I_imp_ind = rank_image[1] if A_imp_ind == i: A_imp_ind = rank_audio[1] nF = nframes[i] if A_imp_ind < nframes.size(0): nFimp = nframes[A_imp_ind] else: nFimp = 16 anchorsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[i][:, :, 0:nF], symfun)) Iimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[I_imp_ind], audio_outputs[i][:, :, 0:nF], symfun)) Aimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[A_imp_ind][:, :, 0:nFimp], symfun)) A2I_simdif = margin + Iimpsim - anchorsim if (A2I_simdif.data > 0).all(): loss = loss + A2I_simdif I2A_simdif = margin + Aimpsim - anchorsim if (I2A_simdif.data > 0).all(): loss = loss + I2A_simdif loss = loss / n return loss
def combined_random_hard_negative_loss(image_outputs, audio_outputs, negatives_output, nframes, margin, symfun): """ Computes the triplet margin ranking loss for each anchor image/caption pair using both the hardes negative in the positive images batch, and also the specific negative for the image. Returns the highest of the two losses for each sample """ # I = image_outputs.view(image_outputs.size(0), embedding_dim) # A = audio_outputs.view(audio_outputs.size(0), embedding_dim) n = image_outputs.size(0) with torch.no_grad(): N = image_outputs.size(0) similarity_loss = torch.zeros(N, N, requires_grad=False).type( image_outputs.data.type()) D = image_outputs.size(1) H = image_outputs.size(2) W = image_outputs.size(3) T = audio_outputs.size(3) image_outputs_hard = image_outputs.detach() audio_outputs_hard = audio_outputs.detach() image_outputs_hard = image_outputs_hard.view(N, 1, D, H, W).expand( N, N, D, H, W).contiguous().view(-1, D, H, W) audio_outputs_hard = audio_outputs_hard.view(1, N, D, 1, T).expand( N, N, D, 1, T).contiguous().view(-1, D, 1, T) match_hard = utils.compute_matchmap_vectorized( image_outputs_hard, audio_outputs_hard).view(N, N, H, W, T) match_hard, _ = match_hard.max(3) match_hard, _ = match_hard.max(2) for i in range(N): similarity_loss[:, i] = match_hard[:, i, 0:nframes[i]].mean(1) loss = torch.zeros(1, requires_grad=True).type(image_outputs.data.type()) for i in range(n): if n >= 2: _, rank_image = torch.topk(similarity_loss[:, i], 2) _, rank_audio = torch.topk(similarity_loss[i, :], 2) I_imp_ind = rank_image[0] A_imp_ind = rank_audio[0] if I_imp_ind == i: I_imp_ind = rank_image[1] if A_imp_ind == i: A_imp_ind = rank_audio[1] I_imp_ind = max(min(image_outputs.size(0) - 1, I_imp_ind), 0) A_imp_ind = max(min(image_outputs.size(0) - 1, A_imp_ind), 0) else: I_imp_ind = 0 A_imp_ind = 0 # I_imp_ind = max(min(image_outputs.size(0)-1,I_imp_ind),0) # A_imp_ind =max(min(image_outputs.size(0)-1,A_imp_ind),0) nF = nframes[i] if A_imp_ind < nframes.size(0): nFimp = nframes[A_imp_ind] else: nFimp = 16 anchorsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[i][:, :, 0:nF], symfun)) Iimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[I_imp_ind], audio_outputs[i][:, :, 0:nF], symfun)) Aimpsim = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[A_imp_ind][:, :, 0:nFimp], symfun)) A2I_simdif = margin + Iimpsim - anchorsim anchorsim_neg = utils.matchmap_sim( utils.compute_matchmap(image_outputs[i], audio_outputs[i][:, :, 0:nF], symfun)) Aimpsim_neg = utils.matchmap_sim( utils.compute_matchmap(negatives_output[i], audio_outputs[i][:, :, 0:nF], symfun)) I2A_simdif_neg = margin + Aimpsim_neg - anchorsim_neg if (A2I_simdif.data > 0).all(): loss = loss + A2I_simdif I2A_simdif = margin + Aimpsim - anchorsim if (I2A_simdif.data > 0).all(): loss = loss + torch.max(I2A_simdif, I2A_simdif_neg) loss = loss / n return loss
def test_recall(trainer): """ This experiment computes one positive and num_fakes corresponding negatives with the GAN, gives the recall of the system in distinguishing the positive from the negatives. Similar to the test_recall_selected, but selecting the negatives online, not using the pre-selected (and better) negatives. """ number_recall = 200 num_fakes = 9 if not trainer.args.use_cpu: torch.cuda.synchronize() recall1_meter = utils.AverageMeter() recall5_meter = utils.AverageMeter() # Switch to evaluate mode trainer.model.eval() with torch.no_grad(): for i, (image_input, audio_input, negatives, nframes, path, _) in enumerate(trainer.loaders['test']): if i % 50 == 0: print(f'Starting batch {i}') if i * image_input.size(0) > number_recall: break for j in range(image_input.size(0)): score_vector = torch.FloatTensor(num_fakes + 1) if not trainer.args.loading_image: v_init = trainer.z[0] z_img = torch.FloatTensor(1, v_init.shape[0]) z_img[0, :] = trainer.z[int(path[j])] image_input = trainer.generator.generate_images(z_img, intervention=None) image_input = utils.transform(image_input) else: image_input = image_input.cuda() pos_image = image_input[0, :, :, :] model_output = trainer.model(image_input, audio_input, []) image_output = model_output[0] audio_output = model_output[1] nF = nframes[j] matchmap = utils.compute_matchmap(image_output[0], audio_output[0][:, :, :nF]) real_score = utils.matchmap_sim(matchmap) score_vector[0] = real_score matchmap = matchmap.data.cpu().numpy().copy() matchmap = matchmap.transpose(2, 0, 1) # l, h, w matchmap = matchmap / matchmap.max() matchmap_image = matchmap.max(axis=0) threshold = 0.95 ind_max = np.argmax(matchmap) ind_h = (ind_max % (matchmap.shape[2] * matchmap.shape[1])) // matchmap.shape[1] ind_w = (ind_max % (matchmap.shape[2] * matchmap.shape[1])) % matchmap.shape[1] for fake_id in range(num_fakes): binary_mask = matchmap_image > (threshold * matchmap_image.max()) binary_mask = utils.geodesic_dilation(binary_mask, (ind_h, ind_w)) norm = 0 threshold_random = 0.95 p = 0.4 while norm < threshold_random: with torch.no_grad(): intervention = {} for layer_n in trainer.layer_list_all: layer_size = trainer.layers_dict[layer_n]['size'] layer_dim = trainer.layers_dict[layer_n]['depth'] ablation, replacement = trainer.get_ablation_replacement(params=[layer_dim, True, p], option='random') ablation_final = cv2.resize(binary_mask, (layer_size, layer_size)) ablation_final = np.tile(ablation_final, (layer_dim, 1, 1)).astype(np.float32) ablation_final = torch.cuda.FloatTensor(ablation_final) ablation_final = ablation.view(layer_dim, 1, 1).expand_as( ablation_final) * ablation_final intervention[layer_n] = (ablation_final, replacement) z_img = trainer.z[int(path[j])] z_img = z_img[np.newaxis, :].detach() neg_img = trainer.generator.generate_images(z_img, intervention=intervention).detach() neg_img_t = utils.transform(neg_img).detach() binary_mask = cv2.resize(binary_mask, (128, 128)) bmask = torch.Tensor(binary_mask).cuda() bmask = bmask.view(1, 128, 128).expand(3, 128, 128) norm = (neg_img_t[0, :, :, :] - pos_image[:, :, :].detach()) norm = norm * bmask norm = torch.norm(torch.norm(torch.norm(norm, dim=2), dim=1), dim=0) norm_normalized = norm / torch.norm( torch.norm(torch.norm(pos_image[:, :, :].detach() * bmask, dim=2), dim=1), dim=0) norm = norm_normalized.item() if random.random() > 0.2: p = p + 0.05 else: threshold_random = threshold_random - 0.01 model_output = trainer.model(neg_img_t, audio_input, []) image_output = model_output[0] audio_output = model_output[1] score_vector[1 + fake_id] = utils.matchmap_sim( utils.compute_matchmap(image_output[0], audio_output[0][:, :, :nF])) _, ids = score_vector.topk(10) ids = ids.cpu().numpy() ids = np.where(ids == 0)[0] A_foundind = ids[0] if A_foundind == 0: recall1_meter.update(1) else: recall1_meter.update(0) if A_foundind < 5: recall5_meter.update(1) else: recall5_meter.update(0) # print('Recall 1: {0}'.format(recall1_meter.avg)) # print('Recall 5: {0}'.format(recall5_meter.avg)) print('Recall 1: {0}'.format(recall1_meter.avg)) print('Recall 5: {0}'.format(recall5_meter.avg)) return recall1_meter.avg
def generate_active_learning(trainer): """ Generate active learning samples, selecting the positive/negative pairs in which the model has the highest error The saved information is: - Current clusters. They are necessary to generate the same negative (as they select the mask) - jpg negative images. Only needed to get the captions. Not needed for training (as they will be GAN-generated) - Information to generate the negative images (masks and units, and the associated paths) This information is saved in {args.active_learning_path}/{trainer.args.name_checkpoint}_{str(time.time())}/ When training with these images, it is recommended to start from the same checkpoint used for obtaining them. After this, the next steps before running again the system with the new samples are: - Collecting captions of the active learning samples (the negatives; the positives already have captions) - Adding the new collected samples to the dataset. The can be added in a separate folder, and only modifying the name_list_{}.txt files is enough. Note that the noise ID (and thus the name of the file) will already exist (for the positive one), so save the new ones in an "active" subfolder. """ assert len( trainer.layer_list_all ) == 1, 'Active learning is only implemented for a single layer ablations' trainer.clusterer.save_results = True clus, mean_clust, std_clust, _ = trainer.clusterer.create_clusters( iteration=0) trainer.clusters = torch.FloatTensor(clus).cuda() trainer.mean_clust = torch.FloatTensor(mean_clust) trainer.std_clust = torch.FloatTensor(std_clust) trainer.cluster_counts = 1 / trainer.clusters.max(1)[0] trainer.clusters_unit = trainer.cluster_counts.view(trainer.clusters.size(0), 1).expand_as(trainer.clusters) * \ trainer.clusters trainer.clusterer.name_with_images_clusters() trainer.clusterer.name_clusters() trainer.optimize_neurons() if not trainer.args.use_cpu: torch.cuda.synchronize() data_time = utils.AverageMeter() # Switch to train mode trainer.model.eval() active_learning_name = os.path.join( trainer.args.active_learning_path, f'{trainer.args.name_checkpoint}_{str(time.time())}') end = time.time() all_loss = [] all_hmap = [] all_hmap_eval = [] all_units = [] all_paths = [] for batch_id, (image_input, audio_input, neg_images, nframes, path, image_raw) in \ enumerate(trainer.loaders['train']): print(batch_id) # Measure data loading time data_time.update(time.time() - end) if not trainer.args.use_cpu: audio_input = audio_input.cuda(async=True) if not trainer.args.loading_image: if trainer.args.active_learning: path_ints = [p.split('/')[-1] for p in path] else: path_ints = path v_init = trainer.z[int(path_ints[0])] z_img = torch.FloatTensor(image_input.size(0), v_init.shape[0]) for k in range(image_input.size(0)): z_img[k, :] = trainer.z[int(path_ints[k])] image_input = trainer.generator.generate_images(z_img, intervention=None) image_input = utils.transform(image_input).detach() else: image_input = image_input.cuda() neg_images = neg_images.cuda() model_output = trainer.model(image_input, audio_input, []) image_output = model_output[0] audio_output = model_output[1] neg_images = [] pooling_ratio = round(audio_input.size(3) / audio_output.size(3)) nframes.div_(pooling_ratio) binary_mask_0 = None if trainer.loss_type == 'negatives_edited' or trainer.loss_type == 'negatives_both': limits = np.zeros((image_input.size(0), 2)) for i in range(image_input.size(0)): pos_image = image_input[i, :, :, :] nF = nframes[i] matchmap = utils.compute_matchmap(image_output[i], audio_output[i][:, :, :nF]) positive_score = utils.matchmap_sim(matchmap).detach() matchmap = matchmap.data.cpu().numpy().copy() matchmap = matchmap.transpose(2, 0, 1) # l, h, w matchmap = matchmap / (matchmap.max() + 1e-10) matchmap_image = matchmap.max(axis=0) threshold = 0.95 # ind_max = np.argmax(matchmap_image) ind_max = np.argmax(matchmap) ind_t = ind_max // (matchmap.shape[2] * matchmap.shape[1]) ind_h = (ind_max % (matchmap.shape[2] * matchmap.shape[1]) ) // matchmap.shape[1] ind_w = (ind_max % (matchmap.shape[2] * matchmap.shape[1]) ) % matchmap.shape[1] limits[i, 0] = ind_t limits[i, 1] = ind_t + 1 v = (image_output[i][:, ind_h, ind_w] - trainer.mean_clust.cuda()) / (trainer.std_clust.cuda() + 1e-8) normalized_clusters = np.matmul( trainer.clusters.cpu(), v.detach().cpu().numpy().transpose()) sorted_val = -np.sort(-normalized_clusters[:]) sorted_val = np.clip(sorted_val, 0, 4) prob_samples = sorted_val / np.sum(sorted_val) sorted_id = np.argsort(-normalized_clusters[:]) cluster_id = sorted_id[0] norm = 0 threshold_random = 0.95 # The number of units to be ablated grows if we cannot generate a good (changed) negative # The following numbers are the starting number of units to change num_units_dict = { 'layer2': 30, 'layer3': 30, 'layer4': 140, 'layer5': 30, 'layer6': 30 } thresold_heatmap = threshold count = 0 binary_mask_eval = matchmap_image > (thresold_heatmap * matchmap_image.max()) binary_mask_eval = utils.geodesic_dilation( binary_mask_eval, (ind_h, ind_w)) binary_mask_eval = cv2.resize(binary_mask_eval, (128, 128)) all_hmap_eval.append(binary_mask_eval) bmask = torch.Tensor(binary_mask_eval).cuda() bmask = bmask.view(1, 128, 128).expand(3, 128, 128) all_paths.append(path_ints[i]) while norm < threshold_random: with torch.no_grad(): binary_mask = matchmap_image > (thresold_heatmap * matchmap_image.max()) binary_mask = utils.geodesic_dilation( binary_mask, (ind_h, ind_w)) if binary_mask_0 is None: binary_mask_0 = cv2.resize(binary_mask, (224, 224)) z_img = trainer.z[int(path_ints[i])] z_img = z_img[np.newaxis, :] _ = trainer.generator.generate_images(z_img) intervention = {} for layer_n in trainer.layer_list_all: # This will only be one layer units_ids = trainer.layers_units[layer_n][ cluster_id][:num_units_dict[layer_n]] layer_size = trainer.layers_dict[layer_n]['size'] layer_dim = trainer.layers_dict[layer_n]['depth'] ablation, replacement = trainer.get_ablation_replacement( params=[layer_dim, units_ids], option='specific') ablation_final = cv2.resize( binary_mask, (layer_size, layer_size)) ablation_final = np.tile(ablation_final, (layer_dim, 1, 1)).astype( np.float32) ablation_final = torch.cuda.FloatTensor( ablation_final) ablation_final = ablation.view( layer_dim, 1, 1).expand_as(ablation_final) * ablation_final intervention[layer_n] = (ablation_final, replacement) neg_img = trainer.generator.generate_images( z_img, intervention=intervention).detach() neg_img_t = utils.transform(neg_img).detach() binary_mask = cv2.resize(binary_mask, (128, 128)) norm = (neg_img_t[0, :, :, :] - pos_image.detach()) norm_im = torch.norm(norm, dim=0) norm = norm * bmask im_dif = norm norm = torch.norm(torch.norm(torch.norm(norm, dim=2), dim=1), dim=0) norm_normalized = norm / torch.norm(torch.norm( torch.norm(pos_image.detach() * bmask, dim=2), dim=1), dim=0) norm = norm_normalized.item() for layer_n in trainer.layer_list_all: num_units_dict[layer_n] = num_units_dict[ layer_n] + 40 # increase units to change thresold_heatmap = thresold_heatmap - 0.1 threshold_random = threshold_random - 0.05 cluster_id = np.random.choice(sorted_id, size=(1), p=prob_samples)[0] count = count + 1 neg_images.append(neg_img.detach()) all_hmap.append(binary_mask) all_units.append(units_ids) neg_images = torch.cat(neg_images) neg_images_t = utils.transform(neg_images) image_output_neg, _, _ = trainer.model(neg_images_t, None, []) loss_list = trainer.vectorized_negatives_loss( image_output, audio_output, image_output_neg, nframes) loss_list = [x.detach() for x in loss_list] all_loss.extend(loss_list) all_loss = [x.view(1, -1) for x in all_loss] all_loss = torch.cat(all_loss).view(-1, 1) _, ind = all_loss.topk(3000, 0) ind = [x.item() for x in ind] a_units = [all_units[i] for i in ind] a_paths = [all_paths[i] for i in ind] a_hmaps = [all_hmap[i] for i in ind] a_hmaps_eval = [all_hmap_eval[i] for i in ind] torch.save(a_units, os.path.join(active_learning_name, 'units.pth')) torch.save(a_paths, os.path.join(active_learning_name, 'a_paths.pth')) torch.save(a_hmaps, os.path.join(active_learning_name, 'a_hmaps.pth')) torch.save(a_hmaps_eval, os.path.join(active_learning_name, 'a_hmaps_eval.pth')) os.makedirs(os.path.join(active_learning_name, 'images'), exist_ok=True) os.makedirs(os.path.join(active_learning_name, 'hm'), exist_ok=True) for j in range(len(a_units)): path = a_paths[j] units_ids = a_units[j] binary_mask = a_hmaps[j] layer_n = trainer.layer_list_all[0] layer_size = trainer.layers_dict[layer_n]['size'] layer_dim = trainer.layers_dict[layer_n]['depth'] intervention = {} ablation, replacement = trainer.get_ablation_replacement( params=[layer_dim, units_ids], option='specific') ablation_final = cv2.resize(binary_mask, (layer_size, layer_size)) ablation_final = np.tile(ablation_final, (layer_dim, 1, 1)).astype(np.float32) ablation_final = torch.cuda.FloatTensor(ablation_final) ablation_final = ablation.view( layer_dim, 1, 1).expand_as(ablation_final) * ablation_final intervention[layer_n] = (ablation_final, replacement) z_img = trainer.z[int(path)] z_img = z_img[np.newaxis, :] neg_img = trainer.generator.generate_images( z_img, intervention=intervention).detach() neg_im = neg_img[0, :, :, :].cpu().numpy().transpose(1, 2, 0) neg_im = neg_im.astype(np.uint8) neg_im = Image.fromarray(neg_im.astype('uint8'), 'RGB') draw = ImageDraw.Draw(neg_im) hm = a_hmaps_eval[j] rows = np.any(hm, axis=1) cols = np.any(hm, axis=0) rmin, rmax = np.where(rows)[0][[0, -1]] cmin, cmax = np.where(cols)[0][[0, -1]] draw.rectangle(((cmin, rmin), (cmax, rmax)), outline='red') neg_im.save(os.path.join(active_learning_name, 'images', f'{j}_hn.jpg')) binary_mask_eval = cv2.resize(a_hmaps_eval[j], (128, 128)) mask_im = binary_mask_eval * 255 mask_im = mask_im.astype(np.uint8) mask_im = mask_im.reshape((128, 128, 1)) mask_im = np.concatenate((mask_im, mask_im, mask_im), axis=2) mask_im = Image.fromarray(mask_im.astype('uint8'), 'RGB') mask_im.save(os.path.join(active_learning_name, 'hm', f'{j}_hn.jpg'))