Esempio n. 1
0
def engine_boxes(loader: Any, checkpoint: Dict[str, Any]):
    checkpoint['model'].eval()
    df = {'Id': [], 'index': [], 'score': [], 'box': []}
    start = time.time()
    all_preds = []
    all_labels = []
    map_scores = []
    
    org_size = (1024, 1024)
    new_size = (224, 224)

    with torch.set_grad_enabled(False):
        for iter_num, data in enumerate(loader):
            scores, indices, boxes = checkpoint['model'](data[0].cuda().float().unsqueeze(dim=0))
            scores, order = torch.sort(scores)

            scores = scores.tolist()
            indices = indices.tolist()
            indices = [indices[i] for i in order]
            boxes = boxes.tolist()
            boxes = [boxes[i] for i in order]

            df['Id'] += [data[3]] * len(scores)
            df['score'] += scores
            df['index'] += indices
            df['box'] += boxes
            
            all_labels.append(data[2])
            all_preds.append(max(scores + [0.0]))
            
            if data[2] == 0:
                continue
            
            ground_map = box_to_map(loader.df[loader.df.Id == data[3]].box.to_list(), np.zeros(org_size))
            predicted_map = box_to_map(boxes, np.zeros(org_size), scores)
            ground_map = cv2.resize(ground_map, new_size, interpolation=cv2.INTER_NEAREST).clip(0, 1)
            predicted_map = cv2.resize(predicted_map, new_size, interpolation=cv2.INTER_AREA).clip(0, 1)
            map_scores.append(roc_auc_score(ground_map.flatten(), predicted_map.flatten()))

            speed = iter_num // (time.time() - start)
            print('Epoch:', checkpoint['epoch'], 'Iter:', iter_num, 'Score:', np.mean(map_scores), 'Speed:', int(speed), 'img/s', end='\r', flush=True)

    df = pd.DataFrame.from_dict(df)
    rmetric = roc_auc_score(all_labels, all_preds)
    wild_metric = np.mean(map_scores)
    return df, rmetric, wild_metric
Esempio n. 2
0
File: engine.py Progetto: Sharut/wsl
def engine_boxes(loader: Any, checkpoint: Dict[str, Any]):
    checkpoint['model'].eval()
    df = {'Id': [], 'index': [], 'score': [], 'box': []}
    start = time.time()
    all_preds = []
    all_labels = []
    map_scores = []

    org_size = (1024, 1024)
    new_size = (224, 224)

    threshold = 0.1
    box_labels = []
    box_scores = []

    with torch.set_grad_enabled(False):
        for iter_num, data in enumerate(loader):
            scores, indices, boxes = checkpoint['model'](
                data[0].cuda().float().unsqueeze(dim=0))
            scores, order = torch.sort(scores)

            scores = scores.tolist()
            indices = indices.tolist()
            indices = [indices[i] for i in order]
            boxes = boxes.tolist()
            boxes = [boxes[i] for i in order]

            df['Id'] += [data[3]] * len(scores)
            df['score'] += scores
            df['index'] += indices
            df['box'] += boxes

            all_labels.append(data[2])
            all_preds.append(max(scores + [0.0]))

            ground_map = box_to_map(
                loader.df[loader.df.Id == data[3]].box.to_list(),
                np.zeros(org_size))
            predicted_map = box_to_map(boxes, np.zeros(org_size), scores)
            ground_map = cv2.resize(ground_map,
                                    new_size,
                                    interpolation=cv2.INTER_NEAREST).clip(
                                        0, 1)
            predicted_map = cv2.resize(predicted_map,
                                       new_size,
                                       interpolation=cv2.INTER_AREA).clip(
                                           0, 1)
            if data[2] != 0:
                map_scores.append(
                    average_precision_score(ground_map.flatten(),
                                            predicted_map.flatten()))

            if len(boxes) > 1:
                box = boxes[-1]
                h, w = (box[0] + box[2] / 2) * new_size[0] / org_size[0], (
                    box[1] + box[3] / 2) * new_size[1] / org_size[1]
                box = [[
                    max(0, int(h - new_size[0] / 14)),
                    max(0, int(w - new_size[1] / 14)),
                    min(new_size[0], int(h + new_size[0] / 14)),
                    min(new_size[1], int(w + new_size[1] / 14))
                ]]
                score = scores[-1:]
                predicted_map = box_to_map(box, np.zeros(new_size), score)
            else:
                predicted_map = np.zeros(new_size)
            overlap = np.sum(np.multiply(
                ground_map, predicted_map)) / np.sum(predicted_map)

            if data[2] == 1 and overlap < threshold:
                box_labels += [0, 1]
                box_scores += [np.max(predicted_map), 0]
            else:
                box_labels += [data[2]]
                box_scores += [np.max(predicted_map)]

            speed = iter_num // (time.time() - start)
            print('Epoch:',
                  checkpoint['epoch'],
                  'Iter:',
                  iter_num,
                  'Score:',
                  np.mean(map_scores),
                  'Speed:',
                  int(speed),
                  'img/s',
                  end='\r',
                  flush=True)

    df = pd.DataFrame.from_dict(df)
    rmetric = roc_auc_score(all_labels, all_preds)
    wild_metric = np.mean(map_scores)
    pointwise = average_precision_score(box_labels, box_scores)
    return df, rmetric, wild_metric, pointwise
Esempio n. 3
0
def main(name: str, start: int, plot: bool):

    if name == 'all':
        model_dirs = wsl_model_dir.glob('rsna*')
    else:
        if 'rsna' in name:
            model_dirs = wsl_model_dir.glob(f'*{name}*')
        else:
            model_dirs = wsl_model_dir.glob(f'rsna*{name}*')

    model_dirs = list(model_dirs)
    num_model_dirs = 50
    print(f'Number of potential model directory matches = {len(model_dirs)}, but doing top {num_model_dirs} models for now.')
    model_dirs = model_dirs[start:start + num_model_dirs]

    if plot:
        ncolors = 256
        color_array = plt.get_cmap('viridis')(range(ncolors))
        # change alpha values
        color_array[:, -1] = np.linspace(1.0, 0.0, ncolors)

    for m_idx, model_dir in enumerate(model_dirs):

        if 'debug' in str(model_dir):  # Debugging model
            print('Debugging model')
            continue

        elif not (model_dir / 'configs.json').exists():  # Model not completed
            print('Model not completed')
            continue

        else:
            with open(model_dir / 'configs.json') as f:
                configs = json.load(f)
            dataset = Loader(data=configs['data'],
                             split='test',
                             extension=configs['extension'],
                             classes=configs['classes'],
                             column=configs['column'],
                             variable_type=configs['variable_type'])
            print('Number of images -', len(dataset))

        print(f'Model {m_idx} : {model_dir}')

        if configs['data'] in known_tasks:
            task = known_tasks[configs['data']]

        checkpoint = torch.load(model_dir / 'best.pt', map_location='cuda:0' if torch.cuda.is_available() else 'cpu')
        checkpoint['model'].gradient = None
        checkpoint['model'].eval()

        # currently hardcoded, should ideally be inferred from image
        org_size = (1024, 1024)
        new_size = (224, 224)

        all_scores = defaultdict(list)

        GD = BackProp(checkpoint['model'])
        GBP = BackProp(checkpoint['model'], True)

        start_time = time.time()

        for idx, data in enumerate(dataset):
            checkpoint['model'].zero_grad()
            name, img, label = data
            label = label.squeeze().cuda()

            if label != 1:
                continue

            # Make the ground map
            if task == 'detect':
                ground_map = box_to_map(dataset.df[dataset.df.Id == name].box.to_list(), np.zeros(org_size))
            elif task == 'segment':
                ground_map = np.zeros(org_size)
                eps = dataset.df[dataset.df.Id == name].EncodedPixels.to_list()
                for ep in eps:
                    ground_map += rle2mask(ep, np.zeros(org_size)).T

            ground_map = cv2.resize(ground_map, new_size, interpolation=cv2.INTER_NEAREST).clip(0, 1)

            # Make the saliency map
            checkpoint['model'].get_map = True
            if configs['wildcat']:
                _, _, wild, handle = checkpoint['model'](img.unsqueeze(dim=0).cuda().float())
                handle.remove()
                wild = torch.max(wild, dim=1)[0]
                wild = wild.squeeze().cpu().data.numpy()
                wild = (wild - wild.min()) / (wild.max() - wild.min())
                wild = cv2.resize(wild, new_size, interpolation=cv2.INTER_NEAREST)
            else:
                wild = np.zeros_like(ground_map)

            gcam = GD.generate_cam(img.unsqueeze(dim=0).cuda().float()).squeeze()

            checkpoint['model'].get_map = False
            grad = GD.generate_gradients(img.unsqueeze(dim=0).cuda().float())
            ig = GD.generate_integrated_gradients(img.unsqueeze(dim=0).cuda().float(), 25)

            sg = GD.generate_smooth_grad(img.unsqueeze(dim=0).cuda().float(), 5, 0.1, 0)
            sig = GD.generate_smooth_grad(img.unsqueeze(dim=0).cuda().float(), 5, 0.1, 10)

            gbp = GBP.generate_gradients(img.unsqueeze(dim=0).cuda().float())
            ggcam = np.multiply(gcam, gbp)

            all_scores['WILD'].append(aupr(ground_map.flatten(), wild.flatten()))
            all_scores['GRAD'].append(aupr(ground_map.flatten(), grad.flatten()))
            all_scores['SG'].append(aupr(ground_map.flatten(), sg.flatten()))
            all_scores['IG'].append(aupr(ground_map.flatten(), ig.flatten()))
            all_scores['SIG'].append(aupr(ground_map.flatten(), sig.flatten()))
            all_scores['GBP'].append(aupr(ground_map.flatten(), gbp.flatten()))
            all_scores['GCAM'].append(aupr(ground_map.flatten(), gcam.flatten()))
            all_scores['GGCAM'].append(aupr(ground_map.flatten(), ggcam.flatten()))

            if plot:
                row, col = 2, 5
                map_names = [['XRAY', 'WILD', 'GRAD', 'GCAM', 'GGCAM'], ['MASK', 'GBP', 'SG', 'IG', 'SIG']]
                maps = [[img, wild, grad, gcam, ggcam], [ground_map, gbp, sg, ig, sig]]
                x = LinearSegmentedColormap.from_list(name='rainbow', colors=color_array)
                plt.register_cmap(cmap=x)

                fig, ax = plt.subplots(row, col, figsize=(18, 8))
                for i in range(row):
                    for j in range(col):
                        ax[i, j].imshow(np.transpose(img, (1, 2, 0)))
                        if not (i == 0 and j == 0):
                            ax[i, j].imshow(maps[i][j], alpha=0.8, cmap='rainbow')
                        ax[i, j].text(0, 220, map_names[i][j], fontsize='x-large', color='white', weight='bold', bbox=dict(fill=True, linewidth=0))
                        ax[i, j].axis('off')
                plt.subplots_adjust(wspace=0.05, hspace=0.05)
                plt.savefig(f'{wsl_plot_dir}/saliency_{name}.png', dpi=300, bbox_inches='tight')
                plt.show()
                plt.close()

            print_str = f'{idx}: | '
            for key, value in all_scores.items():
                print_str += f'{key}-{int(np.mean(value) * 100)} | '
            print_str += str(round((time.time() - start_time) / (idx + 1), 2)) + ' s/img'
            print(print_str, end='\r')

        for key in all_scores.keys():
            configs[key] = np.mean(all_scores[key])
            print(key, ' ', configs[key])

        with open(model_dir / 'configs.json', 'w') as fp:
            json.dump(configs, fp)
Esempio n. 4
0
def main(name: str, task: str):

    if name == 'all':
        models = wsl_model_dir.glob('*')
    else:
        models = wsl_model_dir.glob(f'*{name}*')

    models = list(models)
    models.reverse()
    print('Number of potential model matches =', len(models))
    all_configs = []

    for m, path in enumerate(models):

        if 'debug' in str(path):  # Debugging model
            print('Debugging model')
            continue
        elif 'wild' not in str(path):
            continue
        elif not (path / 'configs.json').exists():  # Model not completed
            print('Model not completed')
            continue
        else:
            with open(path / 'configs.json') as f:
                configs = json.load(f)
                print(configs)
            dataset = Loader(data=configs['data'],
                             split='valid',
                             extension=configs['extension'],
                             classes=configs['classes'],
                             column=configs['column'],
                             regression=configs['regression'])

        print(f'Model {m} : {path}')
        if configs['data'] in known_tasks:
            task = known_tasks[configs['data']]

        checkpoint = torch.load(
            path / 'best.pt',
            map_location='cuda:0' if torch.cuda.is_available() else 'cpu')
        checkpoint['model'] = checkpoint['model'].module
        checkpoint['model'].get_map = True
        checkpoint['model'].eval()

        org_size = (1024, 1024)
        new_size = (224, 224)
        sigmoid = torch.nn.Sigmoid().cuda()
        all_scores = []

        with torch.set_grad_enabled(False):
            for idx, data in enumerate(dataset):
                img, label = data
                name = dataset.names[idx]
                labels = dataset.labels[idx]

                predicted_map = checkpoint['model'](
                    img.unsqueeze(dim=0).cuda().float()).squeeze(dim=0)
                predicted_map = sigmoid(
                    predicted_map.sum(dim=0)).cpu().data.numpy()

                score = []
                for i, label in enumerate(labels):
                    if label == 0:
                        continue

                    if task == 'detect':
                        ground_map = box_to_map(
                            dataset.df[dataset.df.Id == name].box.to_list(),
                            np.zeros(org_size))
                    elif task == 'segment':
                        ground_map = np.zeros(org_size)
                        eps = dataset.df[dataset.df.Id ==
                                         name].EncodedPixels.to_list()
                        for ep in eps:
                            ground_map += rle2mask(ep, np.zeros(org_size)).T
                    else:
                        print('Ground truth not available.')

                    # plt.imshow(ground_map)
                    ground_map = cv2.resize(
                        ground_map, new_size,
                        interpolation=cv2.INTER_NEAREST).clip(0, 1)
                    re_pred_map = cv2.resize(predicted_map[i],
                                             new_size,
                                             interpolation=cv2.INTER_AREA)
                    score.append(
                        roc_auc_score(ground_map.flatten(),
                                      re_pred_map.flatten()))

                all_scores += score
                if (len(all_scores) + 1) % 32 == 0:
                    print('Idx:', idx, 'Mean:', np.mean(all_scores), end='\r')

        configs['wild_metric'] = np.mean(all_scores)
        print('Wild Metric:', configs['wild_metric'])
        with open(path / 'configs.json', 'w') as fp:
            json.dump(configs, fp)
Esempio n. 5
0
def main(name: str, start: int, plot: bool):

    if name == 'all':
        model_dirs = wsl_model_dir.glob('*')
    else:
        model_dirs = wsl_model_dir.glob(f'*{name}*')

    model_dirs = list(model_dirs)
    model_dirs = model_dirs[start:]

    print('Number of potential model directory matches =', len(model_dirs))

    if plot:
        ncolors = 256
        color_array = plt.get_cmap('viridis')(range(ncolors))
        # change alpha values
        color_array[:, -1] = np.linspace(1.0, 0.0, ncolors)

    for m_idx, model_dir in enumerate(model_dirs):

        if 'debug' in str(model_dir):  # Debugging model
            print('Debugging model')
            continue

        elif not (model_dir / 'configs.json').exists():  # Model not completed
            print('Model not completed')
            continue

        else:
            with open(model_dir / 'configs.json') as f:
                configs = json.load(f)
            dataset = Loader(data=configs['data'],
                             split='test',
                             extension=configs['extension'],
                             classes=configs['classes'],
                             column=configs['column'],
                             regression=configs['regression'])
            print('Number of images -', len(dataset))

        print(f'Model {m_idx} : {model_dir}')

        if configs['data'] in known_tasks:
            task = known_tasks[configs['data']]

        checkpoint = torch.load(
            model_dir / 'best.pt',
            map_location='cuda:0' if torch.cuda.is_available() else 'cpu')
        checkpoint['model'] = checkpoint['model'].module
        checkpoint['model'].gradient = None
        checkpoint['model'].eval()

        org_size = (1024, 1024)
        new_size = (224, 224)

        all_scores = defaultdict(list)

        GD = BackProp(checkpoint['model'])
        GBP = BackProp(checkpoint['model'], True)

        start_time = time.time()

        for idx, data in enumerate(dataset):
            print(
                f'{idx} speed-{(time.time() - start_time) // (idx + 1)} s/img',
                end='\r')
            checkpoint['model'].zero_grad()
            name, img, label = data
            label = label.squeeze().cuda()

            if label != 1:
                continue

            # Make the ground map
            if task == 'detect':
                ground_map = box_to_map(
                    dataset.df[dataset.df.Id == name].box.to_list(),
                    np.zeros(org_size))
            elif task == 'segment':
                ground_map = np.zeros(org_size)
                eps = dataset.df[dataset.df.Id == name].EncodedPixels.to_list()
                for ep in eps:
                    ground_map += rle2mask(ep, np.zeros(org_size)).T

            ground_map = cv2.resize(ground_map,
                                    new_size,
                                    interpolation=cv2.INTER_NEAREST).clip(
                                        0, 1)

            # Make the saliency map
            if configs['wildcat']:
                checkpoint['model'].get_map = True
                with torch.set_grad_enabled(False):
                    wild, _, handle = checkpoint['model'](
                        img.unsqueeze(dim=0).cuda().float())
                    handle.remove_hook()
                    wild = wild.squeeze().cpu().data.numpy()
                wild = (wild - wild.min()) / (wild.max() - wild.min())
                wild = cv2.resize(wild,
                                  new_size,
                                  interpolation=cv2.INTER_NEAREST)

                all_scores['WILD'].append(
                    aupr(ground_map.flatten(), wild.flatten()))

                if plot:
                    plt.figure(figsize=(4, 12))
                    x = LinearSegmentedColormap.from_list(name='rainbow',
                                                          colors=color_array)
                    plt.register_cmap(cmap=x)
                    plt.subplot(1, 3, 1)
                    plt.imshow(np.transpose(img, (1, 2, 0)))
                    plt.subplot(1, 3, 2)
                    plt.imshow(ground_map, alpha=0.8, cmap='rainbow')
                    plt.subplot(1, 3, 2)
                    plt.imshow(wild, alpha=0.8, cmap='rainbow')
                    plt.savefig(f'{wsl_plot_dir}/wild_{name}.png',
                                dpi=300,
                                bbox_inches='tight')
                    plt.show()
                    plt.close()

            else:
                checkpoint['model'].get_map = False
                grad = GD.generate_gradients(
                    img.unsqueeze(dim=0).cuda().float())
                ig = GD.generate_integrated_gradients(
                    img.unsqueeze(dim=0).cuda().float(), 100)

                sg = GD.generate_smooth_grad(
                    img.unsqueeze(dim=0).cuda().float(), 5, 0.3, 0)
                sig = GD.generate_smooth_grad(
                    img.unsqueeze(dim=0).cuda().float(), 5, 0.3, 0)

                gbp = GBP.generate_gradients(
                    img.unsqueeze(dim=0).cuda().float())

                checkpoint['model'].get_map = True
                gcam = GD.generate_cam(
                    img.unsqueeze(dim=0).cuda().float()).squeeze()
                ggcam = np.multiply(gcam, gbp)

                all_scores['GRAD'].append(
                    aupr(ground_map.flatten(), grad.flatten()))
                all_scores['SG'].append(
                    aupr(ground_map.flatten(), sg.flatten()))
                all_scores['IG'].append(
                    aupr(ground_map.flatten(), ig.flatten()))
                all_scores['SIG'].append(
                    aupr(ground_map.flatten(), sig.flatten()))
                all_scores['GBP'].append(
                    aupr(ground_map.flatten(), gbp.flatten()))
                all_scores['GCAM'].append(
                    aupr(ground_map.flatten(), gcam.flatten()))
                all_scores['GGCAM'].append(
                    aupr(ground_map.flatten(), ggcam.flatten()))

                if plot:
                    row, col = range(2), range(4)
                    map_names = [['MASK', 'GRAD', 'SG', 'IG'],
                                 ['SIG', 'GCAM', 'GBP', 'GGCAM']]
                    maps = [[ground_map, grad, sg, ig],
                            [sig, gcam, gbp, ggcam]]
                    x = LinearSegmentedColormap.from_list(name='rainbow',
                                                          colors=color_array)
                    plt.register_cmap(cmap=x)

                    fig, ax = plt.subplots(2, 4, figsize=(18, 8))
                    for i in row:
                        for j in col:
                            ax[i, j].imshow(np.transpose(img, (1, 2, 0)))
                            ax[i, j].imshow(maps[i][j],
                                            alpha=0.8,
                                            cmap='rainbow')
                            ax[i, j].text(0,
                                          220,
                                          map_names[i][j],
                                          fontsize='x-large',
                                          color='white',
                                          weight='bold',
                                          bbox=dict(fill=True, linewidth=0))
                            ax[i, j].axis('off')
                    plt.subplots_adjust(wspace=0.05, hspace=0.05)
                    plt.savefig(f'{wsl_plot_dir}/saliency_{name}.png',
                                dpi=300,
                                bbox_inches='tight')
                    plt.show()
                    plt.close()

            del data

        for key in all_scores.keys():
            configs[key] = np.mean(all_scores[key])
            print(key, ' ', configs[key])

        with open(path / 'configs.json', 'w') as fp:
            json.dump(configs, fp)
Esempio n. 6
0
def main(name: str,
		 task: str,
		 store: bool):

	if name == 'all':
		models = wsl_model_dir.glob('*')
	else:
		models = wsl_model_dir.glob(f'*{name}*')
	models = list(models)
	device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") 

	print('Number of potential model matches =', len(models))
	all_configs = []
	for m, path in enumerate(models):

		if 'debug' in str(path):  # Debugging models
			print('Debugging model')
			continue
		elif 'vgg19' not in str(path):  # Model is not wildcat
			print('Model is not densenet')
			continue
		elif 'wildcat' in str(path):  # Model is not wildcat
			print('Model is wildcat')
			continue
		elif not (path / 'configs.json').exists():  # Model not completed
			print('Model not completed')
			continue
		else:
			with open(path / 'configs.json') as f:
				configs = json.load(f)
				print(configs)
				# if configs['pretrained'] == False:
				# 	continue
			dataset = Loader(data=configs['data'],
							 split='test',
							 extension=configs['extension'],
							 classes=configs['classes'],
							 column=configs['column'],
							 regression=configs['regression'])

		# print(f'Model {m} : {path}')
		try:
			checkpoint = torch.load(path / 'best.pt', map_location='cuda:0' if torch.cuda.is_available() else 'cpu')
		except:
			continue
		# checkpoint = torch.load(path / 'best.pt', map_location='cpu')
		checkpoint['model'] = checkpoint['model'].module
		checkpoint['model'].get_map = True
		checkpoint['model'].eval()
		# summary(checkpoint['model'],(3,224,224))
		org_size = (1024, 1024)
		new_size = (224, 224)
		sigmoid = torch.nn.Sigmoid().cuda()
		all_scores = defaultdict(list)

		VBP = vanilla_backprop.VanillaBackprop(checkpoint['model'])
		IG = integrated_gradients.IntegratedGradients(checkpoint['model'])
		GBP = guided_backprop.GuidedBackprop(checkpoint['model'])
		GCAM = gradcam.GradCam(checkpoint['model'],target_layer=34)
		# with torch.set_grad_enabled(False):
		print(len(dataset))
		for idx, data in tqdm(enumerate(dataset)):
			img, label = data
			name = dataset.names[idx]
			labels = dataset.labels[idx]

			saliency_label = 1

			for i, label in enumerate(labels):
				if label == 0:
					saliency_label = 0
					break

			if saliency_label == 0:
				continue
			# saliency_label = torch.tensor(saliency_label)
			saliency_label = torch.tensor(saliency_label).to(device)

			vanilla_grads = VBP.generate_gradients(img.unsqueeze(dim=0).cuda().float(), saliency_label)
			grayscale_vanilla_grads = vanilla_backprop.convert_to_grayscale(vanilla_grads)
			# print(np.shape(grayscale_vanilla_grads))
			# vanilla_backprop.save_gradient_images(grayscale_vanilla_grads, '/data/2015P002510/nishanth/WSL/wsl/wsl/Example_maps/GRAD')
			integrated_grads = IG.generate_integrated_gradients(img.unsqueeze(dim=0).cuda().float(), saliency_label, 100)
			grayscale_integrated_grads = integrated_gradients.convert_to_grayscale(integrated_grads)
			# vanilla_backprop.save_gradient_images(grayscale_integrated_grads, '/data/2015P002510/nishanth/WSL/wsl/wsl/Example_maps/IG')
			guided_grads = GBP.generate_gradients(img.unsqueeze(dim=0).cuda().float(), saliency_label)
			grayscale_guided_grads = guided_backprop.convert_to_grayscale(guided_grads)
			# vanilla_backprop.save_gradient_images(grayscale_guided_grads, '/data/2015P002510/nishanth/WSL/wsl/wsl/Example_maps/GBP')
			smooth_grad_mask = smooth_grad.generate_smooth_grad(VBP, img.unsqueeze(dim=0).cuda().float(), saliency_label, 5, 0.3)
			grayscale_smooth_grad = smooth_grad.convert_to_grayscale(smooth_grad_mask)

			smooth_grad_mask = smooth_grad.generate_smooth_grad(IG, img.unsqueeze(dim=0).cuda().float(), saliency_label, 5, 0.3)
			grayscale_smooth_ig = smooth_grad.convert_to_grayscale(smooth_grad_mask)
			cam = GCAM.generate_cam(img.unsqueeze(dim=0).cuda().float(), saliency_label)
			# grayscale_cam = guided_backprop.convert_to_grayscale(cam)

			cam_gb = guided_gradcam.guided_grad_cam(cam, guided_grads)
			grayscale_cam_gb = guided_gradcam.convert_to_grayscale(cam_gb)
			# vanilla_backprop.save_gradient_images(cam, '/data/2015P002510/nishanth/WSL/wsl/wsl/Example_maps/GCAM')

			# # Save mask2
			# save_class_activation_images(img, cam, '/data/2015P002510/nishanth/WSL/wsl/wsl/Example_maps/GCAM_color')
			# score = []
			# np.save('/data/2015P002510/nishanth/WSL/wsl/wsl/AUPRC_scores/{}_{}.npy'.format('GRAD','resnet18'),np.zeros((2,2))) #test

			for i, label in enumerate(labels):
				if label == 0:
					continue

				if task == 'detect':
					ground_map = box_to_map(dataset.df[dataset.df.Id == name].box.to_list(),
											np.zeros(org_size))
					ground_map = cv2.resize(ground_map, new_size, interpolation=cv2.INTER_NEAREST).clip(0, 1)

					all_scores['GRAD'].append(aupr(grayscale_vanilla_grads,ground_map))
					all_scores['SG'].append(aupr(grayscale_smooth_grad,ground_map))
					all_scores['IG'].append(aupr(grayscale_integrated_grads,ground_map))
					all_scores['SIG'].append(aupr(grayscale_smooth_ig,ground_map))
					all_scores['GBP'].append(aupr(grayscale_guided_grads,ground_map))
					all_scores['GCAM'].append(aupr(cam,ground_map))
					all_scores['GGCAM'].append(aupr(grayscale_cam_gb,ground_map))
					# all_scores['GRAD'].append(aupr(cv2.resize(grayscale_vanilla_grads, new_size, interpolation=cv2.INTER_AREA),ground_map))
				
				elif task == 'segment':
					ground_map = np.zeros(org_size)
					eps = dataset.df[dataset.df.Id == name].EncodedPixels.to_list()
					for ep in eps:
						ground_map += rle2mask(ep, np.zeros(org_size)).T
				else:
					print('Ground truth not available.')

		for key in all_scores.keys():
			print(key, ' ', np.mean(all_scores[key]))
			np.save('/data/2015P002510/nishanth/WSL/wsl/wsl/AUPRC_scores/{}_{}.npy'.format(key,'vgg_test'),all_scores[key])


		configs['wild'] = np.mean(all_scores)
		
		all_configs.append(configs)

	df = pd.DataFrame.from_dict(all_configs)
	print(df)
	time = datetime.datetime.now().strftime('%H_%d_%m')
	if store:
		df.to_csv(wsl_summary_dir / f'wild_{time}')