def vis_grid(self, writer, dataset, image, target, pred, region_prop, pred_softmax, global_step, split): image = image.squeeze() image = np.transpose(image, axes=[1, 2, 0]) image *= (0.229, 0.224, 0.225) image += (0.485, 0.456, 0.406) image *= 255.0 image = image.astype(np.uint8) region_prop = region_prop.squeeze() target = target.squeeze() seg_mask = target == 2 target = decode_segmap(target, dataset=dataset) pred = pred.squeeze() pred_softmax = np.max(pred_softmax.squeeze(), axis=0) pred_softmax = seg_mask * pred_softmax pred = decode_segmap(pred, dataset=dataset).squeeze() fig = plt.figure(figsize=(7, 25), dpi=150) ax1 = fig.add_subplot(511) ax1.imshow(image) ax2 = fig.add_subplot(512) ax2.imshow(pred) ax3 = fig.add_subplot(513) ax3.imshow(target) ax4 = fig.add_subplot(514) ax4.imshow(region_prop, cmap='plasma') ax5 = fig.add_subplot(515) ax5.imshow(pred_softmax, cmap='plasma') writer.add_image(split, figure_to_image(fig), global_step) plt.clf()
def visualize_results(train_dataset, valid_dataset, model, device): model.eval() if valid_dataset is None: valid_dataset = train_dataset image_original, pr_mask, gt_mask, pr, pr2, cl1 = process_visualize_image(train_dataset, device, model) gt_out = np.array(gt_mask).astype(np.uint8) dataset_name = train_dataset.name() segmap_gt = decode_segmap(gt_out, dataset=dataset_name) #V,H = compute_edge_mask(torch.unsqueeze(image_original,0), sigma=0.1) if valid_dataset is not None: image_original_val, pr_mask_val, gt_mask_val, pr, pr2, cl = process_visualize_image(valid_dataset, device, model) gt_val_out = np.array(gt_mask_val).astype(np.uint8) segmap_gt_val = decode_segmap(gt_val_out, dataset=dataset_name) visualize_images( image_train = image_original, gt_train=segmap_gt, predict_train=decode_segmap(pr_mask.cpu().numpy(), dataset_name), image_valid=image_original_val, gt_val=segmap_gt_val, predict_valid = decode_segmap(pr_mask_val.cpu().numpy(), dataset_name), # pr=pr.cpu().numpy(), # pr2=pr2.cpu().numpy(), cl0=cl[0].cpu().numpy(), cl1=cl[1].cpu().numpy(), cl2=cl[2].cpu().numpy() # edgeV=V.squeeze().cpu().numpy(), # edgeH=H.squeeze().cpu().numpy() )
def save_images(self, pred_dict): predictions = pred_dict['images'] ious = pred_dict['ious'] targets = pred_dict['targets'] inputs = pred_dict['inputs'] assert (len(predictions) == len(ious)) imgfolder = self.directory + "//test/" if not os.path.exists(imgfolder): os.mkdir(imgfolder) for predi in range(len(predictions)): str_iou = "{:.05f}".format(ious[predi]) imgname = os.path.join(imgfolder, str(predi) + "_" + str(str_iou) + ".png") segmap = decode_segmap(predictions[predi], dataset='semantic_body') segmap_target = decode_segmap(targets[predi], dataset='semantic_body') img_tmp = np.transpose(inputs[predi], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) save = np.zeros((segmap.shape[0], segmap.shape[1] * 3, 3), dtype=np.uint8) save[0:segmap.shape[0], 0:segmap.shape[1], :] = segmap save[0:segmap.shape[0], segmap.shape[1]:segmap.shape[1] * 2, :] = segmap_target save[0:segmap.shape[0], segmap.shape[1] * 2:, :] = img_tmp cv2.imwrite(imgname, save)
def display_prediction_grid(image, target, pred, dataset='cityscapes'): segmap = decode_segmap(target.numpy().squeeze(), dataset=dataset) segmap_pred = decode_segmap(pred.squeeze(), dataset=dataset) images = [image[:, :, :3], image[:, :, 3:].squeeze(), segmap, segmap_pred] labels = ['RGB', 'Depth', 'GT', 'Prediction'] display_grid(images, labels)
def multi_scale_predict(self): import cv2 test_dir = r'/home/lab/ygy/rssrai2019/datasets/image/test_crop' files = os.listdir(test_dir) self.model.eval() tbar = tqdm(files, desc='\r') for i, filename in enumerate(tbar): image_predict_prob_list = [] image_ori = Image.open(os.path.join(test_dir, filename)) # 预测原图 sample_ori = image_ori.copy() pred = self.predict(sample_ori)[0] ori_pred = np.argmax(pred, axis=0) image_predict_prob_list.append(pred) # 预测旋转三个 angle_list = [90, 180, 270] for angle in angle_list: img_rotate = image_ori.rotate(angle, Image.BILINEAR) pred = self.predict(img_rotate)[0] pred = pred.transpose((1, 2, 0)) m_rotate = cv2.getRotationMatrix2D((200, 200), 360.0 - angle, 1) pred = cv2.warpAffine(pred, m_rotate, (400, 400)) pred = pred.transpose((2, 0, 1)) image_predict_prob_list.append(pred) # 预测竖直翻转 img_flip = image_ori.transpose(Image.FLIP_TOP_BOTTOM) pred = self.predict(img_flip)[0] pred = cv2.flip(pred, 0) image_predict_prob_list.append(pred) # 预测水平翻转 img_flip = image_ori.transpose(Image.FLIP_LEFT_RIGHT) pred = self.predict(img_flip)[0] pred = cv2.flip(pred, 1) image_predict_prob_list.append(pred) # 求和平均 final_predict_prob = sum(image_predict_prob_list) / len( image_predict_prob_list) final_pred = np.argmax(final_predict_prob, axis=0) rgb_ori = decode_segmap(ori_pred, self.args.dataset) rgb = decode_segmap(final_pred, self.args.dataset) pred_img = Image.fromarray(final_pred, mode='1') rgb_ori_img = Image.fromarray(rgb_ori, mode='RGB') rgb_img = Image.fromarray(rgb, mode='RGB') pred_img.save( os.path.join(self.args.vis_logdir, 'raw_train_id', filename)) rgb_ori_img.save( os.path.join(self.args.vis_logdir, 'vis_color_ori', filename)) rgb_img.save( os.path.join(self.args.vis_logdir, 'vis_color', filename))
def testing(self): tbar = tqdm(self.test_loader, desc='\r') self.model.eval() #train_pattern # plt.ion() with torch.no_grad(): for i, sample in enumerate(tbar): image, label = sample['image'], sample['label'] if self.args.cuda: image = image.cuda() output = self.model( image) #(shape: (batch_size=1, n_class=21, img_h, img_w) #data visualization output = output.data.cpu().numpy( ) #(shape: (batch_size=1, n_class=21, img_h, img_w) pred_label_imgs = np.argmax( output, axis=1) #(shape: (batch_size=1, img_h, img_w) pred_label_imgs = pred_label_imgs.astype(np.uint8) for index in range(pred_label_imgs.shape[0]): pred_label_img = pred_label_imgs[ index] #(shape: (img_h, img_w) result = decode_segmap(pred_label_img, dataset='pascal', plot=False) tmp = label[index].data.cpu().numpy() tmp = tmp.astype(np.uint8) segmap = decode_segmap(tmp, dataset='pascal') img = image[index] #(shape: (3, img_h, img_w) img = img.data.cpu().numpy() img = np.transpose(img, (1, 2, 0)) #(shape: (img_h, img_w, 3) img *= (0.229, 0.224, 0.225) img += (0.485, 0.456, 0.406) img *= 255.0 img = img.astype(np.uint8) # plt.figure() plt.title('display') plt.subplot(311) plt.imshow(img) plt.subplot(312) plt.imshow(segmap) plt.subplot(313) plt.imshow(result) plt.pause(0.7) plt.clf()
def generate_seg_vis(model_cfg_paths, dir='imgs', cfg_options=[]): failed = [] for cfg_filepath in model_cfg_paths: try: model_dir = os.path.dirname(cfg_filepath) img_dir = os.path.join(model_dir, dir) if not os.path.exists(img_dir): os.mkdir(img_dir) cfg = match_cfg_versions(cfg_filepath) cfg.merge_from_list([ 'SYSTEM.GPU_IDS', [0], 'CHECKPOINT.RESUME', True, 'CHECKPOINT.DIRECTORY', model_dir, ]) cfg.merge_from_list(cfg_options) model = load_model(cfg) model.eval() dataset = make_dataset(cfg, 'val') for ii in range(0, 10): sample = dataset[ii] image, target, id = sample['image'], sample['label'], sample[ 'id'] img_tmp = np.ascontiguousarray( dataset.loader.invert_normalization( sample['image'].squeeze())) gt = sample['label'].numpy() tmp = np.array(gt).astype(np.uint8) gt_segmap = decode_segmap(tmp, dataset=cfg.DATASET.NAME) print(id) pred = run_image(cfg, image.unsqueeze(0), model) segmap = decode_segmap(pred.squeeze(), dataset=cfg.DATASET.NAME) plt.imsave('{}/{}.png'.format(img_dir, ii), segmap) plt.imsave('{}/{}_original.png'.format(img_dir, ii), img_tmp) plt.imsave('{}/{}_gt.png'.format(img_dir, ii), gt_segmap) except Exception as e: print(e) traceback.print_exc() failed.append(cfg_filepath) print("Failed models: ".format("\n".join(failed)))
def vis_segmentation(self,image, seg_map): plt.figure(figsize=(30, 50)) grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1]) plt.subplot(grid_spec[0]) plt.imshow(image) plt.axis('off') plt.title('input image') plt.subplot(grid_spec[1]) seg_map = seg_map.squeeze() seg_map = np.array(seg_map).astype(np.uint8) seg_image = decode_segmap(seg_map,dataset='vaihingen') plt.imshow(seg_image) plt.axis('off') plt.title('segmentation map') plt.subplot(grid_spec[2]) plt.imshow(image) plt.imshow(seg_image, alpha=0.7) plt.axis('off') plt.title('segmentation overlay') #plt.show() #seg_image = seg_image.astype(np.float32) maxi = seg_image.max() seg_image = seg_image/maxi*255 seg_image = seg_image.astype(np.uint8) #image = Image.fromarray(image) seg_image=Image.fromarray(seg_image) #images=Image.blend(image,seg_image,0.4) seg_image.save('D:\\Segmentation\\test_image\\DFPEPD\\image5.png')
def visualization(self): self.combine_net_model.eval() tbar = tqdm(test_files, desc='\r') for i, filename in enumerate(tbar): image = Image.open(os.path.join(test_dir, filename)) # UNet_multi_scale_predict unt_pred = self.unet_multi_scale_predict(image) # UNetNested_multi_scale_predict unetnested_pred = self.unetnested_multi_scale_predict(image) net_input = torch.cat([unt_pred, unetnested_pred], 1) with torch.no_grad(): output = self.combine_net_model(net_input) pred = output.data.cpu().numpy()[0] pred = np.argmax(pred, axis=0) rgb = decode_segmap(pred, self.args.dataset) pred_img = Image.fromarray(pred, mode='L') rgb_img = Image.fromarray(rgb, mode='RGB') pred_img.save( os.path.join(self.args.vis_logdir, 'raw_train_id', filename)) rgb_img.save( os.path.join(self.args.vis_logdir, 'vis_color', filename))
def visualization_png(edge_hor, edge_ver, pred, pred_mp, image_name=None, accuracy=None, save_dir=None, enable_save_all=False): edge_hor = edge_hor.unsqueeze(2).repeat(1, 1, 3).cpu().numpy().astype(np.float32) edge_ver = edge_ver.unsqueeze(2).repeat(1, 1, 3).cpu().numpy().astype(np.float32) segmap = (decode_segmap(pred, 'pascal') * 255).astype(np.uint8) segmap_mp = (decode_segmap(pred_mp, 'pascal') * 255).astype(np.uint8) edge_hor = edge_hor / edge_hor.max() * 255 edge_ver = edge_ver / edge_ver.max() * 255 edge_hor, edge_ver = edge_hor.astype(np.uint8), edge_ver.astype(np.uint8) img_list = [ transforms.ToTensor()(Image.fromarray(edge_hor).convert('RGB')), transforms.ToTensor()(Image.fromarray(edge_ver).convert('RGB')), transforms.ToTensor()(Image.fromarray(segmap).convert('RGB')), transforms.ToTensor()(Image.fromarray(segmap_mp).convert('RGB')) ] valid_visual = torch.stack(img_list, 0) valid_visual = vutils.make_grid(valid_visual, nrow=2, padding=0) if (image_name is not None) and (accuracy is not None) and (save_dir is not None): save_path = os.path.join( save_dir, '{}_edge_{:.2f}.png'.format(image_name, accuracy)) vutils.save_image(valid_visual, save_path) if enable_save_all: save_dir = os.path.join(save_dir, image_name) os.makedirs(save_dir) if (not os.path.exists(save_dir)) else None vutils.save_image(img_list[0], os.path.join(save_dir, 'Edge_Hor.png')) vutils.save_image(img_list[1], os.path.join(save_dir, 'Edge_Ver.png')) vutils.save_image(img_list[2], os.path.join(save_dir, 'Unary_Seg_Scaled.png')) vutils.save_image(img_list[3], os.path.join(save_dir, 'Final_Seg_Scaled.png'))
def decode_segmentation(root, save_root): image_names = os.listdir(root) image_names.sort() for image_name in image_names: print('processing {} ...'.format(image_name)) label = cv2.imread(os.path.join(root, image_name), 0) print(label.shape) label_color = decode_segmap(label, dataset='pascal', plot=False) #label_color.save(os.path.join(save_root, image_name)) cv2.imwrite(os.path.join(save_root, image_name), np.array(label_color * 255, np.uint8))
def printer(images, type): if type == 'label': for idx, label_mask in enumerate(images): plt.title('masks{}'.format(idx)) plt.imshow(decode_segmap(np.array(label_mask), 'penn')) plt.show() elif type == 'image': sample = torchvision.utils.make_grid(images, normalize=True) sample = sample.numpy().transpose((1, 2, 0)) plt.title('images') plt.imshow(sample) plt.show() elif type == 'mix': for idx, label in enumerate(images['label']): plt.title('mix{}'.format(idx)) mask = decode_segmap(np.array(label), 'penn') image = torchvision.utils.make_grid(images['image'][idx], normalize=True) image = image.numpy().transpose((1, 2, 0)) plt.title('images') plt.imshow(mask + image) plt.show()
def validation(self, epoch): # self.model.load_state_dict(torch.load('model60epoch_visdrone')) ####################### self.model.eval() self.evaluator.reset() tbar = tqdm(self.val_loader, desc='\r') test_loss = 0.0 for i, sample in enumerate(tbar): image, target = sample['image'], sample['label'] if self.args.cuda: image, target = image.cuda(), target.cuda() with torch.no_grad(): output = self.model(image) loss = self.criterion(output, target) test_loss += loss.item() tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1))) pred = output.data.cpu().numpy() target = target.cpu().numpy() trgt = target * 255###################################() if epoch % 5 == 0: img1 = trgt[0, :, :]############################ cv2.imwrite('visdrone/result/target_epoch_' + str(epoch) + '_' + str(i * self.btch_size) + '.png', img1) pred = np.argmax(pred, axis=1) prd = pred * 255############################ if epoch % 5 == 0: img1 = prd[0, :, :]############################### img1 = decode_segmap(img1, dataset='coco') cv2.imwrite('visdrone/result/predict_epoch_' + str(epoch) + '_' + str(i * self.btch_size) + '.png', img1) # Add batch sample into evaluator self.evaluator.add_batch(target, pred) # Fast test during the training Acc = self.evaluator.Pixel_Accuracy() Acc_class = self.evaluator.Pixel_Accuracy_Class() mIoU = self.evaluator.Mean_Intersection_over_Union() FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union() # self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch) # self.writer.add_scalar('val/mIoU', mIoU, epoch) # self.writer.add_scalar('val/Acc', Acc, epoch) # self.writer.add_scalar('val/Acc_class', Acc_class, epoch) # self.writer.add_scalar('val/fwIoU', FWIoU, epoch) print('Validation:') print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0])) print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU)) print('Loss: %.3f' % test_loss)
def test(args): kwargs = {'num_workers': 1, 'pin_memory': True} _, val_loader, _, nclass = make_data_loader(args, **kwargs) checkpoint = torch.load(args.ckpt) if checkpoint is None: raise ValueError device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') model = DeepLab(num_classes=nclass, backbone='resnet', output_stride=16, sync_bn=True, freeze_bn=False) model.load_state_dict(checkpoint['state_dict']) model.eval() model.to(device) torch.set_grad_enabled(False) tbar = tqdm(val_loader) num_img_tr = len(val_loader) for i, sample in enumerate(tbar): x1, x2, y1, y2 = [ int(item) for item in sample['img_meta']['bbox_coord'] ] # bbox coord w, h = x2 - x1, y2 - y1 img = sample['img_meta']['image'].squeeze().cpu().numpy() img_w, img_h = img.shape[:2] inputs = sample['image'].cuda() output = model(inputs).squeeze().cpu().numpy() pred = np.argmax(output, axis=0) result = decode_segmap(pred, dataset=args.dataset, plot=False) result = imresize(result, (w, h)) result_padding = np.zeros(img.shape, dtype=np.uint8) result_padding[y1:y2, x1:x2] = result result = img // 2 + result_padding * 127 result[result > 255] = 255 plt.imsave( os.path.join('run', args.dataset, 'deeplab-resnet', 'output', str(i)), result)
def visualization(self): self.model.eval() tbar = tqdm(self.test_loader, desc='\r') for i, sample in enumerate(tbar): image = sample['image'] img_path = sample['img_path'] if self.args.cuda: image = image.cuda() with torch.no_grad(): output = self.model(image) tbar.set_description('Vis image:') pred = output.data.cpu().numpy() pred = np.argmax(pred, axis=1)[0] rgb = decode_segmap(pred, self.args.dataset) pred_img = Image.fromarray(pred, mode='L') rgb_img = Image.fromarray(rgb, mode='RGB') filename = os.path.basename(img_path[0]) pred_img.save( os.path.join(self.args.vis_logdir, 'raw_train_id', filename)) rgb_img.save( os.path.join(self.args.vis_logdir, 'vis_color', filename))
def save_demo_result(self, pred_dict, image_list, root_folder): predictions = pred_dict['images'] ious = pred_dict['ious'] all_idx = np.arange(11) imgfolder = os.path.join(root_folder, "mask//mask_refine_network") initmaskfolder = os.path.join(root_folder, "mask") if not os.path.exists(imgfolder): os.mkdir(imgfolder) for predi in range(len(predictions)): pred_labels = np.unique(predictions[predi]) iou = ious[predi] iou[~np.isin(all_idx, pred_labels)] = -1 initmask = cv2.imread( os.path.join(initmaskfolder, image_list[predi])) h, w, c = initmask.shape imgname = os.path.join(imgfolder, image_list[predi]) segmap, segmap_gray = decode_segmap(predictions[predi], dataset='semantic_body') segmap_gray = cv2.resize(segmap_gray, (w, h), interpolation=cv2.INTER_NEAREST) cv2.imwrite(imgname, segmap_gray) scorename, _ = os.path.splitext(imgname) np.savetxt(scorename + ".txt", iou[1:], fmt="%.6f")
args.base_size = 513 args.crop_size = 513 comp5421_val = Comp5421Segmentation(args, split='val') dataloader = DataLoader(comp5421_val, batch_size=4, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='comp5421') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
import matplotlib.pyplot as plt composed_transforms_tr = transforms.Compose([ tr.RandomHorizontalFlip(), tr.ScaleNRotate(rots=(-15, 15), scales=(.75, 1.5)), tr.FixedResize(size=512), tr.ToTensor()]) voc_train = VOCSegmentation(split='train', transform=composed_transforms_tr) dataloader = DataLoader(voc_train, batch_size=2, shuffle=True, num_workers=2) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['gt'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) tmp = np.squeeze(tmp, axis=0) segmap = decode_segmap(tmp) img_tmp = np.transpose(img[jj], axes=[1, 2, 0]).astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break plt.show(block=True)
args.base_size = 1024 args.crop_size = 512 args.event_dim = 2 apolloscape_train = ApolloscapeRGBEvent(args, split='train') dataloader = DataLoader(apolloscape_train, batch_size=2, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='apolloscapeevent') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(311) plt.imshow(img_tmp) plt.subplot(312) plt.imshow(segmap) if ii == 2: break
args.base_size = 513 args.crop_size = 513 voc_train = VOCSegmentation(args, split="train") dataloader = DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample["image"].numpy() gt = sample["label"].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset="pascal") img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title("display") plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
import pdb parser = argparse.ArgumentParser() args = parser.parse_args() args.base_dir = "/Users/nopz/Google Drive/voc/VOCdevkit/VOC2012" args.base_size = 513 args.crop_size = 513 args.dataset = "pascal" voc_train = VOCSegmentation(args, base_dir=args.base_dir, split="train") dl = data.DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=0) pdb.set_trace() for i, sample in enumerate(dl): for j in range(sample["image"].size()[0]): img = sample["image"].numpy() gt = sample["label"].numpy() tmp = np.array(gt[j]).astype(np.uint8) segmap = decode_segmap(tmp, dataset=args.dataset) img_tmp = np.transpose(img[j], axes=[1, 2, 0]) img_tmp += (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) im.imsave("sgmap.jpg", segmap) print(img_tmp.size) if i == 1: break
parser = argparse.ArgumentParser() args = parser.parse_args() args.base_size = 513 args.crop_size = 513 cityscapes_train = CityscapesSegmentation(args, split='train') dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='cityscapes') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
parser = argparse.ArgumentParser() args = parser.parse_args() args.base_size = 513 args.crop_size = 513 coco_val = COCOSegmentation(args, split='val', year='2017') dataloader = DataLoader(coco_val, batch_size=4, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='coco') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
args.base_size = 600 args.crop_size = 600 Vaihingen_train = Vaihingen(args, split='val') dataloader = DataLoader(Vaihingen_train, batch_size=5, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='vaihingen') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
args.crop_size = 512 cityscapes_train = VisdroneSegmentation(args, split='train') dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap( tmp, dataset='cityscapes' ) #Decode segmentation class labels into a color image img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
args.base_size = 513 args.crop_size = 513 pascal_voc_val = pascal.VOCSegmentation(args, split='val') sbd = sbd.SBDSegmentation(args, split=['train', 'val']) pascal_voc_train = pascal.VOCSegmentation(args, split='train') dataset = CombineDBs([pascal_voc_train, sbd], excluded=[pascal_voc_val]) dataloader = torch.utils.data.DataLoader(dataset, batch_size=2, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='pascal') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break plt.show(block=True)
parser = argparse.ArgumentParser() args = parser.parse_args() args.base_size = 513 args.crop_size = 513 voc_train = VOCSegmentation(args, split='train') dataloader = DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='pascal') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
def adv_train(train_loader, model, criterion, optimizer, epoch, args, info, writer, dataset, eval_score=None, print_freq=10): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() scores = AverageMeter() # switch to train mode model.train() end = time.time() for i, (input, target) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) #Get attacked image adv_img = PGD_attack(input, target, model, criterion, args.epsilon, args.steps, args.dataset, args.step_size, info, using_noise=True) # input = input.cuda() # print('diff', (adv_img.data-input) / (args.epsilon)) if type(criterion) in [torch.nn.modules.loss.L1Loss, torch.nn.modules.loss.MSELoss]: target = target.float() # TODO: adversarial training clean_input = input input = adv_img.data if torch.cuda.is_available(): input = input.cuda() target = target.cuda() target = target.cuda() clean_input = clean_input.cuda() input_var = torch.autograd.Variable(input) clean_input_var = torch.autograd.Variable(clean_input) target_var = torch.autograd.Variable(target) if args.debug: # Debug visualisations print("\nVisualising here.\n", # Add things to print here ) # print("Max in difference is ", np.max(diff_img)) arr_clean_img = clean_input_var.clone().cpu().numpy() if torch.cuda.is_available() else clean_input_var.clone().numpy() arr_adv_img = input_var.clone().cpu().numpy() if torch.cuda.is_available() else input_var.clone().numpy() vis_clean_img = np.moveaxis(np.copy(arr_clean_img)[0], 0, 2) vis_adv_img = np.moveaxis(np.copy(arr_adv_img)[0], 0, 2) vis_clean_img_tr = np.moveaxis(back_transform(np.copy(arr_clean_img), info)[0],0,2) # np.copy is necessary to make sure that the original arrays are not modified. vis_adv_img_tr = np.moveaxis(back_transform(np.copy(arr_adv_img), info)[0],0,2) # np.copy is necessary to make sure that the original arrays are not modified. diff_img1 = vis_adv_img - vis_clean_img diff_img2 = vis_adv_img_tr - vis_clean_img_tr f, axarr = plt.subplots(1, 6) axarr[0].imshow(vis_clean_img) axarr[1].imshow(vis_adv_img) axarr[2].imshow(vis_clean_img_tr) axarr[3].imshow(vis_adv_img_tr) # print() axarr[4].imshow(diff_img1) axarr[5].imshow(diff_img2) plt.show() print("__max diff is__",np.max(diff_img1)*255,np.max(diff_img2)*255) # compute output output = model(input_var)[0] loss = criterion(output, target_var) losses.update(loss.data.item(), input.size(0)) if eval_score is not None: if target_var.size(0)>0: scores.update(eval_score(output, target_var), input.size(0)) else: print("0 size!") clean_score = 0 # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if args.debug: print_freq = 10 if i % print_freq == 0: # Convert target and prediction to rgb images to visualise class_prediction = torch.argmax(output, dim=1) decoded_target = decode_segmap(target[0].cpu().numpy() if torch.cuda.is_available() else target[0].numpy(), dataset) decoded_target = np.moveaxis(decoded_target, 2, 0) decoded_class_prediction = decode_segmap( class_prediction[0].cpu().numpy() if torch.cuda.is_available() else class_prediction[0].numpy(), dataset) decoded_class_prediction = np.moveaxis(decoded_class_prediction, 2, 0) logger.info('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Score {top1.val:.3f} ({top1.avg:.3f})'.format( epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses, top1=scores)) #TODO: Get training accuracy for non adv images. (forward the clean data.) # compute output for the case when clean image is passed. clean_output = model(clean_input_var)[0] clean_loss = criterion(clean_output, target_var) if eval_score is not None: if target_var.size(0) > 0: clean_score = eval_score(clean_output, target_var) else: print("0 size!") clean_score = 0 # print('img max m,in', torch.max(input_var[0]), torch.min(input_var[0])) # print('img max m,in', torch.max(clean_input_var[0]), torch.min(clean_input_var[0])) # input_var[0] = (back_transform(input_var[0], info) * 255).long() writer.add_image('Image/adv image ', back_transform(input_var, info)[0]) writer.add_image('Image/clean image ', back_transform(clean_input_var, info)[0]) writer.add_image('Image/image target ', decoded_target) writer.add_image('Image/image prediction ', decoded_class_prediction) writer.add_scalar('Adv_Train/Score', scores.val, epoch*len(train_loader)+i) writer.add_scalar('Adv_Train/Loss', losses.val, epoch*len(train_loader)+i) writer.add_scalar('Adv_Train/Clean_Score', clean_score, epoch*len(train_loader)+i) writer.add_scalar('Adv_Train/Clean_Loss', clean_loss, epoch*len(train_loader)+i) if args.debug and i == print_freq: break
parser = argparse.ArgumentParser() args = parser.parse_args() args.base_size = 513 args.crop_size = 513 coco_val = COCOSegmentation(args, split="val", year="2017") dataloader = DataLoader(coco_val, batch_size=4, shuffle=True, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample["image"].numpy() gt = sample["label"].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset="coco") img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title("display") plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
tr.ToTensor()]) voc_train = VOCSegmentation(split='train', transform=composed_transforms_tr) dataloader = DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=2) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) tmp = np.squeeze(tmp, axis=0) segmap = decode_segmap(tmp) img_tmp = np.transpose(img[jj], axes=[1, 2, 0]).astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break plt.show(block=True) # import torch # # composed_transforms_tr = transforms.Compose([
args.positive_only = False siim_train = SIIMDataset(args, split='train') print('Number of training images: ', len(siim_train)) dataloader = DataLoader(siim_train, batch_size=2, shuffle=True, num_workers=2) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='siim') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
args.base_size = 513 args.crop_size = 513 cityscapes_train = CityscapesSegmentation(args, split='train') dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='cityscapes') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) if ii == 1: break
dataset = FashionDataset(args, Path.db_root_dir('fashion_clothes'), 'train', 'clothes') dataloader = DataLoader(dataset, batch_size=1, shuffle=False, num_workers=0) for ii, sample in enumerate(dataloader): for jj in range(sample["image"].size()[0]): img = sample['image'].numpy() gt = sample['label'].numpy() tmp = np.array(gt[jj]).astype(np.uint8) segmap = decode_segmap(tmp, dataset='fashion_clothes') img_tmp = np.transpose(img[jj], axes=[1, 2, 0]) img_tmp *= (0.229, 0.224, 0.225) img_tmp += (0.485, 0.456, 0.406) img_tmp *= 255.0 img_tmp = img_tmp.astype(np.uint8) plt.figure() plt.title('display') plt.subplot(211) plt.imshow(img_tmp) plt.subplot(212) plt.imshow(segmap) break plt.show(block=True)