def vis_grid(self, writer, dataset, image, target, pred, region_prop,
pred_softmax, global_step, split):
image = image.squeeze()
image = np.transpose(image, axes=[1, 2, 0])
image *= (0.229, 0.224, 0.225)
image += (0.485, 0.456, 0.406)
image *= 255.0
image = image.astype(np.uint8)
region_prop = region_prop.squeeze()
target = target.squeeze()
seg_mask = target == 2
target = decode_segmap(target, dataset=dataset)
pred = pred.squeeze()
pred_softmax = np.max(pred_softmax.squeeze(), axis=0)
pred_softmax = seg_mask * pred_softmax
pred = decode_segmap(pred, dataset=dataset).squeeze()
fig = plt.figure(figsize=(7, 25), dpi=150)
ax1 = fig.add_subplot(511)
ax1.imshow(image)
ax2 = fig.add_subplot(512)
ax2.imshow(pred)
ax3 = fig.add_subplot(513)
ax3.imshow(target)
ax4 = fig.add_subplot(514)
ax4.imshow(region_prop, cmap='plasma')
ax5 = fig.add_subplot(515)
ax5.imshow(pred_softmax, cmap='plasma')
writer.add_image(split, figure_to_image(fig), global_step)
plt.clf()
def visualize_results(train_dataset, valid_dataset, model, device):
model.eval()
if valid_dataset is None:
valid_dataset = train_dataset
image_original, pr_mask, gt_mask, pr, pr2, cl1 = process_visualize_image(train_dataset, device, model)
gt_out = np.array(gt_mask).astype(np.uint8)
dataset_name = train_dataset.name()
segmap_gt = decode_segmap(gt_out, dataset=dataset_name)
#V,H = compute_edge_mask(torch.unsqueeze(image_original,0), sigma=0.1)
if valid_dataset is not None:
image_original_val, pr_mask_val, gt_mask_val, pr, pr2, cl = process_visualize_image(valid_dataset, device, model)
gt_val_out = np.array(gt_mask_val).astype(np.uint8)
segmap_gt_val = decode_segmap(gt_val_out, dataset=dataset_name)
visualize_images(
image_train = image_original,
gt_train=segmap_gt,
predict_train=decode_segmap(pr_mask.cpu().numpy(), dataset_name),
image_valid=image_original_val,
gt_val=segmap_gt_val,
predict_valid = decode_segmap(pr_mask_val.cpu().numpy(), dataset_name),
# pr=pr.cpu().numpy(),
# pr2=pr2.cpu().numpy(),
cl0=cl[0].cpu().numpy(),
cl1=cl[1].cpu().numpy(),
cl2=cl[2].cpu().numpy()
# edgeV=V.squeeze().cpu().numpy(),
# edgeH=H.squeeze().cpu().numpy()
)
def save_images(self, pred_dict):
predictions = pred_dict['images']
ious = pred_dict['ious']
targets = pred_dict['targets']
inputs = pred_dict['inputs']
assert (len(predictions) == len(ious))
imgfolder = self.directory + "//test/"
if not os.path.exists(imgfolder):
os.mkdir(imgfolder)
for predi in range(len(predictions)):
str_iou = "{:.05f}".format(ious[predi])
imgname = os.path.join(imgfolder,
str(predi) + "_" + str(str_iou) + ".png")
segmap = decode_segmap(predictions[predi], dataset='semantic_body')
segmap_target = decode_segmap(targets[predi],
dataset='semantic_body')
img_tmp = np.transpose(inputs[predi], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
save = np.zeros((segmap.shape[0], segmap.shape[1] * 3, 3),
dtype=np.uint8)
save[0:segmap.shape[0], 0:segmap.shape[1], :] = segmap
save[0:segmap.shape[0],
segmap.shape[1]:segmap.shape[1] * 2, :] = segmap_target
save[0:segmap.shape[0], segmap.shape[1] * 2:, :] = img_tmp
cv2.imwrite(imgname, save)
def display_prediction_grid(image, target, pred, dataset='cityscapes'):
segmap = decode_segmap(target.numpy().squeeze(), dataset=dataset)
segmap_pred = decode_segmap(pred.squeeze(), dataset=dataset)
images = [image[:, :, :3], image[:, :, 3:].squeeze(), segmap, segmap_pred]
labels = ['RGB', 'Depth', 'GT', 'Prediction']
display_grid(images, labels)
def multi_scale_predict(self):
import cv2
test_dir = r'/home/lab/ygy/rssrai2019/datasets/image/test_crop'
files = os.listdir(test_dir)
self.model.eval()
tbar = tqdm(files, desc='\r')
for i, filename in enumerate(tbar):
image_predict_prob_list = []
image_ori = Image.open(os.path.join(test_dir, filename))
# 预测原图
sample_ori = image_ori.copy()
pred = self.predict(sample_ori)[0]
ori_pred = np.argmax(pred, axis=0)
image_predict_prob_list.append(pred)
# 预测旋转三个
angle_list = [90, 180, 270]
for angle in angle_list:
img_rotate = image_ori.rotate(angle, Image.BILINEAR)
pred = self.predict(img_rotate)[0]
pred = pred.transpose((1, 2, 0))
m_rotate = cv2.getRotationMatrix2D((200, 200), 360.0 - angle,
1)
pred = cv2.warpAffine(pred, m_rotate, (400, 400))
pred = pred.transpose((2, 0, 1))
image_predict_prob_list.append(pred)
# 预测竖直翻转
img_flip = image_ori.transpose(Image.FLIP_TOP_BOTTOM)
pred = self.predict(img_flip)[0]
pred = cv2.flip(pred, 0)
image_predict_prob_list.append(pred)
# 预测水平翻转
img_flip = image_ori.transpose(Image.FLIP_LEFT_RIGHT)
pred = self.predict(img_flip)[0]
pred = cv2.flip(pred, 1)
image_predict_prob_list.append(pred)
# 求和平均
final_predict_prob = sum(image_predict_prob_list) / len(
image_predict_prob_list)
final_pred = np.argmax(final_predict_prob, axis=0)
rgb_ori = decode_segmap(ori_pred, self.args.dataset)
rgb = decode_segmap(final_pred, self.args.dataset)
pred_img = Image.fromarray(final_pred, mode='1')
rgb_ori_img = Image.fromarray(rgb_ori, mode='RGB')
rgb_img = Image.fromarray(rgb, mode='RGB')
pred_img.save(
os.path.join(self.args.vis_logdir, 'raw_train_id', filename))
rgb_ori_img.save(
os.path.join(self.args.vis_logdir, 'vis_color_ori', filename))
rgb_img.save(
os.path.join(self.args.vis_logdir, 'vis_color', filename))
def testing(self):
tbar = tqdm(self.test_loader, desc='\r')
self.model.eval() #train_pattern
# plt.ion()
with torch.no_grad():
for i, sample in enumerate(tbar):
image, label = sample['image'], sample['label']
if self.args.cuda:
image = image.cuda()
output = self.model(
image) #(shape: (batch_size=1, n_class=21, img_h, img_w)
#data visualization
output = output.data.cpu().numpy(
) #(shape: (batch_size=1, n_class=21, img_h, img_w)
pred_label_imgs = np.argmax(
output, axis=1) #(shape: (batch_size=1, img_h, img_w)
pred_label_imgs = pred_label_imgs.astype(np.uint8)
for index in range(pred_label_imgs.shape[0]):
pred_label_img = pred_label_imgs[
index] #(shape: (img_h, img_w)
result = decode_segmap(pred_label_img,
dataset='pascal',
plot=False)
tmp = label[index].data.cpu().numpy()
tmp = tmp.astype(np.uint8)
segmap = decode_segmap(tmp, dataset='pascal')
img = image[index] #(shape: (3, img_h, img_w)
img = img.data.cpu().numpy()
img = np.transpose(img,
(1, 2, 0)) #(shape: (img_h, img_w, 3)
img *= (0.229, 0.224, 0.225)
img += (0.485, 0.456, 0.406)
img *= 255.0
img = img.astype(np.uint8)
# plt.figure()
plt.title('display')
plt.subplot(311)
plt.imshow(img)
plt.subplot(312)
plt.imshow(segmap)
plt.subplot(313)
plt.imshow(result)
plt.pause(0.7)
plt.clf()
def generate_seg_vis(model_cfg_paths, dir='imgs', cfg_options=[]):
failed = []
for cfg_filepath in model_cfg_paths:
try:
model_dir = os.path.dirname(cfg_filepath)
img_dir = os.path.join(model_dir, dir)
if not os.path.exists(img_dir):
os.mkdir(img_dir)
cfg = match_cfg_versions(cfg_filepath)
cfg.merge_from_list([
'SYSTEM.GPU_IDS',
[0],
'CHECKPOINT.RESUME',
True,
'CHECKPOINT.DIRECTORY',
model_dir,
])
cfg.merge_from_list(cfg_options)
model = load_model(cfg)
model.eval()
dataset = make_dataset(cfg, 'val')
for ii in range(0, 10):
sample = dataset[ii]
image, target, id = sample['image'], sample['label'], sample[
'id']
img_tmp = np.ascontiguousarray(
dataset.loader.invert_normalization(
sample['image'].squeeze()))
gt = sample['label'].numpy()
tmp = np.array(gt).astype(np.uint8)
gt_segmap = decode_segmap(tmp, dataset=cfg.DATASET.NAME)
print(id)
pred = run_image(cfg, image.unsqueeze(0), model)
segmap = decode_segmap(pred.squeeze(),
dataset=cfg.DATASET.NAME)
plt.imsave('{}/{}.png'.format(img_dir, ii), segmap)
plt.imsave('{}/{}_original.png'.format(img_dir, ii), img_tmp)
plt.imsave('{}/{}_gt.png'.format(img_dir, ii), gt_segmap)
except Exception as e:
print(e)
traceback.print_exc()
failed.append(cfg_filepath)
print("Failed models: ".format("\n".join(failed)))
def vis_segmentation(self,image, seg_map):
plt.figure(figsize=(30, 50))
grid_spec = gridspec.GridSpec(1, 4, width_ratios=[6, 6, 6, 1])
plt.subplot(grid_spec[0])
plt.imshow(image)
plt.axis('off')
plt.title('input image')
plt.subplot(grid_spec[1])
seg_map = seg_map.squeeze()
seg_map = np.array(seg_map).astype(np.uint8)
seg_image = decode_segmap(seg_map,dataset='vaihingen')
plt.imshow(seg_image)
plt.axis('off')
plt.title('segmentation map')
plt.subplot(grid_spec[2])
plt.imshow(image)
plt.imshow(seg_image, alpha=0.7)
plt.axis('off')
plt.title('segmentation overlay')
#plt.show()
#seg_image = seg_image.astype(np.float32)
maxi = seg_image.max()
seg_image = seg_image/maxi*255
seg_image = seg_image.astype(np.uint8)
#image = Image.fromarray(image)
seg_image=Image.fromarray(seg_image)
#images=Image.blend(image,seg_image,0.4)
seg_image.save('D:\\Segmentation\\test_image\\DFPEPD\\image5.png')
def visualization(self):
self.combine_net_model.eval()
tbar = tqdm(test_files, desc='\r')
for i, filename in enumerate(tbar):
image = Image.open(os.path.join(test_dir, filename))
# UNet_multi_scale_predict
unt_pred = self.unet_multi_scale_predict(image)
# UNetNested_multi_scale_predict
unetnested_pred = self.unetnested_multi_scale_predict(image)
net_input = torch.cat([unt_pred, unetnested_pred], 1)
with torch.no_grad():
output = self.combine_net_model(net_input)
pred = output.data.cpu().numpy()[0]
pred = np.argmax(pred, axis=0)
rgb = decode_segmap(pred, self.args.dataset)
pred_img = Image.fromarray(pred, mode='L')
rgb_img = Image.fromarray(rgb, mode='RGB')
pred_img.save(
os.path.join(self.args.vis_logdir, 'raw_train_id', filename))
rgb_img.save(
os.path.join(self.args.vis_logdir, 'vis_color', filename))
def visualization_png(edge_hor,
edge_ver,
pred,
pred_mp,
image_name=None,
accuracy=None,
save_dir=None,
enable_save_all=False):
edge_hor = edge_hor.unsqueeze(2).repeat(1, 1,
3).cpu().numpy().astype(np.float32)
edge_ver = edge_ver.unsqueeze(2).repeat(1, 1,
3).cpu().numpy().astype(np.float32)
segmap = (decode_segmap(pred, 'pascal') * 255).astype(np.uint8)
segmap_mp = (decode_segmap(pred_mp, 'pascal') * 255).astype(np.uint8)
edge_hor = edge_hor / edge_hor.max() * 255
edge_ver = edge_ver / edge_ver.max() * 255
edge_hor, edge_ver = edge_hor.astype(np.uint8), edge_ver.astype(np.uint8)
img_list = [
transforms.ToTensor()(Image.fromarray(edge_hor).convert('RGB')),
transforms.ToTensor()(Image.fromarray(edge_ver).convert('RGB')),
transforms.ToTensor()(Image.fromarray(segmap).convert('RGB')),
transforms.ToTensor()(Image.fromarray(segmap_mp).convert('RGB'))
]
valid_visual = torch.stack(img_list, 0)
valid_visual = vutils.make_grid(valid_visual, nrow=2, padding=0)
if (image_name is not None) and (accuracy is not None) and (save_dir
is not None):
save_path = os.path.join(
save_dir, '{}_edge_{:.2f}.png'.format(image_name, accuracy))
vutils.save_image(valid_visual, save_path)
if enable_save_all:
save_dir = os.path.join(save_dir, image_name)
os.makedirs(save_dir) if (not os.path.exists(save_dir)) else None
vutils.save_image(img_list[0],
os.path.join(save_dir, 'Edge_Hor.png'))
vutils.save_image(img_list[1],
os.path.join(save_dir, 'Edge_Ver.png'))
vutils.save_image(img_list[2],
os.path.join(save_dir, 'Unary_Seg_Scaled.png'))
vutils.save_image(img_list[3],
os.path.join(save_dir, 'Final_Seg_Scaled.png'))
def decode_segmentation(root, save_root):
image_names = os.listdir(root)
image_names.sort()
for image_name in image_names:
print('processing {} ...'.format(image_name))
label = cv2.imread(os.path.join(root, image_name), 0)
print(label.shape)
label_color = decode_segmap(label, dataset='pascal', plot=False)
#label_color.save(os.path.join(save_root, image_name))
cv2.imwrite(os.path.join(save_root, image_name),
np.array(label_color * 255, np.uint8))
def printer(images, type):
if type == 'label':
for idx, label_mask in enumerate(images):
plt.title('masks{}'.format(idx))
plt.imshow(decode_segmap(np.array(label_mask), 'penn'))
plt.show()
elif type == 'image':
sample = torchvision.utils.make_grid(images, normalize=True)
sample = sample.numpy().transpose((1, 2, 0))
plt.title('images')
plt.imshow(sample)
plt.show()
elif type == 'mix':
for idx, label in enumerate(images['label']):
plt.title('mix{}'.format(idx))
mask = decode_segmap(np.array(label), 'penn')
image = torchvision.utils.make_grid(images['image'][idx],
normalize=True)
image = image.numpy().transpose((1, 2, 0))
plt.title('images')
plt.imshow(mask + image)
plt.show()
def validation(self, epoch):
# self.model.load_state_dict(torch.load('model60epoch_visdrone')) #######################
self.model.eval()
self.evaluator.reset()
tbar = tqdm(self.val_loader, desc='\r')
test_loss = 0.0
for i, sample in enumerate(tbar):
image, target = sample['image'], sample['label']
if self.args.cuda:
image, target = image.cuda(), target.cuda()
with torch.no_grad():
output = self.model(image)
loss = self.criterion(output, target)
test_loss += loss.item()
tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
pred = output.data.cpu().numpy()
target = target.cpu().numpy()
trgt = target * 255###################################()
if epoch % 5 == 0:
img1 = trgt[0, :, :]############################
cv2.imwrite('visdrone/result/target_epoch_' + str(epoch) + '_' + str(i * self.btch_size) + '.png', img1)
pred = np.argmax(pred, axis=1)
prd = pred * 255############################
if epoch % 5 == 0:
img1 = prd[0, :, :]###############################
img1 = decode_segmap(img1, dataset='coco')
cv2.imwrite('visdrone/result/predict_epoch_' + str(epoch) + '_' + str(i * self.btch_size) + '.png', img1)
# Add batch sample into evaluator
self.evaluator.add_batch(target, pred)
# Fast test during the training
Acc = self.evaluator.Pixel_Accuracy()
Acc_class = self.evaluator.Pixel_Accuracy_Class()
mIoU = self.evaluator.Mean_Intersection_over_Union()
FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()
# self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch)
# self.writer.add_scalar('val/mIoU', mIoU, epoch)
# self.writer.add_scalar('val/Acc', Acc, epoch)
# self.writer.add_scalar('val/Acc_class', Acc_class, epoch)
# self.writer.add_scalar('val/fwIoU', FWIoU, epoch)
print('Validation:')
print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU))
print('Loss: %.3f' % test_loss)
def test(args):
kwargs = {'num_workers': 1, 'pin_memory': True}
_, val_loader, _, nclass = make_data_loader(args, **kwargs)
checkpoint = torch.load(args.ckpt)
if checkpoint is None:
raise ValueError
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
model = DeepLab(num_classes=nclass,
backbone='resnet',
output_stride=16,
sync_bn=True,
freeze_bn=False)
model.load_state_dict(checkpoint['state_dict'])
model.eval()
model.to(device)
torch.set_grad_enabled(False)
tbar = tqdm(val_loader)
num_img_tr = len(val_loader)
for i, sample in enumerate(tbar):
x1, x2, y1, y2 = [
int(item) for item in sample['img_meta']['bbox_coord']
] # bbox coord
w, h = x2 - x1, y2 - y1
img = sample['img_meta']['image'].squeeze().cpu().numpy()
img_w, img_h = img.shape[:2]
inputs = sample['image'].cuda()
output = model(inputs).squeeze().cpu().numpy()
pred = np.argmax(output, axis=0)
result = decode_segmap(pred, dataset=args.dataset, plot=False)
result = imresize(result, (w, h))
result_padding = np.zeros(img.shape, dtype=np.uint8)
result_padding[y1:y2, x1:x2] = result
result = img // 2 + result_padding * 127
result[result > 255] = 255
plt.imsave(
os.path.join('run', args.dataset, 'deeplab-resnet', 'output',
str(i)), result)
def visualization(self):
self.model.eval()
tbar = tqdm(self.test_loader, desc='\r')
for i, sample in enumerate(tbar):
image = sample['image']
img_path = sample['img_path']
if self.args.cuda:
image = image.cuda()
with torch.no_grad():
output = self.model(image)
tbar.set_description('Vis image:')
pred = output.data.cpu().numpy()
pred = np.argmax(pred, axis=1)[0]
rgb = decode_segmap(pred, self.args.dataset)
pred_img = Image.fromarray(pred, mode='L')
rgb_img = Image.fromarray(rgb, mode='RGB')
filename = os.path.basename(img_path[0])
pred_img.save(
os.path.join(self.args.vis_logdir, 'raw_train_id', filename))
rgb_img.save(
os.path.join(self.args.vis_logdir, 'vis_color', filename))
def save_demo_result(self, pred_dict, image_list, root_folder):
predictions = pred_dict['images']
ious = pred_dict['ious']
all_idx = np.arange(11)
imgfolder = os.path.join(root_folder, "mask//mask_refine_network")
initmaskfolder = os.path.join(root_folder, "mask")
if not os.path.exists(imgfolder):
os.mkdir(imgfolder)
for predi in range(len(predictions)):
pred_labels = np.unique(predictions[predi])
iou = ious[predi]
iou[~np.isin(all_idx, pred_labels)] = -1
initmask = cv2.imread(
os.path.join(initmaskfolder, image_list[predi]))
h, w, c = initmask.shape
imgname = os.path.join(imgfolder, image_list[predi])
segmap, segmap_gray = decode_segmap(predictions[predi],
dataset='semantic_body')
segmap_gray = cv2.resize(segmap_gray, (w, h),
interpolation=cv2.INTER_NEAREST)
cv2.imwrite(imgname, segmap_gray)
scorename, _ = os.path.splitext(imgname)
np.savetxt(scorename + ".txt", iou[1:], fmt="%.6f")
args.base_size = 513
args.crop_size = 513
comp5421_val = Comp5421Segmentation(args, split='val')
dataloader = DataLoader(comp5421_val,
batch_size=4,
shuffle=True,
num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='comp5421')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
import matplotlib.pyplot as plt
composed_transforms_tr = transforms.Compose([
tr.RandomHorizontalFlip(),
tr.ScaleNRotate(rots=(-15, 15), scales=(.75, 1.5)),
tr.FixedResize(size=512),
tr.ToTensor()])
voc_train = VOCSegmentation(split='train',
transform=composed_transforms_tr)
dataloader = DataLoader(voc_train, batch_size=2, shuffle=True, num_workers=2)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['gt'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
tmp = np.squeeze(tmp, axis=0)
segmap = decode_segmap(tmp)
img_tmp = np.transpose(img[jj], axes=[1, 2, 0]).astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
plt.show(block=True)
args.base_size = 1024
args.crop_size = 512
args.event_dim = 2
apolloscape_train = ApolloscapeRGBEvent(args, split='train')
dataloader = DataLoader(apolloscape_train,
batch_size=2,
shuffle=True,
num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='apolloscapeevent')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(311)
plt.imshow(img_tmp)
plt.subplot(312)
plt.imshow(segmap)
if ii == 2:
break
args.base_size = 513
args.crop_size = 513
voc_train = VOCSegmentation(args, split="train")
dataloader = DataLoader(voc_train,
batch_size=5,
shuffle=True,
num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample["image"].numpy()
gt = sample["label"].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset="pascal")
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title("display")
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
import pdb
parser = argparse.ArgumentParser()
args = parser.parse_args()
args.base_dir = "/Users/nopz/Google Drive/voc/VOCdevkit/VOC2012"
args.base_size = 513
args.crop_size = 513
args.dataset = "pascal"
voc_train = VOCSegmentation(args, base_dir=args.base_dir, split="train")
dl = data.DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=0)
pdb.set_trace()
for i, sample in enumerate(dl):
for j in range(sample["image"].size()[0]):
img = sample["image"].numpy()
gt = sample["label"].numpy()
tmp = np.array(gt[j]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset=args.dataset)
img_tmp = np.transpose(img[j], axes=[1, 2, 0])
img_tmp += (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
im.imsave("sgmap.jpg", segmap)
print(img_tmp.size)
if i == 1:
break
parser = argparse.ArgumentParser()
args = parser.parse_args()
args.base_size = 513
args.crop_size = 513
cityscapes_train = CityscapesSegmentation(args, split='train')
dataloader = DataLoader(cityscapes_train, batch_size=2, shuffle=True, num_workers=2)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='cityscapes')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
parser = argparse.ArgumentParser()
args = parser.parse_args()
args.base_size = 513
args.crop_size = 513
coco_val = COCOSegmentation(args, split='val', year='2017')
dataloader = DataLoader(coco_val, batch_size=4, shuffle=True, num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='coco')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
args.base_size = 600
args.crop_size = 600
Vaihingen_train = Vaihingen(args, split='val')
dataloader = DataLoader(Vaihingen_train,
batch_size=5,
shuffle=True,
num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='vaihingen')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
args.crop_size = 512
cityscapes_train = VisdroneSegmentation(args, split='train')
dataloader = DataLoader(cityscapes_train,
batch_size=2,
shuffle=True,
num_workers=2)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(
tmp, dataset='cityscapes'
) #Decode segmentation class labels into a color image
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
args.base_size = 513
args.crop_size = 513
pascal_voc_val = pascal.VOCSegmentation(args, split='val')
sbd = sbd.SBDSegmentation(args, split=['train', 'val'])
pascal_voc_train = pascal.VOCSegmentation(args, split='train')
dataset = CombineDBs([pascal_voc_train, sbd], excluded=[pascal_voc_val])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=2, shuffle=True, num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='pascal')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
plt.show(block=True)
parser = argparse.ArgumentParser()
args = parser.parse_args()
args.base_size = 513
args.crop_size = 513
voc_train = VOCSegmentation(args, split='train')
dataloader = DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='pascal')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
def adv_train(train_loader, model, criterion, optimizer, epoch, args, info, writer, dataset,
eval_score=None, print_freq=10):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
scores = AverageMeter()
# switch to train mode
model.train()
end = time.time()
for i, (input, target) in enumerate(train_loader):
# measure data loading time
data_time.update(time.time() - end)
#Get attacked image
adv_img = PGD_attack(input, target, model, criterion, args.epsilon, args.steps, args.dataset,
args.step_size, info, using_noise=True)
# input = input.cuda()
# print('diff', (adv_img.data-input) / (args.epsilon))
if type(criterion) in [torch.nn.modules.loss.L1Loss,
torch.nn.modules.loss.MSELoss]:
target = target.float()
# TODO: adversarial training
clean_input = input
input = adv_img.data
if torch.cuda.is_available():
input = input.cuda()
target = target.cuda()
target = target.cuda()
clean_input = clean_input.cuda()
input_var = torch.autograd.Variable(input)
clean_input_var = torch.autograd.Variable(clean_input)
target_var = torch.autograd.Variable(target)
if args.debug:
# Debug visualisations
print("\nVisualising here.\n",
# Add things to print here
)
# print("Max in difference is ", np.max(diff_img))
arr_clean_img = clean_input_var.clone().cpu().numpy() if torch.cuda.is_available() else clean_input_var.clone().numpy()
arr_adv_img = input_var.clone().cpu().numpy() if torch.cuda.is_available() else input_var.clone().numpy()
vis_clean_img = np.moveaxis(np.copy(arr_clean_img)[0], 0, 2)
vis_adv_img = np.moveaxis(np.copy(arr_adv_img)[0], 0, 2)
vis_clean_img_tr = np.moveaxis(back_transform(np.copy(arr_clean_img), info)[0],0,2) # np.copy is necessary to make sure that the original arrays are not modified.
vis_adv_img_tr = np.moveaxis(back_transform(np.copy(arr_adv_img), info)[0],0,2) # np.copy is necessary to make sure that the original arrays are not modified.
diff_img1 = vis_adv_img - vis_clean_img
diff_img2 = vis_adv_img_tr - vis_clean_img_tr
f, axarr = plt.subplots(1, 6)
axarr[0].imshow(vis_clean_img)
axarr[1].imshow(vis_adv_img)
axarr[2].imshow(vis_clean_img_tr)
axarr[3].imshow(vis_adv_img_tr)
# print()
axarr[4].imshow(diff_img1)
axarr[5].imshow(diff_img2)
plt.show()
print("__max diff is__",np.max(diff_img1)*255,np.max(diff_img2)*255)
# compute output
output = model(input_var)[0]
loss = criterion(output, target_var)
losses.update(loss.data.item(), input.size(0))
if eval_score is not None:
if target_var.size(0)>0:
scores.update(eval_score(output, target_var), input.size(0))
else:
print("0 size!")
clean_score = 0
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if args.debug:
print_freq = 10
if i % print_freq == 0:
# Convert target and prediction to rgb images to visualise
class_prediction = torch.argmax(output, dim=1)
decoded_target = decode_segmap(target[0].cpu().numpy() if torch.cuda.is_available() else target[0].numpy(),
dataset)
decoded_target = np.moveaxis(decoded_target, 2, 0)
decoded_class_prediction = decode_segmap(
class_prediction[0].cpu().numpy() if torch.cuda.is_available() else class_prediction[0].numpy(),
dataset)
decoded_class_prediction = np.moveaxis(decoded_class_prediction, 2, 0)
logger.info('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Score {top1.val:.3f} ({top1.avg:.3f})'.format(
epoch, i, len(train_loader), batch_time=batch_time,
data_time=data_time, loss=losses, top1=scores))
#TODO: Get training accuracy for non adv images. (forward the clean data.)
# compute output for the case when clean image is passed.
clean_output = model(clean_input_var)[0]
clean_loss = criterion(clean_output, target_var)
if eval_score is not None:
if target_var.size(0) > 0:
clean_score = eval_score(clean_output, target_var)
else:
print("0 size!")
clean_score = 0
# print('img max m,in', torch.max(input_var[0]), torch.min(input_var[0]))
# print('img max m,in', torch.max(clean_input_var[0]), torch.min(clean_input_var[0]))
# input_var[0] = (back_transform(input_var[0], info) * 255).long()
writer.add_image('Image/adv image ', back_transform(input_var, info)[0])
writer.add_image('Image/clean image ', back_transform(clean_input_var, info)[0])
writer.add_image('Image/image target ', decoded_target)
writer.add_image('Image/image prediction ', decoded_class_prediction)
writer.add_scalar('Adv_Train/Score', scores.val, epoch*len(train_loader)+i)
writer.add_scalar('Adv_Train/Loss', losses.val, epoch*len(train_loader)+i)
writer.add_scalar('Adv_Train/Clean_Score', clean_score, epoch*len(train_loader)+i)
writer.add_scalar('Adv_Train/Clean_Loss', clean_loss, epoch*len(train_loader)+i)
if args.debug and i == print_freq:
break
parser = argparse.ArgumentParser()
args = parser.parse_args()
args.base_size = 513
args.crop_size = 513
coco_val = COCOSegmentation(args, split="val", year="2017")
dataloader = DataLoader(coco_val, batch_size=4, shuffle=True, num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample["image"].numpy()
gt = sample["label"].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset="coco")
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title("display")
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
parser = argparse.ArgumentParser()
args = parser.parse_args()
args.base_size = 513
args.crop_size = 513
coco_val = COCOSegmentation(args, split='val', year='2017')
dataloader = DataLoader(coco_val, batch_size=4, shuffle=True, num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='coco')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
tr.ToTensor()])
voc_train = VOCSegmentation(split='train',
transform=composed_transforms_tr)
dataloader = DataLoader(voc_train, batch_size=5, shuffle=True, num_workers=2)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
tmp = np.squeeze(tmp, axis=0)
segmap = decode_segmap(tmp)
img_tmp = np.transpose(img[jj], axes=[1, 2, 0]).astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
plt.show(block=True)
# import torch
#
# composed_transforms_tr = transforms.Compose([
args.positive_only = False
siim_train = SIIMDataset(args, split='train')
print('Number of training images: ', len(siim_train))
dataloader = DataLoader(siim_train,
batch_size=2,
shuffle=True,
num_workers=2)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='siim')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
args.base_size = 513
args.crop_size = 513
cityscapes_train = CityscapesSegmentation(args, split='train')
dataloader = DataLoader(cityscapes_train,
batch_size=2,
shuffle=True,
num_workers=2)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='cityscapes')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
if ii == 1:
break
dataset = FashionDataset(args, Path.db_root_dir('fashion_clothes'),
'train', 'clothes')
dataloader = DataLoader(dataset,
batch_size=1,
shuffle=False,
num_workers=0)
for ii, sample in enumerate(dataloader):
for jj in range(sample["image"].size()[0]):
img = sample['image'].numpy()
gt = sample['label'].numpy()
tmp = np.array(gt[jj]).astype(np.uint8)
segmap = decode_segmap(tmp, dataset='fashion_clothes')
img_tmp = np.transpose(img[jj], axes=[1, 2, 0])
img_tmp *= (0.229, 0.224, 0.225)
img_tmp += (0.485, 0.456, 0.406)
img_tmp *= 255.0
img_tmp = img_tmp.astype(np.uint8)
plt.figure()
plt.title('display')
plt.subplot(211)
plt.imshow(img_tmp)
plt.subplot(212)
plt.imshow(segmap)
break
plt.show(block=True)
