def read(self, frame_count):
if self.fps > 0.0:
t0 = time.time()
w, h = self.frame_size
if self._bg is None:
buf = np.zeros((h, w, 3), np.uint8)
else:
buf = self._bg.copy()
self.render(buf, frame_count)
if self._noise > 0.0:
noise = np.zeros((h, w, 3), np.int8)
cv2.randn(noise, np.zeros(3), np.ones(3)*255*self._noise)
buf = cv2.add(buf, noise, dtype=cv2.CV_8UC3)
if self.fps > 0.0:
t1 = time.time()
#sleep the difference between the desired fps and our current fps
#actually sleep a little less than the differnce because of jitter
ddt = (1/self.fps) - (t1 - t0)
if ddt > 0:
time.sleep(ddt)
return True, buf
def add_noise(image, channel, mean=0, sigma=1):
noise = np.zeros(img.shape[:-1], dtype='uint8')
cv2.randn(noise, mean, sigma)
noisy_image = np.copy(image)
noisy_image[:,:,channel] += noise
noisy_image = noisy_image.astype('uint8')
return noisy_image
def getNoised(image):
noise = np.zeros(image.shape)
cv2.randn(noise, 0, 5)
noise = noise.astype(np.int32)
noise[noise < 0] = 0
res = image.astype(np.int32) + noise
res[res < 0] = 0
res[res > 255] = 255
return res.astype(np.uint8)
def add_noise(img):
"""Adds noise to inputted image.
input: img - A numpy array representing an image.
output: (numpy array) - New image with noise added.
"""
noise = np.zeros(img.shape, dtype=np.uint8)
cv2.randn(noise, 0, 150)
new_img = img + noise
return new_img
def read(self, dst=None):
"""reads the video"""
width, heigth = self.frame_size
if self.fallback is None:
buf = np.zeros((heigth, width, 3), np.uint8)
else:
buf = self.fallback.copy()
if self.noise > 0.0:
noise = np.zeros((heigth, width, 3), np.int8)
cv2.randn(noise, np.zeros(3), np.ones(3)*255*self.noise)
buf = cv2.add(buf, noise, dtype=cv2.CV_8UC3)
return True, buf
def add_noise(image):
image_normalized = np.zeros(image.shape)
noise = np.zeros(image.shape)
cv2.normalize(image, image_normalized, 0, 1, cv2.NORM_MINMAX, cv2.CV_64F)
cv2.randn(noise, 0, random.random()/5)
image_normalized = image_normalized + noise
cv2.normalize(image_normalized, image_normalized, 0, 1, cv2.NORM_MINMAX, cv2.CV_64F)
cv2.normalize(image_normalized, image_normalized, 0, 255, cv2.NORM_MINMAX, cv2.CV_64F)
image_normalized = image_normalized + random.randint(-50,100)
image_normalized[image_normalized > 255] = 255
image_normalized[image_normalized < 0] = 0
image_normalized = cv2.GaussianBlur(image_normalized, (9,9), 0)
image_normalized = image_normalized.astype(np.uint8, copy=False)
return image_normalized
def read(self, dst=None):
w, h = self.frame_size
if self.bg is None:
buf = np.zeros((h, w, 3), np.uint8)
else:
buf = self.bg.copy()
self.render(buf)
if self.noise > 0.0:
noise = np.zeros((h, w, 3), np.int8)
cv2.randn(noise, np.zeros(3), np.ones(3) * 255 * self.noise)
buf = cv2.add(buf, noise, dtype=cv2.CV_8UC3)
return True, buf
def main():
img = cv2.imread('data/messi5.jpg',cv2.IMREAD_UNCHANGED)
noise = img.copy()
cv2.randn(noise,30,10)
imgNoised = img + noise
cv2.namedWindow('raw', cv2.WINDOW_NORMAL)
cv2.imshow('raw', img)
cv2.namedWindow('test', cv2.WINDOW_NORMAL)
cv2.imshow('test', imgNoised)
k = cv2.waitKey(0)
if k == 27: # wait for ESC key to exit
cv2.destroyAllWindows()
elif k == ord('s'): # wait for 's' key to save and exit
cv2.imwrite('test.png',img)
cv2.destroyAllWindows()
def getNextFrame(self):
img = self.sceneBg.copy()
if self.foreground is not None:
self.currentCenter = (self.center[0] + self.getXOffset(self.time), self.center[1] + self.getYOffset(self.time))
img[self.currentCenter[0]:self.currentCenter[0]+self.foreground.shape[0],
self.currentCenter[1]:self.currentCenter[1]+self.foreground.shape[1]] = self.foreground
else:
self.currentRect = self.initialRect + np.int( 30*cos(self.time) + 50*sin(self.time/3))
if self.deformation:
self.currentRect[1:3] += int(self.h/20*cos(self.time))
cv.fillConvexPoly(img, self.currentRect, (0, 0, 255))
self.time += self.timeStep
if self.noise:
noise = np.zeros(self.sceneBg.shape, np.int8)
cv.randn(noise, np.zeros(3), np.ones(3)*255*self.noise)
img = cv.add(img, noise, dtype=cv.CV_8UC3)
return img
def remove_bg(file,output_file=None,show_intermediate=False,
sobel_ksize=(5,5),gauss_ksize=(9,9), high_bg_noise=False,
gray_threshold=30,erosion_ksize=(0,0),new_bg_color=(255,255,0)):
# 1) Load the source image
img = cv2.imread(file)
# if show_intermediate:
# plt.imshow(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))
# plt.show()
# 2) Convert to Grayscale
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# if show_intermediate:
# plt.imshow(img_gray, cmap='gray')
# plt.show()
# 2.5) Invert the grayscale image
img_negate = 255 - img_gray
# if show_intermediate:
# plt.imshow(img_negate, cmap='gray')
# plt.show()
# 3) Get the Sobel-filtered negative
# Get the horizontal Sobel
#!!!!!!! PARAMETER !!!!!!!!! (ksize)
img_sobel_h_64f = cv2.Sobel(img_negate,cv2.CV_64F,1,0,ksize=sobel_ksize[0])
img_sobel_h_64f_abs = np.absolute(img_sobel_h_64f)
img_sobel_h_8u = np.uint8(img_sobel_h_64f_abs)
# Get the vertical Sobel
img_sobel_v_64f = cv2.Sobel(img_negate,cv2.CV_64F,0,1,ksize=sobel_ksize[1])
img_sobel_v_64f_abs = np.absolute(img_sobel_v_64f)
img_sobel_v_8u = np.uint8(img_sobel_v_64f_abs)
# Bitwise OR those two
img_sobel = cv2.bitwise_or(img_sobel_h_8u,img_sobel_v_8u)
#print(img_sobel[:10,:10])
# if show_intermediate:
# plt.imshow(img_sobel, cmap='gray')
# plt.show()
# 4) Gaussian blur the Sobel
#!!!!!!! PARAMETER !!!!!!!!! (gauss kernel size)
img_gauss = cv2.GaussianBlur(img_sobel, gauss_ksize, 0)
# if show_intermediate:
# plt.imshow(img_gauss, cmap='gray')
# plt.show()
# 4.5) Negate again (OPTIONAL)
if high_bg_noise == True:
img_gauss = 255 - img_gauss
# 5) Threshold-filter to drop away (to pure black) anything that's not white-ish
threshold = gray_threshold #!!!!!!! PARAMETER !!!!!!!!! (threshold)
ret,img_tozero = cv2.threshold(img_gauss,threshold,255,cv2.THRESH_TOZERO)
# if show_intermediate:
# plt.imshow(img_tozero, cmap='gray')
# plt.show()
# 5.5) Convert from single-channel (grayscale) back to BGR
img_tozero_bgr = cv2.cvtColor(img_tozero, cv2.COLOR_GRAY2BGR)
# if show_intermediate:
# plt.imshow(img_tozero_bgr)
# plt.show()
# 6) Local adaptive threshold - looks for lone pixels in 5x5 areas and removes those which are not in a cluster
# TODO: implement
# 7) Flood fill - fills
# TODO: update to flood fill from all of the corners
mask = np.zeros((img_tozero.shape[0]+2,img_tozero.shape[1]+2),np.uint8)
diff = (6,6,6)
cv2.floodFill(image=img_tozero_bgr,mask=mask,seedPoint=(0,0),
newVal=(255,0,255),loDiff=diff,upDiff=diff, flags=4)
cv2.floodFill(image=img_tozero_bgr,mask=mask,seedPoint=(img_tozero.shape[1]-1,0),
newVal=(255,0,255),loDiff=diff,upDiff=diff, flags=4)
cv2.floodFill(image=img_tozero_bgr,mask=mask,seedPoint=(img_tozero.shape[1]-1,img_tozero.shape[0]-1),
newVal=(255,0,255),loDiff=diff,upDiff=diff, flags=4)
cv2.floodFill(image=img_tozero_bgr,mask=mask,seedPoint=(0,img_tozero.shape[0]-1),
newVal=(255,0,255),loDiff=diff,upDiff=diff, flags=4)
# if show_intermediate:
# plt.imshow(img_tozero_bgr)
# plt.show()
# 8) Resulting Mask
# if show_intermediate:
# plt.imshow(mask, cmap='gray')
# plt.show()
mask = mask[1:-1,1:-1]
mask = cv2.bitwise_xor(mask,np.ones(mask.shape,np.uint8)) # swap 0s/1s
# insert erosion here
kernel = np.ones((5,5),np.uint8)
mask = cv2.erode(mask,kernel,iterations=1)
foreground_pass_mask = cv2.cvtColor(mask*255,cv2.COLOR_GRAY2BGR)
background_pass_mask = cv2.bitwise_not(foreground_pass_mask)
# 9) get new background
if type(new_bg_color) == tuple and len(new_bg_color)==3:
new_bg = np.full(img.shape,new_bg_color,dtype=np.uint8)
elif new_bg_color == 'white_noise':
new_bg = np.random.randint(256, size=img.shape).astype(np.uint8)
elif new_bg_color == 'normal_noise':
new_bg = cv2.randn(np.zeros(img.shape,dtype=np.uint8),(127,127,127),(100,100,100))
print('made it here')
else:
print('Using white background')
new_bg = np.ones(img.shape) * 255
background = cv2.bitwise_and(new_bg,background_pass_mask)
# 10) Get Foreground
foreground = cv2.bitwise_and(img,foreground_pass_mask)
# plt.imshow(foreground)
# 11) Add Foreground to New Background (Output Image)
output_img = foreground + background
plt.imshow(output_img)
plt.show()
if show_intermediate:
titles = ['1) Original Image', '2) Negate', '3) Sobel Filter',
'4) Gaussian Blur', '5) Threshold', '6) Local Adaptive Threshold',
'7) Flood Fill', '8) OR Mask', '9) Background',
'10) Foreground', '11) New Image']
images = [img, img_negate, img_sobel, img_gauss, img_tozero, img_tozero,
img_tozero_bgr, background_pass_mask, background,
foreground, output_img]
for i in [0,6,7,8,9,10]:
plt.subplot(3,4,i+1),plt.imshow(cv2.cvtColor(images[i],cv2.COLOR_BGR2RGB))
plt.title(titles[i])
plt.xticks([]),plt.yticks([])
for i in [1,2,3,4,5]:
plt.subplot(3,4,i+1),plt.imshow(images[i], cmap='gray')
plt.title(titles[i])
plt.xticks([]),plt.yticks([])
plt.show()
if output_file:
print(output_file)
cv2.imwrite(output_file,output_img)
# imports
import numpy as np
import cv2
import matplotlib.pyplot as plt
img = cv2.imread('images/saturn.png', 0)
############################################## REMOVING NOISE WITH GAUSSIAN BLUR
# fist create a noisy image
# create an empty np array full of zeros the same size as the image
im = np.zeros(img.shape[:2],
np.uint8) # do not use original image it overwrites it
mean = 0 # the gaussian mean
sigma_noise = 30 # the gaussian sigma
gaussian = cv2.randn(im, mean, sigma_noise) # create the random distribution
img_noisy = cv2.add(img, gaussian) # add the noise to the original image
# clean up the noise
gauss_noise_remv = cv2.GaussianBlur(img_noisy, (21, 21), 2)
# plot original, noisy and gaussian noise removal together
plt.figure(figsize=(10, 8))
plt.subplot(131), plt.imshow(img, cmap='gray', interpolation='bicubic')
plt.title('Original'), plt.xticks([]), plt.yticks([])
plt.subplot(132), plt.imshow(img_noisy, cmap='gray', interpolation='bicubic')
plt.title('Noisy'), plt.xticks([]), plt.yticks([])
plt.subplot(133), plt.imshow(gauss_noise_remv,
cmap='gray',
interpolation='bicubic')
plt.title('Gaussian Noise Removal'), plt.xticks([]), plt.yticks([])
def generate_data(folder):
pixel = 32
if os.path.isfile('training-data.pkl') == False:
imgnames = []
for fname in glob.glob(folder+'*.jpg'):
imgnames.append(fname)
labels = []
data = []
pbar = ProgressBar()
for i in pbar(range(len(imgnames))):
img = cv2.imread(imgnames[i])
#Bug with cv2.imread, not able to read some images (8)
if img == None:
continue
labels.append(int(imgnames[i][7])-1)
#Resizing to pixel x pixel
img_small = cv2.resize(img, (pixel,pixel), \
interpolation = cv2.INTER_AREA)
#Flatten the image
data.append(img_small.reshape(-1))
#Data aumgentation
#1. Flip image vertically
labels.append(int(imgnames[i][7])-1)
flip_img = cv2.flip(img,1)
flip_img = cv2.resize(flip_img, (pixel,pixel), \
interpolation = cv2.INTER_AREA)
data.append(flip_img.reshape(-1))
#2. Add gaussian noise
labels.append(int(imgnames[i][7])-1)
noise = np.zeros([img.shape[0], img.shape[1], img.shape[2]])
m = (50,50,50)
s = (50,50,50)
cv2.randn(noise,m,s)
noise_img = img + noise
noise_img = cv2.resize(noise_img, (pixel,pixel), \
interpolation = cv2.INTER_AREA)
data.append(noise_img.reshape(-1))
#3. Increasing contrast
labels.append(int(imgnames[i][7])-1)
img_yuv = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
img_yuv[:,:,0] = cv2.equalizeHist(img_yuv[:,:,0])
contrast_img = cv2.cvtColor(img_yuv, cv2.COLOR_YUV2BGR)
contrast_img = cv2.resize(contrast_img, (pixel,pixel), \
interpolation = cv2.INTER_AREA)
data.append(contrast_img.reshape(-1))
#4. Crop image
labels.append(int(imgnames[i][7])-1)
crop_img = img[int(0.1*img.shape[0]):img.shape[0]-int(0.1*img.shape[0]) \
,int(0.1*img.shape[1]) :img.shape[1]-int(0.1*img.shape[0]), :]
crop_img = cv2.resize(crop_img, (pixel,pixel), \
interpolation = cv2.INTER_AREA)
data.append(crop_img.reshape(-1))
#5. Rotate image by 20-degrees
labels.append(int(imgnames[i][7])-1)
M = cv2.getRotationMatrix2D((img.shape[1]/2,img.shape[0]/2),20,1)
rot_img = cv2.warpAffine(img,M,(img.shape[1],img.shape[0]))
rot_img = cv2.resize(rot_img, (pixel,pixel), \
interpolation = cv2.INTER_AREA)
data.append(rot_img.reshape(-1))
# Subtract mean and normalization
mean = np.mean(data, axis=0)
sd = np.std(data, axis = 0)
data = np.asarray(data, dtype = np.float32)
data -= mean
data /= sd
#Generating one-hot labels
classes = 3
labels = np.asarray(labels, dtype = np.int32)
one_hot = np.zeros((labels.shape[0], classes))
one_hot[np.arange(labels.shape[0]), labels] = 1
with open('training-data.pkl', 'wb') as f:
pickle.dump([data, labels, one_hot, mean, sd], f)
else:
with open('training-data.pkl', 'rb') as f:
data, labels, one_hot, mean, sd = pickle.load(f)
print('Number of images in class-1: {}'.format((labels == 0).sum()))
print('Number of images in class-2: {}'.format((labels == 1).sum()))
print('Number of images in class-3: {}'.format((labels == 2).sum()))
#Make sure data dimensions are correct
assert data.shape[0] == one_hot.shape[0]
total = data.shape[0]
#Shuffling data
temp = zip(data,one_hot)
np.random.shuffle(temp)
data, one_hot = zip(*temp)
data = np.asarray(data, dtype = np.float32)
one_hot = np.asarray(one_hot, dtype = np.int32)
#Splitting into train and test (3:1)
trainX = data[:int(round(total*0.75)),:]
trainY = one_hot[:int(round(total*0.75))]
testX = data[int(round(total*0.75)):,:]
testY = one_hot[int(round(total*0.75)):]
#Some dimension checks
assert trainX.shape[0] == trainY.shape[0]
assert testX.shape[0] == testY.shape[0]
print('Size of training set: {}'.format(trainX.shape[0]))
print('Size of test set: {}'.format(testX.shape[0]))
return trainX, trainY, testX, testY
def Integrated_Mask(img,
blurred_img,
model,
category,
max_iterations=15,
integ_iter=20,
tv_beta=2,
l1_coeff=0.01 * 300,
tv_coeff=0.2 * 300,
size_init=112,
use_cuda=1):
########################
# IGOS: using integrated gradient descent to find the smallest and smoothest area that maximally decrease the
# output of a deep model
# Parameters:
# -------------
# img: the original input image
# blurred_img: the baseline for the input image
# model: the model that you want to visualize
# category: the classification target that you want to visualize (category=-1 means the top 1 classification label)
# max_iterations: the max iterations for the integrated gradient descent
# integ_iter: how many points you want to use when computing the integrated gradients
# tv_beta: which norm you want to use for the total variation term
# l1_coeff: parameter for the L1 norm
# tv_coeff: parameter for the total variation term
# size_init: the resolution of the mask that you want to generate
# use_cuda: use gpu (1) or not (0)
####################################################
# preprocess the input image and the baseline image
img = preprocess_image(img, use_cuda, require_grad=False)
blurred_img = preprocess_image(blurred_img, use_cuda, require_grad=False)
resize_size = img.data.shape
resize_wh = (img.data.shape[2], img.data.shape[3])
if use_cuda:
zero_img = Variable(torch.zeros(resize_size).cuda(),
requires_grad=False)
else:
zero_img = Variable(torch.zeros(resize_size), requires_grad=False)
# initialize the mask
mask_init = np.ones((size_init, size_init), dtype=np.float32)
mask = numpy_to_torch(mask_init, use_cuda, requires_grad=True)
if use_cuda:
upsample = torch.nn.UpsamplingBilinear2d(size=resize_wh).cuda()
else:
upsample = torch.nn.UpsamplingBilinear2d(size=resize_wh)
# You can choose any optimizer
# The optimizer doesn't matter, because we don't need optimizer.step(), we just use it to compute the gradient
optimizer = torch.optim.Adam([mask], lr=0.1)
#optimizer = torch.optim.SGD([mask], lr=0.1)
target = torch.nn.Softmax(dim=1)(model(img))
if use_cuda:
category_out = np.argmax(target.cpu().data.numpy())
else:
category_out = np.argmax(target.data.numpy())
# if category=-1, choose the original top 1 category as the one that you want to visualize
if category == -1:
category = category_out
print("Category with highest probability", category_out)
print("Category want to generate mask", category)
print("Optimizing.. ")
curve1 = np.array([])
curve2 = np.array([])
curvetop = np.array([])
# Integrated gradient descent
alpha = 0.0001
beta = 0.2
for i in range(max_iterations):
upsampled_mask = upsample(mask)
# The single channel mask is used with an RGB image,
# so the mask is duplicated to have 3 channels
upsampled_mask = \
upsampled_mask.expand(1, 3, upsampled_mask.size(2), \
upsampled_mask.size(3))
# the l1 term and the total variation term
loss1 = l1_coeff * torch.mean(torch.abs(1 - mask)) + \
tv_coeff * tv_norm(mask, tv_beta)
loss_all = loss1.clone()
# compute the perturbed image
perturbated_input_base = img.mul(upsampled_mask) + \
blurred_img.mul(1 - upsampled_mask)
for inte_i in range(integ_iter):
# Use the mask to perturbated the input image.
integ_mask = 0.0 + ((inte_i + 1.0) / integ_iter) * upsampled_mask
perturbated_input_integ = img.mul(integ_mask) + \
blurred_img.mul(1 - integ_mask)
# add noise
noise = np.zeros((resize_wh[0], resize_wh[1], 3), dtype=np.float32)
noise = noise + cv2.randn(noise, 0, 0.2)
noise = numpy_to_torch(noise, use_cuda, requires_grad=False)
perturbated_input = perturbated_input_integ + noise
new_image = perturbated_input
outputs = torch.nn.Softmax(dim=1)(model(new_image))
loss2 = outputs[0, category]
loss_all = loss_all + loss2 / 20.0
# compute the integrated gradients for the given target,
# and compute the gradient for the l1 term and the total variation term
optimizer.zero_grad()
loss_all.backward()
whole_grad = mask.grad.data.clone()
loss2_ori = torch.nn.Softmax(dim=1)(
model(perturbated_input_base))[0, category]
loss_ori = loss1 + loss2_ori
if i == 0:
if use_cuda:
curve1 = np.append(curve1, loss1.data.cpu().numpy())
curve2 = np.append(curve2, loss2_ori.data.cpu().numpy())
curvetop = np.append(curvetop, loss2_ori.data.cpu().numpy())
else:
curve1 = np.append(curve1, loss1.data.numpy())
curve2 = np.append(curve2, loss2_ori.data.numpy())
curvetop = np.append(curvetop, loss2_ori.data.numpy())
if use_cuda:
loss_oridata = loss_ori.data.cpu().numpy()
else:
loss_oridata = loss_ori.data.numpy()
# LINE SEARCH with revised Armijo condition
step = 200.0
MaskClone = mask.data.clone()
MaskClone -= step * whole_grad
MaskClone = Variable(MaskClone, requires_grad=False)
MaskClone.data.clamp_(0, 1) # clamp the value of mask in [0,1]
mask_LS = upsample(MaskClone) # Here the direction is the whole_grad
Img_LS = img.mul(mask_LS) + \
blurred_img.mul(1 - mask_LS)
outputsLS = torch.nn.Softmax(dim=1)(model(Img_LS))
loss_LS = l1_coeff * torch.mean(torch.abs(1 - MaskClone)) + \
tv_coeff * tv_norm(MaskClone, tv_beta) + outputsLS[0, category]
if use_cuda:
loss_LSdata = loss_LS.data.cpu().numpy()
else:
loss_LSdata = loss_LS.data.numpy()
new_condition = whole_grad**2 # Here the direction is the whole_grad
new_condition = new_condition.sum()
new_condition = alpha * step * new_condition
while loss_LSdata > loss_oridata - new_condition.cpu().numpy():
step *= beta
MaskClone = mask.data.clone()
MaskClone -= step * whole_grad
MaskClone = Variable(MaskClone, requires_grad=False)
MaskClone.data.clamp_(0, 1)
mask_LS = upsample(MaskClone)
Img_LS = img.mul(mask_LS) + \
blurred_img.mul(1 - mask_LS)
outputsLS = torch.nn.Softmax(dim=1)(model(Img_LS))
loss_LS = l1_coeff * torch.mean(torch.abs(1 - MaskClone)) + \
tv_coeff * tv_norm(MaskClone, tv_beta) + outputsLS[0, category]
if use_cuda:
loss_LSdata = loss_LS.data.cpu().numpy()
else:
loss_LSdata = loss_LS.data.numpy()
new_condition = whole_grad**2 # Here the direction is the whole_grad
new_condition = new_condition.sum()
new_condition = alpha * step * new_condition
if step < 0.00001:
break
mask.data -= step * whole_grad
#######################################################################################################
if use_cuda:
curve1 = np.append(curve1, loss1.data.cpu().numpy())
curve2 = np.append(curve2, loss2_ori.data.cpu().numpy())
else:
curve1 = np.append(curve1, loss1.data.numpy())
curve2 = np.append(curve2, loss2_ori.data.numpy())
mask.data.clamp_(0, 1)
if use_cuda:
maskdata = mask.data.cpu().numpy()
else:
maskdata = mask.data.numpy()
maskdata = np.squeeze(maskdata)
maskdata, imgratio = topmaxPixel(maskdata, 40)
maskdata = np.expand_dims(maskdata, axis=0)
maskdata = np.expand_dims(maskdata, axis=0)
###############################################
if use_cuda:
Masktop = torch.from_numpy(maskdata).cuda()
else:
Masktop = torch.from_numpy(maskdata)
# Use the mask to perturbated the input image.
Masktop = Variable(Masktop, requires_grad=False)
MasktopLS = upsample(Masktop)
Img_topLS = img.mul(MasktopLS) + \
blurred_img.mul(1 - MasktopLS)
outputstopLS = torch.nn.Softmax(dim=1)(model(Img_topLS))
loss_top1 = l1_coeff * torch.mean(torch.abs(1 - Masktop)) + \
tv_coeff * tv_norm(Masktop, tv_beta)
loss_top2 = outputstopLS[0, category]
if use_cuda:
curvetop = np.append(curvetop, loss_top2.data.cpu().numpy())
else:
curvetop = np.append(curvetop, loss_top2.data.numpy())
if max_iterations > 3:
if i == int(max_iterations / 2):
if np.abs(curve2[0] - curve2[i]) <= 0.001:
print('Adjust Parameter l1_coeff at iteration:',
int(max_iterations / 2))
l1_coeff = l1_coeff / 10
elif i == int(max_iterations / 1.25):
if np.abs(curve2[0] - curve2[i]) <= 0.01:
print('Adjust Parameters l1_coeff again at iteration:',
int(max_iterations / 1.25))
l1_coeff = l1_coeff / 5
#######################################################################################
upsampled_mask = upsample(mask)
if use_cuda:
mask = mask.data.cpu().numpy().copy()
else:
mask = mask.data.numpy().copy()
return mask, upsampled_mask, imgratio, curvetop, curve1, curve2, category
name = "Marlboro_adv"
for i, angle in enumerate([90, 180]):
rotated = np.rot90(
temp0, k=i + 1) # NB: rotate not good here, turns into float!
listTemplate.append((name, rotated))
try:
image = io.imread(file, 0) ########## reading image
image = image[0:500, 165:500]
name1 = file.split("/")[-1]
name1 = name1.split(".")[0]
im = image
noise = np.empty_like(im, dtype="int8")
level = 10
cv2.randn(noise, (0), (level)) # Matrix element are 0 in average
imageNoise = cv2.add(im, noise, dtype=cv2.CV_8U)
Hits_Noise = matchTemplates(listTemplate,
imageNoise,
N_object=90,
score_threshold=0.001,
method=cv2.TM_CCOEFF_NORMED,
maxOverlap=.08)
H = Hits_Noise
df = Hits_Noise.reset_index()
w = df["BBox"].str[2]
h = df["BBox"].str[3]
def compute_heatmap(model,
original_img,
params,
mask_init,
use_cuda=False,
gpu_id=0,
verbose=False):
'''Compute image heatmaps according to: https://arxiv.org/abs/1704.03296
Interpretable Explanations of Black Boxes by Meaningful Perturbation
Params:
model : deep neural network or other black box model; e.g. VGG
params : namedtuple/recordclass of settings
original_img : input image, RGB-8bit
mask_init : init heatmap
use_cuda : enable/disable GPU usage
'''
# scale between 0 and 1 with 32-bit color depth
img = np.float32(original_img) / 255
# generate a perturbated version of the input image
blurred_img_numpy = cv2.GaussianBlur(img, (11, 11), 10)
# prepare image to feed to the model
img = utils.preprocess_image(img, use_cuda,
gpu_id=gpu_id) # original image
blurred_img = utils.preprocess_image(
blurred_img_numpy, use_cuda,
gpu_id=gpu_id) # blurred version of input image
mask = utils.numpy_to_torch(mask_init, use_cuda=use_cuda,
gpu_id=gpu_id) # init mask
upsample = torch.nn.Upsample(size=params.target_shape, mode='bilinear')
blur = utils.BlurTensor(use_cuda, gpu_id=gpu_id)
if use_cuda:
upsample = upsample.cuda(gpu_id)
# optimize only the heatmap
optimizer = torch.optim.Adam([mask], lr=params.learning_rate)
# compute the target output
target_preds = model(img)
targets = torch.nn.Softmax(dim=1)(target_preds)
category, target_prob, label = utils.get_class_info(targets)
if verbose:
print("Category with highest probability:",
(label, category, target_prob))
if params.target_id is not None:
if category != params.target_id:
print("Wrong classification! Skipping")
return None
loss_history = []
if verbose:
print("Optimizing.. ")
for i in range(params.max_iterations):
# upsample the mask and use it
# the mask is duplicated to have 3 channels since it is
# single channel and is used with a 224*224 RGB image
# NOTE: the upsampled mask is only used to compute the
# perturbation on the input image
upsampled_mask = upsample(mask)
if params.blur:
upsampled_mask = blur(upsampled_mask, 5)
upsampled_mask = upsampled_mask.expand(1, 3, *params.target_shape)
# use the (upsampled) mask to perturbated the input image
# blend the median blurred image and the original (scaled) image
# accordingly to the current (upsampled) mask
perturbated_input = img.mul(upsampled_mask) + \
blurred_img.mul(1 - upsampled_mask)
# gaussian noise with is added to the preprocssed image
# at each iteration, inspired by google's smooth gradient
# https://arxiv.org/abs/1706.03825
# https://pair-code.github.io/saliency/
noise = np.zeros(params.target_shape + (3, ), dtype=np.float32)
if params.noise_sigma != 0:
noise = noise + cv2.randn(noise, 0., params.noise_sigma)
noise = utils.numpy_to_torch(noise, use_cuda=use_cuda, gpu_id=gpu_id)
noisy_perturbated_input = perturbated_input + noise * params.noise_scale
# compute current prediction
preds = model(noisy_perturbated_input)
outputs = torch.nn.Softmax(dim=1)(preds)
# compute the loss and use the regularizers
class_loss = outputs[0, category]
l1_loss = params.l1_coeff * l1_reg(mask)
tv_loss = params.tv_coeff * tv_reg(mask, params.tv_beta)
lasso_loss = params.lasso_coeff * lasso_reg(mask)
less_loss = params.less_coeff * less_reg(preds, target_preds)
losses = [class_loss, l1_loss, tv_loss, lasso_loss, less_loss]
total_loss = np.sum(losses)
# convert loss tensors to scalars
losses = [total_loss.data.cpu().squeeze().numpy()[0]
] + [l.data.cpu().numpy()[0] for l in losses]
loss_history.append(losses)
# update the optimization process
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
# optional: clamping seems to give better results
# should be useless, but numerical s**t happens
mask.data.clamp_(0, 1)
# upsample the computed final mask
upsampled_mask = upsample(mask)
if params.blur:
upsampled_mask = blur(upsampled_mask, 5)
perturbated_input = img.mul(upsampled_mask) + \
blurred_img.mul(1 - upsampled_mask)
# compute the prediction probabilities before
# and after the perturbation and masking
outputs = torch.nn.Softmax(dim=1)(model(perturbated_input))
output_prob = outputs[0, category].data.cpu().squeeze().numpy()[0]
# compute the prediction on the completely blurred image
outputs = torch.nn.Softmax(dim=1)(model(blurred_img))
blurred_prob = outputs[0, category].data.cpu().squeeze().numpy()[0]
return upsampled_mask, blurred_img_numpy, target_prob, output_prob, blurred_prob, np.asarray(
loss_history), category
def read(self, dst=None):
noise = np.zeros(self.render.sceneBg.shape, np.int8)
cv2.randn(noise, np.zeros(3), np.ones(3)*255*self.noise)
return True, cv2.add(self.render.getNextFrame(), noise, dtype=cv2.CV_8UC3)
def Preprocessing(mode, in_path, out_path, filters_dir):
k_names = os.listdir(filters_dir)
kernels = [filters_dir + n for n in k_names]
InitDf = pd.read_csv(in_path)
path = InitDf[mode + 'Path'].tolist()
ids = InitDf[mode + 'ID'].tolist()
form = InitDf[mode + 'Form'].tolist()
k = 0
for j, i, f in zip(path, ids, form):
k += 1
print('%.2f%%' % (100 * k / InitDf.shape[0]), end="\r")
img = cv2.imread(j, 0)
inv = np.bitwise_not(img)
y = 730
x = 230
h = 2048
w = 2048
cropped = inv[y:y + h, x:x + w]
resized = cv2.resize(cropped, (1024, 1024))
horizontal = np.split(resized, 4, axis=1)
for idx, h in enumerate(horizontal, start=1):
vertical = np.split(h, 4, axis=0)
for ind, v in enumerate(vertical, start=1):
thv, denv = cv2.threshold(v, 55, 255, cv2.THRESH_TOZERO)
filtered = []
for kern in kernels:
kernel = cv2.imread(kern, 0)
ksum = np.sum(np.multiply(denv, kernel))
filtered.append(ksum)
if 0 not in filtered:
vc = v.copy()
noise = cv2.randn(vc, 0, 15)
vn = v + noise
cv2.imwrite(
out_path + str(i) + '-' + str(f) + '-' + str(idx) +
str(ind) + '.png', vn)
else:
continue
print(mode + ' preprocessing done: 100%')
def Add_gaussian_Noise(src, mean, sigma):
noiseArr = src.copy()
cv2.randn(noiseArr, mean, sigma)
cv2.add(src, noiseArr, src)
return noiseArr
def Add_gaussian_Noise(srcArr, mean, sigma): Noiseimg = srcArr.copy() cv2.randn(Noiseimg,mean,sigma) cv2.add(srcArr, Noiseimg, srcArr)
def ApplyNoise(self, img, mu, std):
h, w = img.shape[:2]
noise = np.zeros((h, w), np.uint8)
cv2.randn(noise, mu, std)
return img + noise
import numpy as np
import imutils
import glob
import cv2
template = cv2.imread('COD_template2.jpg')
template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
template = cv2.Canny(template, 50, 200)
(tH, tW) = template.shape[:2]
cv2.imshow("Template", template)
image = cv2.imread('COD1.jpg')
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
noise = gray.copy()
cv2.randn(noise,(100),(250)) #Mean = 0, Variance = 1
#gray = gray + noise
#cv2.imshow("With-Noise",gray)
found = None
for scale in np.linspace(0.2, 1.0, 20)[::-1]:
resized = imutils.resize(gray, width = int(gray.shape[1] * scale))
r = gray.shape[1] / float(resized.shape[1])
if resized.shape[0] < tH or resized.shape[1] < tW:
break
edged = cv2.Canny(resized, 50, 200)
result = cv2.matchTemplate(edged, template, cv2.TM_CCOEFF)
(_, maxVal, _, maxLoc) = cv2.minMaxLoc(result)
if found is None or maxVal > found[0]:
def main():
"""Run code/call functions to solve problems."""
print "Problem 1"
## 1-a
## Read images
L = cv2.imread(os.path.join('input', 'pair0-L.png'), 0) * (1 / 255.0) # grayscale, scale to [0.0, 1.0]
R = cv2.imread(os.path.join('input', 'pair0-R.png'), 0) * (1 / 255.0)
## Compute disparity (using method disparity_ssd defined in disparity_ssd.py)
start = time.time()
D_L = normalize(disparity_ssd(L, R, window_size=13))
D_R = normalize(disparity_ssd(R, L, window_size=13))
end = time.time()
print "Number of seconds is %s" % (end - start)
## TODO: Save output images (D_L as output/ps3-1-a-1.png and D_R as output/ps3-1-a-2.png)
## Note: They may need to be scaled/shifted before saving to show results properly
cv2.imwrite(os.path.join(output_dir, 'ps3-1-a-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-1-a-2.png'), D_R)
print "Problem 2"
## 2
## TODO: Apply disparity_ssd() to pair1-L.png and pair1-R.png (in both directions)
L = cv2.imread(os.path.join('input', 'pair1-L.png'), 0) * (1 / 255.0) # grayscale, scale to [0.0, 1.0]
R = cv2.imread(os.path.join('input', 'pair1-R.png'), 0) * (1 / 255.0)
start = time.time()
D_L = normalize(disparity_ssd(L, R))
D_R = normalize(disparity_ssd(R, L))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-2-a-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-2-a-2.png'), D_R)
print "Problem 3"
# 3
# TODO: Apply disparity_ssd() to noisy versions of pair1 images
L = cv2.imread(os.path.join('input', 'pair1-L.png'), 0) * (1 / 255.0) # grayscale, scale to [0.0, 1.0]
R = cv2.imread(os.path.join('input', 'pair1-R.png'), 0) * (1 / 255.0)
noisy_L = np.copy(L)
noise = np.zeros(L.shape, dtype='float')
cv2.randn(noise, 0, 15)
noise *= (1 / 255.0)
noisy_L += noise
#noisy_L = normalize(noisy_L)
start = time.time()
D_L = normalize(disparity_ssd(noisy_L, R))
D_R = normalize(disparity_ssd(R, noisy_L))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-3-a-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-3-a-2.png'), D_R)
# TODO: Boost contrast in one image and apply again
L = cv2.imread(os.path.join('input', 'pair1-L.png'), 0) * (1 / 255.0) # grayscale, scale to [0.0, 1.0]
R = cv2.imread(os.path.join('input', 'pair1-R.png'), 0) * (1 / 255.0)
L *= 1.1
start = time.time()
D_L = normalize(disparity_ssd(L, R))
D_R = normalize(disparity_ssd(R, L))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-3-b-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-3-b-2.png'), D_R)
print "Problem 4"
# 4
# TODO: Implement disparity_ncorr() and apply to pair1 images (original, noisy and contrast-boosted)
L = cv2.imread(os.path.join('input', 'pair1-L.png'), 0)
R = cv2.imread(os.path.join('input', 'pair1-R.png'), 0)
start = time.time()
D_L = normalize(disparity_ncorr(L,R))
D_R = normalize(disparity_ncorr(R,L, window_size=15))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-4-a-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-4-a-2.png'), D_R)
print "Part b"
noise = np.zeros(L.shape, dtype=L.dtype)
cv2.randn(noise, 0, 15)
noisy_L = np.copy(L) + noise
start = time.time()
D_L = normalize(disparity_ncorr(noisy_L,R))
D_R = normalize(disparity_ncorr(R,noisy_L))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-4-b-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-4-b-2.png'), D_R)
print "Part b: 3-4"
L *= 1.1
start = time.time()
D_L = normalize(disparity_ncorr(L, R))
D_R = normalize(disparity_ncorr(R, L))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-4-b-3.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-4-b-4.png'), D_R)
print "Problem 5"
# 5
# TODO: Apply stereo matching to pair2 images, try pre-processing the images for best results
L = cv2.imread(os.path.join('input', 'pair2-L.png'), 0)
R = cv2.imread(os.path.join('input', 'pair2-R.png'), 0)
L *= 1.1
R *= 1.1
start = time.time()
D_L = normalize(disparity_ncorr(L,R))
D_R = normalize(disparity_ncorr(R,L, window_size=15))
end = time.time()
print "Number of seconds is %s" % (end - start)
cv2.imwrite(os.path.join(output_dir, 'ps3-5-a-1.png'), D_L)
cv2.imwrite(os.path.join(output_dir, 'ps3-5-a-2.png'), D_R)
#Subtract shifted image from original for interesting results
diff = green -dst;
#Save difference as image
cv2.imwrite('Output/diff_left_conv_2px.png',diff);
utils.showAndWait(diff)
"""
Number 5 ///////////////////////////////////////////
"""
#Read in our image
im1 = cv2.imread('Input/img1.png');
#Establish our noise filter
filt = np.zeros((im1.shape[0],im1.shape[1]),np.uint8);
cv2.randn(filt,(0),(50));
#Apply noise filter to green channel(BGR channel 1)
for i in range(im1.shape[0]):
for j in range(im1.shape[1]):
if int(im1[i][j][1]) + int(filt[i][j]) < 255:
im1[i][j][1] += filt[i][j];
#Display Image
utils.showAndWait(im1)
#Save image
cv2.imwrite('Output/gaussian_green.png',im1);
#Refresh im1 to original image
im1 = cv2.imread('Input/img1.png');
#New noise filter, this may be a lot of blue noise (really high sigma <stdDev>)
#I'm not sure if it's because blue noise is actually
#hard for people to see, or just because I'm colorblind
def gaussian_noise(img, var): noise = np.empty_like(img) cv.randn(noise, 0, var) return noise
import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt
img = cv.imread('maxresdefault.jpg')
noise = np.zeros((738, 1568, 3), 'uint8')
noise = cv.randn(noise, 0, 5) #mean and noise
kernel = np.ones(
(5, 5), np.float32
) / 25 # 5*5 array composing => 1/25 is the weight of each element
img = img + noise
dst = cv.bilateralFilter(img, 9, 500, 500)
plt.subplot(121), plt.imshow(img), plt.title('Original+noise')
plt.xticks([]), plt.yticks([])
plt.subplot(122), plt.imshow(dst), plt.title('Bilateral Filter')
plt.xticks([]), plt.yticks([])
plt.show()
while (cv.waitKey(1) != 'q'):
pass
cv.destroyAllWindows()
currentROI = np.empty(image_ROI[i].shape)
cv2.normalize(image_ROI[i], currentROI, 255, 0, cv2.NORM_MINMAX)
# print "current roi"
# print currentROI
# print np.max(currentROI)
# currentROI = (image_ROI[i] / maxROIvalue) * 255
# currentROI = np.uint8(currentROI)
for run in range(0,6):
psnr = []
errors = []
noiseScale = []
noise_scale = 5
noise_static = np.empty(currentROI.shape)
cv2.randn(noise_static, 0, 0.2)
while noise_scale <= 200:
noise = noise_static * noise_scale
# print "noise"
# print noise
image = np.empty(currentROI.shape)
cv2.add(noise, currentROI, image)
cv2.normalize(image, image, 255, 0, cv2.NORM_MINMAX)
psnr.append(getPSNR(image, currentROI))
# print "image"
# print image
# plt.imshow(currentROI)
# plt.show()
def Integrated_Mask(img,
blurred_img,
model,
category,
max_iterations=15,
integ_iter=20,
tv_beta=2,
l1_coeff=0.01 * 300,
tv_coeff=0.2 * 300,
size_init=112,
use_cuda=1):
img = preprocess_image(img, use_cuda, require_grad=False)
blurred_img = preprocess_image(blurred_img, use_cuda, require_grad=False)
resize_size = img.data.shape
resize_wh = (img.data.shape[2], img.data.shape[3])
#print('resize_size:', resize_size)
#print('resize_wh:', resize_wh)
if use_cuda:
zero_img = Variable(torch.zeros(resize_size).cuda(),
requires_grad=False)
else:
zero_img = Variable(torch.zeros(resize_size), requires_grad=False)
#print('zero_img:', type(zero_img))
mask_init = np.ones((size_init, size_init), dtype=np.float32)
mask = numpy_to_torch(mask_init, use_cuda, requires_grad=True)
#mask_base = np.zeros((size_init, size_init), dtype=np.float32)
#mask_base = numpy_to_torch(mask_base, use_cuda, requires_grad=False)
if use_cuda:
upsample = torch.nn.UpsamplingBilinear2d(size=resize_wh).cuda()
else:
upsample = torch.nn.UpsamplingBilinear2d(size=resize_wh)
# You can choose any optimizer
# The optimizer doesn't matter, because we don't need optimizer.step(), we just to compute the gradient
optimizer = torch.optim.Adam([mask], lr=0.1)
target = torch.nn.Softmax()(model(img))
if use_cuda:
category_out = np.argmax(target.cpu().data.numpy())
else:
category_out = np.argmax(target.data.numpy())
if category == -1:
category = category_out
print("Category with highest probability", category_out)
print("Category want to generate mask", category)
print("Optimizing.. ")
curve1 = np.array([])
curve2 = np.array([])
curvetop = np.array([])
#curve_total = np.array([])
#alpha = 0.25
alpha = 0.0001
#alpha = 0.00005
#beta = 0.9
beta = 0.2
for i in range(max_iterations):
upsampled_mask = upsample(mask)
# The single channel mask is used with an RGB image,
# so the mask is duplicated to have 3 channel,
upsampled_mask = \
upsampled_mask.expand(1, 3, upsampled_mask.size(2), \
upsampled_mask.size(3))
loss1 = l1_coeff * torch.mean(torch.abs(1 - mask)) + \
tv_coeff * tv_norm(mask, tv_beta)
loss_all = loss1.clone()
perturbated_input_base = img.mul(upsampled_mask) + \
blurred_img.mul(1 - upsampled_mask)
for inte_i in range(integ_iter):
############################### Use the mask to perturbated the input image.
integ_mask = 0.0 + ((inte_i + 1.0) / integ_iter) * upsampled_mask
perturbated_input_integ = img.mul(integ_mask) + \
blurred_img.mul(1 - integ_mask)
noise = np.zeros((resize_wh[0], resize_wh[1], 3), dtype=np.float32)
noise = noise + cv2.randn(noise, 0, 0.2)
#noise = noise + cv2.randn(noise, (0, 0, 0), (0.1, 0.1, 0.1))
noise = numpy_to_torch(noise, use_cuda, requires_grad=False)
perturbated_input = perturbated_input_integ + noise
####################################################################################
#new_image = 0.5* blurred_img + (inte_i / 20.0) * perturbated_input
new_image = perturbated_input
outputs = torch.nn.Softmax()(model(new_image))
loss2 = outputs[0, category]
loss_all = loss_all + loss2 / 20.0
optimizer.zero_grad()
loss_all.backward()
whole_grad = mask.grad.data.clone()
loss2_ori = torch.nn.Softmax()(model(perturbated_input_base))[0,
category]
loss_ori = loss1 + loss2_ori
if i == 0:
if use_cuda:
curve1 = np.append(curve1, loss1.data.cpu().numpy())
curve2 = np.append(curve2, loss2_ori.data.cpu().numpy())
curvetop = np.append(curvetop, loss2_ori.data.cpu().numpy())
#curve_total = np.append(curve_total, loss_ori.data.cpu().numpy())
else:
curve1 = np.append(curve1, loss1.data.numpy())
curve2 = np.append(curve2, loss2_ori.data.numpy())
curvetop = np.append(curvetop, loss2_ori.data.numpy())
#curve_total = np.append(curve_total, loss_ori.data.numpy())
if use_cuda:
loss_oridata = loss_ori.data.cpu().numpy()
else:
loss_oridata = loss_ori.data.numpy()
######################################################LINE SEARCH
step = 200.0
#directionLS = torch.div(whole_grad, np.abs(whole_grad.sum()))
MaskClone = mask.data.clone()
MaskClone -= step * whole_grad
#MaskClone -= step * directionLS
MaskClone = Variable(MaskClone, requires_grad=False)
MaskClone.data.clamp_(0, 1)
mask_LS = upsample(MaskClone) # Here the direction is the whole_grad
Img_LS = img.mul(mask_LS) + \
blurred_img.mul(1 - mask_LS)
outputsLS = torch.nn.Softmax()(model(Img_LS))
loss_LS = l1_coeff * torch.mean(torch.abs(1 - MaskClone)) + \
tv_coeff * tv_norm(MaskClone, tv_beta) + outputsLS[0, category]
if use_cuda:
loss_LSdata = loss_LS.data.cpu().numpy()
else:
loss_LSdata = loss_LS.data.numpy()
new_condition = whole_grad**2 # Here the direction is the whole_grad
#new_condition = torch.mul(whole_grad, directionLS)
new_condition = new_condition.sum()
new_condition = alpha * step * new_condition
#while loss_LS.data.cpu().numpy() > loss_ori.data.cpu().numpy() - new_condition:
while loss_LSdata > loss_oridata - new_condition:
step *= beta
#directionLS = torch.div(whole_grad, np.abs(whole_grad.sum()))
MaskClone = mask.data.clone()
MaskClone -= step * whole_grad
#MaskClone -= step * directionLS
MaskClone = Variable(MaskClone, requires_grad=False)
MaskClone.data.clamp_(0, 1)
mask_LS = upsample(MaskClone)
Img_LS = img.mul(mask_LS) + \
blurred_img.mul(1 - mask_LS)
outputsLS = torch.nn.Softmax()(model(Img_LS))
loss_LS = l1_coeff * torch.mean(torch.abs(1 - MaskClone)) + \
tv_coeff * tv_norm(MaskClone, tv_beta) + outputsLS[0, category]
if use_cuda:
loss_LSdata = loss_LS.data.cpu().numpy()
else:
loss_LSdata = loss_LS.data.numpy()
new_condition = whole_grad**2 # Here the direction is the whole_grad
#new_condition = torch.mul(whole_grad, directionLS)
new_condition = new_condition.sum()
new_condition = alpha * step * new_condition
if step < 0.00001:
break
#print('loss_LSdata:', loss_LSdata)
print('step:', step)
mask.data -= step * whole_grad
#######################################################################################################
#mask.data -= learning_rate * whole_grad
#mask.data -= 50*whole_grad
if use_cuda:
curve1 = np.append(curve1, loss1.data.cpu().numpy())
curve2 = np.append(curve2, loss2_ori.data.cpu().numpy())
else:
curve1 = np.append(curve1, loss1.data.numpy())
curve2 = np.append(curve2, loss2_ori.data.numpy())
# Optional: clamping seems to give better results
mask.data.clamp_(0, 1)
if use_cuda:
maskdata = mask.data.cpu().numpy()
else:
maskdata = mask.data.numpy()
#print('mask:', mask)
#print('mask_min:', np.min(maskdata))
#print('mask_min_index:', np.unravel_index(maskdata.argmin(), maskdata.shape))
#print(maskdata.shape)
maskdata = np.squeeze(maskdata)
maskdata, imgratio = topmaxPixel(maskdata, 40)
maskdata = np.expand_dims(maskdata, axis=0)
maskdata = np.expand_dims(maskdata, axis=0)
###############################################
# Masktop = maskdata
if use_cuda:
Masktop = torch.from_numpy(maskdata).cuda()
else:
Masktop = torch.from_numpy(maskdata)
# Use the mask to perturbated the input image.
Masktop = Variable(Masktop, requires_grad=False)
MasktopLS = upsample(Masktop)
# MasktopLS = \
# MasktopLS.expand(1, 3, MasktopLS.size(2), \
# MasktopLS.size(3))
Img_topLS = img.mul(MasktopLS) + \
blurred_img.mul(1 - MasktopLS)
outputstopLS = torch.nn.Softmax()(model(Img_topLS))
loss_top1 = l1_coeff * torch.mean(torch.abs(1 - Masktop)) + \
tv_coeff * tv_norm(Masktop, tv_beta)
loss_top2 = outputstopLS[0, category]
if use_cuda:
curvetop = np.append(curvetop, loss_top2.data.cpu().numpy())
else:
curvetop = np.append(curvetop, loss_top2.data.numpy())
if max_iterations > 3:
if i == int(max_iterations / 2):
if np.abs(curve2[0] - curve2[i]) <= 0.001:
print('Adjust Parameter l1_coeff at iteration:',
int(max_iterations / 2))
l1_coeff = l1_coeff / 10
elif i == int(max_iterations / 1.25):
if np.abs(curve2[0] - curve2[i]) <= 0.01:
print('Adjust Parameters l1_coeff again at iteration:',
int(max_iterations / 1.25))
l1_coeff = l1_coeff / 5
#######################################################################################
upsampled_mask = upsample(mask)
if use_cuda:
mask = mask.data.cpu().numpy().copy()
else:
mask = mask.data.numpy().copy()
return mask, upsampled_mask, imgratio, curvetop, curve1, curve2, category
def image_data_augmentation(mat, w, h, pleft, ptop, swidth, sheight, flip,
dhue, dsat, dexp, gaussian_noise, blur, truth):
try:
img = mat
oh, ow, _ = img.shape
pleft, ptop, swidth, sheight = int(pleft), int(ptop), int(swidth), int(
sheight)
# crop
src_rect = [pleft, ptop, swidth + pleft, sheight + ptop] # x1,y1,x2,y2
img_rect = [0, 0, ow, oh]
new_src_rect = rect_intersection(src_rect, img_rect) # 交集
dst_rect = [
max(0, -pleft),
max(0, -ptop),
max(0, -pleft) + new_src_rect[2] - new_src_rect[0],
max(0, -ptop) + new_src_rect[3] - new_src_rect[1]
]
# cv2.Mat sized
if (src_rect[0] == 0 and src_rect[1] == 0
and src_rect[2] == img.shape[0]
and src_rect[3] == img.shape[1]):
sized = cv2.resize(img, (w, h), cv2.INTER_LINEAR)
else:
cropped = np.zeros([sheight, swidth, 3])
cropped[:, :, ] = np.mean(img, axis=(0, 1))
cropped[dst_rect[1]:dst_rect[3], dst_rect[0]:dst_rect[2]] = \
img[new_src_rect[1]:new_src_rect[3], new_src_rect[0]:new_src_rect[2]]
# resize
sized = cv2.resize(cropped, (w, h), cv2.INTER_LINEAR)
# flip
if flip:
# cv2.Mat cropped
sized = cv2.flip(
sized, 1) # 0 - x-axis, 1 - y-axis, -1 - both axes (x & y)
# HSV augmentation
# cv2.COLOR_BGR2HSV, cv2.COLOR_RGB2HSV, cv2.COLOR_HSV2BGR, cv2.COLOR_HSV2RGB
if dsat != 1 or dexp != 1 or dhue != 0:
if img.shape[2] >= 3:
hsv_src = cv2.cvtColor(sized.astype(np.float32),
cv2.COLOR_RGB2HSV) # RGB to HSV
hsv = cv2.split(hsv_src)
hsv[1] *= dsat
hsv[2] *= dexp
hsv[0] += 179 * dhue
hsv_src = cv2.merge(hsv)
sized = cv2.cvtColor(
hsv_src,
cv2.COLOR_HSV2RGB) # HSV to RGB (the same as previous)
else:
sized *= dexp
# cv2.imshow(window_name.str(), sized)
# cv2.waitKey(0)
if blur:
if blur == 1:
dst = cv2.GaussianBlur(sized, (17, 17), 0)
# cv2.bilateralFilter(sized, dst, 17, 75, 75)
else:
ksize = (blur / 2) * 2 + 1
dst = cv2.GaussianBlur(sized, (ksize, ksize), 0)
# cv2.medianBlur(sized, dst, ksize)
# cv2.bilateralFilter(sized, dst, ksize, 75, 75)
# sharpen
# cv2.Mat img_tmp
# cv2.GaussianBlur(dst, img_tmp, cv2.Size(), 3)
# cv2.addWeighted(dst, 1.5, img_tmp, -0.5, 0, img_tmp)
# dst = img_tmp
# std::cout << " blur num_boxes = " << num_boxes << std::endl
if blur == 1:
img_rect = [0, 0, sized.cols, sized.rows]
for b in truth:
left = (b.x - b.w / 2.) * sized.shape[1]
width = b.w * sized.shape[1]
top = (b.y - b.h / 2.) * sized.shape[0]
height = b.h * sized.shape[0]
roi(left, top, width, height)
roi = roi & img_rect
dst[roi[0]:roi[0] + roi[2],
roi[1]:roi[1] + roi[3]] = sized[roi[0]:roi[0] + roi[2],
roi[1]:roi[1] + roi[3]]
sized = dst
if gaussian_noise:
# noise = cv2.Mat(sized.size(), sized.type())
noise = np.array(sized.shape)
gaussian_noise = min(gaussian_noise, 127)
gaussian_noise = max(gaussian_noise, 0)
cv2.randn(noise, 0, gaussian_noise) # mean and variance
sized = sized + noise
# cv2.normalize(sized_norm, sized_norm, 0.0, 255.0, cv2.NORM_MINMAX, sized.type())
# cv2.imshow("source", sized)
# cv2.imshow("gaussian noise", sized_norm)
# cv2.waitKey(0)
# sized = sized_norm
# char txt[100]
# sprintf(txt, "blur = %d", blur)
# cv2.putText(sized, txt, cv2.Point(100, 100), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1.7, CV_RGB(255, 0, 0), 1, CV_AA)
except:
print("OpenCV can't augment image: " + str(w) + " x " + str(h))
sized = mat
return sized
def generate_noise_image(self, img, amt=5):
# Get a noise profile
while 1:
index = get_rand_int(0, len(self.triplets) - 1)
triplet = self.triplets[index]
if triplet[2] == "full": break
# Load image
anchor_index = triplet[0]
pos_index = triplet[1]
anchor_image = cv2.imread(
self.root + "tracks_cropped/" + get_padded(anchor_index) + ".jpg",
0)
pos_image = cv2.imread(
self.root + "references/" + get_padded(pos_index) + ".png", 0)
pos_image = cv2.resize(pos_image,
(anchor_image.shape[1], anchor_image.shape[0]))
pos_image = pos_image.astype(np.float32) / 255.
anchor_image = anchor_image.astype(np.float32) / 255.
noise_profile = np.abs(pos_image - anchor_image)
noise_profile = anchor_image
#noise_profile = cv2.flip(noise_profile, -1)
# Resize clean image
clean = img.copy()
clean = clean.astype(np.float32) / 255.
noise_profile_res = cv2.resize(noise_profile,
(clean.shape[1], clean.shape[0]))
clean_noise = clean
# Add noise
for i in range(0, amt):
gauss_noise = clean_noise.copy()
cv2.randn(gauss_noise, 0, 0.07)
clean_noise = gauss_noise + clean_noise
# clean_noise = skimage.util.random_noise(clean_noise, mode='gaussian', seed=None, clip=True)
#clean_noise = skimage.util.random_noise(clean_noise, mode='Poisson', seed=None, clip=True)
translation_matrix = np.float32([[1, 0,
get_rand_int(-2, 2, 100)],
[0, 1,
get_rand_int(-2, 2, 100)]])
noise_profile_res = cv2.warpAffine(
noise_profile_res, translation_matrix,
(noise_profile_res.shape[1], noise_profile_res.shape[0]))
translation_matrix = np.float32([[1, 0,
get_rand_int(-2, 2, 100)],
[0, 1,
get_rand_int(-2, 2, 100)]])
clean_noise = cv2.warpAffine(
clean_noise, translation_matrix,
(clean_noise.shape[1], clean_noise.shape[0]))
translation_matrix = np.float32(
[[get_rand_int(1, 2, 100), 0,
get_rand_int(-2, 2, 100)],
[0, get_rand_int(1, 2, 100),
get_rand_int(-2, 2, 100)]])
weird_noise = cv2.warpAffine(
clean_noise, translation_matrix,
(clean_noise.shape[1], clean_noise.shape[0]))
val = 0.1
clean_noise = noise_profile_res * val + clean_noise * (1 - val) + (
weird_noise - 1) * 0.1
# Return
clean_noise = (clean_noise * 255).astype(np.uint8)
return clean_noise
def noise(image):
uniform = np.zeros(image.shape, np.uint8)
cv2.randn(uniform, (0), (99))
return cv2.add(image, uniform)
def gauss_noise_jittering(im_sample, value):
noise_map = np.zeros(im_sample.shape)
std = np.array([np.random.uniform(0, value)] * 3)
cv2.randn(noise_map, 0, std)
# noise_map = np.random.normal(0.0, np.random.uniform(0, value), im_sample.shape)
return cv2.add(im_sample, noise_map, dtype=cv2.CV_8U)
def gauss_noise_jittering(im_sample, value):
noise_map = np.zeros(im_sample.shape)
cv2.randn(noise_map, 0, np.random.uniform(0, value))
# noise_map = np.random.normal(0.0, np.random.uniform(0, value), im_sample.shape)
return cv2.add(im_sample, noise_map, dtype=cv2.CV_8U)
def image_data_augmentation(mat, truth, w, h, pleft, ptop, swidth, sheight, flip=0,
dhue=0, dsat=1, dexp=1, gaussian_noise=0, blur=0):
"""
1. get final crop_box that we want to crop
2. crop image patch from original image
3. execute image augmentation to cropped image patch
"""
try:
img = mat
oh, ow, _ = img.shape
pleft, ptop, swidth, sheight = int(pleft), int(ptop), int(swidth), int(sheight)
# crop
src_rect = [pleft, ptop, swidth + pleft, sheight + ptop] # x1,y1,x2,y2
img_rect = [0, 0, ow, oh]
new_src_rect = rect_intersection(src_rect, img_rect) # 交集
# (x1, y1, x2, y2)
dst_rect = [max(0, -pleft), max(0, -ptop), max(0, -pleft) + new_src_rect[2] - new_src_rect[0],
max(0, -ptop) + new_src_rect[3] - new_src_rect[1]]
# cv2.Mat sized
if src_rect[0] == 0 and src_rect[1] == 0 and src_rect[2] == img.shape[0] and src_rect[3] == img.shape[1]:
sized = cv2.resize(img, (w, h), cv2.INTER_LINEAR)
else:
cropped = np.zeros([sheight, swidth, 3])
# cropped 初始化为img 像素的均值， 相当于global average pooling
cropped[:, :, ] = np.mean(img, axis=(0, 1))
cropped[dst_rect[1]:dst_rect[3], dst_rect[0]:dst_rect[2]] = \
img[new_src_rect[1]:new_src_rect[3], new_src_rect[0]:new_src_rect[2]]
# resize
sized = cv2.resize(cropped, (w, h), cv2.INTER_LINEAR)
# flip
if flip:
# cv2.Mat cropped
sized = cv2.flip(sized, 1) # 0 - x-axis, 1 - y-axis, -1 - both axes (x & y)
# HSV augmentation
# cv2.COLOR_BGR2HSV, cv2.COLOR_RGB2HSV, cv2.COLOR_HSV2BGR, cv2.COLOR_HSV2RGB
if dsat != 1 or dexp != 1 or dhue != 0:
if img.shape[2] >= 3:
hsv_src = cv2.cvtColor(sized.astype(np.float32), cv2.COLOR_RGB2HSV) # RGB to HSV
hsv = cv2.split(hsv_src)
hsv[1] *= dsat
hsv[2] *= dexp
hsv[0] += 179 * dhue
hsv_src = cv2.merge(hsv)
sized = np.clip(cv2.cvtColor(hsv_src, cv2.COLOR_HSV2RGB), 0, 255) # HSV to RGB (the same as previous)
else:
sized *= dexp
if blur:
# if blur == 1:
# dst = cv2.GaussianBlur(sized, (17, 17), 0)
# # cv2.bilateralFilter(sized, dst, 17, 75, 75)
# else:
ksize = int((blur / 2) * 2 + 1)
dst = cv2.GaussianBlur(sized, (ksize, ksize), 0)
# if blur == 1:
# img_rect = [0, 0, sized.shape[1], sized.shape[0]]
# for b in truth:
# left = b[0] * sized.shape[1]
# width = (b[2] - b[0]) * sized.shape[1]
# top = b[1] * sized.shape[0]
# height = (b[3] - b[1]) * sized.shape[0]
# roi = [left, top, width, height]
# roi = rect_intersection(roi, img_rect)
# dst[roi[0]:roi[0] + roi[2], roi[1]:roi[1] + roi[3]] = sized[roi[0]:roi[0] + roi[2],
# roi[1]:roi[1] + roi[3]]
sized = dst
if gaussian_noise:
noise = np.array(sized.shape)
gaussian_noise = min(gaussian_noise, 127)
gaussian_noise = max(gaussian_noise, 0)
cv2.randn(noise, 0, gaussian_noise) # mean and variance
sized = sized + noise
except:
print("OpenCV can't augment image: " + str(w) + " x " + str(h))
sized = mat
return sized
## take elements of the channels with increment of -1 meaning to reverse it so from BGR it will be RGB
plt.imshow(img)
plt.xticks([]), plt.yticks([]) # to hide tick values on X and Y axis
plt.show()
#construct Salt and Pepper Noise
salt_noise = add_salt_and_pepper(img, 0.2)
print salt_noise.shape
cv2.imshow("Salt and Pepper Noise", salt_noise)
#construct gaussian noise
gauss_nois = np.zeros(img.shape, dtype=np.uint8)
m = (500, 500, 500)
s = (500, 500, 500)
cv2.randn(gauss_nois, m, s)
noised_img = img + gauss_nois
cv2.imshow("gauss_nois", noised_img)
# #construct avg filter
blur = cv2.blur(img, (5, 5))
cv2.imshow("Avg Blurred", blur)
#construct gaussian filter
gblur = cv2.GaussianBlur(img, (5, 5), 0)
cv2.imshow("Gaussian Blurred", gblur)
#draw All
plt.figure()
plt.subplot(221), plt.imshow(salt_noise), plt.title('Salt and Pepper Noise')
plt.xticks([]), plt.yticks([])
def make_noise(self):
noise = np.zeros([512, 512, 1], dtype=np.uint8)
noise = cv2.randn(noise, 0, 255)
noise = np.asarray(noise / 255, dtype=np.uint8)
noise = np.expand_dims(noise, axis=0)
return noise
#reading the images
#image = cv2.imread('lowfreq.png',0)
image = cv2.imread('highfreq.png',0)
cv2.imshow('Original image TASK 1', image)
#########################TASK 1#####################################################
#create a matrix of zeroes from image size, then fill it with gaussian noise
rows, cols = image.shape[:2]
gaus_noise = np.zeros((rows, cols), dtype=np.uint8)
mean = 128
standard_deviation = 156
# standard_deviation = 56
cv2.randn(gaus_noise, mean, standard_deviation)
#add noise to OG image
gaus_noise = (gaus_noise * 0.5).astype(np.uint8)
noise_img = cv2.add(image, gaus_noise)
cv2.imshow('Img with gaussian noise', noise_img)
#apply gaussian blur on OG image
filt_img = cv2.GaussianBlur(noise_img, (3, 3), cv2.BORDER_DEFAULT)
cv2.imshow("Img with gaussian noise - filtered", filt_img)
#sharpen image (also sharpens the noise)
sharp = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
sharp_img = cv2.filter2D(image, -1, sharp)
cv2.imshow("Img with gaussian noise - sharpened", sharp_img)
def add_noise_randomly(img):
aux = img.copy()
cv2.randn(aux, aux.mean(), aux.std() / 5)
cv2.add(img, aux, aux, mask=None)
return aux
def noise(default_Pic):
noise_Pic = copy.deepcopy(default_Pic) #make copy of Pic
cv2.randn(noise_Pic,0,20) #apply gaussian noise on an image and store in noise
noise_Pic = impluse(noise_Pic+default_Pic,.001) #apply SP noise on image that has gaussian noise
return noise_Pic
import cv2
import numpy
image = cv2.imread('labka.jpg')
gauss = numpy.zeros(image.shape, numpy.uint8)
mean = (0, 0, 0)
sigma = (50, 50, 50)
cv2.randn(gauss, mean, sigma)
new_image = cv2.add(image, gauss)
cv2.imshow('original', image)
cv2.imshow('gaussian noise', new_image)
cv2.imwrite('gaussian_noise_labka.jpg', new_image)
key = cv2.waitKey(0) & 0xFF
cv2.waitKey(0)
cv2.destroyAllWindows()
def read(self, dst=None):
noise = np.zeros(self.render.sceneBg.shape, np.int8)
cv.randn(noise, np.zeros(3), np.ones(3)*255*self.noise)
return True, cv.add(self.render.getNextFrame(), noise, dtype=cv.CV_8UC3)
def compute_heatmap_using_superpixels(model,
original_img,
params,
mask_init=None,
use_cuda=False,
gpu_id=0,
verbose=False):
img = np.float32(original_img) / 255
blurred_img_numpy = cv2.GaussianBlur(img, (11, 11), 10)
# associate at each pixel the id of the corresponding superpixel
segm_img = slic(img.copy()[:, :, ::-1],
n_segments=2000,
compactness=10,
sigma=0.5)
s2p = Superpixel2Pixel(segm_img, use_cuda, gpu_id=gpu_id)
# generate superpixel initialization image
nb_segms = np.max(segm_img) + 1
segm_init = np.zeros((nb_segms, ), dtype=np.float32)
if mask_init is None:
segm_init = segm_init + 0.5
else:
for i in range(nb_segms):
segm_init[i] = np.mean(mask_init[segm_img == i])
# segm_init[i] = 0.5 if segm_init[i] < 0.5 else segm_init[i]
blur = utils.BlurTensor(use_cuda, gpu_id=gpu_id)
# create superpixel image mask
if use_cuda:
segm = Variable(torch.from_numpy(segm_init).cuda(gpu_id),
requires_grad=True)
else:
segm = Variable(torch.from_numpy(segm_init), requires_grad=True)
img = utils.preprocess_image(img, use_cuda,
gpu_id=gpu_id) # original image
blurred_img = utils.preprocess_image(
blurred_img_numpy, use_cuda,
gpu_id=gpu_id) # blurred version of input image
optimizer = torch.optim.Adam([segm], lr=params.learning_rate)
target_preds = model(img)
targets = torch.nn.Softmax(dim=1)(target_preds)
category, target_prob, label = utils.get_class_info(targets)
if verbose:
print("Category with highest probability:",
(label, category, target_prob))
loss_history = []
if verbose:
print("Optimizing.. ")
for i in range(params.max_iterations):
upsampled_mask = s2p(segm).unsqueeze(0).unsqueeze(0)
if params.blur:
upsampled_mask = blur(upsampled_mask, 5)
upsampled_mask = upsampled_mask.expand(1, 3, *params.target_shape)
perturbated_input = img.mul(upsampled_mask) + \
blurred_img.mul(1 - upsampled_mask)
noise = np.zeros(params.target_shape + (3, ), dtype=np.float32)
if params.noise_sigma != 0:
noise = noise + cv2.randn(noise, 0., params.noise_sigma)
noise = utils.numpy_to_torch(noise, use_cuda=use_cuda, gpu_id=gpu_id)
noisy_perturbated_input = perturbated_input + noise * params.noise_scale
preds = model(noisy_perturbated_input)
outputs = torch.nn.Softmax(dim=1)(preds)
current_mask = segm # upsampled_mask
class_loss = outputs[0, category]
l1_loss = params.l1_coeff * l1_reg(current_mask)
tv_loss = params.tv_coeff * tv_reg(upsampled_mask, params.tv_beta)
lasso_loss = params.lasso_coeff * lasso_reg(current_mask)
less_loss = params.less_coeff * less_reg(preds, target_preds)
losses = [class_loss, l1_loss, tv_loss, lasso_loss, less_loss]
total_loss = np.sum(losses)
losses = [total_loss.data.cpu().squeeze().numpy()[0]
] + [l.data.cpu().numpy()[0] for l in losses]
loss_history.append(losses)
optimizer.zero_grad()
total_loss.backward()
optimizer.step()
segm.data.clamp_(0, 1)
if params.blur:
upsampled_mask = blur(upsampled_mask, 5)
perturbated_input = img.mul(upsampled_mask) + \
blurred_img.mul(1 - upsampled_mask)
outputs = torch.nn.Softmax(dim=1)(model(perturbated_input))
output_prob = outputs[0, category].data.cpu().squeeze().numpy()[0]
outputs = torch.nn.Softmax(dim=1)(model(blurred_img))
blurred_prob = outputs[0, category].data.cpu().squeeze().numpy()[0]
return upsampled_mask, blurred_img_numpy, target_prob, output_prob, blurred_prob, np.asarray(
loss_history), category
def run(img_path,
tv_beta=3,
lr=0.1,
max_iterations=500,
l1_coefficient=0.01,
tv_coefficient=0.2):
"""
run main training script
:param str img_path:
:param int tv_beta:
:param float lr: learning rate
:param int max_iterations:
:param float l1_coefficient:
:param float tv_coefficient:
:return:
"""
model = load_model()
original_img = cv2.imread(img_path, 1)
original_img = cv2.resize(original_img, (224, 224))
img = np.float32(original_img)
blurred_img1 = cv2.GaussianBlur(img, (11, 11), 5)
blurred_img2 = np.float32(cv2.medianBlur(original_img, 11))
blurred_img_numpy = (blurred_img1 + blurred_img2) / 2
mask_init = np.ones((28, 28), dtype=np.float32)
# Convert to torch variables
img = preprocess_image(img)
blurred_img = preprocess_image(blurred_img2)
mask = numpy_to_torch(mask_init)
if use_cuda:
upsample = torch.nn.Upsample(size=(224, 224)).cuda()
else:
upsample = torch.nn.Upsample(size=(224, 224))
optimizer = torch.optim.Adam([mask], lr=lr)
target = softmax(model(img), dim=1)
# gpu -> cpu
target = target.cpu().data.numpy()
category = np.argmax(target)
prob = np.max(target) * 100
print("-" * 30 +
"\nCategory with highest probability: {category} - {prob:.2f} %".
format(**locals()))
print("Optimizing.. ")
for i in range(max_iterations):
upsampled_mask = upsample(mask)
# The single channel mask is used with an RGB image,
# so the mask is duplicated to have 3 channel,
upsampled_mask = \
upsampled_mask.expand(1, 3, upsampled_mask.size(2), upsampled_mask.size(3))
# Use the mask to perturbated the input image.
perturbated_input = img.mul(upsampled_mask) + blurred_img.mul(
1 - upsampled_mask)
noise = np.zeros((224, 224, 3), dtype=np.float32)
# add noise to zero vector.
# in the paper, noise variance is 4. but in this code, input image is normalized (i.e. devided by 255)
# so we set 4 / 255 ~ 0.2
noise = noise + cv2.randn(noise, 0, 0.2)
noise = numpy_to_torch(noise)
perturbated_input = perturbated_input + noise
output_i = model(perturbated_input)
prob_i = softmax(output_i, dim=1)
loss = l1_coefficient * torch.mean(
torch.abs(1 - mask)) + tv_coefficient * tv_norm(
mask, tv_beta) + prob_i[0, int(category)]
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Optional: clamping seems to give better results
mask.data.clamp_(0, 1)
logger.info("finish!")
# mask を (224, 224) に up sampling しその後 cpu に dump してから shape = (1, 224, 224) に変換
upsampled_mask = upsample(mask).cpu().data.numpy()[0]
input_filename = os.path.splitext(ntpath.basename(img_path))[0]
perturbated, heat_map, mask, cam = generate_masked_images(
upsampled_mask, original_img, blurred_img_numpy)
pred_masked = softmax(model(preprocess_image(perturbated * 255)), dim=1)
pred_masked = pred_masked.cpu().data.numpy()[0, :]
prob_masked = pred_masked[category] * 100
logger.info("-" * 30 +
"\nafter masked probability: {prob_masked:.2f} %".format(
**locals()))
# 入力された画像のファイル名を先頭に付けて保存する
for img, name in zip([perturbated, heat_map, mask, cam],
["perturbated", "heat_map", "mask", "cam"]):
fname_i = input_filename + "_" + name + ".png"
save(img, file_name=fname_i, save_dir="output")
# -*- coding: utf-8 -*-
import numpy as np;
#from numpy import insert;
import cv2;
import scipy;
from scipy import misc;
from scipy.misc import imread;
#imag=scipy.misc.imread('F:\program file\canopy\Home_Work\lena_gray.png',1);
imag_origin=cv2.imread('F:\program file\canopy\Project\lena.jpg',0);
[row,col]=imag_origin.shape;
#im=np.zeros(shape=(row,col),dtype=np.uint8);
noise=np.zeros(shape=(row,col),dtype=np.uint8);
cv2.randn(noise,(0),(20));
#print noise;
imag=np.zeros(shape=(row,col));
for i in range(0,row):
for j in range(0,col):
imag[i][j]=imag_origin.item((i,j))+noise.item((i,j));
cv2.imwrite('le_image.png',imag_origin);
cv2.imwrite('le_image_noise.png',imag);
[row,col]=imag.shape;
row2=int(row*0.5);
col2=int(col*0.5);
