def estimateFeatureTranslation(startX, startY, Ix, Iy, img1, img2): #Convert from RGB to grayscale gray1 = rgb2gray(img1) gray2 = rgb2gray(img2) # Get size of image and allocate space for matrices in "culmination" [h, w] = gray1.shape LHS_summation = np.zeros((2,2)) #make this an np.empty after shown to work RHS_summation = np.zeros((2, 1)) # Get partial with respect to time It = gray2 - gray1 x = np.arange(0, w) y = np.arange(0, h) f_Ix = interp2d(x, y, Ix) f_Iy = interp2d(x, y, Iy) f_It = interp2d(x, y, It) # Calculate boundaries of window (normally 10x10 except when on boundary) #startXWin = np.round(startX).astype(int) #startYWin = np.round(startY).astype(int) windowMinX = startX - 9 if startX - 9 > 0 else startX windowMaxX = startX + 10 if startX + 10 < w else w - startX windowMinY = startY - 9 if startY - 9 > 0 else startY windowMaxY = startY + 10 if startY + 10 < h else h - startY windowH = np.round(windowMaxY - windowMinY).astype(int) windowW = np.round(windowMaxX - windowMinX).astype(int) # Get window points from Ix, Iy, and It #Ix_window = Ix[windowMinY: windowMaxY, windowMinX: windowMaxX] #Iy_window = Iy[windowMinY: windowMaxY, windowMinX: windowMaxX] #It_window = It[windowMinY: windowMaxY, windowMinX: windowMaxX] Ix_window = np.zeros((windowH, windowW)) Ix_window = getInterpolatedWindow(Ix_window, windowMinX, windowMinY, f_Ix) Iy_window = np.zeros((windowH, windowW)) Iy_window = getInterpolatedWindow(Iy_window, windowMinX, windowMinY, f_Iy) It_window = np.zeros((windowH, windowW)) It_window = getInterpolatedWindow(It_window, windowMinX, windowMinY, f_It) #Fill in culmination matrices LHS_summation[0][0] = np.sum(np.multiply(Ix_window, Ix_window)) LHS_summation[0][1] = np.sum(np.multiply(Ix_window, Iy_window)) LHS_summation[1][0] = np.sum(np.multiply(Ix_window, Iy_window)) LHS_summation[1][1] = np.sum(np.multiply(Iy_window, Iy_window)) RHS_summation[0][0] = -1.0 * np.sum(np.multiply(Ix_window, It_window)) RHS_summation[1][0] = -1.0 * np.sum(np.multiply(Iy_window, It_window)) #Solve for newX and newY LHS_inverse = np.linalg.inv(LHS_summation) u, v = LHS_inverse.dot(RHS_summation) newX = startX + u newY = startY + v return newX, newY
def estimateAllTranslation(startXs, startYs, img1, img2):
#Convert from RGB to grayscale
grey1 = rgb2gray(img1)
#find the number of total features
[numFeatures, numFaces] = startXs.shape
#instantiate the output
newXs = np.zeros((numFeatures, numFaces))
newYs = np.zeros((numFeatures, numFaces))
#Get the gradients of the first image
#[Ix, Iy] = np.gradient(grey1)
Ix = cv2.Sobel(grey1, cv2.CV_64F, 1, 0, ksize=5)
Iy = cv2.Sobel(grey1, cv2.CV_64F, 0, 1, ksize=5)
#Loop through all the faces
for face in range(0, numFaces):
#Loop through all the features
for feature in range(0, numFeatures):
#get the current feature for the current face
startX = startXs[feature, face]
startY = startYs[feature, face]
# Estimate the translation for the current feature
[newX, newY] = estimateFeatureTranslation(startX, startY, Ix, Iy,
img1, img2)
#append the newX and newY for the current feature to the list of new featuers
newXs[feature, face] = newX
newYs[feature, face] = newY
return newXs, newYs
def estimateFeatureTranslation(startX, startY, Ix, Iy, img1, img2):
#TODO: Your code here
# if np.ndim(img1) == 3:
# img1 = rgb2gray(img1)
# if np.ndim(img2) == 3:
# img2 = rgb2gray(img2)
win_width = 10
win_height = 10
nr = win_height / 2
nc = win_width / 2
newX = []
newY = []
# keep within image boundaries
if startY - nr > 0 and startX - nc > 0 and startY + nr < img1.shape[
0] and startX + nc < img1.shape[1]:
# extract an image patch (10 x 10 window) from first image (centered on startX, startY)
im1_patch = getPatch(img1, startX, startY, win_width, win_height)
# use starting location of search in second image and patches to track a feature point from image 1 to 2
p = [0, 0, 0, 0, 2, 3]
delta_p = 7
threshold = 0.01
x, y = np.meshgrid(range(0,
np.shape(im1_patch)[0]),
range(0,
np.shape(im1_patch)[1]))
patch_center = np.hstack(
(np.shape(im1_patch)[0] / 2, np.shape(im1_patch)[1] / 2))
x = x - patch_center[0]
y = y - patch_center[1]
It = rgb2gray(img2 - img1)
It_patch = getPatch(It, startX, startY, win_width, win_height)
Ix_patch = getPatch(Ix, startX, startY, win_width, win_height)
Iy_patch = getPatch(Iy, startX, startY, win_width, win_height)
Ixx_patch = Ix_patch * Ix_patch
Iyy_patch = Iy_patch * Iy_patch
Ixy_patch = Ix_patch * Iy_patch
Ixt_patch = Ix_patch * It_patch
Iyt_patch = Iy_patch * It_patch
a = np.array([[np.sum(Ixx_patch), np.sum(Ixy_patch)],
[np.sum(Ixy_patch), np.sum(Iyy_patch)]])
b = -np.array([[np.sum(Ixt_patch)], [np.sum(Iyt_patch)]])
x = np.linalg.solve(a, b)
u = x.item(0)
v = x.item(1)
newX = startX + u
newY = startY + v
# while np.linalg.norm(delta_p) > threshold: # L2 norm of vector delta_p
# # affine matrix
# W_p = np.vstack(( [1+p[0], p[2], p[4]], [p[1], 1+p[3], p[5]] ))
# # warp img
# img2_patch = getPatch(img2, startX, startY, win_width, win_height);
# img2_warped = cv2.warpAffine(img2_patch,W_p,(img2_patch.shape[1],img2_patch.shape[0]))
# # subtract warp from original (temporal gradient)
# img_err = im1_patch - img2_warped
# # img_err = getPatch(im_temp, startX, startY, win_width, win_height);
# # img_err = im1_patch - img_warped
# # break if outside image boundaries
# if p[4]>img2.shape[0]-1 or p[5]>img2.shape[1]-1 or p[4]<0 or p[5]<0 :
# break
# # warp gradient
# Ix_patch = getPatch(Ix, startX, startY, win_width, win_height);
# Iy_patch = getPatch(Iy, startX, startY, win_width, win_height);
# Ix_warped = cv2.warpAffine(Ix_patch,W_p,(win_width, win_height))
# Iy_warped = cv2.warpAffine(Iy_patch,W_p,(win_width, win_height))
# # evaluate Jacobian of the warp
# x_coord = x.flatten()
# x_coord = np.array([x_coord])
# x_coord = x_coord.T
# y_coord = y.flatten()
# y_coord = np.array([y_coord])
# y_coord = y_coord.T
# Jacobian_x = np.hstack(([x_coord, np.zeros((np.shape(x_coord))), y_coord, np.zeros((np.shape(x_coord))), np.ones((np.shape(x_coord))), np.zeros((np.shape(x_coord)))]))
# Jacobian_y = np.hstack(([np.zeros((np.shape(x_coord))), x_coord, np.zeros((np.shape(x_coord))), y_coord, np.zeros((np.shape(x_coord))), np.ones((np.shape(x_coord)))]))
# # compute steepest descent
# I_steepest = np.zeros((im1_patch.size, 6))
# Ix_warped = Ix_warped.flatten()
# Iy_warped = Iy_warped.flatten()
# for j in range(0, x.size):
# Jacobian = np.vstack((Jacobian_x[j,:], Jacobian_y[j,:]))
# grad = np.hstack((Ix_warped[j], Iy_warped[j]))
# I_steepest[j,range(0,6)] = grad.dot(Jacobian)
# # compute Hessian
# H = np.zeros((6,6))
# for j in range(0, x.size):
# H = H + I_steepest[j][:].flatten().dot(I_steepest[j][:])
# # multipy steepest descend with error
# final = np.zeros((6,1))
# img_err = img_err.flatten()
# for j in range(0, x.size):
# I_steep = I_steepest[j][:]
# I_steep = np.array([I_steep])
# I_steep = I_steep.T
# final = final + I_steep.dot(img_err[j])
# # compute delta_p
# delta_p = np.linalg.lstsq(H, final)[0]
# # update p
# p = p + delta_p.flatten()
#
# newX = p[5]
# newY = p[4]
# else:
# newX = [];
# newY = [];
return newX, newY
def faceTracking(rawVideo):
#TODO: Your code here
ind = 0
trackedVideo = []
nFrames = len(rawVideo)
# detect faces
bbox = detectFace(rawVideo[0])
# detect features
gray_im = rgb2gray(rawVideo[0])
x, y = getFeatures(gray_im, bbox)
for f in range(0, x.shape[1]):
x[:, f] = x[:, f] + bbox[f][0][1] # row coords
y[:, f] = y[:, f] + bbox[f][0][0] # col coords
for i in range(0, nFrames - 1):
if x.shape[0] < 15:
# detect faces
bbox = detectFace(rawVideo[0])
# detect features
gray_im = rgb2gray(rawVideo[0])
x, y = getFeatures(gray_im, bbox)
for f in range(0, x.shape[1]):
x[:, f] = x[:, f] + bbox[f][0][1] # row coords
y[:, f] = y[:, f] + bbox[f][0][0] # col coords
# get frames
img1 = rawVideo[ind]
img2 = rawVideo[ind + 1]
# track features from first frame to second using KLT procedure
newX, newY = estimateAllTranslation(x, y, img1, img2)
# apply resulting transformation
newXs, newYs, bbox = applyGeometricTransformation(
x, y, newX, newY, bbox)
# apply tracked features and bounding box to frames, update output array
for f in range(0, newXs.shape[1]):
im = cv2.rectangle(img2, (int(bbox[f][0][0]), int(bbox[f][0][1])),
(int(bbox[f][2][0]), int(bbox[f][2][1])),
(0, 0, 0), 3)
for j in range(0, len(newXs)):
im = cv2.circle(im, (int(newYs[j][f]), int(newXs[j][f])), 1,
(0, 0, 255), 2)
trackedVideo.append(im)
ind = ind + 1
x = newXs
y = newYs
trackedVideo = np.array(trackedVideo)
return trackedVideo
# handle any extra rows by assigning it to last worker
if image_x % (size - 1) != 0 and rank == (size - 1):
image_y = image_x // (size - 1) + image_x % (size - 1)
else:
image_y = image_x // (size - 1)
image_y_bottom = image_x // (size - 1) + image_x % (size - 1)
################################################################################################
######## Split Up and Send Out Initial Data : Begin ########
# master sends out partitions of data, slaves receive the data
if rank == 0:
# read in image, strip rgb values, convert to numpy array
img = plt.imread("test_image3.jpeg")
gray = rgb2gray(img)
clean_image = np.matrix.round(gray).astype(int)
for i in range(1, size):
# if last worker, handle extra rows
if i == (size - 1):
data_send = clean_image[image_y * (i - 1):, :image_x]
else:
data_send = clean_image[image_y * (i - 1):image_y * (i), :image_x]
comm.Send(data_send, dest=i)
print("Master: Image partition sent to slave %d." % i)
sys.stdout.flush()
else:
#allocate space for incoming data
data_recv = np.empty((image_y, image_x), dtype='int')
comm.Recv(data_recv, source=0)
print("Slave %d: Image partition received from master." % rank)
from getFeatures import getFeatures
from estimateAllTranslation import estimateAllTranslation
from applyGeometricTransformation import applyGeometricTransformation
from helper import rgb2gray
import numpy as np
import cv2
ind = 0
trackedVideo = []
nFrames = len(rawVideo)
# detect faces
bbox = detectFace(rawVideo[0])
# detect features
gray_im = rgb2gray(rawVideo[0])
x, y = getFeatures(gray_im, bbox)
for f in range(0, x.shape[1]):
x[:, f] = x[:, f] + bbox[f][0][1] # row coords
y[:, f] = y[:, f] + bbox[f][0][0] # col coords
# detect faces
bbox = detectFace(rawVideo[0])
# detect features
gray_im = rgb2gray(rawVideo[0])
x, y = getFeatures(gray_im, bbox)
for f in range(0, x.shape[1]):
x[:, f] = x[:, f] + bbox[f][0][1] # row coords
break
# Find face
dets = detector(frame, 0)
# For each face
for i, d in enumerate(dets):
# Crop the image to the face
crop = frame[d.top():d.bottom(), d.left():d.right()]
# Resize for CNN
newimg = cv2.resize(crop, (64, 64))
# Turn to grayscale
gray = rgb2gray(np.array(newimg))
# Reshape for CNN input
reshape = np.reshape(gray, (1, 64, 64, 1))
# Predict output and print informative prompt
prediction = new_model.predict(reshape)
if prediction[0][1] > 0.85:
print("Smiling")
elif prediction[0][0] > 0.85:
print("Not smiling")
else:
print("Unsure")
# Show image
Created on Tue Nov 21 13:57:25 2017
@author: rajivpatel-oconnor
"""
from detectFace import detectFace
from getFeatures import getFeatures
from estimateAllTranslation import estimateAllTranslation
from applyGeometricTransformation import applyGeometricTransformation
from helper import rgb2gray, overlay_points
import matplotlib.pyplot as plt
import cv2
#create all the images
color_img1 = plt.imread('./data/easy/TheMartian0.jpg')
img1 = rgb2gray(color_img1)
color_img2 = plt.imread('./data/easy/TheMartian130.jpg')
img2 = rgb2gray(color_img2)
#find the bounding boxes
bbox = detectFace(color_img1)
#find the positions of all the features
startXs, startYs = getFeatures(img1, bbox)
#overlay_points(img1, startXs, startYs, 'postGetFeatures_TheMartian1')
#draw the bounding boxes
first = bbox[0,0,:]
second = bbox[0,3,:]
cv2.rectangle(img1,(first[0],first[1]), (second[0],second[1]),color=(0,255,0))
def estimateFeatureTranslation_new(startX, startY, Ix, Iy, img1, img2):
#TODO: Your code here
if np.ndim(img1) == 3:
img1 = rgb2gray(img1)
if np.ndim(img2) == 3:
img2 = rgb2gray(img2)
win_width = 10
win_height = 10
nr = win_height/2
nc = win_width/2
im1_patch = getPatch(img1, startY, startX, win_width, win_height);
newX = []
newY = []
sXtemp = startX
sYtemp = startY
x, y = np.meshgrid(range(0,np.shape(im1_patch)[0]),range(0,np.shape(im1_patch)[1]))
patch_center = np.hstack((np.shape(im1_patch)[0]/2,np.shape(im1_patch)[1]/2))
x = x-patch_center[0]
y = y-patch_center[1]
x = x+startX
y = y+startY
im2_patch = getPatch(img2, startY, startX, win_width, win_height);
Ix_patch = getPatch(Ix, startY, startX, win_width, win_height);
Iy_patch = getPatch(Iy, startY, startX, win_width, win_height);
# W1 = scipy.interpolate.interp2d(img1, x, y, kind='linear')
# W2 = scipy.interpolate.interp2d(im2_patch, x, y, kind='linear')
# dx1 = scipy.interpolate.interp2d(Ix_patch, x, y, kind='linear')
# dy1 = scipy.interpolate.interp2d(Iy_patch, x, y, kind='linear')
W1 = im1_patch
W2 = im2_patch
dx1 = Ix_patch
dy1 = Iy_patch
It = W2-W1
I1x = dx1**2
I1y = dy1**2
I1xy = dx1*dy1
SigX1 = np.sum(I1x)
SigY1 = np.sum(I1y)
SigXY1 = np.sum(I1xy)
SigXIT = np.sum(dx1*It)
SigYIT = np.sum(dy1*It)
M = np.vstack(((SigX1, SigXY1),(SigXY1, SigY1)))
if np.linalg.cond(M) < 1/sys.float_info.epsilon: # if M is not singular
a = np.array([[SigX1,SigXY1],[SigXY1,SigY1]])
b = -np.array([[SigXIT],[SigYIT]])
UVMat = np.linalg.solve(a,b)
u = UVMat.item(0)
v = UVMat.item(1)
old_xrange = x
old_yrange = y
new_xrange = x
new_yrange = y
oldscore = np.sum((W2-W1)**2)
stopscore = 9999
count = 1
xtemp = 0
ytemp = 0
startX = []
startY = []
while stopscore > .1 and count < 40 :
#recompute window, matrix, u,v
startX = new_xrange[0,x.shape[0]/2]
startY = new_yrange[x.shape[1]/2,0]
Wprime = getPatch(img2, startY, startX, win_width, win_height);
if (Wprime.shape!=W1.shape):
break
It = Wprime-W1
SigXIT = np.sum(dx1*It)
SigYIT = np.sum(dy1*It)
M = np.vstack(((SigX1, SigXY1),(SigXY1, SigY1)))
a = np.array([[SigX1,SigXY1],[SigXY1,SigY1]])
b = -np.array([[SigXIT],[SigYIT]])
UVMat = np.linalg.solve(a,b)
u = UVMat.item(0)
v = UVMat.item(1)
new_xrange_t = old_xrange + u
new_yrange_t = old_yrange + v
newscore = np.sum((W2-Wprime)**2)
#in case loop goes past optimal (which is not w/in range of
#stopping condition)
if (oldscore < newscore):
break
stopscore = np.absolute(oldscore-newscore)
oldscore = newscore
old_xrange = new_xrange
old_yrange = new_yrange
new_xrange = new_xrange_t
new_yrange = new_yrange_t
#keep track of path
xtemp = xtemp + u
ytemp = ytemp + v
count = count+1
if np.any(np.absolute(xtemp)<win_width) and np.any(np.absolute(ytemp)<win_width) :
newX = sXtemp + xtemp
newY = sYtemp + ytemp
else:
newX = sXtemp
newY = sYtemp
else:
newX = sXtemp
newY = sYtemp
return newX, newY
def adaptive_hist_eq_cuda(img, slider_len):
""" Apply sliding window adaptive histogram equalization to an image
for improved local contrast. """
# make a copy of original to replace pixels
final_img = pycuda.driver.mem_alloc(img.nbytes)
n = len(img)
m = len(img[0])
gap = int(slider_len[0]// 2) # left and right shifts
for i in range(gap):
for j in range(gap, m-gap):
center_pixel_val = np.int32(img[i, j])
img_subset = copy(img[:i+gap,j-gap:j+gap])
n_temp = np.int32(len(img_subset))
m_temp = np.int32(len(img_subset[0]))
img_temp = pycuda.driver.mem_alloc(img_subset.nbytes)
pycuda.driver.memcpy_htod(img_temp, img_subset)
window_hist(img_temp, n_temp, m_temp, center_pixel_val, np.int32(0), np.int32(0), final_img, np.int32(i), np.int32(j), block=(32,32,1), grid=(2,2))
for i in range(n-gap, n):
for j in range(gap, m-gap):
center_pixel_val = img[i, j]
window_hist(img[i-gap:n,j-gap:j+gap], center_pixel_val, None, final_img, [i,j])
for i in range(gap, n-gap):
for j in range(gap):
center_pixel_val = img[i, j]
window_hist(img[i-gap:i+gap,:j+gap], center_pixel_val, None, final_img, (i,j))
for i in range(gap, n-gap):
for j in range(m-gap, m):
center_pixel_val = img[i, j]
window_hist(img[i-gap:i+gap,j-gap:m], center_pixel_val, None, final_img, (i,j))
for i in range(gap):
for j in range(gap):
center_pixel_val = img[i, j]
window_hist(img[:i+gap,:j+gap], center_pixel_val, None, final_img, (i,j))
for i in range(n-gap, n):
for j in range(m-gap, m):
center_pixel_val = img[i, j]
window_hist(img[i-gap:,j-gap:], center_pixel_val, None, final_img, (i,j))
for i in range(n-gap, n):
for j in range(gap):
center_pixel_val = img[i, j]
window_hist(img[i-gap:,:j+gap], center_pixel_val, None, final_img, (i,j))
for i in range(gap):
for j in range(m-gap, m):
center_pixel_val = img[i, j]
window_hist(img[:i+gap,j-gap:], center_pixel_val, None, final_img, (i,j))
# for each pixel in the center of the image, apply adaptive histogram equalization
for i in range(gap, n - gap):
for j in range(gap, m - gap):
center_pixel_val = img[i, j]
window_hist(img[i-gap:i+gap, j-gap:j+gap], center_pixel_val, slider_len, final_img, (i,j))
return final_img.astype(int)
if __name__ == "__main__":
source_module = pycuda.compiler.SourceModule \
(
"""
__global__ void window_hist(
unsigned int* img , int n ,
int m , int center_pixel_val ,
int slider_X, int slider_Y ,
unsigned int* final_img ,
int index_X,
int index_Y)
{
int pixel_freq[255];
int pdf[255];
int cdf[255];
int pixel_count;
if(slider_X != 0 && slider_Y){
pixel_count = slider_X * slider_Y;
}
else{
pixel_count = n * m;
slider_X = n;
slider_Y = m;
}
for(int i =0; i<slider_X; i++){
for(int j = 0; i<slider_Y; j++){
pixel_freq[ img[i,j] ]++;
}
}
for(int k = 0; k<255; k++){
pdf[ k ] = pixel_freq[k] / pixel_count;
}
int prev = 0;
for(int l = 0; l<255 ; l++){
cdf[l] = prev + pdf[l];
prev = cdf[l];
int temp = (cdf[l] * 250);
//int temp1 = (int)(( temp - floor(temp) > 0.5 ) ? ceil(temp) : floor(temp));
cdf[l] = temp;
if( l == center_pixel_val){
break;
}
final_img[index_X*n + index_Y] = cdf[center_pixel_val];
}
}
"""
)
img = plt.imread("test_image3.jpeg")
gray = rgb2gray(img)
clean_image = np.matrix.round(gray).astype(np.int32)
plt.figure(figsize=(13,7))
window_hist = source_module.get_function("window_hist")
final_img = adaptive_hist_eq_cuda(clean_image, (1, 1))