def sharpen(im, kernel):
if im.dtype != np.uint8:
print('Fix data type to uint8')
return
#Convert to float and scale to 255
im = dip.im_to_float(im)
im *= 255
im_lap = im
im_lap = dip.convolve2d(im_lap, kernel)
# Shift up to remove negative values
im_lap = im_lap - np.min(im_lap)
# Rescale to range of integers
im_lap = 255 * (im_lap / np.max(im_lap))
#crop to original size
im_lap = dip.resample(im_lap, np.array([[1, 0], [0, 1]]), crop=True, crop_size=(im.shape[0], im.shape[1]))
# Add laplacian back in, normalize, convert to uint8
im_n = im + im_lap
im_n = im_n - np.min(im_n)
im_n = 255 * im_n / np.max(im_n)
im_n = im_n.astype(np.uint8)
im_lap = im_lap.astype(np.uint8)
return im_n, im_lap
def horn_schuck(u, v, dIx, dIy, dIt, param):
# Set constants
avg_kernel = np.array([[1 / 12, 1 / 6, 1 / 12], [1 / 6, 0, 1 / 6],
[1 / 12, 1 / 6, 1 / 12]])
if param is None:
num_iterations = 100
else:
num_iterations = param
alpha = 1
for i in range(num_iterations):
# Iteratively filter the u and v arrays with averaging filters
u_avg = dip.convolve2d(u, avg_kernel, mode='same')
v_avg = dip.convolve2d(v, avg_kernel, mode='same')
# Compute flow vectors constrained by local averages and the optical
# flow constaints
# ============================ EDIT THIS PART =========================
denominator = np.power(dIx, 2) + np.power(dIy, 2) + np.power(alpha, 2)
numerator = dIx * u_avg + dIy * v_avg + dIt
u = u_avg - (dIx * numerator) / denominator
v = v_avg - (dIy * numerator) / denominator
# Compute and display the nan and inf ratios
u_nan_ratio = float(np.sum(np.isnan(u)) / u.size)
v_nan_ratio = float(np.sum(np.isnan(v)) / v.size)
u_inf_ratio = float(np.sum(np.isinf(u)) / u.size)
v_inf_ratio = float(np.sum(np.isinf(v)) / v.size)
print('Estimated Flow nan ratio: u = {:.2f}, v = {:.2f}'.format(
u_nan_ratio, v_nan_ratio))
print('Estimated Flow inf ratio: u = {:.2f}, v = {:.2f}'.format(
u_inf_ratio, v_inf_ratio))
# Remove nan values from u and v
u[np.isnan(u)] = 0
v[np.isnan(v)] = 0
return u, v
#Filter
SIZE = 25
filterSize = (SIZE, SIZE)
minor = 5**2
mild = 25**2
severe = 100**2
minorWindow = dip.window_2d(filterSize, window_type='gaussian', variance=minor)
mildWindow = dip.window_2d(filterSize, window_type='gaussian', variance=mild)
severeWindow = dip.window_2d(filterSize,
window_type='gaussian',
variance=severe)
#Output of the Filter
minorX = dip.convolve2d(X, minorWindow, mode='same', boundary='wrap')
mildX = dip.convolve2d(X, mildWindow, mode='same', boundary='wrap')
severeX = dip.convolve2d(X, severeWindow, mode='same', boundary='wrap')
#Laplacian Filtering
LaplaceKernel = np.array([[0, 1, 0], [1, -4, 1], [0, 1, 0]], dtype=np.float)
edgeMinorX = dip.convolve2d(X, LaplaceKernel, mode='same', boundary='wrap')
edgeMildX = dip.convolve2d(X, LaplaceKernel, mode='same', boundary='wrap')
edgeSevereX = dip.convolve2d(X, LaplaceKernel, mode='same', boundary='wrap')
#PSNR Calculations
PSNR_minor = dip.metrics.PSNR(dip.float_to_im(X / 255),
dip.float_to_im(edgeMinorX / 255))
PSNR_mild = dip.metrics.PSNR(dip.float_to_im(X / 255),
def run_optical_flow(filepath_ind: int, OF_alg: int, param: int=100,
display: bool=True):
frame_1 = dip.im_read(filepaths[filepath_ind] + 'frame1.png')[:, :, :3]
frame_2 = dip.im_read(filepaths[filepath_ind] + 'frame2.png')[:, :, :3]
residual = np.abs(frame_1.astype(float) - frame_2.astype(float)) \
.astype(np.uint8)
frame_1_gray = dip.rgb2gray(frame_1)
frame_2_gray = dip.rgb2gray(frame_2)
PSNR_val = dip.PSNR(frame_1_gray, frame_2_gray)
if display:
# Plot the initial images
dip.figure()
dip.subplot(1, 3, 1)
dip.imshow(frame_1)
dip.title('Frame 1', fontsize='x-small')
dip.subplot(1, 3, 2)
dip.imshow(frame_2)
dip.title('Frame 2', fontsize='x-small')
dip.subplot(1, 3, 3)
dip.imshow(residual)
dip.title('Residual - PSNR: {:.2f} dB'.format(PSNR_val), fontsize='x-small')
# Convert to grayscale for analysis
frame_1 = dip.im_to_float(frame_1_gray)
frame_2 = dip.im_to_float(frame_2_gray)
start_time = default_timer()
# ============================ EDIT THIS PART =============================
mask_x = np.array([[-1, 1], [-1,1]])
mask_y = np.array([[-1, -1], [1,1]])
mask_t_2 = np.array([[-1, -1], [-1,-1]])
mask_t_1 = np.array([[1, 1], [1,1]])
dIx = dip.convolve2d(frame_1, mask_x, mode='same', like_matlab=True)
dIy = dip.convolve2d(frame_1, mask_y, mode='same', like_matlab=True)
dIt = dip.convolve2d(frame_1, mask_t_1, mode='same', like_matlab=True) + dip.convolve2d(frame_2, mask_t_2, mode='same', like_matlab=True)
# ==========!!!!! DO NOT EDIT ANYTHING BELOW THIS !!!!!====================
# Instantiate blank u and v matrices
u = np.zeros_like(frame_1)
v = np.zeros_like(frame_1)
if 0 == OF_alg:
print('The optical flow is estimated using Horn-Schuck...')
u, v = horn_schuck(u, v, dIx, dIy, dIt, param)
elif 1 == OF_alg:
print('The optical flow is estimated using Lucas-Kanade...')
u, v = lucas_kanade(u, v, dIx, dIy, dIt, param)
else:
raise ValueError('OF_alg must be either 0 or 1')
end_time = default_timer()
# Determine run time
duration = end_time - start_time
clock = [int(duration // 60), int(duration % 60)]
print('Flow estimation time was {} minutes and {} seconds'
.format(*clock))
# Downsample for better visuals
stride = 10
m, n = frame_1.shape
x, y = np.meshgrid(range(n), range(m))
x = x.astype('float64')
y = y.astype('float64')
# Downsampled u and v
u_ds = u[::stride, ::stride]
v_ds = v[::stride, ::stride]
# Coords for downsampled u and v
x_ds = x[::stride, ::stride]
y_ds = y[::stride, ::stride]
# Estimated flow
estimated_flow = np.stack((u, v), axis=2)
# Read file for ground truth flow
ground_truth_flow = read_flow_file(filepaths[filepath_ind] + 'flow1_2.flo')
u_gt_orig = ground_truth_flow[:, :, 0]
v_gt_orig = ground_truth_flow[:, :, 1]
u_gt = np.where(np.isnan(u_gt_orig), 0, u_gt_orig)
v_gt = np.where(np.isnan(v_gt_orig), 0, v_gt_orig)
# Downsampled u_gt and v_gt
u_gt_ds = u_gt[::stride, ::stride]
v_gt_ds = v_gt[::stride, ::stride]
if display:
# Plot the optical flow field
dip.figure()
dip.subplot(2, 2, 1)
dip.imshow(frame_2, 'gray')
dip.quiver(x_ds, y_ds, u_ds, v_ds, color='r')
dip.title('Estimated', fontsize='x-small')
dip.subplot(2, 2, 2)
dip.imshow(frame_2, 'gray')
dip.quiver(x_ds, y_ds, u_gt_ds, v_gt_ds, color='r')
dip.title('Ground truth', fontsize='x-small')
# Draw colored velocity flow maps
dip.subplot(2, 2, 3)
dip.imshow(flow_to_color(estimated_flow))
dip.title('Estimated', fontsize='x-small')
dip.subplot(2, 2, 4)
dip.imshow(flow_to_color(ground_truth_flow))
dip.title('Ground truth', fontsize='x-small')
# Normalization for metric computations
normalize = lambda im: (im - np.min(im)) / (np.max(im) - np.min(im))
un = normalize(u)
un_gt = normalize(u_gt)
un_gt[np.isnan(u_gt_orig)] = 1
vn = normalize(v)
vn_gt = normalize(v_gt)
vn_gt[np.isnan(v_gt_orig)] = 1
# Error calculations and displays
EPE = ((un - un_gt) ** 2 + (vn - vn_gt) ** 2) ** 0.5
AE = np.arccos(((un * un_gt) + (vn * vn_gt) + 1) /
(((un + vn + 1) * (un_gt + vn_gt + 1)) ** 0.5))
EPE_nan_ratio = np.sum(np.isnan(EPE)) / EPE.size
AE_nan_ratio = np.sum(np.isnan(AE)) / AE.size
EPE_inf_ratio = np.sum(np.isinf(EPE)) / EPE.size
AE_inf_ratio = np.sum(np.isinf(AE)) / AE.size
print('Error nan ratio: EPE={:.2f}, AE={:.2f}'
.format(EPE_nan_ratio, AE_nan_ratio))
print('Error inf ratio: EPE={:.2f}, AE={:.2f}'
.format(EPE_inf_ratio, AE_inf_ratio))
EPE_avg = np.mean(EPE[~np.isnan(EPE)])
AE_avg = np.mean(AE[~np.isnan(AE)])
print('EPE={:.2f}, AE={:.2f}'.format(EPE_avg, AE_avg))
if display:
dip.show()
return clock, EPE_avg, AE_avg
def scale(img: np.ndarray):
for i in range(img.shape[0]):
for j in range(img.shape[1]):
if img[i, j] > 1:
img[i, j] = 1
if img[i, j] < 0:
img[i, j] = 0
return img
img = dip.im_read('barbara.png')
Gaussian = 1 / 16 * np.array([[1, 2, 1], [2, 4, 2], [1, 2, 1]])
img1 = dip.convolve2d(img, Gaussian, mode='same')
img2 = dip.convolve2d(img1, Gaussian, mode='same')
img3 = dip.convolve2d(img2, Gaussian, mode='same')
Lap_kernel = np.array([[0, 1, 0], [1, -4, 1], [0, 1, 0]])
ELap_kernel = np.array([[1, 1, 1], [1, -8, 1], [1, 1, 1]])
Lap1 = dip.convolve2d(img1, Lap_kernel, mode='same')
Lap2 = dip.convolve2d(img2, Lap_kernel, mode='same')
Lap3 = dip.convolve2d(img3, Lap_kernel, mode='same')
ELap1 = dip.convolve2d(img1, ELap_kernel, mode='same')
ELap2 = dip.convolve2d(img2, ELap_kernel, mode='same')
ELap3 = dip.convolve2d(img3, ELap_kernel, mode='same')
sharpen1 = scale(img1 - Lap1)
def computeGradientDMD(img, xi, yi, scale, stride, blkRadii):
pts1 = xi
pts2 = yi
numSamp = np.shape(xi)[1]
sampPerScale = numSamp / scale
# Constrain points within appropriate bounds
pts1[np.where(pts1 > blkRadii - scale + 1)] = blkRadii - scale + 1
pts1[np.where(pts1 < -blkRadii)] = -blkRadii
pts2[np.where(pts2 > blkRadii - scale + 1)] = blkRadii - scale + 1
pts2[np.where(pts2 < -blkRadii)] = -blkRadii
pts1 = pts1 + blkRadii + 1
pts2 = pts2 + blkRadii + 1
blkSize = 2 * blkRadii + 1
r, c = np.shape(img)
effr = r - blkSize
effc = c - blkSize
IGrad_feat = 5 # 5 descriptors in BIGD vector
Gx = sobel(img, axis=1)
Gy = sobel(img, axis=0)
featureimg_total = np.zeros((r, c, IGrad_feat))
featureimg_total[..., 0] = img
featureimg_total[..., 1] = Gx
featureimg_total[..., 2] = np.abs(Gx)
featureimg_total[..., 3] = Gy
featureimg_total[..., 4] = np.abs(Gy)
numFV = int((np.shape(img)[0] / stride - blkRadii) *
(np.shape(img)[1] / stride - blkRadii))
v_new = np.zeros(
(numFV, numSamp * IGrad_feat
)) # Number of feature vectors by number of features per vector
pts1 = pts1.astype(int)
pts2 = pts2.astype(int)
for j in range(IGrad_feat):
ftimg = featureimg_total[..., j]
v = np.zeros(
(numFV, numSamp)
) # Number of feature vectors by number of sampled points in patch
# Compute integral images
itimg = np.cumsum(ftimg, 0)
itimg = np.cumsum(itimg, 1)
iimg = np.zeros((np.shape(itimg)[0] + 2, np.shape(itimg)[1] + 2))
iimg[1:-1, 1:-1] = itimg
# Compute the normalization constant for each block
normMat = np.sqrt(
dip.convolve2d(np.power(ftimg, 2),
np.ones(((blkRadii * 2) + 1, (blkRadii * 2) + 1))))
normMat = normMat[2 * blkRadii:-2 * blkRadii,
2 * blkRadii:-2 * blkRadii]
normMat[np.where(normMat == 0)] = 1e-10
for i in range(numSamp):
mbSize = int(np.floor((i + sampPerScale) / sampPerScale))
# Integral image coordinates for computing the sum of the pixel values
# of size mbSize wrt pts1
iiPt1 = iimg[pts1[0, i] + mbSize:pts1[0, i] + effr + mbSize + 1,
pts1[1, i] + mbSize:pts1[1, i] + effc + mbSize + 1]
iiPt2 = iimg[pts1[0,i] + mbSize : pts1[0,i] + effr + mbSize + 1, \
pts1[1,i] : pts1[1,i]+effc + 1]
iiPt3 = iimg[pts1[0,i] : pts1[0,i] + effr + 1, \
pts1[1,i]+ mbSize : pts1[1,i]+effc+ mbSize + 1]
iiPt4 = iimg[pts1[0,i] : pts1[0,i] + effr + 1,\
pts1[1,i] : pts1[1,i] + effc + 1]
blockSum1 = iiPt4 + iiPt1 - iiPt2 - iiPt3
iiPt1 = iimg[pts2[0,i] + mbSize : pts2[0,i]+effr + mbSize + 1, \
pts2[1,i] + mbSize : pts2[1,i]+effc + mbSize + 1]
iiPt2 = iimg[pts2[0,i] + mbSize : pts2[0,i] + effr + mbSize + 1, \
pts2[1,i] : pts2[1,i]+effc + 1]
iiPt3 = iimg[pts2[0,i] : pts2[0,i] + effr + 1, \
pts2[1,i]+ mbSize : pts2[1,i]+effc+ mbSize + 1]
iiPt4 = iimg[pts2[0,i] : pts2[0,i] + effr + 1,\
pts2[1,i] : pts2[1,i] + effc + 1]
blockSum2 = iiPt4 + iiPt1 - iiPt2 - iiPt3
# Average intensities
blockSum1 = blockSum1 / (mbSize * mbSize)
blockSum2 = blockSum2 / (mbSize * mbSize)
#Block difference
diffImg = ((blockSum1 - blockSum2)) / normMat
# Samples at appropriate points
selectedGrid = diffImg[0:-1:stride, 0:-1:stride]
v[:, i] = selectedGrid.flatten()
v_new[:, j * numSamp:(j + 1) * numSamp] = v
return v_new.T