def extract(self,
root_path,
save_path):
flows_bvx = []
flows_bvy = []
flows_fvx = []
flows_fvy = []
if(os.path.isfile(save_path)):
self.logger.info("Output file {} exists.".format(save_path))
else:
dset = BaseDataset(root_path)
self.logger.info('Precomputing the optical flows...')
for f in np.arange(1, len(dset)):
self.logger.info('{}/{}'.format(f, len(dset)))
im1 = dset[f-1]['image'] / 255.
im2 = dset[f]['image'] / 255.
fvx, fvy, _ = pyflow.coarse2fine_flow(im1,
im2,
self.alpha,
self.ratio,
self.minWidth,
self.nOuterFPIterations,
self.nInnerFPIterations,
self.nSORIterations,
0)
bvx, bvy, _ = pyflow.coarse2fine_flow(im2,
im1,
self.alpha,
self.ratio,
self.minWidth,
self.nOuterFPIterations,
self.nInnerFPIterations,
self.nSORIterations,
0)
flows_bvx.append(bvx.astype(np.float32))
flows_bvy.append(bvy.astype(np.float32))
flows_fvx.append(fvx.astype(np.float32))
flows_fvy.append(fvy.astype(np.float32))
flows_bvx = np.asarray(flows_bvx).transpose(1, 2, 0)
flows_bvy = np.asarray(flows_bvy).transpose(1, 2, 0)
flows_fvx = np.asarray(flows_fvx).transpose(1, 2, 0)
flows_fvy = np.asarray(flows_fvy).transpose(1, 2, 0)
self.logger.info('Optical flow calculations done')
self.logger.info('Saving optical flows to {}'.format(save_path))
data = dict()
data['bvx'] = flows_bvx
data['bvy'] = flows_bvy
data['fvx'] = flows_fvx
data['fvy'] = flows_fvy
np.savez(save_path, **data)
self.logger.info('Done.')
def extract(self, root_path, save_path):
flows_bvx = []
flows_bvy = []
flows_fvx = []
flows_fvy = []
paths = [
os.path.join(save_path, 'flows_{}.npy'.format(f))
for f in ['fvx', 'fvy', 'bvx', 'bvy']
]
exists = [os.path.exists(p) for p in paths]
if (np.sum(exists) == 4):
self.logger.info("Flows are already computed.")
else:
dset = BaseDataset(root_path)
self.logger.info('Precomputing the optical flows...')
for f in np.arange(1, len(dset)):
self.logger.info('{}/{}'.format(f, len(dset)))
im1 = dset[f - 1]['image'] / 255.
im2 = dset[f]['image'] / 255.
fvx, fvy, _ = pyflow.coarse2fine_flow(im1, im2, self.alpha,
self.ratio,
self.minWidth,
self.nOuterFPIterations,
self.nInnerFPIterations,
self.nSORIterations, 0)
bvx, bvy, _ = pyflow.coarse2fine_flow(im2, im1, self.alpha,
self.ratio,
self.minWidth,
self.nOuterFPIterations,
self.nInnerFPIterations,
self.nSORIterations, 0)
flows_bvx.append(bvx.astype(np.float32))
flows_bvy.append(bvy.astype(np.float32))
flows_fvx.append(fvx.astype(np.float32))
flows_fvy.append(fvy.astype(np.float32))
bvx = np.asarray(bvx).transpose(1, 2, 0)
bvy = np.asarray(bvy).transpose(1, 2, 0)
fvx = np.asarray(fvx).transpose(1, 2, 0)
fvy = np.asarray(fvy).transpose(1, 2, 0)
self.logger.info('Optical flow calculations done')
self.logger.info('Saving optical flows to {}'.format(save_path))
np.save(os.path.join(save_path, 'flows_fvx.npy'), fvx)
np.save(os.path.join(save_path, 'flows_fvy.npy'), fvy)
np.save(os.path.join(save_path, 'flows_bvx.npy'), bvx)
np.save(os.path.join(save_path, 'flows_bvy.npy'), bvy)
self.logger.info('Done.')
def get_dense_optical_flow(image1, image2):
"""compute a vector field indicating how each position in image1 flows into image2?
"""
if image1.shape[-1] > 1:
image1 = cv2.cvtColor(image1, cv2.COLOR_RGBA2GRAY)[..., np.newaxis]
if image2.shape[-1] > 1:
image2 = cv2.cvtColor(image2, cv2.COLOR_RGBA2GRAY)[..., np.newaxis]
if OPTFLOW_METHOD == "Farneback":
flow = cv2.calcOpticalFlowFarneback(img_to_UINT8(image1), img_to_UINT8(image2), None, \
pyr_scale=0.5, levels=15, #min(get_max_mip_level(image1), get_max_mip_level(image2)), \
winsize=11, iterations=5, poly_n=5, poly_sigma=1.1, flags=0)
elif OPTFLOW_METHOD == "Brox":
u, v, _ = pyflow.coarse2fine_flow(img_to_FP64(image1),
img_to_FP64(image2),
alpha=alpha,
ratio=0.875,
minWidth=window,
nOuterFPIterations=12,
nInnerFPIterations=1,
nSORIterations=40,
colType=1) #alpha 0.012
flow = np.concatenate((u[..., np.newaxis], v[..., np.newaxis]),
axis=-1).astype(np.float32)
#print("flow_shape", flow.shape, "dtype", flow.dtype)
return flow
def calc_OF(prvs, next):
next = cv2.resize(next, (224, 224), interpolation=cv2.INTER_AREA)
prvs = cv2.resize(prvs, (224, 224), interpolation=cv2.INTER_AREA)
prvs = np.array(prvs, dtype=float) / 255.0
next = np.array(next, dtype=float) / 255.0
u, v, im2W = pyflow.coarse2fine_flow(prvs, next, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
max_flow = 8
scale = 128 / max_flow
mag_flow = np.sqrt(np.square(flow[..., 0]) + np.square(flow[..., 1]))
flow = flow * scale
flow = flow + 128
flow[flow < 0] = 0
flow[flow > 255] = 255
mag_flow = mag_flow * scale
mag_flow = mag_flow + 128
mag_flow[mag_flow < 0] = 0
mag_flow[mag_flow > 255] = 255
flow_img = np.dstack([flow, mag_flow[..., np.newaxis]])
flow_img = flow_img.astype(int)
# cv2.imwrite(os.path.join("temp.png"),flow_img)
#x = cv2.imread("temp.png")
#y = cv2.resize(x,(224,224),interpolation = cv2.INTER_AREA)
return flow_img
def optical_flow_movement_score(video):
T,h,w,c=video.shape
score=np.zeros((T,h,w))
#flow options
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 5
nInnerFPIterations = 1
nSORIterations = 25
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
for t in range(1,T):
u, v, im2W = pyflow.coarse2fine_flow(
video[t-1,:,:,:], video[t,:,:,:], alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
score[t,:,:]=np.sqrt(u**2+v**2)
mean=score.mean()
std=score.std()
threshold = mean - 0.1* std
print("Optical flow treshold: %f" % threshold)
score[score<threshold]=0
score= ndimage.grey_erosion(score,(1, 20, 20))
score= ndimage.gaussian_filter(score, sigma=(0,3,3), order=0)
return score
def get_opfl_frame(image_1, image_2, save_path, frame_num):
im1 = np.array(Image.open(image_1)).astype(float) / 255
im2 = np.array(Image.open(image_2)).astype(float) / 255
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 1 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
u, v, im2W = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
save_path = save_path + '/' + str(frame_num) + '.png'
cv2.imwrite(save_path, rgb)
def create_flows(im1, im2, wrapp_file, flow_file):
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
#np.save('examples/outFlow.npy', flow)
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imwrite(flow_file, rgb)
cv2.imwrite(wrapp_file, im2W[:, :, ::-1] * 255)
def save_of(im1, im2, fname):
# im1 = np.array(Image.open('examples/car1.jpg'))
# im2 = np.array(Image.open('examples/car2.jpg'))
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
# np.save('examples/outFlow.npy', flow)
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
# print('save of to %s' % fname)
save_img(rgb, fname)
def get_opticalflow_nor(image0, image1):
s = time.time()
alpha = 0.012
ratio = 0.75
minWidth = 10
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 1
# help(pyflow)
u, v, im2W = pyflow.coarse2fine_flow(image0, image1, alpha, ratio,
minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, image0.shape[0], image0.shape[1], image0.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
# create output image
hsv = np.zeros((64, 720, 3), dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
hsv[:, :, 2] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
return rgb
def flow_twoimages(im1path, im2path, outpath, outimage=False):
im1 = np.array(Image.open(im1path))
im2 = np.array(Image.open(im2path))
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
np.save(outpath, flow)
if outimage:
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imwrite(outpath + '.png', rgb)
cv2.imwrite(outpath + '.jpg', im2W[:, :, ::-1] * 255)
def get_flow(im1, im2):
# Resizing this won't work, it causes a dimension mismatch
# width, height = im1.size
# width = int(width / 2)
# height = int(height / 2)
# im1 = im1.resize((width, height), Image.BICUBIC)
# im2 = im2.resize((width, height), Image.BICUBIC)
im1 = np.array(im1)
im2 = np.array(im2)
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
#flow = rescale_flow(flow,0,1)
return flow
def createFlow(base, savebase, impath1, impath2):
im1 = np.array(Image.open(os.path.join(base, impath1)))
im2 = np.array(Image.open(os.path.join(base, impath2)))
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
#s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
#e = time.time()
#print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' % (
# e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
import cv2
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imwrite(os.path.join(savebase, impath2), rgb)
def create_frame_flow(prev_frame, current_frame):
height = np.size(prev_frame, 0)
width = np.size(prev_frame, 1)
b_height = math.floor((height - 11) / n)
b_width = math.floor((width - 11) / m)
alpha = 0.0026
ratio = 0.6
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
para = []
[vx, vy, warpI2] = pyflow.coarse2fine_flow(np.float64(prev_frame),
np.float64(current_frame),
alpha,
ratio,
minWidth,
nOuterFPIterations,
nInnerFPIterations,
nSORIterations,
colType=1)
flow_magnitude = np.power(np.square(vx) + np.square(vy), 0.5)
return [flow_magnitude, vx, vy]
def interpolation(im1, im2, ts):
uForward, vForward, im2WForward = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations, colType)
uBackward, vBackward, im2WBackward = pyflow.coarse2fine_flow(
im2, im1, alpha, ratio, minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations, colType)
forward = np.concatenate((uForward[..., None], vForward[..., None]),
axis=2)
backward = np.concatenate((uBackward[..., None], vBackward[..., None]),
axis=2)
interpolated = []
for t in ts:
uFlow, vFlow = pyflow.splat_motions(uForward, vForward, uBackward,
vBackward, im1, im2, t)
flow = np.concatenate((uFlow[..., None], vFlow[..., None]), axis=2)
flow = cv2.GaussianBlur(flow, (11, 11), 10)
interp = pyflow.color_transfer(im1, im2, forward, backward, flow, t)
interpolated.append(interp)
return interpolated
def generate_optical_flow(self, previous_frame):
'''
Generate the Dense Optical Flow between the current frame, and the previous frame
Parameters:
@previous_frame : A parameter of type Frame, that contains the previous frame
'''
'''
Flow Options
'''
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
assert (self.dense_optical_flow_vector == False
), "Optical Flow already generated for the current frame."
# Cropping the previous_frame
previous_frame = crop_to_height(previous_frame)
# Resizing the previous frame
previous_frame = cv2.resize(previous_frame,
(image_height, image_width))
# Numpy Arrays
previous_temporal_image = previous_frame.astype(float) / 255.
next_temporal_image = self.placeholder_pixel_values_array.astype(
float) / 255.
# previous_temporal_image = previous_temporal_image.astype(np.float32)
# next_temporal_image = next_temporal_image.astype(np.float32)
# print("ACTUAL:")
# print(len(self.pixel_values_array),"x",len(self.pixel_values_array[0]), "x", len(self.pixel_values_array[0][0][0]))
# Adding new method
u, v, im2W = pyflow.coarse2fine_flow(previous_temporal_image,
next_temporal_image, alpha, ratio,
minWidth, nOuterFPIterations,
nInnerFPIterations,
nSORIterations, colType)
temporal_image = np.concatenate((u[..., None], v[..., None]), axis=2)
temporal_image = process_flow(previous_temporal_image, temporal_image)
# temporal_image_grayscale = cv2.cvtColor(temporal_image, cv2.COLOR_BGR2GRAY)
temporal_image_after_reshape = np.reshape(
temporal_image, (1, image_width, image_height, 1))
self.placeholder_dense_optical_flow_vector = temporal_image_after_reshape
def pyflow_optical_flow(im1: np.ndarray, im2: np.ndarray):
import pyflow
# Flow Options:
alpha = 0.012
ratio = 0.75
with suppress_stdout_stderr():
u, v, _ = pyflow.coarse2fine_flow(
im1.astype(float) / 255,
im2.astype(float) / 255, alpha, ratio)
return np.concatenate((u[..., None], v[..., None]), axis=2)
def interpolation(im1, im2, ts):
## be careful about the ordering
uBackward, vBackward, im2WBackward = pyflow.coarse2fine_flow(
im2, im1, alpha, ratio, minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations, colType)
interpolated = []
for t in ts:
ut = uBackward * t
vt = vBackward * t
interp = apply_flow(im1, ut, vt)
interpolated.append(interp)
return interpolated
def opt_flow2(image1, image2):
image1 = image1.astype(float) / 255.
image2 = image2.astype(float) / 255.
image1 = np.expand_dims(image1, axis=2)
image2 = np.expand_dims(image2, axis=2)
# for full res
#alpha = 0.012
#ratio = 0.5
#minWidth = 10
#nOuterFPIterations = 2
#nInnerFPIterations = 2
#nSORIterations = 3
alpha = 0.015
ratio = 0.5
minWidth = 10
nOuterFPIterations = 2
nInnerFPIterations = 2
nSORIterations = 3
colType = 1 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(image1, image2, alpha, ratio,
minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, image1.shape[0], image1.shape[1], image1.shape[2]))
flow = np.concatenate(
(-u[..., None], -v[..., None]),
axis=2) # reversed direction, from image 2 flow to image 1
gamma = 3
hsv = np.zeros([image1.shape[0], image1.shape[1], 3])
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 1, cv2.NORM_MINMAX)
hsv[..., 2] = 255 * hsv[..., 2]**(1 / gamma)
hsv = np.uint8(hsv)
img_flow = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
return mag, ang / np.pi * 180, img_flow
def video2motion_level(vid,queue):
fps = vid.get_meta_data()['fps']
num_frames = vid._meta['nframes']
trim_borders, step_btwn_frames = get_valid_frame_range(num_frames)
flow_mag = np.zeros(num_frames- 2 * trim_borders - 1,dtype=np.float32);
print(flow_mag.shape)
#print(fps, num_frames)
try:
previous_frame = vid.get_data(trim_borders)
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
flow_mag_idx = 0
for frame_idx in np.arange(trim_borders+1, num_frames-trim_borders):
current_frame = vid.get_data(frame_idx)
if np.array_equal(current_frame, previous_frame):
## If both frames are identical, optical flow is zero
flow_mag[flow_mag_idx] = 0;
else:
im1 = previous_frame.astype(float) / 255.
im2 = current_frame.astype(float) / 255.
u, v, im2W = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
flow_mag[flow_mag_idx] = np.linalg.norm(mag);
flow_mag_idx +=1
previous_frame = current_frame;
except:
queue.put(None)
return None
## Visualization code
#plt.bar(np.arange(trim_borders+1, num_frames-trim_borders), flow_mag);
#plt.savefig('freq.png')
queue.put(flow_mag / np.sum(flow_mag))
def optical_flow(loader, viz, cont):
n_images = loader.__len__()
for index in range(n_images):
print("Estimating flow " + str(index + 1) + " / " + str(n_images))
images, _, _ = loader.__getitem__(index)
img_path = loader.files[loader.split][index].rstrip()
flow_path = re.sub('.png$', '.npy', img_path)
flow_path = re.sub('leftImg8bit', 'flow', flow_path)
if cont and os.path.isfile(flow_path):
continue
flow_dir = flow_path.rpartition("/")[0]
if not os.path.exists(flow_dir):
os.makedirs(flow_dir)
first = np.array(np.swapaxes(np.swapaxes(images[:3, :, :], 0, 1), 1,
2),
dtype=float,
order='c')
second = np.array(np.swapaxes(np.swapaxes(images[3:, :, :], 0, 1), 1,
2),
dtype=float,
order='c')
s = time.time()
u, v, _ = pyflow.coarse2fine_flow(first, second, alpha, ratio,
minWidth, nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, first.shape[0], first.shape[1], first.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
np.save(flow_path, flow)
if viz:
first_img_path = re.sub('_flow.npy$', '.png', flow_path)
cv2.imwrite(first_img_path, first)
hsv = np.zeros(first.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
flow_img_path = re.sub('.npy$', '.png', flow_path)
cv2.imwrite(flow_img_path, rgb)
def compute_pyflow(vid, vidpath):
# Check vid path to load video
pathname, ext = os.path.splitext(vidpath)
flowpath = pathname + '_pyflow.npy'
_, name = os.path.split(flowpath)
print('Checking for', name)
if os.path.exists(flowpath):
flow_mags = np.load(flowpath)
if flow_mags.shape == vid.shape:
print('file found')
return flow_mags
print('Flow mag shape does not match')
else:
print(name, 'does not exist. Computing from scratch.')
# calculate flow from scratch
# vid must be double from 0 to 1
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 1 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
rows, cols, n_frames = vid.shape
flow_mags = np.zeros(vid.shape)
update = int(n_frames / 10)
frame1 = vid[..., 0, np.newaxis].copy(order='C')
for i in range(1, n_frames):
frame2 = vid[..., i, np.newaxis].copy(order='C')
u, v, im2W = pyflow.coarse2fine_flow(frame1, frame2, alpha, ratio,
minWidth, nOuterFPIterations,
nInnerFPIterations,
nSORIterations, colType)
mag = (u**2 + v**2)**.5
flow_mags[..., i] = cv2.normalize(mag, None, 0, 1.0, cv2.NORM_MINMAX)
frame1 = frame2.copy()
if i % update == 0:
print(int(100 * i / n_frames), '% complete')
flow_mags[..., 0] = flow_mags[..., 1]
np.save(flowpath, flow_mags)
return flow_mags
def optical_flow(data_values, idx_2, idx_1):
im1 = np.zeros((16, 16, 1))
im2 = np.zeros((16, 16, 1))
im1[:, :, 0] = data_values[:, :, idx_1]
im2[:, :, 0] = data_values[:, :, idx_2]
# im1[:, :, 1] = 0
# im2[:, :, 1] = 0
# im1[:, :, 2] = 1
# im2[:, :, 2] = 1
# im1 = im1.astype(float) / 255.
# im2 = im2.astype(float) / 255.
alpha = 0.012
ratio = 0.75
minWidth = 4
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 1
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations, nSORIterations,
colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
np.save('./outFlow.npy', flow)
plt.imshow(flow[:, :, 0], interpolation='nearest')
plt.show()
plt.imshow(flow[:, :, 1], interpolation='nearest')
plt.show()
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
# hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
# hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
# rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
# cv2.imwrite('./outFlow_new.png', hsv)
plt.imshow(hsv[:, :, 0], interpolation='nearest')
plt.show()
def ComputeCeLiu(target, source):
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
params = [
alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations
]
vx, vy, _ = pyflow.coarse2fine_flow(target, source, alpha, ratio, minWidth,
nOuterFPIterations, nInnerFPIterations,
nSORIterations)
flow = np.stack((vx, vy))
flow = np.swapaxes(np.swapaxes(flow, 0, 2), 0, 1)
return flow
def get_flow(im1, im2):
im1 = np.array(im1)
im2 = np.array(im2)
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
#flow = rescale_flow(flow,0,1)
return flow
def coarse2fine_flow(im1,im2):
im1_f = im1.astype("float") / 255.0
im2_f = im2.astype("float") / 255.0
#check if image is grayscale
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
if(im1_f.ndim < 3):
im1_f = im1_f[:, :, np.newaxis]
im2_f = im2_f[:, :, np.newaxis]
colType = 1
elif(im1_f.shape[2] < 3):
colType = 1
u, v, im2W = pyflow.coarse2fine_flow( im1_f, im2_f, alpha, ratio, minWidth,
nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
im_warped = (im2W[:, :, ::-1] * 255).astype("uint8")
return flow, im_warped
def optical_flow(framepath, frames):
dataset_root = framepath
output_root = join(framepath, 'Optical')
if not os.path.exists(output_root):
os.mkdir(output_root)
dir_frames = join(dataset_root, 'Frames')
for j in range(0, frames - 1):
im1 = np.array(Image.open(join(dir_frames, '{:d}'.format(j) + '.jpg')))
im2 = np.array(
Image.open(join(dir_frames, '{:d}'.format(j + 1) + '.jpg')))
im1 = cv2.resize(im1, (120, 68))
im2 = cv2.resize(im2, (120, 68))
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations,
nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
print(j)
cv2.imwrite(join(output_root, '{}'.format(j) + '.png'), rgb)
def calculate_flow(im1,im2):
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
u, v, im2W = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
np.save(str(frame)+ '_frame.npy', flow)
return flow,im2W
def calcOpticalFlow(self, im1, im2, count):
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations,
nSORIterations, colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
return flow
def brocks_wrapper(self, pos, sample, output):
im1 = sample[0]
im2 = sample[1]
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
im1 = np.stack((im1, ), -1)
im2 = np.stack((im2, ), -1)
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 5
nOuterFPIterations = 1
nInnerFPIterations = 1
nSORIterations = 1
colType = 1 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations,
nInnerFPIterations,
nSORIterations, colType)
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
output.put([pos, flow])
im2 = np.array(Image.open('examples/car2.jpg'))
im1 = im1.astype(float) / 255.
im2 = im2.astype(float) / 255.
# Flow Options:
alpha = 0.012
ratio = 0.75
minWidth = 20
nOuterFPIterations = 7
nInnerFPIterations = 1
nSORIterations = 30
colType = 0 # 0 or default:RGB, 1:GRAY (but pass gray image with shape (h,w,1))
s = time.time()
u, v, im2W = pyflow.coarse2fine_flow(im1, im2, alpha, ratio, minWidth,
nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
e = time.time()
print('Time Taken: %.2f seconds for image of size (%d, %d, %d)' %
(e - s, im1.shape[0], im1.shape[1], im1.shape[2]))
flow = np.concatenate((u[..., None], v[..., None]), axis=2)
np.save('examples/outFlow.npy', flow)
if args.viz:
import cv2
hsv = np.zeros(im1.shape, dtype=np.uint8)
hsv[:, :, 0] = 255
hsv[:, :, 1] = 255
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
absolute_coorindates[y][x][0] = x
absolute_coorindates[y][x][1] = y
j = 0
for i in range(10):
source = destination
destination = np.array(Image.open("gray/%05i.tiff" % (i + 1)))
im1 = source.astype(float) / 255.
im2 = destination.astype(float) / 255.
im1 = np.reshape(im1, list(im1.shape) + [1])
im2 = np.reshape(im2, list(im2.shape) + [1])
u, v, im2W = pyflow.coarse2fine_flow(
im1, im2, alpha, ratio, minWidth, nOuterFPIterations, nInnerFPIterations,
nSORIterations, colType)
relative_flow = np.concatenate((u[..., None], v[..., None]), axis=2)
flow = absolute_coorindates + relative_flow
Image.fromarray(source).save("interpolate/%05i.tiff" % j)
j += 1
for ti in np.linspace(0, 1, NFRAMES + 1, endpoint=False)[1:]:
back = - ti * flow
forward = (1 - ti) * flow
bf = trans(source, back)
af = trans(destination, forward)
frame = (1 - ti) * bf + ti * af
