def measurePictures(self):
prvs = cv2.cvtColor(self.f1,cv2.COLOR_BGR2GRAY)
ret, self.frame1 = cv2.imencode('.png', self.f1)
ret = cv2.imwrite('frame1.png', self.f1)
next = cv2.cvtColor(self.f2,cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
bgr = self.flow2hsv(flow)
(res, mx, my, ms) = self.flow2measure(flow)
self.showmeasure(bgr, res, mx, my, ms)
ret, self.flow12 = cv2.imencode('.png', bgr)
ret = cv2.imwrite('flow12.png', bgr)
ret, self.frame2 = cv2.imencode('.png', self.f2)
ret = cv2.imwrite('frame2.png', self.f2)
prvs = next
next = cv2.cvtColor(self.f3,cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
bgr = self.flow2hsv(flow)
(res, mx, my, ms) = self.flow2measure(flow)
self.showmeasure(bgr, res, mx, my, ms)
ret, self.flow23 = cv2.imencode('.png', bgr)
ret = cv2.imwrite('flow23.png', bgr)
ret, self.frame3 = cv2.imencode('.png', self.f3)
ret = cv2.imwrite('frame3.png', self.f3)
def improved_farneback(previous_frame, frame):
global_motion = estimate_global_motion(previous_frame, frame)
camera_moved = norm2(*global_motion) > 2
if camera_moved:
print 'corrected', global_motion
previous_frame, frame = apply_translation(previous_frame, frame, global_motion)
flow = cv2.calcOpticalFlowFarneback(previous_frame, frame, 0.5, 1, 3, 15, 3, 5, 1)
else:
flow = cv2.calcOpticalFlowFarneback(previous_frame, frame, 0.5, 1, 3, 15, 3, 5, 1)
return generate_flow_map(flow)
def opticalFlowCalc(input, input_pre, input_hsv):
flow = cv2.calcOpticalFlowFarneback(input, input_pre, 0.5, 3, 15, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
input_hsv[...,0] = ang*180/np.pi/2
input_hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
output = cv2.cvtColor(input_hsv,cv2.COLOR_HSV2BGR)
return output, input
def read_video(vid_file, interval):
vid = cv2.VideoCapture(vid_file)
ret, frame1 = vid.read()
prev = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
dx_list = []
dy_list = []
count = 0
while True:
ret, frame2 = vid.read()
if not ret:
break
next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
if np.mod(count, interval) == 0:
flow = cv2.calcOpticalFlowFarneback(prev, next, 0.5, 3, 15, 3, 5, 1.2, 0)
dx = crop_optic_flow(flow[...,0])
dy = crop_optic_flow(flow[...,1])
dx_list.append(dx)
dy_list.append(dy)
count += 1
prev = next
dx_list = np.array(dx_list)[None,...]
dy_list = np.array(dy_list)[None,...]
return np.vstack([dx_list, dy_list]).swapaxes(1,0)
def Main():
save_file_root = "C:\\Users\\Camera\\Desktop\\Video_Editing_Tools\\Analysis\\Set 8\\Limb_Movement"
load_video_filename = "C:\\Users\\Camera\\Desktop\\Video_Editing_Tools\\Analysis\\Set 8\\Median.avi"
# load_video_filename = "C:\\Users\\Camera\\Desktop\\GtHUb\\Two-Cameras\\Data\\AG052014-01\\21072014\\Trial5\\PS3_Vid83.avi"
cap = cv2.VideoCapture(load_video_filename)
ret, prvs = cap.read()
prvs = cv2.cvtColor(prvs, cv2.COLOR_BGR2GRAY)
frames = int(cap.get(cv.CV_CAP_PROP_FRAME_COUNT))
# Select mask in which we should look for the initial good points to track - a.k.a. select limb to track
pl.figure()
pl.title("Select mask")
pl.imshow(prvs, cmap=mpl_cm.Greys_r)
pts = []
while not len(pts):
pts = pl.ginput(0)
pl.close()
path = mpl_path.Path(pts)
mask = np.zeros(np.shape(prvs), dtype=np.uint8)
for ridx, row in enumerate(mask):
for cidx, pt in enumerate(row):
if path.contains_point([cidx, ridx]):
mask[ridx, cidx] = 1
for n in range(frames):
ret, next = cap.read()
next = cv2.cvtColor(next, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 3, 10, 3, 5, 1.2, 0)
distt = 3
threshold = 100
new_mask = np.zeros(np.shape(prvs), dtype=np.uint8)
for ridx, row in enumerate(mask):
for cidx, pt in enumerate(row):
if mask[ridx, cidx] == 1:
y = ridx + int(round(flow[ridx, cidx, 1]))
# print int(round(flow[ridx,cidx,1]))
x = cidx + int(round(flow[ridx, cidx, 0]))
if int(round(flow[ridx, cidx, 1])) < 10 and int(round(flow[ridx, cidx, 0])) < 10:
tmask = next[y - distt : y + distt + 1, x - distt : x + distt + 1]
# print tmask
idx = np.where(tmask > threshold)
# print idx[0]+y-distt
new_mask[idx[0] + y - distt, idx[1] + x - distt] = 1
mask = new_mask
next *= new_mask
cv2.imwrite("C:\\Users\\Camera\\Desktop\\TEST.png", next)
cv2.imshow("Limb", next)
k = cv2.waitKey(30) & 0xFF
if k == 27:
break
prvs = next
cv2.destroyAllWindows()
cap.release()
def __estimate_motion(self, imgs, win=5, eig_th=1e-4):
img0 = imgs[0]
img1 = imgs[1]
img0_gray = cv2.cvtColor(img0, cv2.COLOR_BGR2GRAY)
img1_gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)
w, h = np.meshgrid(range(img0.shape[1]), range(img0.shape[0]))
loc0 = (np.vstack((w.flatten(), h.flatten())).T).astype('float32')
flow = cv2.calcOpticalFlowFarneback(img0_gray, img1_gray,
0.5, 0, win, 3, 5, 1.2, 0)
flow = (np.vstack((flow[:, :, 0].flatten(),
flow[:, :, 1].flatten())).T).astype('float32')
# Removing irrelevant flows
minEig = cv2.cornerMinEigenVal(img0_gray, blockSize=win * 3,
borderType=cv2.BORDER_REPLICATE)
loc0_of = loc0[minEig.flatten() > eig_th, :]
loc1_of = flow[minEig.flatten() > eig_th, :] + loc0_of
# Surf-based match
loc0_sf, loc1_sf = self.__calc_surf(imgs)
loc0_all = np.vstack((loc0_of, loc0_sf))
loc1_all = np.vstack((loc1_of, loc1_sf))
hom = cv2.findHomography(loc0_all, loc1_all, cv2.cv.CV_RANSAC, 1)[0]
gm = cv2.perspectiveTransform(np.array([loc0]), hom)[0] - loc0
lm = flow - gm
gm = gm[:, 0] * 1j + gm[:, 1]
lm = lm[:, 0] * 1j + lm[:, 1]
lm[minEig.flatten() < eig_th] = 0
return gm, lm
def demo(self):
'Demo for Optical flow, draw flow vectors and write to video'
with CaptureElement(self.video_path) as ce, WriteElement(self.video_path) as we:
pbar = startBar("Computing flow vectors", len(ce.frames))
hsv = np.zeros_like(ce.frames[0])
hsv[...,1] = 255
for n, img in enumerate(ce.frames):
# First run
if n < 1:
prev_frame = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
else:
current_frame = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prev_frame, current_frame, 0.5, 3, 15, 3, 5, 1.2, 0)
# Reformatting for display
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
we.write_frame(rgb)
prev_frame = current_frame.copy()
pbar.update(n)
pbar.finish()
def optical_flow_dense():
to_grayscale = lambda f: cv2.cvtColor(f, cv2.COLOR_BGR2GRAY)
params = dict(
pyr_scale=0.5,
levels=3,
winsize=15,
iterations=3,
poly_n=5,
poly_sigma=1.1,
flags=0
)
frames = gen_frames()
_frame = next(frames)
p_frame = to_grayscale(_frame)
hsv = np.zeros_like(_frame)
hsv[...,1] = 255
for frame in frames:
glayed = to_grayscale(frame)
# calculate optical flow
flow = cv2.calcOpticalFlowFarneback(p_frame, glayed, None, **params) # type: np.ndarray?
if flow is None:
print("none")
continue
# optical flow's magnitudes and angles
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) # magnitude to 0-255 scale
frame = cv2.cvtColor(hsv,cv2.COLOR_HSV2RGB)
p_frame = glayed.copy()
yield frame
def process(frame):
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
frame = cv2.pyrDown(frame)
if d['prev'] is None:
d['prev'] = frame
# Computes a dense optical flow
# using the Gunnar Farneback’s algorithm
flow = cv2.calcOpticalFlowFarneback(d['prev'],
frame,
pyr_scale=0.5,
levels=1,
winsize=10,
iterations=1,
poly_n=5,
poly_sigma=1.1,
flags=0)
# Calculates the magnitude and angle of 2D vectors
mag, _ = cv2.cartToPolar(flow[..., 0], flow[..., 1])
# threshold it
_, mag = cv2.threshold(mag, 3, 255, cv2.THRESH_TOZERO)
# store the current frame
d['prev'] = frame
return mag
def run_simple(self):
u'''Simple prediction'''
if len(self.datapath) >= 2:
# Use only two previous images
af_img = io.imread(self.datapath[0])
bf_img = io.imread(self.datapath[1])
#af_img = io.imread(r'./viptrafficof_02.png')
#bf_img = io.imread(r'./viptrafficof_03.png')
# Convert to gray image
af_gray = color.rgb2gray(af_img)
bf_gray = color.rgb2gray(bf_img)
# Calculate density flow
# Small -> WHY?
flow = cv2.calcOpticalFlowFarneback(bf_gray, af_gray, \
0.5, 6, 20, 10, 5, 1.2, 0)
print flow.shape, flow[:, :, 0].min(), flow[:, :, 1].max()
self.before = bf_gray
self.after = af_gray
#self.result = self.current
self.result = transform(af_img, flow)
# Color code the result for better visualization of optical flow.
# Direction corresponds to Hue value of the image.
# Magnitude corresponds to Value plane
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv = np.zeros_like(af_img)
hsv[...,1] = 255
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
self.optical = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
def warp_probs(self):
data = self.data_to_warp; assert( data.dtype == np.uint8 )
prevgray = data[:,:,0]
show_figures = False
self.warps = np.zeros(self.size.tolist() + [2],dtype=np.float32)
if self.dpWarp_verbose:
print('Warping %d slices' % (self.size[2]-1,)); t = time.time()
for z in range(1,self.size[2]):
#if self.dpWarp_verbose:
# print('Warping to slice %d / %d' % (z,self.size[2])); t = time.time()
gray = data[:,:,z]
#flow = cv2.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
flow = cv2.calcOpticalFlowFarneback(prevgray, gray, None, 0.8, 8, 16, 10, 7, 1.5, 0)
self.warps[:,:,z,:] = flow
if show_figures:
#cv2.imshow('flow', draw_flow(gray, flow))
cv2.imshow('prevgray',prevgray); cv2.imshow('gray',gray)
cv2.imshow('flow HSV', draw_hsv(flow))
warpedgray = warp_flow(prevgray, flow)
cv2.imshow('warped', warpedgray)
cv2.imshow('diff', np.abs(gray-warpedgray))
cv2.waitKey(0)
prevgray = gray
#if self.dpWarp_verbose:
# print('\tdone in %.4f s' % (time.time() - t, ))
if show_figures: cv2.destroyAllWindows()
if self.dpWarp_verbose:
print('\tdone in %.4f s' % (time.time() - t, ))
def CalculateOpticalFlow(self, prev, nxt):
"""
Function definition
+++++++++++++++++++
.. py:function:: CalculateOpticalFlow(prev, nxt)
Computes a dense optical flow using the Gunnar Farneback’s algorithm using two subsequent
frames.
:param numpy_array prev: the first frame of the two subsequent frames.
:param numpy_array nxt: the second frame of the two subsequent frames.
"""
hsv = np.zeros((prev.shape[0], prev.shape[1], 3))
hsv[...,1] = 255
flow = cv2.calcOpticalFlowFarneback(prev,nxt, 0.5, 3, 15, 3, 5, 1.2, 0)
self.flow = flow
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1], angleInDegrees=1)
hsv[...,0] = ang/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
hsv = np.array(hsv, dtype=np.uint8)
self.motion_image = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
self.magnitude_image = hsv[...,2]
self.direction_image = ang
def flow_frame_farneback(tracks, prevframe, curframe): flow = cv2.calcOpticalFlowFarneback( prevframe, curframe, pyr_scale=0.5, levels=7, winsize=5, iterations=3, poly_n=5, poly_sigma=1.1, flow=None, flags=0 ) cflow = flow.view(np.complex64).squeeze() / framescale angle = np.angle(cflow) % (2*np.pi) magnitude = np.abs(cflow) # HSV #vis = cv2.cvtColor(curframe, cv2.COLOR_GRAY2BGR) vis = np.zeros(flow.shape[:2] + (3,), dtype=np.uint8) vis[:,:,0] = np.uint8(angle / np.pi * 90) vis[:,:,1] = 255 vis[:,:,2] = magnitude / (magnitude + 10) * 255 vis = cv2.cvtColor(vis, cv2.COLOR_HSV2BGR) return vis
def apply_control(self, next, image_out):
flow = cv2.calcOpticalFlowFarneback(self.prvs, next, 0.5, 3, 10, 3, 5, 1.2, 0)
left_flow = self.flow_in_area(flow, image_out, 5, 50, flow_top, flow_bottom, (255, 0, 0))
right_flow = self.flow_in_area(flow, image_out, 270, 315, flow_top, flow_bottom, ( 0, 0, 255))
middle_flow = self.flow_in_area(flow, image_out, 120, 200, flow_top_ref_ground, flow_bottom_ref_ground, (255, 255, 0))
left_flow = left_flow - middle_flow
right_flow = right_flow - middle_flow
#print("left_flow : " + str(left_flow[0]) + " middle : "+str(middle_flow[0])+" right_flow : " + str(right_flow[0]))
print("middle : " + str(middle_flow))
# print("left : " + str(left_flow))
# print("right : " + str(right_flow))
self.moy[0] = (self.i*self.moy[0]+middle_flow[0])/(self.i+1)
self.i+=1
currentmoy = (middle_flow[0]+self.prev+self.prevprev+self.prevprevprev)/4
if(time.clock() - self.time_since_too_near > 0.5):
angular_speed = - angular_scale*(left_flow[0] + right_flow[0])#/abs(middle_flow[0])
if angular_speed < -max_angular_twist : angular_speed = -max_angular_twist
elif angular_speed > max_angular_twist : angular_speed = max_angular_twist
self.go_twist.angular.z = angular_speed
self.display_twist(image_out)
if (abs(currentmoy)<0.0005*self.moy[0]):
self.go_twist.linear.x = -0.1
self.time_since_too_near = time.clock()
self.go_twist.angular.z = max_angular_twist/2
elif (abs(currentmoy)<0.005*self.moy[0]):
self.go_twist.linear.x = 0.05
#elif (abs(currentmoy)<0.05*self.moy[0]):
# self.go_twist.linear.x = 0.1
else:
self.go_twist.linear.x = 0.6
# self.go_twist.linear.x = 0.1* (currentmoy/self.moy[0])**2
self.prevprevprev = self.prevprev
self.prevprev = self.prev
self.prev = middle_flow[0]
return self.go_twist
def video_flow_FB():
cam = cv2.VideoCapture(0)
ret, frame1 = cam.read()
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
while 1:
ret, frame2 = cam.read()
next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 3, 15, 3, 5, 1.2, 0)
imFilter = cv2.GaussianBlur(next, (5, 5), 1.5)
bg_numba = autojit(get_background)
bg = bg_numba(imFilter, flow)
edge_numba = autojit(get_edge)
lap = edge_numba(bg)
lap = get_edge_sobel(bg)
cv2.imshow("flow", bg)
k = cv2.waitKey(30) & 0xFF
if k == 27:
cam.release()
cv2.destroyAllWindows()
break
prvs = next
def loadGT(self):
# case 1: GT already exists
self.gtFile = self.gtFolder + self.imFiles[self.i].replace('.tif', '.bmp')
if os.path.isfile(self.gtFile):
return cv2.imread(self.gtFile, cv2.IMREAD_GRAYSCALE)
# case 2: GT can not be estimated for the first image
if self.i == 0:
return np.zeros((self.im.shape[0], self.im.shape[1]), dtype=np.uint8)
# TODO:
# case 3: GT can not be estimated, because the previous image has no GT
# gtPrevFile = self.gtFolder + self.imFiles[self.i-1].replace('.tif', '.bmp')
# if not os.path.isfile(gtPrevFile):
# return np.zeros((self.im.shape[0], self.im.shape[1]), dtype=np.uint8)
# case 4: GT can be estimated
prevIm = cv2.imread(self.imFolder + self.imFiles[self.i-1], cv2.IMREAD_GRAYSCALE)
currentIm = cv2.cvtColor(self.im, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prevIm, currentIm, \
None, 0.5, 3, 15, 3, 5, 1.2, 0)
#gtPrev = cv2.imread(gtPrevFile, cv2.IMREAD_GRAYSCALE)
gtPrev = self.gt.copy()
gt = np.zeros((self.im.shape[0], self.im.shape[1]), dtype=np.uint8)
for c in self.contours:
r = cv2.boundingRect(c)
i1, j1, i2, j2 = rect2coordinates(r)
fx, fy = self.avgFlow(flow, self.gt, i1, j1, i2, j2)
fi = np.round(fx)
fj = np.round(fy)
gt[i1+fi:i2+fi,j1+fj:j2+fj] = gtPrev[i1:i2,j1:j2]
return gt
def apply(self, stim, show=False):
events = []
for i, f in enumerate(stim):
img = f.data
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
if i == 0:
last_frame = img
total_flow = 0
flow = cv2.calcOpticalFlowFarneback(
last_frame, img, 0.5, 3, 15, 3, 5, 1.2, 0)
flow = np.sqrt((flow ** 2).sum(2))
if show:
cv2.imshow('frame', flow.astype('int8'))
cv2.waitKey(1)
last_frame = img
total_flow = flow.sum()
value = Value(stim, self, {'total_flow': total_flow})
event = Event(onset=f.onset, duration=f.duration, values=[value])
events.append(event)
return events
def video_flow_FB_getpeople():
cam = cv2.VideoCapture(0)
ret, frame1 = cam.read()
prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
while(1):
ret, frame2 = cam.read()
next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs,next, 0.5, 3, 15, 3, 5, 1.2, 0)
imFilter = cv2.GaussianBlur(next,(5,5),1.5)
bg_numba = autojit(get_background)
bg = bg_numba(imFilter,flow)
(cnts, _) = cv2.findContours(bg.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
for c in cnts:
if cv2.contourArea(c) < 8000:
continue
(x, y, w, h) = cv2.boundingRect(c)
cv2.rectangle(frame2, (x, y), (x + w, y + h), (0, 255, 0), 2)
cv2.imshow('flow', frame2)
k = cv2.waitKey(30) & 0xff
if k == 27:
cam.release()
cv2.destroyAllWindows()
break
prvs = next
def MFMGetFM(self, src):
# convert scale
I8U = np.uint8(src) * 255
# calculating optical flows
if self.prev_frame != None:
param1 = pySaliencyMapDefs.farne_pyr_scale
param2 = pySaliencyMapDefs.farne_levels
param3 = pySaliencyMapDefs.farne_winsize
param4 = pySaliencyMapDefs.farne_iterations
param5 = pySaliencyMapDefs.farne_poly_n
param6 = pySaliencyMapDefs.farne_poly_sigma
param7 = pySaliencyMapDefs.farne_flags
flow = cv2.calcOpticalFlowFarneback(prev_frame, I8U, param1, param2, param3, param4, param5, param6, param7)
flowx = flow[..., 0]
flowy = flow[..., 1]
else:
flowx = np.zeros(I8U.shape)
flowy = np.zeros(I8U.shape)
# create Gaussian pyramids
dst_x = self.FMGaussianPyrCSD(flowx)
dst_y = self.FMGaussianPyrCSD(flowy)
# update the current frame
self.prev_frame = np.array(I8U)
# return
return dst_x, dst_y
def mainLoopForCam(self, camNumber):
# Main Loop
self.run = True
frameToShow = 0
while (self.run):
self.curFrames[camNumber] += 1
startTime = time()
capture = self.captureSources[camNumber]
prevGray = np.float32(self.prevGrayFrames[camNumber])
success, frame = capture.read()
if (not success):
print "capture failed on host " + str(camNumber) + " continuing..."
continue
if (self.record):
videoRecorders[camNumber].write(frame)
self.frames[camNumber] = cv2.cvtColor(frame, cv2.cv.CV_BGR2RGB)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prevGray, gray, 0.5, 1, 20, 3, 5, 1.2, 0)
self.prevGrayFrames[camNumber] = gray
# building flow magnitude
fx, fy = flow[:,:,0], flow[:,:,1]
v = np.sqrt(fx*fx+fy*fy)
normalized = np.uint8(np.minimum(v*4, 255))
retval, threshold = cv2.threshold(normalized, 20, 1, cv2.THRESH_BINARY)
self.thresholds[camNumber] = np.dstack((threshold, threshold, threshold))
self.raw_heatmaps[camNumber] += threshold
self.pretty_heatmaps[camNumber] = self.convertToPrettyHeatMap(self.raw_heatmaps[camNumber])
if (self.curFrames[camNumber] % 20) == 0:
np.save("heatmap_backup_" + str(camNumber), self.raw_heatmaps[camNumber].data)
def test_optFlowPyrLK():
# Parameters for lucas kanade optical flow
lk_params = dict( winSize = (15,15),
maxLevel = 2,
criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03))
cap = cv2.VideoCapture(0)
ret,frame = cap.read()
old_col = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame)
hsv[...,1] = 255
while(1):
ret,frame = cap.read()
frame_col = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(old_col,frame_col, 0.5, 3, 15, 3, 5, 1.2, 0)
old_col = frame_col.copy()
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
cv2.imshow('flow',rgb)
cap.release()
def updateData(self):
ret, img = self.cam.read()
p = cv2.cvtColor(self.prev, cv2.COLOR_BGR2GRAY)
i = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(p, i, 0.5, 3, 15, 3, 5, 1.2, 0)
w, h = WIDTH, HEIGHT
step = 16
y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1)
fx, fy = flow[y,x].T
polys = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2)
polys = np.int32(polys + 0.5)
self.prev = img
# fgmask = self.fgbg.apply(img)
# draw = img & fgmask
self.image = QtGui.QImage(img.data, WIDTH, HEIGHT, QtGui.QImage.Format_RGB888)
painter = QtGui.QPainter(self.image)
blue = QtGui.QColor()
blue.setNamedColor("blue")
painter.setPen(QtGui.QPen(blue))
for lines in polys:
p1 = QtCore.QPoint(*lines[0])
p2 = QtCore.QPoint(*lines[1])
l = QtCore.QLine(p1, p2)
painter.drawLine(l)
self.pix.convertFromImage(self.image)
self.repaint()
def process_optical_flow(video_path):
cap = cv2.VideoCapture(video_path)
ret, frame1 = cap.read()
frame1 = imutils.resize(frame1, width=800)
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
prvs = cv2.GaussianBlur(prvs, (21, 21), 0)
hsv = np.zeros_like(frame1)
hsv[..., 1] = 255
while True:
ret, frame2 = cap.read()
frame2 = imutils.resize(frame2, width=800)
next_f = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
next_f = cv2.GaussianBlur(next_f, (21, 21), 0)
# flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 1, 3, 15, 3, 5, 1)
flow = cv2.calcOpticalFlowFarneback(prvs, next_f, 0.5, 3, 15, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imshow('frame2', rgb)
k = cv2.waitKey(30) & 0xff
if k == ord('q'):
break
# elif k == ord('s'):
# cv2.imwrite('opticalfb.png', frame2)
# cv2.imwrite('opticalhsv.png', rgb)
prvs = next_f
cap.release()
cv2.destroyAllWindows()
def dense_opt_flow(n_stills): #dense optical flow
cap = cv2.VideoCapture("test2.avi")
ret, frame1 = cap.read()
prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame1)
hsv[...,1] = 255
while(1):
ret, frame2 = cap.read()
next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
cv2.imshow('frame2',bgr)
k = cv2.waitKey(1) & 0xff
if k == 27:
break
elif k == ord('s'):
cv2.imwrite('opticalfb.png',frame2)
cv2.imwrite('opticalhsv.png',bgr)
prvs = next
cap.release()
cv2.destroyAllWindows()
def optical_flow_video(self, frames, first_frame_idx):
thread_no = (first_frame_idx/self.step_size)
flows = pd.DataFrame(columns=['x', 'y', 'frame'])
for i in range(1,len(frames)):
print("Thread no. %s reporting, first_frame: %s, total frames:%s and working on frame:%s\n"
% (thread_no, first_frame_idx, len(frames), i))
curr = cv2.cvtColor(frames[i],cv2.COLOR_BGR2GRAY)
prev = cv2.cvtColor(frames[i-1],cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prev, curr, None, 0.5, 3, 15, 3, 5, 1.2, 0)
df = pd.DataFrame()
df['x'] = [round(item, 3) for item in flow[:,:,0].flatten()]
df['y'] = [round(item, 3) for item in flow[:,:,1].flatten()]
df['frame'] = first_frame_idx+i
flows = flows.append(df)
flows.to_csv("%s_%s" % (self.outpath, thread_no))
print("Thread no.%s finished" % (thread_no))
def optical_flow(video_path):
cap = cv2.VideoCapture(video_path)
ret, frame1 = cap.read()
prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame1)
hsv[...,1] = 255
while(1):
ret, frame2 = cap.read()
next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY)
#prev, next, scale, levels, winsize, ite, poly_n, poly_sigma
flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 1, 4, 3, 5, 1.1, cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
# flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
cv2.imshow('optflow',bgr)
cv2.imshow('orig', frame2)
k = cv2.waitKey(30) & 0xff
if k == 27:
break
elif k == ord('s'):
cv2.imwrite('opticalfb.png',frame2)
cv2.imwrite('opticalhsv.png',bgr)
prvs = next
cap.release()
cv2.destroyAllWindows()
def calculate_optical_flow(prvs_gray, current_gray):
flow = cv2.calcOpticalFlowFarneback(prvs_gray, current_gray, 0.5, 3, 15, 3, 5, 1.2, 0)
binary_pic = np.zeros(current_gray.shape, np.uint8)
h, w = current_gray.shape
total_threshold=0
for i in xrange(0,h,5):
for j in xrange(0,w,5):
fx, fy = flow[i, j]
total_threshold += abs(fx) + abs(fy)
if abs(fx) + abs(fy) > 0.4:
binary_pic[i, j] = 255;
print('mean threshold is %f'%(total_threshold/(h*w)))
binary_pic_translated = conduct_translation(binary_pic)
contours, hierarchy = cv2.findContours(binary_pic_translated,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
print('contours is %d'%(len(contours)))
largest_area = 0
largest_contour_index = 0
bounding_rect_x, bounding_rect_y, bounding_rect_w, bounding_rect_h = 0, 0, 0, 0
if len(contours) > 0:
for k in xrange(0, len(contours)):
area = cv2.contourArea(contours[k])
if largest_area < area:
largest_area = area
largest_contour_index = k
bounding_rect_x, bounding_rect_y, bounding_rect_w, bounding_rect_h = cv2.boundingRect(contours[k])
return ((bounding_rect_x, bounding_rect_y, bounding_rect_w, bounding_rect_h), largest_area)
else:
print("contours is not found...")
return (-1, -1)
def HSVizualizer(inputDirectory, filenames, outputDirectory):
"""
Args:
inputDirectory: (str)
filenames: (list) filenames to analyze and draw
outputDirectory: (str) Where to place output
Returns:
"""
for filename in filenames:
raw = tiff.imread(inputDirectory+filename)
nframes, frame_width, frame_height = raw.shape
#outstack is in RGB color so we need an extra 3 dimensions
outStack = np.zeros((nframes-1, frame_width, frame_height, 3), dtype='uint8')
for i in xrange(nframes-1):
t1 = time.time()
print "Start frame "+str(i)+"..."+filename
frame = raw[i]
next_frame = raw[i+1]
flow = cv2.calcOpticalFlowFarneback(frame, next_frame, None, 0.5, 3, 8, 4, 7, 1.5, 0)
outStack[i] = draw_hsv(flow)
print "Finish frame "+str(i)+" in "+str(time.time()-t1)+" s."
#print outStack.shape
tiff.imsave(outputDirectory+filename+'_HSV.tif', outStack)
print "All done in "+str(time.time()-t0)+" s"
def getOpticalFlow(self, prevgray, gray):
flow = cv2.calcOpticalFlowFarneback(prevgray, gray , 0.5, 3, 5, 3, 2, 0.4, cv2.OPTFLOW_USE_INITIAL_FLOW)
# cv2.imshow('flow', self.warp_flow(gray, flow))
cv2.imshow('flow', self.draw_flow(gray, flow))
def cv2_array_processor(arr, stopFrame, cv2_params, startFrame=0, frameSamplingInterval=1):
"""
:param arr: numpy array of shape (nFrames, x, y)
:param stopFrame: frame where to stop the processing
:param cv2_params: (dict) parameters for the cv2.calcOpticalFlowFarneback() function
:param startFrame: frame where to start the processing
:param frameSamplingInterval: process every n-th frame
:return:
two (nFrames-1, x, y) arrays with the u-, and v- components of the velocity vectors
u_out & v_out
"""
assert (stopFrame <= arr.shape[0]) and (startFrame < stopFrame)
u_out = np.zeros_like(arr[startFrame:stopFrame - 1, :, :])
v_out = np.zeros_like(u_out)
for frame in range(startFrame, stopFrame, frameSamplingInterval):
if frame >= (stopFrame - 1):
break
frame_a = arr[frame]
frame_b = arr[frame + 1]
flow = cv2.calcOpticalFlowFarneback(frame_a, frame_b, **cv2_params)
u_out[frame], v_out[frame] = flow[..., 0], flow[..., 1]
return u_out, v_out
prev_gray = cv.cvtColor(first_frame, cv.COLOR_BGR2GRAY)
# Creates an image filled with zero intensities with the same dimensions as the frame
mask = np.zeros_like(first_frame)
# Sets image saturation to maximum
mask[..., 1] = 255
while (cap.isOpened()):
# ret = a boolean return value from getting the frame, frame = the current frame being projected in the video
ret, frame = cap.read()
if ret == False:
break
# Opens a new window and displays the input frame
cv.imshow("input", frame)
# Converts each frame to grayscale - we previously only converted the first frame to grayscale
gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY)
# Calculates dense optical flow by Farneback method
flow = cv.calcOpticalFlowFarneback(prev_gray, gray, None, 0.5, 3, 15, 3, 5,
1.2, 0)
# Computes the magnitude and angle of the 2D vectors
magnitude, angle = cv.cartToPolar(flow[..., 0], flow[..., 1])
# Sets image hue according to the optical flow direction
mask[..., 0] = angle * 180 / np.pi / 2
# Sets image value according to the optical flow magnitude (normalized)
mask[..., 2] = cv.normalize(magnitude, None, 0, 255, cv.NORM_MINMAX)
# Converts HSV to RGB (BGR) color representation
rgb = cv.cvtColor(mask, cv.COLOR_HSV2BGR)
# Opens a new window and displays the output frame
cv.imshow("dense optical flow", rgb)
# Updates previous frame
prev_gray = gray
# Frames are read by intervals of 1 millisecond. The programs breaks out of the while loop when the user presses the 'q' key
if cv.waitKey(1) & 0xFF == ord('q'):
break
def read_avi(avi_path):
global video
global F
# global sample
print avi_path
cap = cv2.VideoCapture(avi_path)
length = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT))
print length
if length < 54:
print "length is less than 50"
rand_list = range(length)
else:
rand_list = random.sample(range(1, length - 1), 50)
rand_list.sort()
length = 50
length = (length - length % F) / F
print rand_list
# sample=sample+length
video = np.zeros((length, F * 8, 240, 320), dtype=np.uint8)
poi = 0
cap.set(1, 0)
ret, img = cap.read()
#img=cv2.resize(img,(320,240))
op_pre_frames = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
pre_frames = img
for i in range(length):
m = 0
for s in range(F):
cap.set(1, rand_list[poi])
ret, img = cap.read()
# print rand_list[poi]
poi = poi + 1
# print np.shape(img)
# print poi
op_next_frames = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(op_pre_frames, op_next_frames,
0.5, 1, 3, 15, 3, 5, 1)
fx = cv2.normalize(flow[..., 0], None, 0, 255, cv2.NORM_MINMAX)
fy = cv2.normalize(flow[..., 1], None, 0, 255, cv2.NORM_MINMAX)
diff_frames = img - pre_frames
#RGB channels
video[i, m] = img[:, :, 0]
m = m + 1
video[i, m] = img[:, :, 1]
m = m + 1
video[i, m] = img[:, :, 2]
m = m + 1
# three difference channels
video[i, m] = diff_frames[:, :, 0]
m = m + 1
video[i, m] = diff_frames[:, :, 1]
m = m + 1
video[i, m] = diff_frames[:, :, 2]
m = m + 1
#optical flow channels
video[i, m] = fx
m = m + 1
video[i, m] = fy
m = m + 1
pre_frames = img
op_pre_frames = op_next_frames
cap.release()
return video
def run_sim(imgFile):
""" imgFile: filename of the image uploaded by the user
"""
print("Processing:", imgFile)
################################### SET UP ################################
###########################################################################
# Using a queue to decouple key press from update action
# pressed_keys = []
# def on_release(key):
# if hasattr(key, 'char'):
# pressed_keys.append(key.char)
#
# cv2.startWindowThread()
# cv2.namedWindow("window", flags=cv2.WND_PROP_FULLSCREEN)
camera = Camera(imgFile=imgFile, camera_index=configuration.camera_index, no_cam_allowed=True)
# initialise run-time configurable Options
# (display name, keys, range, step, initial value)
fullscreen = Options.Cycle('Fullscreen', 'f', ['Window', 'Fullscreen'], configuration.fullscreen)
mirror_screen = Options.Cycle('Mirror Screen', 'g', ['Normal', 'Horizontal', 'Verticle', 'Both'], configuration.mirror_screen)
render_mask = Options.Cycle('Render Mask', 'm', ['false', 'true'], configuration.render_mask)
bg_mode = Options.Cycle('BG', 'b', ['white', 'black', 'hue', 'bg subtract'], configuration.bg_mode)
mask_level = Options.Range('Mask Threshold', ['1','2'], [0, 1], 0.03, configuration.mask_level)
mask_width = Options.Range('Mask Width', ['3','4'], [0, 0.5], 0.01, configuration.mask_width)
optical_flow = Options.Cycle('Optical Flow', 'o', ['false', 'true'], configuration.optical_flow)
sim_res_multiplier = Options.Range('Sim Res', ['9','0'], [0.1, 2.0], 0.1, configuration.sim_res_multiplier)
flow_speed = Options.Range('Flow Speed', ['-','='], [0.02, 1], 0.02, configuration.flow_speed)
flow_direction = Options.Cycle('Flow Direction', 'p', ['right', 'down', 'left', 'up'], configuration.flow_direction)
num_smoke_streams = Options.Range('Smoke Streams', ['[',']'], [1, 50], 1, configuration.num_smoke_streams)
smoke_percentage = Options.Range('Smoke Amount', ['\'','#'], [0.1, 1], 0.1, configuration.smoke_percentage)
debugMode = Options.Cycle('Mode', 'd', ['Normal', 'Debug'], 0)
# add to a list to update and display
options = [fullscreen, mirror_screen, render_mask,
bg_mode, mask_level, mask_width, optical_flow,
sim_res_multiplier, flow_speed, flow_direction, num_smoke_streams, smoke_percentage,
debugMode]
###########################################################################
################################## END SETUP ##############################
fps = FPS_counter(limit=30)
display_counter = 0 # display values for a short time if they change
run = True
# Added a frame counter (skip first frame)
frame = 1
frameJump = 25
frameMax = 227
while(frame <= frameMax):
fps.update()
if display_counter > 0:
display_counter -= fps.last_dt
# Always true on first iteration. Sub-sample the webcam image to fit the
# fluid sim resolution. Update when option changes.
if sim_res_multiplier.get_has_changed(reset_change_flag=True):
sim = configuration.Sim(camera.shape, sim_res_multiplier.current, 0, 0)
sim.mode = debugMode.current
flow = np.zeros((sim.shape[1], sim.shape[0], 2))
# Always True on first iteration. Update fullscreen if option changed
# if fullscreen.get_has_changed(reset_change_flag=True):
# if fullscreen.current:
# cv2.setWindowProperty("window", cv2.WND_PROP_FULLSCREEN, 1)
# else:
# cv2.setWindowProperty("window", cv2.WND_PROP_FULLSCREEN, 0)
if debugMode.get_has_changed(reset_change_flag=True):
sim.mode = debugMode.current
# if flow direction changes, reset, else things get messy
# if flow_direction.get_has_changed(reset_change_flag=True):
# sim.reset()
# camera.reset()
# update input image
camera.update(bg_mode.current, mirror_screen.current,
mask_level.current, mask_width.current)
if optical_flow.current == 1:
flow[:] = cv2.calcOpticalFlowFarneback(cv2.resize(camera.last_grey, sim.shape),
cv2.resize(camera.current_grey, sim.shape),
float(0),
pyr_scale=0.5, levels=3, winsize=15, iterations=3,
poly_n=5, poly_sigma=1.0, flags=0)
scale = np.sqrt(sim._dx[0] * sim._dx[1]) / fps.last_dt
flow *= scale
else:
flow[:] = 0
sim.set_velocity(fps.last_dt, flow_speed.current,
flow_direction.current, num_smoke_streams.current,
smoke_percentage.current)
# copy the webcam generated mask into the boundary
boundary = np.array(cv2.resize(camera.mask, sim.shape).T, dtype=bool)
sim.set_boundary(boundary, flow.T, flow_direction.current)
# update and render the sim
sim.udpate(fps.last_dt)
output = sim.render(camera, render_mask_velocity=(optical_flow.current==1), render_mask=(render_mask.current == 1))
output_shape = np.shape(output)
# add the GUI
text_color = (0, 0, 255) if bg_mode.current == 0 else (255, 255, 0)
if debugMode.current == 0 and display_counter <= 0:
cv2.putText(output, 'd=Debug Mode', (30,output_shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color)
else:
pos = np.array((30,30))
for option in options:
cv2.putText(output, str(option), tuple(pos), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color)
pos = pos + [0,20]
cv2.putText(output, str(fps), (output_shape[1] - 80,output_shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color)
cv2.putText(output, 'q=Quit, r=Reset', (30,output_shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color)
# render the output
# cv2.imshow('window', output)
# Save the output to file every some frames
if frame % frameJump == 0:
ind = int(frame/frameJump)
name = imgFile.split("/")[-1].split(".")[0]
fullPath = "server/out/{}_{}.png".format(name, ind)
cv2.imwrite(fullPath, output)
# for key in pressed_keys:
# # update the options (poll for input, cycle)
# for option in options:
# if option.update(key, fps.last_dt):
# display_counter = 5
#
# # poll for quit, reset
# if key == 'q':
# run = False
# elif key == 'r':
# sim.reset()
# camera.reset()
# pressed_keys[:] = []
# Update frame
frame += 1
import cv2
import numpy as np
cap=cv2.VideoCapture('video2.mp4'); #create a capture object
ret,frame1=cap.read(); #read first frame
prv_gray_frame=cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY); #convert color of image from bgr2gray
hsv=np.zeros_like(frame1); #convert frame 1 to hsv using zeroslike
hsv[...,1]=255; #set saturation to 255
while cap.isOpened():
ret,frame2=cap.read(); #read frame by frame
next_gray_frame=cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY); #convert color of frame from bgr2gray
flow=cv2.calcOpticalFlowFarneback(prv_gray_frame,next_gray_frame,None,0.3,3,15,3,5,1.2,0); #find the optical flow of object
mag,ang=cv2.cartToPolar(flow[...,0],flow[...,1]); #cartTo Plor cordinates to find x and y
hsv[...,0]=ang*180/np.pi/2;
hsv[...,2]=cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX); #normalize te magnitide
rgb=cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR); #convert color of imag from hsv2bgr
cv2.namedWindow('DenseOpticalFlow',cv2.WINDOW_NORMAL); #create a named window
cv2.imshow('DenseOpticalFlow',rgb); #image show
if cv2.waitKey(5)&0xFF==ord('q'):
break;
cv2.destroyWindow('DenseOpticalFlow'); #destroy window
prevgray = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)
show_hsv = True
show_glitch = False
cur_glitch = prev.copy()
while True:
(ret, img) = cam.read()
vis = img.copy()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
"""
Computes a dense optical flow using the Gunnar Farneback’s algorithm.
cv2.calcOpticalFlowFarneback(prev, next, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags) → flow
"""
flow = cv2.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 5, 5, 3,
5, 1.1,
cv2.OPTFLOW_LK_GET_MIN_EIGENVALS)
prevgray = gray
cv2.imshow('flow', draw_flow(gray, flow))
if show_hsv:
gray1 = cv2.cvtColor(draw_hsv(flow), cv2.COLOR_BGR2GRAY)
thresh = cv2.threshold(gray1, 0, 0xFF, cv2.THRESH_BINARY)[1]
thresh = cv2.dilate(thresh, None, iterations=2)
cv2.imshow('thresh', thresh)
##gray2, cnts, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
gray2, cnts, hierarchy = cv2.findContours(thresh.copy(),
cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
# loop over the contours
"""
for c in cnts:
try:
fn = sys.argv[1]
except:
fn = 0
cam = video.create_capture(fn)
ret, prev = cam.read()
prevgray = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY)
show_hsv = False
show_glitch = False
cur_glitch = prev.copy()
while True:
ret, img = cam.read()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prevgray, gray, 0.5, 3, 15, 3, 5,
1.2, 0)
prevgray = gray
cv2.imshow('flow', draw_flow(gray, flow))
if show_hsv:
cv2.imshow('flow HSV', draw_hsv(flow))
if show_glitch:
cur_glitch = warp_flow(cur_glitch, flow)
cv2.imshow('glitch', cur_glitch)
ch = 0xFF & cv2.waitKey(5)
if ch == 27:
break
if ch == ord('1'):
show_hsv = not show_hsv
print 'HSV flow visualization is', ['off', 'on'][show_hsv]
# #tools.save(x, y, u3, v3, mask, 'exp1_001.txt')
# #tools.display_vector_field('exp1_001.txt', scale=100, width=0.0025)
#
# cv.imshow('test',frame_b.astype(np.uint8))
#
# plt.quiver(x,y,u3,v3)
# Calculates dense optical flow by Farneback method
# https://docs.opencv.org/3.0-beta/modules/video/doc/motion_analysis_and_object_tracking.html#calcopticalflowfarneback
# flow = cv.calcOpticalFlowFarneback(prev=prev_gray, next=gray, flow=None, pyr_scale=0.5, levels=3, winsize=25, iterations=3, poly_n=7, poly_sigma=1.5, flags=0)
# LOOKS LIKE WE ARE GETTING SOMEWHERE WITH THE NORMALIZED VERSION
flow = cv.calcOpticalFlowFarneback(prev=prev_gray.astype(np.uint8), next=gray.astype(np.uint), flow=None, pyr_scale=0.5, levels=3, winsize=24, iterations=3, poly_n=7, poly_sigma=1.5, flags=0)
#flow = cv.calcOpticalFlowFarneback(prev=frame_a.astype(np.uint8), next=frame_b.astype(np.uint), flow=None, pyr_scale=0.5, levels=3, winsize=24, iterations=3, poly_n=7, poly_sigma=1.5, flags=0)
flow = flow*10
# Computes the magnitude and angle of the 2D vectors
magnitude, angle = cv.cartToPolar(flow[..., 0], flow[..., 1])
# Sets image hue according to the optical flow direction
mask[..., 0] = angle * 180 / np.pi / 2
# Sets image value according to the optical flow magnitude (normalized)
mask[..., 2] = cv.normalize(magnitude, None, 0, 255, cv.NORM_MINMAX)
# Converts HSV to RGB (BGR) color representation
rgb = cv.cvtColor(mask, cv.COLOR_HSV2BGR)
# Opens a new window and displays the output frame
#rgbS = cv.resize(rgb, (1024, 1224))
arrows.clear()
ret1, frame1 = video.read()
if not ret1:
# End of sequence
print('Empty video file')
sys.exit(2)
frameCounter = 1
while (video.isOpened()):
print('Processing frame {0}/{1}'.format(frameCounter, length), end='\r')
ret3, frame3 = video.read()
if ret3:
prevgray = cv.cvtColor(frame1, cv.COLOR_BGR2GRAY)
gray = cv.cvtColor(frame3, cv.COLOR_BGR2GRAY)
flow = cv.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 3, 30, 3,
5, 1.2, 0)
backFlow = cv.calcOpticalFlowFarneback(gray, prevgray, None, 0.5, 3,
30, 3, 5, 1.2, 0)
# if not helper.showImage(helper.drawFlow(gray, flow)):
# break
warpedFlow = flow * 0.5
backWarpedFlow = backFlow * 0.5
# TODO: use NaNs or something else instead of 0s to indicate no information
# newFrame = np.full((height, width, 3), np.nan)
newFrame = np.zeros((height, width, 3), np.uint8)
warpMapX = np.zeros((height, width), np.float32)
warpMapX[:] = range(width)
warpMapY = np.zeros((height, width), np.float32)
import cv2
import numpy as np
cap = cv2.VideoCapture("vtest.avi")
ret, frame1 = cap.read()
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame1)
hsv[..., 1] = 255
while (1):
ret, frame2 = cap.read()
next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5,
1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imshow('frame2', rgb)
k = cv2.waitKey(30) & 0xff
if k == 27:
break
elif k == ord('s'):
cv2.imwrite('opticalfb.png', frame2)
cv2.imwrite('opticalhsv.png', rgb)
prvs = next
cap.release()
def main(video, device):
# init dict to track time for every stage at each iteration
timers = {
"full pipeline": [],
"reading": [],
"pre-process": [],
"optical flow": [],
"post-process": [],
}
# init video capture with video
cap = cv2.VideoCapture(video)
# get default video FPS
fps = cap.get(cv2.CAP_PROP_FPS)
# get total number of video frames
num_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT)
# read the first frame
ret, previous_frame = cap.read()
# proceed if frame reading was successful
if ret:
# resize frame
frame = cv2.resize(previous_frame, (960, 540))
# convert to gray
previous_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# create hsv output for optical flow
hsv = np.zeros_like(frame)
# set saturation to a maximum value
hsv[..., 1] = 255
while True:
# start full pipeline timer
start_full_time = time.time()
# start reading timer
start_read_time = time.time()
# capture frame-by-frame
ret, current_frame = cap.read()
# end reading timer
end_read_time = time.time()
# add elapsed iteration time
timers["reading"].append(end_read_time - start_read_time)
# if frame reading was not successful, break
if not ret:
break
# start pre-process timer
start_pre_time = time.time()
# resize frame
frame = cv2.resize(current_frame, (960, 540))
# convert to gray
current_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
if device == "cpu":
# end pre-process timer
end_pre_time = time.time()
# add elapsed iteration time
timers["pre-process"].append(end_pre_time - start_pre_time)
# start optical flow timer
start_of = time.time()
# calculate optical flow
flow = cv2.calcOpticalFlowFarneback(
previous_frame,
current_frame,
None,
0.5,
5,
15,
3,
5,
1.2,
0,
)
# end of timer
end_of = time.time()
# add elapsed iteration time
timers["optical flow"].append(end_of - start_of)
else:
# move both frames to GPU
cu_previous = cv2.cuda_GpuMat()
cu_current = cv2.cuda_GpuMat()
cu_previous.upload(previous_frame)
cu_current.upload(current_frame)
# end pre-process timer
end_pre_time = time.time()
# add elapsed iteration time
timers["pre-process"].append(end_pre_time - start_pre_time)
# start optical flow timer
start_of = time.time()
# create optical flow instance
flow = cv2.cuda_FarnebackOpticalFlow.create(
5,
0.5,
False,
15,
3,
5,
1.2,
0,
)
# calculate optical flow
flow = cv2.cuda_FarnebackOpticalFlow.calc(
flow,
cu_previous,
cu_current,
None,
)
# sent result from GPU back to CPU
flow = flow.download()
# end of timer
end_of = time.time()
# add elapsed iteration time
timers["optical flow"].append(end_of - start_of)
# start post-process timer
start_post_time = time.time()
# convert from cartesian to polar coordinates to get magnitude and angle
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
# set hue according to the angle of optical flow
hsv[..., 0] = ang * 180 / np.pi / 2
# set value according to the normalized magnitude of optical flow
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
# convert hsv to bgr
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
# end post-process timer
end_post_time = time.time()
# add elapsed iteration time
timers["post-process"].append(end_post_time - start_post_time)
# end full pipeline timer
end_full_time = time.time()
# add elapsed iteration time
timers["full pipeline"].append(end_full_time - start_full_time)
# visualization
cv2.imshow("original", frame)
cv2.imshow("result", bgr)
k = cv2.waitKey(1)
if k == 27:
break
# update previous_frame value
previous_frame = current_frame
# release the capture
cap.release()
# destroy all windows
cv2.destroyAllWindows()
# print results
print("Number of frames : ", num_frames)
# elapsed time at each stage
print("Elapsed time")
for stage, seconds in timers.items():
print("-", stage, ": {:0.3f} seconds".format(sum(seconds)))
# calculate frames per second
print("Default video FPS : {:0.3f}".format(fps))
of_fps = (num_frames - 1) / sum(timers["optical flow"])
print("Optical flow FPS : {:0.3f}".format(of_fps))
full_fps = (num_frames - 1) / sum(timers["full pipeline"])
print("Full pipeline FPS : {:0.3f}".format(full_fps))
if folder:
files.extend(dirnames)
else:
files.extend(filenames)
break
return files
master_dir = "/home/wangsq/CS4243/DAVIS/"
master_dir = _get_files(master_dir, folder=True)
for subfolder in master_dir:
img_files = _get_files(subfolder)
count = 0
filepath, _ = os.path.split(img_files[0])
for i in range(0, len(img_files) - 1):
prev = get_img(img_files[i], (HEIGHT, WIDTH, 3),
grayscale=True).astype(np.float32)
nxt = get_img(img_files[i + 1], (HEIGHT, WIDTH, 3),
grayscale=True).astype(np.float32)
forward_flow = cv2.calcOpticalFlowFarneback(prev, nxt, None, 0.5, 3,
15, 3, 5, 1.2, 0)
flow.write_flo(filepath + "/flow/flow" + str(count) + ".flo",
forward_flow)
backward_flow = cv2.calcOpticalFlowFarneback(nxt, prev, None, 0.5, 3,
15, 3, 5, 1.2, 0)
flow.write_flo(filepath + "/flow/flow" + str(count + 1) + ".flo",
backward_flow)
count += 2
def internp(frame0, frame1, t=0.5, flow0=None):
'''
:param frame0: beggining frame
:param frame1: ending frame
:return frame_t: an interpolated frame at time t
'''
print('==============================')
print('===== interpolate an intermediate frame at t=', str(t))
print('==============================')
# ==================================================
# ===== 1/ find the optical flow between the two given images: from frame0 to frame1,
# if there is no given flow0, run opencv function to extract it
# ==================================================
if flow0 is None:
i1 = cv2.cvtColor(frame0, cv2.COLOR_BGR2GRAY)
i2 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
flow0 = cv2.calcOpticalFlowFarneback(i1, i2, None, 0.5, 3, 15, 3, 5,
1.2, 0)
# ==================================================
# ===== 2/ find holes in the flow
# ==================================================
holes0 = find_holes(flow0)
pickle.dump(holes0, open('holes0.step2.data',
'wb')) # save your intermediate result
# ====== score
holes0 = pickle.load(open('holes0.step2.data',
'rb')) # load your intermediate result
holes0_step2 = pickle.load(open('holes0.step2.sample',
'rb')) # load sample result
diff = np.sum(np.abs(holes0 - holes0_step2))
print('holes0_step2', diff)
# ==================================================
# ===== 3/ fill in any hole using an outside-in strategy
# ==================================================
flow0 = holefill(flow0, holes0)
pickle.dump(flow0, open('flow0.step3.data',
'wb')) # save your intermediate result
# ====== score
flow0 = pickle.load(open('flow0.step3.data',
'rb')) # load your intermediate result
flow0_step3 = pickle.load(open('flow0.step3.sample',
'rb')) # load sample result
diff = np.sum(np.abs(flow0 - flow0_step3))
print('flow0_step3', diff)
# ==================================================
# ===== 5/ estimate occlusion mask
# ==================================================
occ0, occ1 = occlusions(flow0, frame0, frame1)
pickle.dump(occ0, open('occ0.step5.data',
'wb')) # save your intermediate result
pickle.dump(occ1, open('occ1.step5.data',
'wb')) # save your intermediate result
# ===== score
occ0 = pickle.load(open('occ0.step5.data',
'rb')) # load your intermediate result
occ1 = pickle.load(open('occ1.step5.data',
'rb')) # load your intermediate result
occ0_step5 = pickle.load(open('occ0.step5.sample',
'rb')) # load sample result
occ1_step5 = pickle.load(open('occ1.step5.sample',
'rb')) # load sample result
diff = np.sum(np.abs(occ0_step5 - occ0))
print('occ0_step5', diff)
diff = np.sum(np.abs(occ1_step5 - occ1))
print('occ1_step5', diff)
# ==================================================
# ===== step 6/ blur occlusion mask
# ==================================================
for iblur in range(0, BLUR_OCC):
occ0 = blur(occ0)
occ1 = blur(occ1)
pickle.dump(occ0, open('occ0.step6.data',
'wb')) # save your intermediate result
pickle.dump(occ1, open('occ1.step6.data',
'wb')) # save your intermediate result
# ===== score
occ0 = pickle.load(open('occ0.step6.data',
'rb')) # load your intermediate result
occ1 = pickle.load(open('occ1.step6.data',
'rb')) # load your intermediate result
occ0_step6 = pickle.load(open('occ0.step6.sample',
'rb')) # load sample result
occ1_step6 = pickle.load(open('occ1.step6.sample',
'rb')) # load sample result
diff = np.sum(np.abs(occ0_step6 - occ0))
print('occ0_step6', diff)
diff = np.sum(np.abs(occ1_step6 - occ1))
print('occ1_step6', diff)
# ==================================================
# ===== step 7/ forward-warp the flow to time t to get flow_t
# ==================================================
flow_t = interpflow(flow0, frame0, frame1, t)
#flow_t = pickle.load(open('flow_t.step7.sample', 'rb')) # load sample result
pickle.dump(flow_t, open('flow_t.step7.data',
'wb')) # save your intermediate result
# ====== score
flow_t = pickle.load(open('flow_t.step7.data',
'rb')) # load your intermediate result
flow_t_step7 = pickle.load(open('flow_t.step7.sample',
'rb')) # load sample result
#flow_t = flow_t_step7
diff = np.sum(np.abs(flow_t - flow_t_step7))
print('flow_t_step7', diff)
# ==================================================
# ===== step 8/ find holes in the estimated flow_t
# ==================================================
holes1 = find_holes(flow_t)
pickle.dump(holes1, open('holes1.step8.data',
'wb')) # save your intermediate result
# ====== score
holes1 = pickle.load(open('holes1.step8.data',
'rb')) # load your intermediate result
holes1_step8 = pickle.load(open('holes1.step8.sample',
'rb')) # load sample result
diff = np.sum(np.abs(holes1 - holes1_step8))
print('holes1_step8', diff)
# ===== fill in any hole in flow_t using an outside-in strategy
flow_t = holefill(flow_t, holes1)
pickle.dump(flow_t, open('flow_t.step8.data',
'wb')) # save your intermediate result
# ====== score
flow_t = pickle.load(open('flow_t.step8.data',
'rb')) # load your intermediate result
flow_t_step8 = pickle.load(open('flow_t.step8.sample',
'rb')) # load sample result
diff = np.sum(np.abs(flow_t - flow_t_step8))
print('flow_t_step8', diff)
# ==================================================
# ===== 9/ inverse-warp frame 0 and frame 1 to the target time t
# ==================================================
#frame_t = warpimages(flow_t, frame0, frame1, occ0_step6, occ1_step6, t)
frame_t = warpimages(flow_t, frame0, frame1, occ0, occ1, t)
pickle.dump(frame_t, open('frame_t.step9.data',
'wb')) # save your intermediate result
# ====== score
frame_t = pickle.load(open('frame_t.step9.data',
'rb')) # load your intermediate result
frame_t_step9 = pickle.load(open('frame_t.step9.sample',
'rb')) # load sample result
diff = np.sqrt(
np.mean(
np.square(
frame_t.astype(np.float32) -
frame_t_step9.astype(np.float32))))
print('frame_t', diff)
return frame_t
def capture_dense_optical_flow(path_in,
name_of_videos,
grids,
ROI,
time_interval,
X_BLOCK_SIZE,
Y_BLOCK_SIZE,
if_show=True,
if_normal_ori=True):
'''
Objective: capture optical flow points from video
input:
path_in: dir of training video data, e.g './data/train/videos'
name_of_videos: read the videos in the directory iteratively
'/data/train/videos/name_of_videos'. 'train001'
grids: the corner for each grid created by "make_grid" function
if_show: a boolin defines if to show the video with optical flow and grid
Output:
an n by 6 array [starting x, starting y, magnitude, orientation,
grid of the flow, time interval of the flow]
'''
#set the Farneback parameters
fb_params = dict(
pyr_scale=0.03,
levels=1,
winsize=4,
iterations=3,
poly_n=5,
poly_sigma=1.2,
flags=0) # The video feed is read in as a VideoCapture object
cap = cv2.VideoCapture(join(path_in, name_of_videos + '.avi'))
# ret = a boolean return value from getting the frame, first_frame = the first frame in the entire video sequence
ret, frame = cap.read()
# Converts frame to grayscale because we only need the luminance channel for detecting edges - less computationally expensive
prev_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Creates an image filled with zero intensities with the same dimensions as the frame
mask = np.zeros_like(frame)
# Sets image saturation to maximum
mask[..., 1] = 255
# calculate different thresholds for different rows of magnitude to discard
numbers_y_grid = int(frame.shape[0] / Y_BLOCK_SIZE)
decreasing_rate = 0.88
original_threshold = 1.2
motion_threshold = [original_threshold]
for row in range(1, numbers_y_grid):
motion_threshold.insert(0, motion_threshold[0] * decreasing_rate)
if row % 2 == 0:
decreasing_rate -= .07
# if needs to show the video, portrait the grids
if if_show == True:
color_grid = (0, 255, 0)
grid_mask = np.zeros_like(frame)
num_of_grids = grids.shape[1]
for g in range(num_of_grids):
grid_mask = cv2.line(grid_mask,
(grids[0][g] + X_BLOCK_SIZE, grids[1][g]),
(grids[0][g], grids[1][g]), color_grid, 1)
grid_mask = cv2.line(grid_mask,
(grids[0][g], grids[1][g] + Y_BLOCK_SIZE),
(grids[0][g], grids[1][g]), color_grid, 1)
#for counting which frame is being processing for each video
frame_count = 0
# 20 frames as a set, so record the which set the flows belong to
set_of_frames = 1
flow_points = []
while cap.isOpened():
# ret = a boolean return value from getting the frame, frame = the current frame being projected in the video
# 這個.read()一次就換下一張video 的frame
ret, frame = cap.read()
if not ret:
break
# Converts each frame to grayscale - we previously only converted the first frame to grayscale
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Calculates dense optical flow by Farneback method
# https://docs.opencv.org/3.0-beta/modules/video/doc/motion_analysis_and_object_tracking.html#calcopticalflowfarneback
flow = cv2.calcOpticalFlowFarneback(prev_gray, gray, None, **fb_params)
# Computes the magnitude and angle of the 2D vectors
magnitude, angle = cv2.cartToPolar(flow[..., 0],
flow[..., 1],
angleInDegrees=0)
# to discard the stationary objects, and use different threshold for different rows
for row in range(numbers_y_grid - 1):
# for row in range(int(mask[..., 2].shape[0]/Y_BLOCK_SIZE)):
thr_idx = magnitude[Y_BLOCK_SIZE * row:Y_BLOCK_SIZE *
(row + 1), :] < motion_threshold[row]
magnitude[Y_BLOCK_SIZE * row:Y_BLOCK_SIZE *
(row + 1), :][thr_idx] = 0
# if row < 2:
# magnitude[Y_BLOCK_SIZE*row : Y_BLOCK_SIZE*(row+1), :] = magnitude[Y_BLOCK_SIZE*row : Y_BLOCK_SIZE*(row+1), :] * median_ratio[row]
thr_idx = magnitude[Y_BLOCK_SIZE * (row + 1):, :] < motion_threshold[
-1] # -1 here same as index row+1
magnitude[Y_BLOCK_SIZE * (row + 1):, :][thr_idx] = 0
# magnitude = np.array([m if m > motion_threshold else 0 for m in magnitude.reshape(-1)]).reshape(angle.shape[0], angle.shape[1])
# Sets image hue according to the optical flow direction
mask[..., 0] = angle * 180 / np.pi / 2
# Sets image value according to the optical flow magnitude (normalized)
mask[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX)
# Converts HSV to RGB (BGR) color representation
rgb = cv2.cvtColor(mask, cv2.COLOR_HSV2BGR)
# Opens a new window and displays the output frame
# Updates previous frame
prev_gray = gray.copy()
for roi in ROI:
for y in range(grids[1][roi - 1],
grids[1][roi - 1] + Y_BLOCK_SIZE):
for x in range(grids[0][roi - 1],
grids[0][roi - 1] + X_BLOCK_SIZE):
if motion_threshold[0] < magnitude[y][x] < 15:
if if_normal_ori:
orientation = categorize_by_bin(
mask[..., 0][y][x], 0, 180, 20)
else:
orientation = categorize_to_four(angle[y][x])
flow_points.append([
x, y, magnitude[y][x], orientation, roi,
set_of_frames
])
'''
for y in range(magnitude.shape[0]):
for x in range(magnitude.shape[1]):
if motion_threshold[0] < magnitude[y][x] < 15:
if if_normal_ori:
orientation = categorize_by_bin(mask[..., 0][y][x], 0, 180, 20)
else:
orientation = categorize_to_four(angle[y][x])
which_grid = categorize_flow_points(x, y, grids, X_BLOCK_SIZE, Y_BLOCK_SIZE)
flow_points.append([x, y, magnitude[y][x], orientation,
which_grid, set_of_frames])
'''
frame_count += 1
if (frame_count + 1) % time_interval == 0:
set_of_frames += 1
if if_show == True:
# Opens a new window and displays the input frame
cv2.imshow("input", frame)
# Overlays the optical flow tracks on the original frame
output = cv2.add(rgb, grid_mask)
# Opens a new window and displays the output frame
cv2.imshow("dense optical flow" + name_of_videos, output)
# Frames are read by intervals of 10 milliseconds. The programs breaks out of the while loop when the user presses the 'q' key
if cv2.waitKey(10) & 0xFF == ord('q'):
break
# The following frees up resources and closes all windows
cap.release()
cv2.destroyAllWindows()
return np.array(flow_points)
def extractFrame(self, frame):
frame = self.scale_frame(frame, self.percentage)
#each frame to grayscale
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
if self.prev_gray is None:
self.prev_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
if self.hsv_mask is None:
self.hsv_mask = np.zeros_like(frame)
self.hsv_mask[..., 1] = 255
#calculates optical flow/2D flow vector
flow = cv2.calcOpticalFlowFarneback(self.prev_gray, gray, None, 0.5, 3,
15, 3, 5, 1.2, 0)
#compute magnitude + angle of vector
magnitude, angle = cv2.cartToPolar(flow[..., 0], flow[..., 1])
#set image hue according to optical flow direction (angle)
self.hsv_mask[..., 0] = angle * 180 / np.pi / 2
#set image value according to optical flow magnitude (normalized)
self.hsv_mask[..., 2] = cv2.normalize(magnitude, None, 0, 255,
cv2.NORM_MINMAX)
#convert to rgb
rgb = cv2.cvtColor(self.hsv_mask, cv2.COLOR_HSV2BGR)
#grayscale image
h, s, v1 = cv2.split(rgb)
rgb = v1
img = rgb
_, thresh = cv2.threshold(img, 3, 255, cv2.THRESH_BINARY)
#thresh = self.dilateErodeFrame(thresh, 10, 1)
# Detect the contours in the image
currentCentroids, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
# max_kernel_size = 15
# result_array = np.zeros(shape=(max_kernel_size,max_kernel_size))
# for x in range(max_kernel_size):
# for y in range(max_kernel_size):
# morphed_thresh = cv2.erode(thresh, np.ones((x,x),np.uint8), 1)
# morphed_thresh = cv2.erode(morphed_thresh, np.ones((y,y),np.uint8), 1)
# currentCentroids, _ = cv2.findContours(morphed_thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
# amountOfCentroids = len(currentCentroids)
# result_array[x,y] = amountOfCentroids
# if(amountOfCentroids <= self.MAXIMUM_TRACKABLE_OBJECTS and amountOfCentroids > 0):
# for contour in currentCentroids:
# img_x,img_y,w,h = cv2.boundingRect(contour)
# cv2.rectangle(thresh, (img_x,img_y),(img_x+w,img_y+h), (255,255,0), 1)
# print('contours found: ' + str(len(currentCentroids)))
# print(result_array)
# Draw all the contours
img = cv2.drawContours(img, currentCentroids, -1, (0, 255, 0), 1)
labelDictionary = {}
cv2.imshow("Farneback Optical Flow", thresh)
maxContourLength = 0
# Iterate through all the contours
for contour in currentCentroids:
if cv2.contourArea(contour) > maxContourLength:
maxContourLength = cv2.contourArea(contour)
# Find bounding rectangles
x, y, w, h = cv2.boundingRect(contour)
# Draw the rectangle
print('found contour!')
currentBoundingBox = [x, (x + w), y, (y + h)]
#scale it back to the original frame size
scaledBack_currentBoundingBox = list(
map(self.scaleBack, currentBoundingBox))
labelDictionary = {'baer': scaledBack_currentBoundingBox}
cv2.imshow("image", img)
self.prev_gray = gray
return labelDictionary
import numpy as np
from PIL import Image
import init_auv as auv
im_filename = '*.png'
imdir = '/Users/Mackeprang/Dropbox (Personlig)/Master Thesis/Pictures/20181005_084733.9640_Mission_1'
filenames = auv.imagesFilePath(imdir)
images = []
prev_frame = None
hsv = np.zeros_like(cv2.imread(filenames[0]))
hsv[...,1] = 255
for i,frame in enumerate(filenames):
if auv.image_broken(frame):
continue
img = cv2.imread(frame)
gray = auv.preprocess_image(img,size=None)
if prev_frame is None:
prev_frame = gray
continue
flow = cv2.calcOpticalFlowFarneback(prev_frame, gray, None, 0.5, 3, 30, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / np.pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
cv2.imshow("Match", bgr)
key = cv2.waitKey()
if key & 0xFF == ord('q'):
break
prev_frame = gray.copy()
cv2.destroyAllWindows()
def main(**kwargs):
method = [
'BM1', 'coarse2fine', 'DenseCV', 'HS', 'TVL', 'LK', 'PWCNet', 'BM2'
]
sel_method = 0
pyr = True
show_legend = False
gt_paths = [
curr_dir + "/datasets/of_pred/noc_000045_10.png",
curr_dir + "/datasets/of_pred/noc_000157_10.png"
]
select_image = 0 #0: image 1, 2: image 2
im = cv2.imread(curr_dir + "/datasets/results/LKflow_000157_10.png",
cv2.IMREAD_UNCHANGED)
if select_image == 0:
im1 = cv2.imread(curr_dir + "/datasets/of_pred/000045_10.png",
cv2.IMREAD_UNCHANGED)
im2 = cv2.imread(curr_dir + "/datasets/of_pred/000045_11.png",
cv2.IMREAD_UNCHANGED)
elif select_image == 1:
im1 = cv2.imread(curr_dir + "/datasets/of_pred/000157_10.png",
cv2.IMREAD_UNCHANGED)
im2 = cv2.imread(curr_dir + "/datasets/of_pred/000157_11.png",
cv2.IMREAD_UNCHANGED)
gt = cv2.imread(gt_paths[select_image], cv2.IMREAD_UNCHANGED)
#flow_im,valid_flow = decode_optical_flow(im)
flow_gt, val_gt_flow = decode_optical_flow(gt)
#Como llamar al Block Matching
if sel_method == 1:
flow, im_warped = coarse2fine_flow(im1, im2)
elif sel_method == 2:
flow = cv2.calcOpticalFlowFarneback(im1, im2, None, 0.5, 3, 15, 3, 5,
1.2, 0)
elif sel_method == 3:
if pyr == True:
flow = opticalFlowHSPyr(im1, im2)
else:
flow = opticalFlowHS(im1, im2)
elif sel_method == 4:
if pyr == True:
flow = tvl1_simple(im1, im2)
else:
flow = opticalFlowTVL1Pyr(im1, im2)
elif sel_method == 5:
if pyr == True:
flow = opticalFlowLK(im1, im2)
else:
flow = opticalFlowLKPyr(im1, im2)
elif sel_method == 0:
flow = bm1.obtain_dense_mov(im1, im2, **kwargs)
elif sel_method == 6:
gpu_devices = ['/device:CPU:0']
controller = '/device:CPU:0'
ckpt_path = '/Users/sergi/mcv-m6-2020-team5/src/opflows/tfoptflow/models/pwcnet-lg-6-2-multisteps-chairsthingsmix/pwcnet.ckpt-595000'
nn_opts = deepcopy(_DEFAULT_PWCNET_TEST_OPTIONS)
nn_opts['verbose'] = True
nn_opts['ckpt_path'] = ckpt_path
nn_opts['batch_size'] = 1
nn_opts['gpu_devices'] = gpu_devices
nn_opts['controller'] = controller
nn_opts['use_dense_cx'] = True
nn_opts['use_res_cx'] = True
nn_opts['pyr_lvls'] = 6
nn_opts['flow_pred_lvl'] = 2
nn_opts['adapt_info'] = (1, 376, 1241, 2)
nn = ModelPWCNet(mode='test', options=nn_opts)
nn.print_config()
img_pairs = []
img_pairs.append((cv2.cvtColor(im1, cv2.COLOR_GRAY2RGB),
cv2.cvtColor(im2, cv2.COLOR_GRAY2RGB)))
print(np.asarray(img_pairs).shape)
flow = nn.predict_from_img_pairs(img_pairs,
batch_size=1,
verbose=False)
flow = np.asarray(flow)
flow = np.squeeze(flow, axis=0)
elif sel_method == 7:
block_match2 = bm2.EBMA_searcher(15, 15)
im_warped, flow = block_match2.run(im1, im2)
flow = block_match2.get_original_size(flow, im1.shape[:2])
else:
print("El método seleccionado no es válido.")
if type(flow) is tuple:
movsx, movsy, reliab = flow
flow = np.stack((movsx, movsy), axis=2)
color_plot = colorflow_white(flow)
# cv2.imwrite("pyflow.png",color_plot)
cv2.imshow("color_plot", color_plot)
cv2.waitKey(1)
if (show_legend):
flow_legend = get_plot_legend(256, 256)
color_flow_legend = colorflow_white(flow_legend)
# im_empty = np.zeros((256,256))
# legend_arrow = arrow_flow(flow_legend.astype("float")/8,im_empty, filter_zero=False)
cv2.imshow("color wheel", color_flow_legend)
cv2.waitKey(1)
# arrow_flow(flow,im1)
##metrics
msen, pepn = flowmetrics.flowmetrics(flow, flow_gt, val_gt_flow)
# print('MSEN')
# print(msen)
# print('PEPN')
# print(pepn)
return msen, pepn, flow, color_plot
while (capture.isOpened()):
if cv2.waitKey(1)==32:
decise_v+=1
while(decise_v%2==1):
#print('pause')
if cv2.waitKey(1)==32:
decise_v+=1
stime=time.time()
ret, frame =capture.read()
imgC=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
imgC=cv2.GaussianBlur(imgC,(5,5),0.5)
flow=cv2.calcOpticalFlowFarneback(imgP,imgC,None,**params)
definePass(frame,flow,TH)
imgP=imgC.copy()
max_x,max_y,t = frame.shape
image_hsv=cv2.cvtColor(frame,cv2.COLOR_BGR2HSV)
#hist = cv2.calcHist([hsv], [0, 1], None, [180, 256], [0, 180, 0, 256])
#hist2 = cv2.calcHist([hsv], [1, 2], None, [256, 256], [0, 256, 0, 256])
#mask_white=cv2.inRange(hsv,lower_white,upper_white)
image_mask1=cv2.inRange(image_hsv, line_low1, line_up1)
image_mask2=cv2.inRange(image_hsv, line_low2, line_up2)
image_mask3=cv2.inRange(image_hsv, line_low3, line_up3)
image_mask=image_mask1|image_mask2|image_mask3
show_hsv = False
show_glitch = False
cur_glitch = prev.copy()
while True:
ret, img = cam.read() # 读取视频的下一帧作为光流输入的当前帧
if ret == True: # 判断视频是否结束
if cv2.waitKey(10) == 27:
break
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(
prevgray,
gray,
None,
pyr_scale=0.5,
levels=3,
winsize=15,
iterations=3,
poly_n=5,
poly_sigma=1.5,
flags=cv2.OPTFLOW_LK_GET_MIN_EIGENVALS) # Farnback光流法
prevgray = gray # 计算完光流后，将当前帧存储为下一次计算的前一帧
residual = me.LinearRegression(flow)
cv2.imshow('flow', draw_flow(gray, flow))
cv2.imshow("residual", residual)
# cv2.imshow("mod flow", Mod_flow(flow))
if show_hsv:
cv2.imshow('flow HSV', draw_hsv(flow))
if show_glitch:
cap = cv2.VideoCapture(0)
# Get first frame
ret, first_frame = cap.read()
previous_gray = cv2.cvtColor(first_frame, cv2.COLOR_BGR2GRAY)
hsv = np.zeros_like(first_frame)
hsv[..., 1] = 255
while True:
# Read of video file
ret, frame2 = cap.read()
next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
# Computes the dense optical flow using the Gunnar Farneback’s algorithm
flow = cv2.calcOpticalFlowFarneback(previous_gray, next, None, 0.5, 3, 15,
3, 5, 1.2, 0)
# use flow to calculate the magnitude (speed) and angle of motion
# use these values to calculate the color to reflect speed and angle
magnitude, angle = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = angle * (180 / (np.pi / 2))
hsv[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX)
final = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
# Show our demo of Dense Optical Flow
cv2.imshow('Dense Optical Flow', final)
if cv2.waitKey(1) == 13: #13 is the Enter Key
break
# Store current image as previous image
previous_gray = next
def readData(Filename, data_shape):
data_1 = []
data_2 = []
pic = []
pic_x = []
pic_y = []
for filename in glob.glob(Filename + '/image*.jpg'):
pic.append(filename)
pic.sort()
a = 0
for i in range(len(pic) - 1):
prev = cv2.imread(pic[i])
prev = cv2.resize(prev, (320, 240))
prev = cv2.cvtColor(prev, cv2.COLOR_RGB2GRAY)
#prev = np.multiply(prev, 1/255.0)
#print prev[156][0:100]
cur = cv2.imread(pic[i + 1])
cur = cv2.resize(cur, (320, 240))
cur = cv2.cvtColor(cur, cv2.COLOR_RGB2GRAY)
#cur = np.multiply(cur, 1/255.0)
#print cur.shape
#print cur[156][0:100]
flow = cv2.calcOpticalFlowFarneback(prev, cur, 0.702, 5, 10, 2, 7, 1.5,
cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
#flow = np.array(flow)
#flow = np.multiply(flow, 255)
#flow = cv2.resize(flow, (data_shape[2], data_shape[1]/10))
#array_bound = np.ones((data_shape[2], data_shape[1]/10, 2), dtype=int)*20
# array_bound = np.ones((256, 256, 2), dtype=int)*20
# flow_img = 255*((flow)+array_bound)/(2*20)
# flow_img = np.uint8(flow_img)
# flow_img = cv2.resize(flow_img, (data_shape[2], data_shape[1]/10))
# #aaaa += 1
# #flow = np.multiply(flow_img, 1/255.0)
#
# flow_1 = flow_img.transpose((2,0,1))
# cv2.imwrite('./flow/flow-'+str(a).zfill(4)+'.jpg',flow_1[0,...])
# flow_1 = np.multiply(flow_1, 1/255.0)
# flow_1 = flow_1.tolist()
# pic_x.append(flow_1[0])
# pic_y.append(flow_1[1])
# a += 1
flow_x = flow[..., 0]
flow_y = flow[..., 1]
flow_x = cv2.resize(flow_x, (data_shape[2], data_shape[1] / 10))
flow_y = cv2.resize(flow_y, (data_shape[2], data_shape[1] / 10))
array_bound = np.ones(
(data_shape[2], data_shape[1] / 10), dtype=int) * 20
flow_x_img = 255 * ((flow_x) + array_bound) / (2 * 20)
flow_y_img = 255 * ((flow_y) + array_bound) / (2 * 20)
cv2.imwrite('./flow/flow-' + str(a).zfill(4) + '.jpg', flow_x_img)
a += 1
flow_x_img = np.multiply(flow_x_img, 1 / 255.0)
flow_y_img = np.multiply(flow_y_img, 1 / 255.0)
flow_x_list = flow_x_img.tolist()
flow_y_list = flow_y_img.tolist()
pic_x.append(flow_x_list)
pic_y.append(flow_y_list)
for j in range(len(pic_x) - LEN_SEQ):
data_1_1 = []
for i in range(LEN_SEQ):
idx = j + i
data_1_1.append([pic_x[idx], pic_y[idx]])
data_2.append(0)
data_1.append(data_1_1)
# le = len(pic_x)/LEN_SEQ
# for j in range(2):
# data_1_1 = []
# for i in range(LEN_SEQ):
# ret = random.randint(i*le, (i+1)*le-1)
# data_1_1.append([pic_x[ret], pic_y[ret]])
# data_2.append(0)
# data_1.append(data_1_1)
return (data_1, data_2)
def main():
with tf.Session() as sess:
model_cfg, model_outputs = posenet.load_model(args.model, sess)
output_stride = model_cfg['output_stride']
if args.file is not None:
cap = cv2.VideoCapture(args.file)
else:
cap = cv2.VideoCapture(args.cam_id)
cap.set(3, args.cam_width)
cap.set(4, args.cam_height)
video_dict = np.load('../atharva_old/stair_name_dict.npy',allow_pickle=True).item()
frame_list = []
path_list = []
for video_name in video_dict:
k = video_name[-12:-2]
# video_path = '.' + video_name[:-1]
video_path = '../atharva_old/' + video_name[2:-1]
for j in range(video_dict[video_name]):
path_list.append(video_path + str(j) + '.png')
# print(video_path + str(j))
break # remove this for labelling
# print(path_list)
frame_c = 0
frame_n = len(path_list)
# hasFrame, frame = cap.read()
ret,frame = img_read(path_list,frame_c)
frame_c += 1
if not ret:
print('no frame')
print(path_list[frame_c])
exit()
# print(frame.shape)
start = time.time()
frame_count = 0
prvs = cv2.resize(frame,(224,224))
prvs = cv2.cvtColor(prvs,cv2.COLOR_BGR2GRAY)
hsv = np.zeros((224,224,3),dtype=np.uint8)
hsv[...,1] = 255 #intensity
c = 0
while True:
c += 1
flag = 0
t = time.time()
# hasFrame, frame = cap.read()
ret,frame = img_read(path_list,frame_c)
frame_c += 1
if not ret:
print('no frame')
break
next = cv2.resize(frame,(224,224))
# print(next.shape)
next = cv2.cvtColor(next,cv2.COLOR_BGR2GRAY)
prvs = cv2.medianBlur(prvs,5)
next = cv2.medianBlur(next,5)
# print(prvs.shape,next.shape)
flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5,3,7,4,7,5, 0)
mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
mag = (mag>1.4)*mag
hsv[...,0] = ang*180/np.pi/2 #hue, colour
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) #brightness
up_mask = make_mask(hsv[...,0],130,145) #purple
down_mask = make_mask(hsv[...,0],35,75) #green
left_mask = make_mask(hsv[...,0],165,179) | make_mask(hsv[...,0],1,20)#red
right_mask = make_mask(hsv[...,0],80,100) #blue
hsv_up = apply_mask(hsv,up_mask)
hsv_down = apply_mask(hsv,down_mask)
hsv_left = apply_mask(hsv,left_mask)
hsv_right = apply_mask(hsv,right_mask)
#input_image, display_image, output_scale = posenet.read_cap(
# cap, scale_factor=args.scale_factor, output_stride=output_stride)
f= path_list[frame_c]
input_image, draw_image, output_scale = posenet.read_imgfile(
f, scale_factor=args.scale_factor, output_stride=output_stride)
heatmaps_result, offsets_result, displacement_fwd_result, displacement_bwd_result = sess.run(
model_outputs,
feed_dict={'image:0':input_image}
)
pose_scores, keypoint_scores, keypoint_coords = posenet.decode_multi.decode_multiple_poses(
heatmaps_result.squeeze(axis=0),
offsets_result.squeeze(axis=0),
displacement_fwd_result.squeeze(axis=0),
displacement_bwd_result.squeeze(axis=0),
output_stride=output_stride,
max_pose_detections=1,
min_pose_score=0.15)
keypoint_coords *= output_scale #what the hell is this
#for i in range(len(pose_scores)):
for pts in keypoint_coords:
dist1 = [0]
dist2 = [0]
# print(pts.shape)
for i in range(len(mag)):
for j in range(len(mag[0])):
if mag[i,j] > 10:
# pass
# dist1.append(dist_from_line(j,i,pts[7,:],pts[9,:])) # left hand
# dist2.append(dist_from_line(j,i,pts[8,:],pts[10,:]))# right hand
dist1.append(dist_from_pt(j,i,pts[9,:])) # left wrist
dist2.append(dist_from_pt(j,i,pts[10,:])) # right wrist
# if True:
thresh = 140
up_thresh = 34
down_thresh = 16
left_thresh = 24
right_thresh = 28
if (np.mean(dist1) < thresh or np.mean(dist2)<thresh) and (np.mean(dist1) >0 and np.mean(dist2)>0):
# if True:
#original
# print('please print')
# print(np.mean(hsv_right[...,0]))
if np.mean(hsv_up[...,0])>up_thresh and np.mean(mag)>0.07:
# print(np.mean(hsv_up[...,0]))
print('UP',c)
flag = 1
elif np.mean(hsv_down[...,0])>down_thresh and np.mean(mag)>0.07:
# print(np.mean(hsv_down[...,0]))
print('DOWN',c)
flag = 1
elif np.mean(hsv_left[...,0])>left_thresh and np.mean(mag)>0.08:
# print(np.mean(hsv_left[...,0]))
print('LEFT',c)
flag = 1
elif np.mean(hsv_right[...,0])>right_thresh and np.mean(mag)>0.08:
# print(np.mean(hsv_right[...,0]))
print('RIGHT',c)
flag = 1
#modified
# if np.mean(hsv_up[...,0])>38 and np.mean(mag)>0.08:
# print('UP',np.mean(hsv_up[...,0]))
# flag = 1
# if np.mean(hsv_down[...,0])>16.5 and np.mean(mag)>0.08:
# print('DOWN',np.mean(hsv_down[...,0]))
# flag = 1
# if np.mean(hsv_left[...,0])>24 and np.mean(mag)>0.08:
# print('LEFT',c)
# flag = 1
# if np.mean(hsv_right[...,0])>28 and np.mean(mag)>0.08:
# print('RIGHT',c)
# flag = 1
# TODO this isn't particularly fast, use GL for drawing and display someday...
overlay_image = posenet.draw_skel_and_kp(
draw_image, pose_scores, keypoint_scores, keypoint_coords,
min_pose_score=0.15, min_part_score=0.1)
bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
bgr = cv2.medianBlur(bgr,5)
cv2.imshow('flow',bgr)
cv2.imshow('posenet', overlay_image)
prvs = next
frame_count += 1
if cv2.waitKey(1) & 0xFF == ord('q'):
exit()
print('Average FPS: ', frame_count / (time.time() - start))
# ret, old_frame = cap.read()
old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY)
# p0 = cv2.goodFeaturesToTrack(old_gray, mask = None, **feature_params)
# Create a mask image for drawing purposes
# mask = np.zeros_like(old_frame)
while(j < 500):
print j
j += 1
# frame = cv2.imread(video_path + '/img/' + img_seq[j], 1)
frame = cv2.imread('/home/tiago/Desktop/2.jpg', 1)
# frame = cv2.resize(frame, (0,0), fx=0.25, fy=0.25)
frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# calculate optical flow
# p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params)
start = time.time()
flow = cv2.calcOpticalFlowFarneback(old_gray, frame_gray, None, 0.5, 3, 15, 3, 5, 1.2, 0)
print 'took', round(time.time() - start, 2), 'secs to compute OF'
mean = np.mean(flow[:,:, 1 ])
print 'mean', mean
cv2.imshow('1', flow[:, :,1])
cv2.waitKey(-1)
break
# Select good points
# try:
# good_new = p1[st==1]
# good_old = p0[st==1]
# # draw the tracks
# for i,(new,old) in enumerate(zip(good_new,good_old)):
# a,b = new.ravel()
# c,d = old.ravel()
# mask = cv2.line(mask, (a,b),(c,d), color[i].tolist(), 2)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--log_dir", type=str)
parser.add_argument("--images_path", type=str)
parser.add_argument("--images_ext", type=str, default='png')
parser.add_argument("--labels_path", type=str, default='')
parser.add_argument("--labels_ext", type=str, default='png')
parser.add_argument("--labels_col", type=str, default='green')
parser.add_argument("--seg_paths", type=str_to_list, default=[])
parser.add_argument("--seg_ext", type=str, default='png')
parser.add_argument("--seg_root_dir", type=str, default='')
parser.add_argument("--seg_labels", type=str_to_list, default=[])
parser.add_argument(
"--seg_cols",
type=str_to_list,
default=['blue', 'forest_green', 'magenta', 'cyan', 'red'])
parser.add_argument("--out_path", type=str, default='')
parser.add_argument("--out_ext", type=str, default='jpg')
parser.add_argument("--out_size", type=str, default='1920x1080')
parser.add_argument("--fps", type=float, default=30)
parser.add_argument("--codec", type=str, default='H264')
parser.add_argument("--save_path", type=str, default='')
parser.add_argument("--n_classes", type=int)
parser.add_argument("--save_stitched", type=int, default=0)
parser.add_argument("--load_ice_conc_diff", type=int, default=0)
parser.add_argument("--start_id", type=int, default=0)
parser.add_argument("--end_id", type=int, default=-1)
parser.add_argument("--show_img", type=int, default=0)
parser.add_argument("--stitch", type=int, default=0)
parser.add_argument("--stitch_seg", type=int, default=1)
parser.add_argument("--plot_changed_seg_count", type=int, default=0)
parser.add_argument("--normalize_labels", type=int, default=0)
parser.add_argument("--selective_mode", type=int, default=0)
parser.add_argument("--ice_type",
type=int,
default=0,
help='0: combined, 1: anchor, 2: frazil')
parser.add_argument("--enable_plotting",
type=int,
default=1,
help='enable_plotting')
args = parser.parse_args()
images_path = args.images_path
images_ext = args.images_ext
labels_path = args.labels_path
labels_ext = args.labels_ext
labels_col = args.labels_col
seg_paths = args.seg_paths
seg_root_dir = args.seg_root_dir
seg_ext = args.seg_ext
out_path = args.out_path
out_ext = args.out_ext
out_size = args.out_size
fps = args.fps
codec = args.codec
# save_path = args.save_path
n_classes = args.n_classes
end_id = args.end_id
start_id = args.start_id
show_img = args.show_img
stitch = args.stitch
stitch_seg = args.stitch_seg
save_stitched = args.save_stitched
normalize_labels = args.normalize_labels
selective_mode = args.selective_mode
seg_labels = args.seg_labels
seg_cols = args.seg_cols
ice_type = args.ice_type
plot_changed_seg_count = args.plot_changed_seg_count
load_ice_conc_diff = args.load_ice_conc_diff
enable_plotting = args.enable_plotting
ice_types = {
0: 'Ice',
1: 'Anchor Ice',
2: 'Frazil Ice',
}
loc = (5, 120)
size = 8
thickness = 6
fgr_col = (255, 255, 255)
bgr_col = (0, 0, 0)
font_id = 0
video_exts = ['mp4', 'mkv', 'avi', 'mpg', 'mpeg', 'mjpg']
labels_col_rgb = col_bgr[labels_col]
seg_cols_rgb = [col_bgr[seg_col] for seg_col in seg_cols]
ice_type_str = ice_types[ice_type]
print('ice_type_str: {}'.format(ice_type_str))
src_files, src_labels_list, total_frames = read_data(
images_path, images_ext, labels_path, labels_ext)
if end_id < start_id:
end_id = total_frames - 1
if seg_paths:
n_seg_paths = len(seg_paths)
n_seg_labels = len(seg_labels)
if n_seg_paths != n_seg_labels:
raise IOError(
'Mismatch between n_seg_labels: {} and n_seg_paths: {}'.format(
n_seg_labels, n_seg_paths))
if seg_root_dir:
seg_paths = [
os.path.join(seg_root_dir, name) for name in seg_paths
]
if not out_path:
if labels_path:
out_path = labels_path + '_conc'
elif seg_paths:
out_path = seg_paths[0] + '_conc'
if not os.path.isdir(out_path):
os.makedirs(out_path)
# print('Saving results data to {}'.format(out_path))
# if not save_path:
# save_path = os.path.join(os.path.dirname(images_path), 'ice_concentration')
# if not os.path.isdir(save_path):
# os.makedirs(save_path)
# if stitch and save_stitched:
# print('Saving ice_concentration plots to: {}'.format(save_path))
# log_fname = os.path.join(out_path, 'vis_log_{:s}.txt'.format(getDateTime()))
# print('Saving log to: {}'.format(log_fname))
if selective_mode:
label_diff = int(255.0 / n_classes)
else:
label_diff = int(255.0 / (n_classes - 1))
print('label_diff: {}'.format(label_diff))
n_frames = end_id - start_id + 1
print_diff = int(n_frames * 0.01)
labels_img = None
n_cols = len(seg_cols_rgb)
plot_y_label = '{} concentration (%)'.format(ice_type_str)
plot_x_label = 'distance in pixels from left edge'
dists = {}
for _label in seg_labels:
dists[_label] = {
# 'bhattacharyya': [],
'euclidean': [],
'mae': [],
'mse': [],
# 'frobenius': [],
}
plot_title = '{} concentration'.format(ice_type_str)
out_size = tuple([int(x) for x in out_size.split('x')])
write_to_video = out_ext in video_exts
out_width, out_height = out_size
out_seq_name = os.path.basename(out_path)
if enable_plotting:
if write_to_video:
stitched_seq_path = os.path.join(
out_path, '{}.{}'.format(out_seq_name, out_ext))
print('Writing {}x{} output video to: {}'.format(
out_width, out_height, stitched_seq_path))
save_dir = os.path.dirname(stitched_seq_path)
fourcc = cv2.VideoWriter_fourcc(*codec)
video_out = cv2.VideoWriter(stitched_seq_path, fourcc, fps,
out_size)
else:
stitched_seq_path = os.path.join(out_path, out_seq_name)
print('Writing {}x{} output images of type {} to: {}'.format(
out_width, out_height, out_ext, stitched_seq_path))
save_dir = stitched_seq_path
if save_dir and not os.path.isdir(save_dir):
os.makedirs(save_dir)
prev_seg_img = {}
prev_conc_data_y = {}
changed_seg_count = {}
ice_concentration_diff = {}
if load_ice_conc_diff:
for seg_id in seg_labels:
ice_concentration_diff[seg_id] = np.loadtxt(os.path.join(
out_path, '{}_ice_concentration_diff.txt'.format(seg_id)),
dtype=np.float64)
_pause = 0
mae_data_y = []
for seg_id, _ in enumerate(seg_paths):
mae_data_y.append([])
for img_id in range(start_id, end_id + 1):
start_t = time.time()
# img_fname = '{:s}_{:d}.{:s}'.format(fname_templ, img_id + 1, img_ext)
img_fname = src_files[img_id]
img_fname_no_ext = os.path.splitext(img_fname)[0]
src_img_fname = os.path.join(images_path, img_fname)
src_img = imread(src_img_fname)
if src_img is None:
raise SystemError('Source image could not be read from: {}'.format(
src_img_fname))
try:
src_height, src_width = src_img.shape[:2]
except ValueError as e:
print('src_img_fname: {}'.format(src_img_fname))
print('src_img: {}'.format(src_img))
print('src_img.shape: {}'.format(src_img.shape))
print('error: {}'.format(e))
sys.exit(1)
conc_data_x = np.asarray(range(src_width), dtype=np.float64)
plot_data_x = conc_data_x
plot_data_y = []
plot_cols = []
plot_labels = []
stitched_img = src_img
if labels_path:
labels_img_fname = os.path.join(
labels_path, img_fname_no_ext + '.{}'.format(labels_ext))
labels_img_orig = imread(labels_img_fname)
if labels_img_orig is None:
raise SystemError(
'Labels image could not be read from: {}'.format(
labels_img_fname))
labels_height, labels_width = labels_img_orig.shape[:2]
if labels_height != src_height or labels_width != src_width:
raise AssertionError(
'Mismatch between dimensions of source: {} and label: {}'.
format((src_height, src_width), (seg_height, seg_width)))
if len(labels_img_orig.shape) == 3:
labels_img_orig = np.squeeze(labels_img_orig[:, :, 0])
if show_img:
cv2.imshow('labels_img_orig', labels_img_orig)
if normalize_labels:
labels_img = (labels_img_orig.astype(np.float64) /
label_diff).astype(np.uint8)
else:
labels_img = np.copy(labels_img_orig)
if len(labels_img.shape) == 3:
labels_img = labels_img[:, :, 0].squeeze()
conc_data_y = np.zeros((labels_width, ), dtype=np.float64)
for i in range(labels_width):
curr_pix = np.squeeze(labels_img[:, i])
if ice_type == 0:
ice_pix = curr_pix[curr_pix != 0]
else:
ice_pix = curr_pix[curr_pix == ice_type]
conc_data_y[i] = (len(ice_pix) / float(src_height)) * 100.0
conc_data = np.zeros((labels_width, 2), dtype=np.float64)
conc_data[:, 0] = conc_data_x
conc_data[:, 1] = conc_data_y
plot_data_y.append(conc_data_y)
plot_cols.append(labels_col_rgb)
gt_dict = {
conc_data_x[i]: conc_data_y[i]
for i in range(labels_width)
}
if not normalize_labels:
labels_img_orig = (labels_img_orig.astype(np.float64) *
label_diff).astype(np.uint8)
if len(labels_img_orig.shape) == 2:
labels_img_orig = np.stack(
(labels_img_orig, labels_img_orig, labels_img_orig),
axis=2)
stitched_img = np.concatenate((stitched_img, labels_img_orig),
axis=1)
plot_labels.append('GT')
# gt_cl, _ = eval.extract_classes(labels_img_orig)
# print('gt_cl: {}'.format(gt_cl))
mean_seg_counts = {}
seg_count_data_y = []
curr_mae_data_y = []
mean_conc_diff = {}
conc_diff_data_y = []
seg_img_disp_list = []
for seg_id, seg_path in enumerate(seg_paths):
seg_img_fname = os.path.join(
seg_path, img_fname_no_ext + '.{}'.format(seg_ext))
seg_img_orig = imread(seg_img_fname)
seg_col = seg_cols_rgb[seg_id % n_cols]
_label = seg_labels[seg_id]
if seg_img_orig is None:
raise SystemError(
'Seg image could not be read from: {}'.format(
seg_img_fname))
seg_height, seg_width = seg_img_orig.shape[:2]
if seg_height != src_height or seg_width != src_width:
raise AssertionError(
'Mismatch between dimensions of source: {} and seg: {}'.
format((src_height, src_width), (seg_height, seg_width)))
if len(seg_img_orig.shape) == 3:
seg_img_orig = np.squeeze(seg_img_orig[:, :, 0])
if seg_img_orig.max() > n_classes - 1:
seg_img = (seg_img_orig.astype(np.float64) /
label_diff).astype(np.uint8)
seg_img_disp = seg_img_orig
else:
seg_img = seg_img_orig
seg_img_disp = (seg_img_orig.astype(np.float64) *
label_diff).astype(np.uint8)
if len(seg_img_disp.shape) == 2:
seg_img_disp = np.stack(
(seg_img_disp, seg_img_disp, seg_img_disp), axis=2)
ann_fmt = (font_id, loc[0], loc[1], size,
thickness) + fgr_col + bgr_col
put_text_with_background(seg_img_disp,
seg_labels[seg_id],
fmt=ann_fmt)
seg_img_disp_list.append(seg_img_disp)
# eval_cl, _ = eval.extract_classes(seg_img)
# print('eval_cl: {}'.format(eval_cl))
if show_img:
cv2.imshow('seg_img_orig', seg_img_orig)
if len(seg_img.shape) == 3:
seg_img = seg_img[:, :, 0].squeeze()
conc_data_y = np.zeros((seg_width, ), dtype=np.float64)
for i in range(seg_width):
curr_pix = np.squeeze(seg_img[:, i])
if ice_type == 0:
ice_pix = curr_pix[curr_pix != 0]
else:
ice_pix = curr_pix[curr_pix == ice_type]
conc_data_y[i] = (len(ice_pix) / float(src_height)) * 100.0
plot_cols.append(seg_col)
plot_data_y.append(conc_data_y)
if labels_path:
seg_dict = {
conc_data_x[i]: conc_data_y[i]
for i in range(seg_width)
}
# dists['bhattacharyya'].append(bhattacharyya(gt_dict, seg_dict))
dists[_label]['euclidean'].append(euclidean(gt_dict, seg_dict))
dists[_label]['mse'].append(mse(gt_dict, seg_dict))
dists[_label]['mae'].append(mae(gt_dict, seg_dict))
# dists['frobenius'].append(np.linalg.norm(conc_data_y - plot_data_y[0]))
curr_mae_data_y.append(dists[_label]['mae'][-1])
else:
if img_id > 0:
if plot_changed_seg_count:
flow = cv2.calcOpticalFlowFarneback(
prev_seg_img[_label], seg_img, None, 0.5, 3, 15, 3,
5, 1.2, 0)
print('flow: {}'.format(flow.shape))
# # Obtain the flow magnitude and direction angle
# mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
# hsvImg = np.zeros((2160, 3840, 3), dtype=np.uint8)
# hsvImg[..., 1] = 255
# # Update the color image
# hsvImg[..., 0] = 0.5 * ang * 180 / np.pi
# hsvImg[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
# rgbImg = cv2.cvtColor(hsvImg, cv2.COLOR_HSV2BGR)
# rgbImg = resizeAR(rgbImg, width=out_width, height=out_height)
# # Display the resulting frame
# cv2.imshow('dense optical flow', rgbImg)
# k = cv2.waitKey(0)
curr_x, curr_y = (prev_x + flow[..., 0]).astype(
np.int32), (prev_y + flow[..., 1]).astype(np.int32)
seg_img_flow = seg_img[curr_y, curr_x]
changed_seg_count[_label].append(
np.count_nonzero(
np.not_equal(seg_img, prev_seg_img[_label])))
seg_count_data_y.append(changed_seg_count[_label])
mean_seg_counts[_label] = np.mean(
changed_seg_count[_label])
else:
ice_concentration_diff[_label].append(
np.mean(
np.abs(conc_data_y -
prev_conc_data_y[_label])))
conc_diff_data_y.append(ice_concentration_diff[_label])
mean_conc_diff[_label] = np.mean(
ice_concentration_diff[_label])
else:
if plot_changed_seg_count:
prev_x, prev_y = np.meshgrid(range(seg_width),
range(seg_height),
sparse=False,
indexing='xy')
changed_seg_count[_label] = []
else:
ice_concentration_diff[_label] = []
prev_seg_img[_label] = seg_img
prev_conc_data_y[_label] = conc_data_y
# conc_data = np.concatenate([conc_data_x, conc_data_y], axis=1)
if labels_path:
for i, k in enumerate(curr_mae_data_y):
mae_data_y[i].append(k)
n_test_images = img_id + 1
mae_data_X = np.asarray(range(1, n_test_images + 1),
dtype=np.float64)
print('')
# print('mae_data_X:\n {}'.format(pformat(mae_data_X)))
# print('mae_data_y:\n {}'.format(pformat(np.array(mae_data_y).transpose())))
if img_id == end_id:
mae_data_y_arr = np.array(mae_data_y).transpose()
print('mae_data_y:\n {}'.format(
tabulate(mae_data_y_arr,
headers=seg_labels,
tablefmt='plain')))
pd.DataFrame(data=mae_data_y_arr,
columns=seg_labels).to_clipboard(excel=True)
mae_img = getPlotImage(mae_data_X, mae_data_y, plot_cols, 'MAE',
seg_labels, 'frame', 'MAE')
cv2.imshow('mae_img', mae_img)
conc_diff_img = resize_ar(mae_img,
seg_width,
src_height,
bkg_col=255)
else:
if img_id > 0:
n_test_images = img_id
seg_count_data_X = np.asarray(range(1, n_test_images + 1),
dtype=np.float64)
if plot_changed_seg_count:
seg_count_img = getPlotImage(seg_count_data_X,
seg_count_data_y, plot_cols,
'Count', seg_labels, 'frame',
'Changed Label Count')
cv2.imshow('seg_count_img', seg_count_img)
else:
# print('seg_count_data_X:\n {}'.format(pformat(seg_count_data_X)))
# print('conc_diff_data_y:\n {}'.format(pformat(conc_diff_data_y)))
conc_diff_img = getPlotImage(
seg_count_data_X, conc_diff_data_y, plot_cols,
'Mean concentration difference between consecutive frames'
.format(ice_type_str), seg_labels, 'frame',
'Concentration Difference (%)')
# cv2.imshow('conc_diff_img', conc_diff_img)
conc_diff_img = resize_ar(conc_diff_img,
seg_width,
src_height,
bkg_col=255)
else:
conc_diff_img = np.zeros((src_height, seg_width, 3),
dtype=np.uint8)
plot_labels += seg_labels
if enable_plotting:
plot_img = getPlotImage(plot_data_x,
plot_data_y,
plot_cols,
plot_title,
plot_labels,
plot_x_label,
plot_y_label,
legend=0
# ylim=(0, 100)
)
plot_img = resize_ar(plot_img, seg_width, src_height, bkg_col=255)
# plt.plot(conc_data_x, conc_data_y)
# plt.show()
# conc_data_fname = os.path.join(out_path, img_fname_no_ext + '.txt')
# np.savetxt(conc_data_fname, conc_data, fmt='%.6f')
ann_fmt = (font_id, loc[0], loc[1], size,
thickness) + labels_col_rgb + bgr_col
put_text_with_background(src_img,
'frame {}'.format(img_id + 1),
fmt=ann_fmt)
if n_seg_paths == 1:
print('seg_img_disp: {}'.format(seg_img_disp.shape))
print('plot_img: {}'.format(plot_img.shape))
stitched_seg_img = np.concatenate((seg_img_disp, plot_img),
axis=1)
print('stitched_seg_img: {}'.format(stitched_seg_img.shape))
print('stitched_img: {}'.format(stitched_img.shape))
stitched_img = np.concatenate((stitched_img, stitched_seg_img),
axis=0 if labels_path else 1)
elif n_seg_paths == 2:
stitched_img = np.concatenate((
np.concatenate((src_img, conc_diff_img), axis=1),
np.concatenate(seg_img_disp_list, axis=1),
),
axis=0)
elif n_seg_paths == 3:
stitched_img = np.concatenate((
np.concatenate((src_img, plot_img, conc_diff_img), axis=1),
np.concatenate(seg_img_disp_list, axis=1),
),
axis=0)
stitched_img = resize_ar(stitched_img,
width=out_width,
height=out_height)
# print('dists: {}'.format(dists))
if write_to_video:
video_out.write(stitched_img)
else:
stacked_img_path = os.path.join(
stitched_seq_path, '{}.{}'.format(img_fname_no_ext,
out_ext))
cv2.imwrite(stacked_img_path, stitched_img)
cv2.imshow('stitched_img', stitched_img)
k = cv2.waitKey(1 - _pause)
if k == 27:
break
elif k == 32:
_pause = 1 - _pause
end_t = time.time()
sys.stdout.write('\rDone {:d}/{:d} frames. fps: {}'.format(
img_id + 1 - start_id, n_frames, 1.0 / (end_t - start_t)))
sys.stdout.flush()
print()
if enable_plotting and write_to_video:
video_out.release()
if labels_path:
median_dists = {}
mean_dists = {}
mae_data_y = []
for _label in seg_labels:
_dists = dists[_label]
mae_data_y.append(_dists['mae'])
mean_dists[_label] = {k: np.mean(_dists[k]) for k in _dists}
median_dists[_label] = {k: np.median(_dists[k]) for k in _dists}
print('mean_dists:\n{}'.format(pformat(mean_dists)))
print('median_dists:\n{}'.format(pformat(median_dists)))
n_test_images = len(mae_data_y[0])
mae_data_x = np.asarray(range(1, n_test_images + 1), dtype=np.float64)
mae_img = getPlotImage(mae_data_x, mae_data_y, plot_cols, 'MAE',
seg_labels, 'test image', 'Mean Absolute Error')
# plt.show()
cv2.imshow('MAE', mae_img)
k = cv2.waitKey(0)
else:
mean_seg_counts = {}
median_seg_counts = {}
seg_count_data_y = []
mean_conc_diff = {}
median_conc_diff = {}
conc_diff_data_y = []
for seg_id in ice_concentration_diff:
if plot_changed_seg_count:
seg_count_data_y.append(changed_seg_count[seg_id])
mean_seg_counts[seg_id] = np.mean(changed_seg_count[seg_id])
median_seg_counts[seg_id] = np.median(
changed_seg_count[seg_id])
else:
_ice_concentration_diff = ice_concentration_diff[seg_id]
n_test_images = len(_ice_concentration_diff)
conc_diff_data_y.append(_ice_concentration_diff)
mean_conc_diff[seg_id] = np.mean(_ice_concentration_diff)
median_conc_diff[seg_id] = np.median(_ice_concentration_diff)
np.savetxt(os.path.join(
out_path, '{}_ice_concentration_diff.txt'.format(seg_id)),
_ice_concentration_diff,
fmt='%8.4f',
delimiter='\t')
if plot_changed_seg_count:
print('mean_seg_counts:\n{}'.format(pformat(mean_seg_counts)))
print('median_seg_counts:\n{}'.format(pformat(median_seg_counts)))
else:
print('mean_conc_diff:')
for seg_id in mean_conc_diff:
print('{}\t{}'.format(seg_id, mean_conc_diff[seg_id]))
print('median_conc_diff:')
for seg_id in mean_conc_diff:
print('{}\t{}'.format(seg_id, median_conc_diff[seg_id]))
if Path(sys.argv[1]).is_file():
file_name = sys.argv[1]
cap = cv.VideoCapture(file_name) # Capture video from camera
ret, frame1 = cap.read()
prvs = cv.cvtColor(frame1, cv.COLOR_BGR2GRAY)
hsv = np.zeros_like(frame1)
hsv[..., 1] = 255
while True:
ret, frame2 = cap.read()
if not ret: break
next = cv.cvtColor(frame2, cv.COLOR_BGR2GRAY)
# prev, next, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags
flow = cv.calcOpticalFlowFarneback(prvs, next, None, 0.5, 2, 15, 7, 5,
1.1, 0)
mag, ang = cv.cartToPolar(flow[..., 0], flow[..., 1])
vert = ang % (np.pi / 2.0)
vert = vert < (np.pi / 16.)
hsv[..., 0] = ang * 180 / np.pi
nmag = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX)
nmag[vert] = 0.0
hsv[..., 2] = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX)
bgr = cv.cvtColor(hsv, cv.COLOR_HSV2BGR)
cv.imshow('frame2', bgr)
k = cv.waitKey(5) & 0xff
if k == 27:
break
elif k == ord('s'):
def extract_flow_angles(vidFile,
hist_bins,
mag_thresh,
density=False,
crop=240):
'''
Extract optical flow maps from video vidFile for all the frames and put the angles with >mag_threshold in different
bins. The bins vector is the feature representation for the stroke.
Use only the strokes given by list of tuples frame_indx.
Parameters:
------
vidFile: str
complete path to a video
start: int
starting frame number
end: int
ending frame number
hist_bins: 1d np array
bin divisions (boundary values). Used np.linspace(0, 2*PI, 11) for 10 bins
mag_thresh: int
minimum size of the magnitude vectors that are considered (no. of pixels shifted in consecutive frames of OF)
'''
cap = cv2.VideoCapture(vidFile)
if not cap.isOpened():
print("Capture object not opened. Aborting !!")
sys.exit(0)
ret = True
start = 0
end = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
stroke_features = []
prvs, next_ = None, None
m, n = start, end
#print("stroke {} ".format((m, n)))
sum_norm_mag_ang = np.zeros(
(len(hist_bins) - 1)) # for optical flow maxFrames - 1 size
frameNo = m
while ret: # and frameNo < n:
# cap.set(cv2.CAP_PROP_POS_FRAMES, frameNo)
ret, frame1 = cap.read()
if not ret:
# print("Frame not read. Aborting !!")
break
# resize
if frame1.shape[0] < crop or frame1.shape[1] < crop:
frame1 = cv2.resize(frame1, (crop, crop))
if (frameNo - m) == 0: # first frame condition
# resize and then convert to grayscale
if crop is not None:
frame1 = center_crop(frame1, crop)
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
#prvs = scale_and_crop(prvs, scale)
frameNo += 1
continue
if crop is not None:
frame1 = center_crop(frame1, crop)
# resize and then convert to grayscale
next_ = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs, next_, None, 0.5, 3, 15, 3,
5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
#print("Mag > 5 = {}".format(np.sum(mag>THRESH)))
pixAboveThresh = np.sum(mag > mag_thresh)
#use weights=mag[mag>THRESH] to be weighted with magnitudes
#returns a tuple of (histogram, bin_boundaries)
ang_hist = np.histogram(ang[mag > mag_thresh],
bins=hist_bins,
density=density)
stroke_features.append(ang_hist[0])
#sum_norm_mag_ang +=ang_hist[0]
# if not pixAboveThresh==0:
# sum_norm_mag_ang[frameNo-m-1] = np.sum(mag[mag > THRESH])/pixAboveThresh
# sum_norm_mag_ang[(maxFrames-1)+frameNo-m-1] = np.sum(ang[mag > THRESH])/pixAboveThresh
frameNo += 1
prvs = next_
#stroke_features.append(sum_norm_mag_ang/(n-m+1))
cap.release()
#cv2.destroyAllWindows()
stroke_features = np.array(stroke_features)
#Normalize row - wise
#stroke_features = stroke_features/(1+stroke_features.sum(axis=1)[:, None])
return stroke_features
def runIterations(self, testAgent=False, doUpdate=False, iterations=1000):
self.env.seed()
observation = self.env.reset()
observation = rescale(observation, self.zoom)
obs2 = np.copy(observation)
#
all_rewards = []
reseting = 0
otp_flow = 1
if otp_flow:
obs2[0:, 0:, 2] = obs2[0:, 0:, 2] * 0
#pass
for t in range(iterations):
if self.skip_frame_timer == self.skip_frames:
if testAgent == False:
if doUpdate:
self.agent.update(self)
t_before_action = time.time()
action = self.agent.getNextAction(obs2)
self.times["get_action"] += time.time() - t_before_action
elif testAgent == True:
self.env.render()
action = self.agent.getBestAction(obs2)
self.skip_frame_timer = 0
self.skip_frame_timer += 1
self.lastAction = action
prev_state = np.copy(obs2)
observation, reward, done, info = self.env.step(action)
all_rewards.append(reward)
#print(reward)
observation = rescale(observation, self.zoom)
obs2 = np.copy(observation)
#plt.imshow(obs2)
#plt.show()
####OPTICAL FLOW
if reseting == 0:
gray = rgb2gray(obs2)
of_y = cv2.calcOpticalFlowFarneback(rgb2gray(prev_state * 255),
gray * 255, None, 0.5, 3,
5, 3, 5, 1.2, 0)[0:, 0:, 1]
#of = cv2.calcOpticalFlowFarneback(rgb2gray(prev_state)*255,
# gray*255,None,0.5, 3, 5, 3, 5, 1.2, 0)
obs2[0:, 0:, 2] = of_y / 10
#obs2[0:,0:,2] = drawShadow(gray,of)
#obs2[0:,0:,0] = obs2[0:,0:,0]+ of_y*500
#obs2[0:,0:,1] = obs2[0:,0:,1]+ of_y*500
#obs2[0:,0:,2] = obs2[0:,0:,2]+ of_y*500
else:
obs2[0:, 0:, 2] = obs2[0:, 0:, 2] * 0
#pass
reseting = 0
r = reward
#### log rewards
all_rewards.append(r)
####
r = np.clip(reward, -1, 1)
if testAgent == False:
if self.skip_frame_timer == 1:
self.experienceData.addData(prev_state, action, obs2, r,
done)
if done:
self.skip_frame_timer = self.skip_frames
self.env.seed()
observation = self.env.reset()
observation = rescale(observation, self.zoom)
obs2 = np.copy(observation)
if otp_flow:
obs2[0:, 0:, 2] = obs2[0:, 0:, 2] * 0
reseting = 1
if t == self.episode_maxLength: # never
print("episode max length reached")
#self.env.close()
if testAgent == False:
if self.agent.exploreChance > self.agent.exploration_final_eps:
if doUpdate:
self.agent.exploreChance *= 0.8
return all_rewards
if testAgent == True:
return all_rewards
def extract_flow_grid(vidFile, grid_size, crop):
'''
Extract optical flow maps from video vidFile starting from start frame number
to end frame no. The grid based features are flattened and appended.
Parameters:
------
vidFile: str
complete path to a video
grid_size: int
grid size for sampling at intersection points of 2D flow.
Returns:
------
np.array 2D with N x (360/G * 640/G) where G is grid size
'''
cap = cv2.VideoCapture(vidFile)
if not cap.isOpened():
print("Capture object not opened. Aborting !!")
sys.exit(0)
ret = True
start = 0
end = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
stroke_features = []
prvs, next_ = None, None
m, n = start, end
#print("stroke {} ".format((m, n)))
#sum_norm_mag_ang = np.zeros((len(hist_bins)-1)) # for optical flow maxFrames - 1 size
frameNo = m
while ret: # and frameNo <= n:
# cap.set(cv2.CAP_PROP_POS_FRAMES, frameNo)
ret, frame1 = cap.read()
if not ret:
# print("Frame not read. Aborting !!")
break
# resize
if frame1.shape[0] < crop or frame1.shape[1] < crop:
frame1 = cv2.resize(frame1, (crop, crop))
if (frameNo - m) == 0: # first frame condition
# resize and then convert to grayscale
#cv2.imwrite(os.path.join(flow_numpy_path, str(frameNo)+".png"), frame1)
if crop is not None:
frame1 = center_crop(frame1, crop)
prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
#prvs = scale_and_crop(prvs, scale)
frameNo += 1
continue
if crop is not None:
frame1 = center_crop(frame1, crop)
next_ = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
flow = cv2.calcOpticalFlowFarneback(prvs, next_, None, 0.5, 3, 15, 3,
5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
# stack sliced arrays along the first axis (2, 12, 16)
sliced_flow = np.stack(( mag[::grid_size, ::grid_size], \
ang[::grid_size, ::grid_size]), axis=0)
# stroke_features.append(sliced_flow[1, ...].ravel()) # Only angles
#feature = np.array(feature)
stroke_features.append(sliced_flow.ravel()) # Both magnitude and angle
frameNo += 1
prvs = next_
cap.release()
#cv2.destroyAllWindows()
stroke_features = np.array(stroke_features)
#Normalize row - wise
#stroke_features = stroke_features/(1+stroke_features.sum(axis=1)[:, None])
return stroke_features
####----New idea ---
## Once I get the mask of ROI, lets get the optical flow of the mask in each frame and from there we can catagorise activities
# once we get the optical flow we can create another bounding box on the original frame with TAG : Picking / Not Picking
clone = copy.copy(frame)
curr_frame = copy.copy(clone)
curr_frame = cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY)
curr_frame = imutils.resize(curr_frame, width=WIDTH)
#vel_x = None
#vel_y = None
#visual = None
print ('clone_frame', np.shape(clone))
print ('curr_frame', np.shape(curr_frame))
flow = cv2.calcOpticalFlowFarneback(prev_frame, curr_frame, flow=None, pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.5, flags=0)
visual, vel_x, vel_y = draw_flow(curr_frame, flow)
#vel_x_list = []
#vel_y_list = []
#vel_y_list = vel_y_list.append(vel_y)
#vel_x_list = vel_x_list.append(vel_x)
#print ('vel_x', vel_x)
cv2.imshow('visual', visual)
prev_frame = curr_frame
### Idea ENDS--------------
# show the output image
cv2.imshow("Output", clone)
cv2.waitKey(10000)
def crop_and_get_of(video_path, thresh, resize):
optical_flows = []
# Open vide and read first frame
cap = cv2.VideoCapture(video_path)
ret = None
current_frame = None
frame = 0
while (current_frame is None):
if (debug):
print('None')
ret, current_frame = cap.read()
frame += 1
total_frame = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
# Assign current to prev and resize prev
previous_frame = current_frame
previous_frame = cv2.resize(previous_frame, resize)
net = get_obj_detection_model()
x_min, x_max, y_min, y_max, w, h = get_person_bounding_box(
previous_frame, net)
(h, w) = previous_frame.shape[:2]
box_w = x_max - x_min
box_h = y_max - y_min
inc_x_min, inc_x_max, inc_y_min, inc_y_max = (max(
0, int(x_min - 0.5 * box_w)), min(
w, int(x_max + 0.5 * box_w)), max(
0, int(y_min - 0.5 * box_h)), min(h, int(y_max + 0.5 * box_h)))
of_x_min, of_x_max, of_y_min, of_y_max = inc_x_min, inc_x_max, inc_y_min, inc_y_max
# Get bounding box coordinates
if (debug):
print("x_min " + str(x_min))
print("x_max " + str(x_max))
print("x_max - x_min " + str(x_max - x_min))
print("y_min " + str(y_min))
print("y_max " + str(y_max))
print("y_max - y_min " + str(y_max - y_min))
cv2_imshow(previous_frame[y_min:y_max, x_min:x_max])
hsv = zeros_like(previous_frame[of_y_min:of_y_max, of_x_min:of_x_max])
hsv[..., 1] = 255
of_acc = None
a = []
width = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
rec = False
while (cap.isOpened()):
frame += 1
print(str(frame) + '/' + str(total_frame))
current_frame = cv2.resize(current_frame, resize)
current_frame_gray = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY)
previous_frame_gray = cv2.cvtColor(previous_frame, cv2.COLOR_BGR2GRAY)
# NEW MOTION DETECTION
blurred_current = cv2.GaussianBlur(current_frame_gray, (11, 11), 0)
blurred_prev = cv2.GaussianBlur(previous_frame_gray, (11, 11), 0)
blurred_frame_diff = cv2.absdiff(
blurred_current[of_y_min:of_y_max, of_x_min:of_x_max],
blurred_prev[of_y_min:of_y_max, of_x_min:of_x_max])
thr = cv2.threshold(blurred_frame_diff, 25, 255, cv2.THRESH_BINARY)[1]
thr = cv2.dilate(thr, None, iterations=1)
cnts = cv2.findContours(thr, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
sum_area = 0
for c in cnts:
sum_area += cv2.contourArea(c)
sum_area2 = sum_area / (of_x_max - of_x_min) / (of_y_max - of_y_min)
if (sum_area2 > thresh2):
if (rec is False):
rec = True
start = frame
bad = 0
bad += 1
cv2.fastNlMeansDenoising(previous_frame_gray, previous_frame_gray,
20, 5, 21)
cv2.fastNlMeansDenoising(current_frame_gray, current_frame_gray,
20, 5, 21)
if (bad < 18):
ret, current_frame = cap.read()
if (current_frame is None):
break
continue
flow = cv2.calcOpticalFlowFarneback(
previous_frame_gray[of_y_min:of_y_max, of_x_min:of_x_max],
current_frame_gray[of_y_min:of_y_max, of_x_min:of_x_max], None,
0.5, 3, 4, 3, 5, 1.2, 0)
mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1])
hsv[..., 0] = ang * 180 / pi / 2
hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
if of_acc is None:
of_acc = bgr
else:
of_acc += bgr
else:
if of_acc is not None:
r = True
optical_flows.append(of_acc)
#break
rec = False
of_acc = None
previous_frame = current_frame.copy()
ret, current_frame = cap.read()
if (current_frame is None):
break
cap.release()
cv2.destroyAllWindows()
return optical_flows, of_x_min, of_x_max, of_y_min, of_y_max, inc_x_min, inc_x_max, inc_y_min, inc_y_max, w, h
#cv2.putText(draw_roi,"Original Screen", (0,35), cv2.FONT_HERSHEY_PLAIN, 2,(255,255,255),3)
#Set Frame Speed as 60 per second
cap.set(cv2.CAP_PROP_FPS, int(60))
font = cv2.FONT_HERSHEY_SIMPLEX
status_cap, frame = cap.read()
frame = cv2.resize(frame, (0, 0), None, 0.5, 0.5)
if not status_cap:
break
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
if init_flow:
opt_flow = cv2.calcOpticalFlowFarneback(
prev_frame, gray, None, 0.5, 5, 13, 10, 5, 1.1,
cv2.OPTFLOW_FARNEBACK_GAUSSIAN)
init_flow = False
else:
opt_flow = cv2.calcOpticalFlowFarneback(
prev_frame, gray, opt_flow, 0.5, 5, 13, 10, 5, 1.1,
cv2.OPTFLOW_USE_INITIAL_FLOW)
prev_frame = np.copy(gray)
img = display_flow(frame, opt_flow)
if cv2.waitKey(1) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
# Expand dimensions since the model expects images to have shape: [1, None, None, 3]