def measurePictures(self): prvs = cv2.cvtColor(self.f1,cv2.COLOR_BGR2GRAY) ret, self.frame1 = cv2.imencode('.png', self.f1) ret = cv2.imwrite('frame1.png', self.f1) next = cv2.cvtColor(self.f2,cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0) bgr = self.flow2hsv(flow) (res, mx, my, ms) = self.flow2measure(flow) self.showmeasure(bgr, res, mx, my, ms) ret, self.flow12 = cv2.imencode('.png', bgr) ret = cv2.imwrite('flow12.png', bgr) ret, self.frame2 = cv2.imencode('.png', self.f2) ret = cv2.imwrite('frame2.png', self.f2) prvs = next next = cv2.cvtColor(self.f3,cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0) bgr = self.flow2hsv(flow) (res, mx, my, ms) = self.flow2measure(flow) self.showmeasure(bgr, res, mx, my, ms) ret, self.flow23 = cv2.imencode('.png', bgr) ret = cv2.imwrite('flow23.png', bgr) ret, self.frame3 = cv2.imencode('.png', self.f3) ret = cv2.imwrite('frame3.png', self.f3)
def improved_farneback(previous_frame, frame): global_motion = estimate_global_motion(previous_frame, frame) camera_moved = norm2(*global_motion) > 2 if camera_moved: print 'corrected', global_motion previous_frame, frame = apply_translation(previous_frame, frame, global_motion) flow = cv2.calcOpticalFlowFarneback(previous_frame, frame, 0.5, 1, 3, 15, 3, 5, 1) else: flow = cv2.calcOpticalFlowFarneback(previous_frame, frame, 0.5, 1, 3, 15, 3, 5, 1) return generate_flow_map(flow)
def opticalFlowCalc(input, input_pre, input_hsv): flow = cv2.calcOpticalFlowFarneback(input, input_pre, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) input_hsv[...,0] = ang*180/np.pi/2 input_hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) output = cv2.cvtColor(input_hsv,cv2.COLOR_HSV2BGR) return output, input
def read_video(vid_file, interval): vid = cv2.VideoCapture(vid_file) ret, frame1 = vid.read() prev = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) dx_list = [] dy_list = [] count = 0 while True: ret, frame2 = vid.read() if not ret: break next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY) if np.mod(count, interval) == 0: flow = cv2.calcOpticalFlowFarneback(prev, next, 0.5, 3, 15, 3, 5, 1.2, 0) dx = crop_optic_flow(flow[...,0]) dy = crop_optic_flow(flow[...,1]) dx_list.append(dx) dy_list.append(dy) count += 1 prev = next dx_list = np.array(dx_list)[None,...] dy_list = np.array(dy_list)[None,...] return np.vstack([dx_list, dy_list]).swapaxes(1,0)
def Main(): save_file_root = "C:\\Users\\Camera\\Desktop\\Video_Editing_Tools\\Analysis\\Set 8\\Limb_Movement" load_video_filename = "C:\\Users\\Camera\\Desktop\\Video_Editing_Tools\\Analysis\\Set 8\\Median.avi" # load_video_filename = "C:\\Users\\Camera\\Desktop\\GtHUb\\Two-Cameras\\Data\\AG052014-01\\21072014\\Trial5\\PS3_Vid83.avi" cap = cv2.VideoCapture(load_video_filename) ret, prvs = cap.read() prvs = cv2.cvtColor(prvs, cv2.COLOR_BGR2GRAY) frames = int(cap.get(cv.CV_CAP_PROP_FRAME_COUNT)) # Select mask in which we should look for the initial good points to track - a.k.a. select limb to track pl.figure() pl.title("Select mask") pl.imshow(prvs, cmap=mpl_cm.Greys_r) pts = [] while not len(pts): pts = pl.ginput(0) pl.close() path = mpl_path.Path(pts) mask = np.zeros(np.shape(prvs), dtype=np.uint8) for ridx, row in enumerate(mask): for cidx, pt in enumerate(row): if path.contains_point([cidx, ridx]): mask[ridx, cidx] = 1 for n in range(frames): ret, next = cap.read() next = cv2.cvtColor(next, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 3, 10, 3, 5, 1.2, 0) distt = 3 threshold = 100 new_mask = np.zeros(np.shape(prvs), dtype=np.uint8) for ridx, row in enumerate(mask): for cidx, pt in enumerate(row): if mask[ridx, cidx] == 1: y = ridx + int(round(flow[ridx, cidx, 1])) # print int(round(flow[ridx,cidx,1])) x = cidx + int(round(flow[ridx, cidx, 0])) if int(round(flow[ridx, cidx, 1])) < 10 and int(round(flow[ridx, cidx, 0])) < 10: tmask = next[y - distt : y + distt + 1, x - distt : x + distt + 1] # print tmask idx = np.where(tmask > threshold) # print idx[0]+y-distt new_mask[idx[0] + y - distt, idx[1] + x - distt] = 1 mask = new_mask next *= new_mask cv2.imwrite("C:\\Users\\Camera\\Desktop\\TEST.png", next) cv2.imshow("Limb", next) k = cv2.waitKey(30) & 0xFF if k == 27: break prvs = next cv2.destroyAllWindows() cap.release()
def __estimate_motion(self, imgs, win=5, eig_th=1e-4): img0 = imgs[0] img1 = imgs[1] img0_gray = cv2.cvtColor(img0, cv2.COLOR_BGR2GRAY) img1_gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY) w, h = np.meshgrid(range(img0.shape[1]), range(img0.shape[0])) loc0 = (np.vstack((w.flatten(), h.flatten())).T).astype('float32') flow = cv2.calcOpticalFlowFarneback(img0_gray, img1_gray, 0.5, 0, win, 3, 5, 1.2, 0) flow = (np.vstack((flow[:, :, 0].flatten(), flow[:, :, 1].flatten())).T).astype('float32') # Removing irrelevant flows minEig = cv2.cornerMinEigenVal(img0_gray, blockSize=win * 3, borderType=cv2.BORDER_REPLICATE) loc0_of = loc0[minEig.flatten() > eig_th, :] loc1_of = flow[minEig.flatten() > eig_th, :] + loc0_of # Surf-based match loc0_sf, loc1_sf = self.__calc_surf(imgs) loc0_all = np.vstack((loc0_of, loc0_sf)) loc1_all = np.vstack((loc1_of, loc1_sf)) hom = cv2.findHomography(loc0_all, loc1_all, cv2.cv.CV_RANSAC, 1)[0] gm = cv2.perspectiveTransform(np.array([loc0]), hom)[0] - loc0 lm = flow - gm gm = gm[:, 0] * 1j + gm[:, 1] lm = lm[:, 0] * 1j + lm[:, 1] lm[minEig.flatten() < eig_th] = 0 return gm, lm
def demo(self): 'Demo for Optical flow, draw flow vectors and write to video' with CaptureElement(self.video_path) as ce, WriteElement(self.video_path) as we: pbar = startBar("Computing flow vectors", len(ce.frames)) hsv = np.zeros_like(ce.frames[0]) hsv[...,1] = 255 for n, img in enumerate(ce.frames): # First run if n < 1: prev_frame = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) else: current_frame = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prev_frame, current_frame, 0.5, 3, 15, 3, 5, 1.2, 0) # Reformatting for display mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) hsv[...,0] = ang*180/np.pi/2 hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) rgb = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR) we.write_frame(rgb) prev_frame = current_frame.copy() pbar.update(n) pbar.finish()
def optical_flow_dense(): to_grayscale = lambda f: cv2.cvtColor(f, cv2.COLOR_BGR2GRAY) params = dict( pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.1, flags=0 ) frames = gen_frames() _frame = next(frames) p_frame = to_grayscale(_frame) hsv = np.zeros_like(_frame) hsv[...,1] = 255 for frame in frames: glayed = to_grayscale(frame) # calculate optical flow flow = cv2.calcOpticalFlowFarneback(p_frame, glayed, None, **params) # type: np.ndarray? if flow is None: print("none") continue # optical flow's magnitudes and angles mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) hsv[...,0] = ang*180/np.pi/2 hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) # magnitude to 0-255 scale frame = cv2.cvtColor(hsv,cv2.COLOR_HSV2RGB) p_frame = glayed.copy() yield frame
def process(frame): frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) frame = cv2.pyrDown(frame) if d['prev'] is None: d['prev'] = frame # Computes a dense optical flow # using the Gunnar Farneback’s algorithm flow = cv2.calcOpticalFlowFarneback(d['prev'], frame, pyr_scale=0.5, levels=1, winsize=10, iterations=1, poly_n=5, poly_sigma=1.1, flags=0) # Calculates the magnitude and angle of 2D vectors mag, _ = cv2.cartToPolar(flow[..., 0], flow[..., 1]) # threshold it _, mag = cv2.threshold(mag, 3, 255, cv2.THRESH_TOZERO) # store the current frame d['prev'] = frame return mag
def run_simple(self): u'''Simple prediction''' if len(self.datapath) >= 2: # Use only two previous images af_img = io.imread(self.datapath[0]) bf_img = io.imread(self.datapath[1]) #af_img = io.imread(r'./viptrafficof_02.png') #bf_img = io.imread(r'./viptrafficof_03.png') # Convert to gray image af_gray = color.rgb2gray(af_img) bf_gray = color.rgb2gray(bf_img) # Calculate density flow # Small -> WHY? flow = cv2.calcOpticalFlowFarneback(bf_gray, af_gray, \ 0.5, 6, 20, 10, 5, 1.2, 0) print flow.shape, flow[:, :, 0].min(), flow[:, :, 1].max() self.before = bf_gray self.after = af_gray #self.result = self.current self.result = transform(af_img, flow) # Color code the result for better visualization of optical flow. # Direction corresponds to Hue value of the image. # Magnitude corresponds to Value plane mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) hsv = np.zeros_like(af_img) hsv[...,1] = 255 hsv[...,0] = ang*180/np.pi/2 hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) self.optical = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
def warp_probs(self): data = self.data_to_warp; assert( data.dtype == np.uint8 ) prevgray = data[:,:,0] show_figures = False self.warps = np.zeros(self.size.tolist() + [2],dtype=np.float32) if self.dpWarp_verbose: print('Warping %d slices' % (self.size[2]-1,)); t = time.time() for z in range(1,self.size[2]): #if self.dpWarp_verbose: # print('Warping to slice %d / %d' % (z,self.size[2])); t = time.time() gray = data[:,:,z] #flow = cv2.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0) flow = cv2.calcOpticalFlowFarneback(prevgray, gray, None, 0.8, 8, 16, 10, 7, 1.5, 0) self.warps[:,:,z,:] = flow if show_figures: #cv2.imshow('flow', draw_flow(gray, flow)) cv2.imshow('prevgray',prevgray); cv2.imshow('gray',gray) cv2.imshow('flow HSV', draw_hsv(flow)) warpedgray = warp_flow(prevgray, flow) cv2.imshow('warped', warpedgray) cv2.imshow('diff', np.abs(gray-warpedgray)) cv2.waitKey(0) prevgray = gray #if self.dpWarp_verbose: # print('\tdone in %.4f s' % (time.time() - t, )) if show_figures: cv2.destroyAllWindows() if self.dpWarp_verbose: print('\tdone in %.4f s' % (time.time() - t, ))
def CalculateOpticalFlow(self, prev, nxt): """ Function definition +++++++++++++++++++ .. py:function:: CalculateOpticalFlow(prev, nxt) Computes a dense optical flow using the Gunnar Farneback’s algorithm using two subsequent frames. :param numpy_array prev: the first frame of the two subsequent frames. :param numpy_array nxt: the second frame of the two subsequent frames. """ hsv = np.zeros((prev.shape[0], prev.shape[1], 3)) hsv[...,1] = 255 flow = cv2.calcOpticalFlowFarneback(prev,nxt, 0.5, 3, 15, 3, 5, 1.2, 0) self.flow = flow mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1], angleInDegrees=1) hsv[...,0] = ang/2 hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) hsv = np.array(hsv, dtype=np.uint8) self.motion_image = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR) self.magnitude_image = hsv[...,2] self.direction_image = ang
def flow_frame_farneback(tracks, prevframe, curframe): flow = cv2.calcOpticalFlowFarneback( prevframe, curframe, pyr_scale=0.5, levels=7, winsize=5, iterations=3, poly_n=5, poly_sigma=1.1, flow=None, flags=0 ) cflow = flow.view(np.complex64).squeeze() / framescale angle = np.angle(cflow) % (2*np.pi) magnitude = np.abs(cflow) # HSV #vis = cv2.cvtColor(curframe, cv2.COLOR_GRAY2BGR) vis = np.zeros(flow.shape[:2] + (3,), dtype=np.uint8) vis[:,:,0] = np.uint8(angle / np.pi * 90) vis[:,:,1] = 255 vis[:,:,2] = magnitude / (magnitude + 10) * 255 vis = cv2.cvtColor(vis, cv2.COLOR_HSV2BGR) return vis
def apply_control(self, next, image_out): flow = cv2.calcOpticalFlowFarneback(self.prvs, next, 0.5, 3, 10, 3, 5, 1.2, 0) left_flow = self.flow_in_area(flow, image_out, 5, 50, flow_top, flow_bottom, (255, 0, 0)) right_flow = self.flow_in_area(flow, image_out, 270, 315, flow_top, flow_bottom, ( 0, 0, 255)) middle_flow = self.flow_in_area(flow, image_out, 120, 200, flow_top_ref_ground, flow_bottom_ref_ground, (255, 255, 0)) left_flow = left_flow - middle_flow right_flow = right_flow - middle_flow #print("left_flow : " + str(left_flow[0]) + " middle : "+str(middle_flow[0])+" right_flow : " + str(right_flow[0])) print("middle : " + str(middle_flow)) # print("left : " + str(left_flow)) # print("right : " + str(right_flow)) self.moy[0] = (self.i*self.moy[0]+middle_flow[0])/(self.i+1) self.i+=1 currentmoy = (middle_flow[0]+self.prev+self.prevprev+self.prevprevprev)/4 if(time.clock() - self.time_since_too_near > 0.5): angular_speed = - angular_scale*(left_flow[0] + right_flow[0])#/abs(middle_flow[0]) if angular_speed < -max_angular_twist : angular_speed = -max_angular_twist elif angular_speed > max_angular_twist : angular_speed = max_angular_twist self.go_twist.angular.z = angular_speed self.display_twist(image_out) if (abs(currentmoy)<0.0005*self.moy[0]): self.go_twist.linear.x = -0.1 self.time_since_too_near = time.clock() self.go_twist.angular.z = max_angular_twist/2 elif (abs(currentmoy)<0.005*self.moy[0]): self.go_twist.linear.x = 0.05 #elif (abs(currentmoy)<0.05*self.moy[0]): # self.go_twist.linear.x = 0.1 else: self.go_twist.linear.x = 0.6 # self.go_twist.linear.x = 0.1* (currentmoy/self.moy[0])**2 self.prevprevprev = self.prevprev self.prevprev = self.prev self.prev = middle_flow[0] return self.go_twist
def video_flow_FB(): cam = cv2.VideoCapture(0) ret, frame1 = cam.read() prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) while 1: ret, frame2 = cam.read() next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 3, 15, 3, 5, 1.2, 0) imFilter = cv2.GaussianBlur(next, (5, 5), 1.5) bg_numba = autojit(get_background) bg = bg_numba(imFilter, flow) edge_numba = autojit(get_edge) lap = edge_numba(bg) lap = get_edge_sobel(bg) cv2.imshow("flow", bg) k = cv2.waitKey(30) & 0xFF if k == 27: cam.release() cv2.destroyAllWindows() break prvs = next
def loadGT(self): # case 1: GT already exists self.gtFile = self.gtFolder + self.imFiles[self.i].replace('.tif', '.bmp') if os.path.isfile(self.gtFile): return cv2.imread(self.gtFile, cv2.IMREAD_GRAYSCALE) # case 2: GT can not be estimated for the first image if self.i == 0: return np.zeros((self.im.shape[0], self.im.shape[1]), dtype=np.uint8) # TODO: # case 3: GT can not be estimated, because the previous image has no GT # gtPrevFile = self.gtFolder + self.imFiles[self.i-1].replace('.tif', '.bmp') # if not os.path.isfile(gtPrevFile): # return np.zeros((self.im.shape[0], self.im.shape[1]), dtype=np.uint8) # case 4: GT can be estimated prevIm = cv2.imread(self.imFolder + self.imFiles[self.i-1], cv2.IMREAD_GRAYSCALE) currentIm = cv2.cvtColor(self.im, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prevIm, currentIm, \ None, 0.5, 3, 15, 3, 5, 1.2, 0) #gtPrev = cv2.imread(gtPrevFile, cv2.IMREAD_GRAYSCALE) gtPrev = self.gt.copy() gt = np.zeros((self.im.shape[0], self.im.shape[1]), dtype=np.uint8) for c in self.contours: r = cv2.boundingRect(c) i1, j1, i2, j2 = rect2coordinates(r) fx, fy = self.avgFlow(flow, self.gt, i1, j1, i2, j2) fi = np.round(fx) fj = np.round(fy) gt[i1+fi:i2+fi,j1+fj:j2+fj] = gtPrev[i1:i2,j1:j2] return gt
def apply(self, stim, show=False): events = [] for i, f in enumerate(stim): img = f.data img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if i == 0: last_frame = img total_flow = 0 flow = cv2.calcOpticalFlowFarneback( last_frame, img, 0.5, 3, 15, 3, 5, 1.2, 0) flow = np.sqrt((flow ** 2).sum(2)) if show: cv2.imshow('frame', flow.astype('int8')) cv2.waitKey(1) last_frame = img total_flow = flow.sum() value = Value(stim, self, {'total_flow': total_flow}) event = Event(onset=f.onset, duration=f.duration, values=[value]) events.append(event) return events
def video_flow_FB_getpeople(): cam = cv2.VideoCapture(0) ret, frame1 = cam.read() prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY) while(1): ret, frame2 = cam.read() next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs,next, 0.5, 3, 15, 3, 5, 1.2, 0) imFilter = cv2.GaussianBlur(next,(5,5),1.5) bg_numba = autojit(get_background) bg = bg_numba(imFilter,flow) (cnts, _) = cv2.findContours(bg.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) for c in cnts: if cv2.contourArea(c) < 8000: continue (x, y, w, h) = cv2.boundingRect(c) cv2.rectangle(frame2, (x, y), (x + w, y + h), (0, 255, 0), 2) cv2.imshow('flow', frame2) k = cv2.waitKey(30) & 0xff if k == 27: cam.release() cv2.destroyAllWindows() break prvs = next
def MFMGetFM(self, src): # convert scale I8U = np.uint8(src) * 255 # calculating optical flows if self.prev_frame != None: param1 = pySaliencyMapDefs.farne_pyr_scale param2 = pySaliencyMapDefs.farne_levels param3 = pySaliencyMapDefs.farne_winsize param4 = pySaliencyMapDefs.farne_iterations param5 = pySaliencyMapDefs.farne_poly_n param6 = pySaliencyMapDefs.farne_poly_sigma param7 = pySaliencyMapDefs.farne_flags flow = cv2.calcOpticalFlowFarneback(prev_frame, I8U, param1, param2, param3, param4, param5, param6, param7) flowx = flow[..., 0] flowy = flow[..., 1] else: flowx = np.zeros(I8U.shape) flowy = np.zeros(I8U.shape) # create Gaussian pyramids dst_x = self.FMGaussianPyrCSD(flowx) dst_y = self.FMGaussianPyrCSD(flowy) # update the current frame self.prev_frame = np.array(I8U) # return return dst_x, dst_y
def mainLoopForCam(self, camNumber): # Main Loop self.run = True frameToShow = 0 while (self.run): self.curFrames[camNumber] += 1 startTime = time() capture = self.captureSources[camNumber] prevGray = np.float32(self.prevGrayFrames[camNumber]) success, frame = capture.read() if (not success): print "capture failed on host " + str(camNumber) + " continuing..." continue if (self.record): videoRecorders[camNumber].write(frame) self.frames[camNumber] = cv2.cvtColor(frame, cv2.cv.CV_BGR2RGB) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prevGray, gray, 0.5, 1, 20, 3, 5, 1.2, 0) self.prevGrayFrames[camNumber] = gray # building flow magnitude fx, fy = flow[:,:,0], flow[:,:,1] v = np.sqrt(fx*fx+fy*fy) normalized = np.uint8(np.minimum(v*4, 255)) retval, threshold = cv2.threshold(normalized, 20, 1, cv2.THRESH_BINARY) self.thresholds[camNumber] = np.dstack((threshold, threshold, threshold)) self.raw_heatmaps[camNumber] += threshold self.pretty_heatmaps[camNumber] = self.convertToPrettyHeatMap(self.raw_heatmaps[camNumber]) if (self.curFrames[camNumber] % 20) == 0: np.save("heatmap_backup_" + str(camNumber), self.raw_heatmaps[camNumber].data)
def test_optFlowPyrLK(): # Parameters for lucas kanade optical flow lk_params = dict( winSize = (15,15), maxLevel = 2, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) cap = cv2.VideoCapture(0) ret,frame = cap.read() old_col = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY) hsv = np.zeros_like(frame) hsv[...,1] = 255 while(1): ret,frame = cap.read() frame_col = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(old_col,frame_col, 0.5, 3, 15, 3, 5, 1.2, 0) old_col = frame_col.copy() mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) hsv[...,0] = ang*180/np.pi/2 hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) rgb = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR) cv2.imshow('flow',rgb) cap.release()
def updateData(self): ret, img = self.cam.read() p = cv2.cvtColor(self.prev, cv2.COLOR_BGR2GRAY) i = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(p, i, 0.5, 3, 15, 3, 5, 1.2, 0) w, h = WIDTH, HEIGHT step = 16 y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1) fx, fy = flow[y,x].T polys = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2) polys = np.int32(polys + 0.5) self.prev = img # fgmask = self.fgbg.apply(img) # draw = img & fgmask self.image = QtGui.QImage(img.data, WIDTH, HEIGHT, QtGui.QImage.Format_RGB888) painter = QtGui.QPainter(self.image) blue = QtGui.QColor() blue.setNamedColor("blue") painter.setPen(QtGui.QPen(blue)) for lines in polys: p1 = QtCore.QPoint(*lines[0]) p2 = QtCore.QPoint(*lines[1]) l = QtCore.QLine(p1, p2) painter.drawLine(l) self.pix.convertFromImage(self.image) self.repaint()
def process_optical_flow(video_path): cap = cv2.VideoCapture(video_path) ret, frame1 = cap.read() frame1 = imutils.resize(frame1, width=800) prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) prvs = cv2.GaussianBlur(prvs, (21, 21), 0) hsv = np.zeros_like(frame1) hsv[..., 1] = 255 while True: ret, frame2 = cap.read() frame2 = imutils.resize(frame2, width=800) next_f = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY) next_f = cv2.GaussianBlur(next_f, (21, 21), 0) # flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 1, 3, 15, 3, 5, 1) flow = cv2.calcOpticalFlowFarneback(prvs, next_f, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv[..., 0] = ang * 180 / np.pi / 2 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) cv2.imshow('frame2', rgb) k = cv2.waitKey(30) & 0xff if k == ord('q'): break # elif k == ord('s'): # cv2.imwrite('opticalfb.png', frame2) # cv2.imwrite('opticalhsv.png', rgb) prvs = next_f cap.release() cv2.destroyAllWindows()
def dense_opt_flow(n_stills): #dense optical flow cap = cv2.VideoCapture("test2.avi") ret, frame1 = cap.read() prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY) hsv = np.zeros_like(frame1) hsv[...,1] = 255 while(1): ret, frame2 = cap.read() next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) hsv[...,0] = ang*180/np.pi/2 hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR) cv2.imshow('frame2',bgr) k = cv2.waitKey(1) & 0xff if k == 27: break elif k == ord('s'): cv2.imwrite('opticalfb.png',frame2) cv2.imwrite('opticalhsv.png',bgr) prvs = next cap.release() cv2.destroyAllWindows()
def optical_flow_video(self, frames, first_frame_idx): thread_no = (first_frame_idx/self.step_size) flows = pd.DataFrame(columns=['x', 'y', 'frame']) for i in range(1,len(frames)): print("Thread no. %s reporting, first_frame: %s, total frames:%s and working on frame:%s\n" % (thread_no, first_frame_idx, len(frames), i)) curr = cv2.cvtColor(frames[i],cv2.COLOR_BGR2GRAY) prev = cv2.cvtColor(frames[i-1],cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prev, curr, None, 0.5, 3, 15, 3, 5, 1.2, 0) df = pd.DataFrame() df['x'] = [round(item, 3) for item in flow[:,:,0].flatten()] df['y'] = [round(item, 3) for item in flow[:,:,1].flatten()] df['frame'] = first_frame_idx+i flows = flows.append(df) flows.to_csv("%s_%s" % (self.outpath, thread_no)) print("Thread no.%s finished" % (thread_no))
def optical_flow(video_path): cap = cv2.VideoCapture(video_path) ret, frame1 = cap.read() prvs = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY) hsv = np.zeros_like(frame1) hsv[...,1] = 255 while(1): ret, frame2 = cap.read() next = cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY) #prev, next, scale, levels, winsize, ite, poly_n, poly_sigma flow = cv2.calcOpticalFlowFarneback(prvs, next, 0.5, 1, 4, 3, 5, 1.1, cv2.OPTFLOW_FARNEBACK_GAUSSIAN) # flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) hsv[...,0] = ang*180/np.pi/2 hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR) cv2.imshow('optflow',bgr) cv2.imshow('orig', frame2) k = cv2.waitKey(30) & 0xff if k == 27: break elif k == ord('s'): cv2.imwrite('opticalfb.png',frame2) cv2.imwrite('opticalhsv.png',bgr) prvs = next cap.release() cv2.destroyAllWindows()
def calculate_optical_flow(prvs_gray, current_gray): flow = cv2.calcOpticalFlowFarneback(prvs_gray, current_gray, 0.5, 3, 15, 3, 5, 1.2, 0) binary_pic = np.zeros(current_gray.shape, np.uint8) h, w = current_gray.shape total_threshold=0 for i in xrange(0,h,5): for j in xrange(0,w,5): fx, fy = flow[i, j] total_threshold += abs(fx) + abs(fy) if abs(fx) + abs(fy) > 0.4: binary_pic[i, j] = 255; print('mean threshold is %f'%(total_threshold/(h*w))) binary_pic_translated = conduct_translation(binary_pic) contours, hierarchy = cv2.findContours(binary_pic_translated,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) print('contours is %d'%(len(contours))) largest_area = 0 largest_contour_index = 0 bounding_rect_x, bounding_rect_y, bounding_rect_w, bounding_rect_h = 0, 0, 0, 0 if len(contours) > 0: for k in xrange(0, len(contours)): area = cv2.contourArea(contours[k]) if largest_area < area: largest_area = area largest_contour_index = k bounding_rect_x, bounding_rect_y, bounding_rect_w, bounding_rect_h = cv2.boundingRect(contours[k]) return ((bounding_rect_x, bounding_rect_y, bounding_rect_w, bounding_rect_h), largest_area) else: print("contours is not found...") return (-1, -1)
def HSVizualizer(inputDirectory, filenames, outputDirectory): """ Args: inputDirectory: (str) filenames: (list) filenames to analyze and draw outputDirectory: (str) Where to place output Returns: """ for filename in filenames: raw = tiff.imread(inputDirectory+filename) nframes, frame_width, frame_height = raw.shape #outstack is in RGB color so we need an extra 3 dimensions outStack = np.zeros((nframes-1, frame_width, frame_height, 3), dtype='uint8') for i in xrange(nframes-1): t1 = time.time() print "Start frame "+str(i)+"..."+filename frame = raw[i] next_frame = raw[i+1] flow = cv2.calcOpticalFlowFarneback(frame, next_frame, None, 0.5, 3, 8, 4, 7, 1.5, 0) outStack[i] = draw_hsv(flow) print "Finish frame "+str(i)+" in "+str(time.time()-t1)+" s." #print outStack.shape tiff.imsave(outputDirectory+filename+'_HSV.tif', outStack) print "All done in "+str(time.time()-t0)+" s"
def getOpticalFlow(self, prevgray, gray): flow = cv2.calcOpticalFlowFarneback(prevgray, gray , 0.5, 3, 5, 3, 2, 0.4, cv2.OPTFLOW_USE_INITIAL_FLOW) # cv2.imshow('flow', self.warp_flow(gray, flow)) cv2.imshow('flow', self.draw_flow(gray, flow))
def cv2_array_processor(arr, stopFrame, cv2_params, startFrame=0, frameSamplingInterval=1): """ :param arr: numpy array of shape (nFrames, x, y) :param stopFrame: frame where to stop the processing :param cv2_params: (dict) parameters for the cv2.calcOpticalFlowFarneback() function :param startFrame: frame where to start the processing :param frameSamplingInterval: process every n-th frame :return: two (nFrames-1, x, y) arrays with the u-, and v- components of the velocity vectors u_out & v_out """ assert (stopFrame <= arr.shape[0]) and (startFrame < stopFrame) u_out = np.zeros_like(arr[startFrame:stopFrame - 1, :, :]) v_out = np.zeros_like(u_out) for frame in range(startFrame, stopFrame, frameSamplingInterval): if frame >= (stopFrame - 1): break frame_a = arr[frame] frame_b = arr[frame + 1] flow = cv2.calcOpticalFlowFarneback(frame_a, frame_b, **cv2_params) u_out[frame], v_out[frame] = flow[..., 0], flow[..., 1] return u_out, v_out
prev_gray = cv.cvtColor(first_frame, cv.COLOR_BGR2GRAY) # Creates an image filled with zero intensities with the same dimensions as the frame mask = np.zeros_like(first_frame) # Sets image saturation to maximum mask[..., 1] = 255 while (cap.isOpened()): # ret = a boolean return value from getting the frame, frame = the current frame being projected in the video ret, frame = cap.read() if ret == False: break # Opens a new window and displays the input frame cv.imshow("input", frame) # Converts each frame to grayscale - we previously only converted the first frame to grayscale gray = cv.cvtColor(frame, cv.COLOR_BGR2GRAY) # Calculates dense optical flow by Farneback method flow = cv.calcOpticalFlowFarneback(prev_gray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0) # Computes the magnitude and angle of the 2D vectors magnitude, angle = cv.cartToPolar(flow[..., 0], flow[..., 1]) # Sets image hue according to the optical flow direction mask[..., 0] = angle * 180 / np.pi / 2 # Sets image value according to the optical flow magnitude (normalized) mask[..., 2] = cv.normalize(magnitude, None, 0, 255, cv.NORM_MINMAX) # Converts HSV to RGB (BGR) color representation rgb = cv.cvtColor(mask, cv.COLOR_HSV2BGR) # Opens a new window and displays the output frame cv.imshow("dense optical flow", rgb) # Updates previous frame prev_gray = gray # Frames are read by intervals of 1 millisecond. The programs breaks out of the while loop when the user presses the 'q' key if cv.waitKey(1) & 0xFF == ord('q'): break
def read_avi(avi_path): global video global F # global sample print avi_path cap = cv2.VideoCapture(avi_path) length = int(cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)) print length if length < 54: print "length is less than 50" rand_list = range(length) else: rand_list = random.sample(range(1, length - 1), 50) rand_list.sort() length = 50 length = (length - length % F) / F print rand_list # sample=sample+length video = np.zeros((length, F * 8, 240, 320), dtype=np.uint8) poi = 0 cap.set(1, 0) ret, img = cap.read() #img=cv2.resize(img,(320,240)) op_pre_frames = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) pre_frames = img for i in range(length): m = 0 for s in range(F): cap.set(1, rand_list[poi]) ret, img = cap.read() # print rand_list[poi] poi = poi + 1 # print np.shape(img) # print poi op_next_frames = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(op_pre_frames, op_next_frames, 0.5, 1, 3, 15, 3, 5, 1) fx = cv2.normalize(flow[..., 0], None, 0, 255, cv2.NORM_MINMAX) fy = cv2.normalize(flow[..., 1], None, 0, 255, cv2.NORM_MINMAX) diff_frames = img - pre_frames #RGB channels video[i, m] = img[:, :, 0] m = m + 1 video[i, m] = img[:, :, 1] m = m + 1 video[i, m] = img[:, :, 2] m = m + 1 # three difference channels video[i, m] = diff_frames[:, :, 0] m = m + 1 video[i, m] = diff_frames[:, :, 1] m = m + 1 video[i, m] = diff_frames[:, :, 2] m = m + 1 #optical flow channels video[i, m] = fx m = m + 1 video[i, m] = fy m = m + 1 pre_frames = img op_pre_frames = op_next_frames cap.release() return video
def run_sim(imgFile): """ imgFile: filename of the image uploaded by the user """ print("Processing:", imgFile) ################################### SET UP ################################ ########################################################################### # Using a queue to decouple key press from update action # pressed_keys = [] # def on_release(key): # if hasattr(key, 'char'): # pressed_keys.append(key.char) # # cv2.startWindowThread() # cv2.namedWindow("window", flags=cv2.WND_PROP_FULLSCREEN) camera = Camera(imgFile=imgFile, camera_index=configuration.camera_index, no_cam_allowed=True) # initialise run-time configurable Options # (display name, keys, range, step, initial value) fullscreen = Options.Cycle('Fullscreen', 'f', ['Window', 'Fullscreen'], configuration.fullscreen) mirror_screen = Options.Cycle('Mirror Screen', 'g', ['Normal', 'Horizontal', 'Verticle', 'Both'], configuration.mirror_screen) render_mask = Options.Cycle('Render Mask', 'm', ['false', 'true'], configuration.render_mask) bg_mode = Options.Cycle('BG', 'b', ['white', 'black', 'hue', 'bg subtract'], configuration.bg_mode) mask_level = Options.Range('Mask Threshold', ['1','2'], [0, 1], 0.03, configuration.mask_level) mask_width = Options.Range('Mask Width', ['3','4'], [0, 0.5], 0.01, configuration.mask_width) optical_flow = Options.Cycle('Optical Flow', 'o', ['false', 'true'], configuration.optical_flow) sim_res_multiplier = Options.Range('Sim Res', ['9','0'], [0.1, 2.0], 0.1, configuration.sim_res_multiplier) flow_speed = Options.Range('Flow Speed', ['-','='], [0.02, 1], 0.02, configuration.flow_speed) flow_direction = Options.Cycle('Flow Direction', 'p', ['right', 'down', 'left', 'up'], configuration.flow_direction) num_smoke_streams = Options.Range('Smoke Streams', ['[',']'], [1, 50], 1, configuration.num_smoke_streams) smoke_percentage = Options.Range('Smoke Amount', ['\'','#'], [0.1, 1], 0.1, configuration.smoke_percentage) debugMode = Options.Cycle('Mode', 'd', ['Normal', 'Debug'], 0) # add to a list to update and display options = [fullscreen, mirror_screen, render_mask, bg_mode, mask_level, mask_width, optical_flow, sim_res_multiplier, flow_speed, flow_direction, num_smoke_streams, smoke_percentage, debugMode] ########################################################################### ################################## END SETUP ############################## fps = FPS_counter(limit=30) display_counter = 0 # display values for a short time if they change run = True # Added a frame counter (skip first frame) frame = 1 frameJump = 25 frameMax = 227 while(frame <= frameMax): fps.update() if display_counter > 0: display_counter -= fps.last_dt # Always true on first iteration. Sub-sample the webcam image to fit the # fluid sim resolution. Update when option changes. if sim_res_multiplier.get_has_changed(reset_change_flag=True): sim = configuration.Sim(camera.shape, sim_res_multiplier.current, 0, 0) sim.mode = debugMode.current flow = np.zeros((sim.shape[1], sim.shape[0], 2)) # Always True on first iteration. Update fullscreen if option changed # if fullscreen.get_has_changed(reset_change_flag=True): # if fullscreen.current: # cv2.setWindowProperty("window", cv2.WND_PROP_FULLSCREEN, 1) # else: # cv2.setWindowProperty("window", cv2.WND_PROP_FULLSCREEN, 0) if debugMode.get_has_changed(reset_change_flag=True): sim.mode = debugMode.current # if flow direction changes, reset, else things get messy # if flow_direction.get_has_changed(reset_change_flag=True): # sim.reset() # camera.reset() # update input image camera.update(bg_mode.current, mirror_screen.current, mask_level.current, mask_width.current) if optical_flow.current == 1: flow[:] = cv2.calcOpticalFlowFarneback(cv2.resize(camera.last_grey, sim.shape), cv2.resize(camera.current_grey, sim.shape), float(0), pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.0, flags=0) scale = np.sqrt(sim._dx[0] * sim._dx[1]) / fps.last_dt flow *= scale else: flow[:] = 0 sim.set_velocity(fps.last_dt, flow_speed.current, flow_direction.current, num_smoke_streams.current, smoke_percentage.current) # copy the webcam generated mask into the boundary boundary = np.array(cv2.resize(camera.mask, sim.shape).T, dtype=bool) sim.set_boundary(boundary, flow.T, flow_direction.current) # update and render the sim sim.udpate(fps.last_dt) output = sim.render(camera, render_mask_velocity=(optical_flow.current==1), render_mask=(render_mask.current == 1)) output_shape = np.shape(output) # add the GUI text_color = (0, 0, 255) if bg_mode.current == 0 else (255, 255, 0) if debugMode.current == 0 and display_counter <= 0: cv2.putText(output, 'd=Debug Mode', (30,output_shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color) else: pos = np.array((30,30)) for option in options: cv2.putText(output, str(option), tuple(pos), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color) pos = pos + [0,20] cv2.putText(output, str(fps), (output_shape[1] - 80,output_shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color) cv2.putText(output, 'q=Quit, r=Reset', (30,output_shape[0] - 20), cv2.FONT_HERSHEY_SIMPLEX, 0.4, text_color) # render the output # cv2.imshow('window', output) # Save the output to file every some frames if frame % frameJump == 0: ind = int(frame/frameJump) name = imgFile.split("/")[-1].split(".")[0] fullPath = "server/out/{}_{}.png".format(name, ind) cv2.imwrite(fullPath, output) # for key in pressed_keys: # # update the options (poll for input, cycle) # for option in options: # if option.update(key, fps.last_dt): # display_counter = 5 # # # poll for quit, reset # if key == 'q': # run = False # elif key == 'r': # sim.reset() # camera.reset() # pressed_keys[:] = [] # Update frame frame += 1
import cv2 import numpy as np cap=cv2.VideoCapture('video2.mp4'); #create a capture object ret,frame1=cap.read(); #read first frame prv_gray_frame=cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY); #convert color of image from bgr2gray hsv=np.zeros_like(frame1); #convert frame 1 to hsv using zeroslike hsv[...,1]=255; #set saturation to 255 while cap.isOpened(): ret,frame2=cap.read(); #read frame by frame next_gray_frame=cv2.cvtColor(frame2,cv2.COLOR_BGR2GRAY); #convert color of frame from bgr2gray flow=cv2.calcOpticalFlowFarneback(prv_gray_frame,next_gray_frame,None,0.3,3,15,3,5,1.2,0); #find the optical flow of object mag,ang=cv2.cartToPolar(flow[...,0],flow[...,1]); #cartTo Plor cordinates to find x and y hsv[...,0]=ang*180/np.pi/2; hsv[...,2]=cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX); #normalize te magnitide rgb=cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR); #convert color of imag from hsv2bgr cv2.namedWindow('DenseOpticalFlow',cv2.WINDOW_NORMAL); #create a named window cv2.imshow('DenseOpticalFlow',rgb); #image show if cv2.waitKey(5)&0xFF==ord('q'): break; cv2.destroyWindow('DenseOpticalFlow'); #destroy window
prevgray = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY) show_hsv = True show_glitch = False cur_glitch = prev.copy() while True: (ret, img) = cam.read() vis = img.copy() gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) """ Computes a dense optical flow using the Gunnar Farneback’s algorithm. cv2.calcOpticalFlowFarneback(prev, next, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags) → flow """ flow = cv2.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 5, 5, 3, 5, 1.1, cv2.OPTFLOW_LK_GET_MIN_EIGENVALS) prevgray = gray cv2.imshow('flow', draw_flow(gray, flow)) if show_hsv: gray1 = cv2.cvtColor(draw_hsv(flow), cv2.COLOR_BGR2GRAY) thresh = cv2.threshold(gray1, 0, 0xFF, cv2.THRESH_BINARY)[1] thresh = cv2.dilate(thresh, None, iterations=2) cv2.imshow('thresh', thresh) ##gray2, cnts, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) gray2, cnts, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # loop over the contours """ for c in cnts:
try: fn = sys.argv[1] except: fn = 0 cam = video.create_capture(fn) ret, prev = cam.read() prevgray = cv2.cvtColor(prev, cv2.COLOR_BGR2GRAY) show_hsv = False show_glitch = False cur_glitch = prev.copy() while True: ret, img = cam.read() gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prevgray, gray, 0.5, 3, 15, 3, 5, 1.2, 0) prevgray = gray cv2.imshow('flow', draw_flow(gray, flow)) if show_hsv: cv2.imshow('flow HSV', draw_hsv(flow)) if show_glitch: cur_glitch = warp_flow(cur_glitch, flow) cv2.imshow('glitch', cur_glitch) ch = 0xFF & cv2.waitKey(5) if ch == 27: break if ch == ord('1'): show_hsv = not show_hsv print 'HSV flow visualization is', ['off', 'on'][show_hsv]
# #tools.save(x, y, u3, v3, mask, 'exp1_001.txt') # #tools.display_vector_field('exp1_001.txt', scale=100, width=0.0025) # # cv.imshow('test',frame_b.astype(np.uint8)) # # plt.quiver(x,y,u3,v3) # Calculates dense optical flow by Farneback method # https://docs.opencv.org/3.0-beta/modules/video/doc/motion_analysis_and_object_tracking.html#calcopticalflowfarneback # flow = cv.calcOpticalFlowFarneback(prev=prev_gray, next=gray, flow=None, pyr_scale=0.5, levels=3, winsize=25, iterations=3, poly_n=7, poly_sigma=1.5, flags=0) # LOOKS LIKE WE ARE GETTING SOMEWHERE WITH THE NORMALIZED VERSION flow = cv.calcOpticalFlowFarneback(prev=prev_gray.astype(np.uint8), next=gray.astype(np.uint), flow=None, pyr_scale=0.5, levels=3, winsize=24, iterations=3, poly_n=7, poly_sigma=1.5, flags=0) #flow = cv.calcOpticalFlowFarneback(prev=frame_a.astype(np.uint8), next=frame_b.astype(np.uint), flow=None, pyr_scale=0.5, levels=3, winsize=24, iterations=3, poly_n=7, poly_sigma=1.5, flags=0) flow = flow*10 # Computes the magnitude and angle of the 2D vectors magnitude, angle = cv.cartToPolar(flow[..., 0], flow[..., 1]) # Sets image hue according to the optical flow direction mask[..., 0] = angle * 180 / np.pi / 2 # Sets image value according to the optical flow magnitude (normalized) mask[..., 2] = cv.normalize(magnitude, None, 0, 255, cv.NORM_MINMAX) # Converts HSV to RGB (BGR) color representation rgb = cv.cvtColor(mask, cv.COLOR_HSV2BGR) # Opens a new window and displays the output frame #rgbS = cv.resize(rgb, (1024, 1224)) arrows.clear()
ret1, frame1 = video.read() if not ret1: # End of sequence print('Empty video file') sys.exit(2) frameCounter = 1 while (video.isOpened()): print('Processing frame {0}/{1}'.format(frameCounter, length), end='\r') ret3, frame3 = video.read() if ret3: prevgray = cv.cvtColor(frame1, cv.COLOR_BGR2GRAY) gray = cv.cvtColor(frame3, cv.COLOR_BGR2GRAY) flow = cv.calcOpticalFlowFarneback(prevgray, gray, None, 0.5, 3, 30, 3, 5, 1.2, 0) backFlow = cv.calcOpticalFlowFarneback(gray, prevgray, None, 0.5, 3, 30, 3, 5, 1.2, 0) # if not helper.showImage(helper.drawFlow(gray, flow)): # break warpedFlow = flow * 0.5 backWarpedFlow = backFlow * 0.5 # TODO: use NaNs or something else instead of 0s to indicate no information # newFrame = np.full((height, width, 3), np.nan) newFrame = np.zeros((height, width, 3), np.uint8) warpMapX = np.zeros((height, width), np.float32) warpMapX[:] = range(width) warpMapY = np.zeros((height, width), np.float32)
import cv2 import numpy as np cap = cv2.VideoCapture("vtest.avi") ret, frame1 = cap.read() prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) hsv = np.zeros_like(frame1) hsv[..., 1] = 255 while (1): ret, frame2 = cap.read() next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv[..., 0] = ang * 180 / np.pi / 2 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) cv2.imshow('frame2', rgb) k = cv2.waitKey(30) & 0xff if k == 27: break elif k == ord('s'): cv2.imwrite('opticalfb.png', frame2) cv2.imwrite('opticalhsv.png', rgb) prvs = next cap.release()
def main(video, device): # init dict to track time for every stage at each iteration timers = { "full pipeline": [], "reading": [], "pre-process": [], "optical flow": [], "post-process": [], } # init video capture with video cap = cv2.VideoCapture(video) # get default video FPS fps = cap.get(cv2.CAP_PROP_FPS) # get total number of video frames num_frames = cap.get(cv2.CAP_PROP_FRAME_COUNT) # read the first frame ret, previous_frame = cap.read() # proceed if frame reading was successful if ret: # resize frame frame = cv2.resize(previous_frame, (960, 540)) # convert to gray previous_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # create hsv output for optical flow hsv = np.zeros_like(frame) # set saturation to a maximum value hsv[..., 1] = 255 while True: # start full pipeline timer start_full_time = time.time() # start reading timer start_read_time = time.time() # capture frame-by-frame ret, current_frame = cap.read() # end reading timer end_read_time = time.time() # add elapsed iteration time timers["reading"].append(end_read_time - start_read_time) # if frame reading was not successful, break if not ret: break # start pre-process timer start_pre_time = time.time() # resize frame frame = cv2.resize(current_frame, (960, 540)) # convert to gray current_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if device == "cpu": # end pre-process timer end_pre_time = time.time() # add elapsed iteration time timers["pre-process"].append(end_pre_time - start_pre_time) # start optical flow timer start_of = time.time() # calculate optical flow flow = cv2.calcOpticalFlowFarneback( previous_frame, current_frame, None, 0.5, 5, 15, 3, 5, 1.2, 0, ) # end of timer end_of = time.time() # add elapsed iteration time timers["optical flow"].append(end_of - start_of) else: # move both frames to GPU cu_previous = cv2.cuda_GpuMat() cu_current = cv2.cuda_GpuMat() cu_previous.upload(previous_frame) cu_current.upload(current_frame) # end pre-process timer end_pre_time = time.time() # add elapsed iteration time timers["pre-process"].append(end_pre_time - start_pre_time) # start optical flow timer start_of = time.time() # create optical flow instance flow = cv2.cuda_FarnebackOpticalFlow.create( 5, 0.5, False, 15, 3, 5, 1.2, 0, ) # calculate optical flow flow = cv2.cuda_FarnebackOpticalFlow.calc( flow, cu_previous, cu_current, None, ) # sent result from GPU back to CPU flow = flow.download() # end of timer end_of = time.time() # add elapsed iteration time timers["optical flow"].append(end_of - start_of) # start post-process timer start_post_time = time.time() # convert from cartesian to polar coordinates to get magnitude and angle mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) # set hue according to the angle of optical flow hsv[..., 0] = ang * 180 / np.pi / 2 # set value according to the normalized magnitude of optical flow hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) # convert hsv to bgr bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) # end post-process timer end_post_time = time.time() # add elapsed iteration time timers["post-process"].append(end_post_time - start_post_time) # end full pipeline timer end_full_time = time.time() # add elapsed iteration time timers["full pipeline"].append(end_full_time - start_full_time) # visualization cv2.imshow("original", frame) cv2.imshow("result", bgr) k = cv2.waitKey(1) if k == 27: break # update previous_frame value previous_frame = current_frame # release the capture cap.release() # destroy all windows cv2.destroyAllWindows() # print results print("Number of frames : ", num_frames) # elapsed time at each stage print("Elapsed time") for stage, seconds in timers.items(): print("-", stage, ": {:0.3f} seconds".format(sum(seconds))) # calculate frames per second print("Default video FPS : {:0.3f}".format(fps)) of_fps = (num_frames - 1) / sum(timers["optical flow"]) print("Optical flow FPS : {:0.3f}".format(of_fps)) full_fps = (num_frames - 1) / sum(timers["full pipeline"]) print("Full pipeline FPS : {:0.3f}".format(full_fps))
if folder: files.extend(dirnames) else: files.extend(filenames) break return files master_dir = "/home/wangsq/CS4243/DAVIS/" master_dir = _get_files(master_dir, folder=True) for subfolder in master_dir: img_files = _get_files(subfolder) count = 0 filepath, _ = os.path.split(img_files[0]) for i in range(0, len(img_files) - 1): prev = get_img(img_files[i], (HEIGHT, WIDTH, 3), grayscale=True).astype(np.float32) nxt = get_img(img_files[i + 1], (HEIGHT, WIDTH, 3), grayscale=True).astype(np.float32) forward_flow = cv2.calcOpticalFlowFarneback(prev, nxt, None, 0.5, 3, 15, 3, 5, 1.2, 0) flow.write_flo(filepath + "/flow/flow" + str(count) + ".flo", forward_flow) backward_flow = cv2.calcOpticalFlowFarneback(nxt, prev, None, 0.5, 3, 15, 3, 5, 1.2, 0) flow.write_flo(filepath + "/flow/flow" + str(count + 1) + ".flo", backward_flow) count += 2
def internp(frame0, frame1, t=0.5, flow0=None): ''' :param frame0: beggining frame :param frame1: ending frame :return frame_t: an interpolated frame at time t ''' print('==============================') print('===== interpolate an intermediate frame at t=', str(t)) print('==============================') # ================================================== # ===== 1/ find the optical flow between the two given images: from frame0 to frame1, # if there is no given flow0, run opencv function to extract it # ================================================== if flow0 is None: i1 = cv2.cvtColor(frame0, cv2.COLOR_BGR2GRAY) i2 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) flow0 = cv2.calcOpticalFlowFarneback(i1, i2, None, 0.5, 3, 15, 3, 5, 1.2, 0) # ================================================== # ===== 2/ find holes in the flow # ================================================== holes0 = find_holes(flow0) pickle.dump(holes0, open('holes0.step2.data', 'wb')) # save your intermediate result # ====== score holes0 = pickle.load(open('holes0.step2.data', 'rb')) # load your intermediate result holes0_step2 = pickle.load(open('holes0.step2.sample', 'rb')) # load sample result diff = np.sum(np.abs(holes0 - holes0_step2)) print('holes0_step2', diff) # ================================================== # ===== 3/ fill in any hole using an outside-in strategy # ================================================== flow0 = holefill(flow0, holes0) pickle.dump(flow0, open('flow0.step3.data', 'wb')) # save your intermediate result # ====== score flow0 = pickle.load(open('flow0.step3.data', 'rb')) # load your intermediate result flow0_step3 = pickle.load(open('flow0.step3.sample', 'rb')) # load sample result diff = np.sum(np.abs(flow0 - flow0_step3)) print('flow0_step3', diff) # ================================================== # ===== 5/ estimate occlusion mask # ================================================== occ0, occ1 = occlusions(flow0, frame0, frame1) pickle.dump(occ0, open('occ0.step5.data', 'wb')) # save your intermediate result pickle.dump(occ1, open('occ1.step5.data', 'wb')) # save your intermediate result # ===== score occ0 = pickle.load(open('occ0.step5.data', 'rb')) # load your intermediate result occ1 = pickle.load(open('occ1.step5.data', 'rb')) # load your intermediate result occ0_step5 = pickle.load(open('occ0.step5.sample', 'rb')) # load sample result occ1_step5 = pickle.load(open('occ1.step5.sample', 'rb')) # load sample result diff = np.sum(np.abs(occ0_step5 - occ0)) print('occ0_step5', diff) diff = np.sum(np.abs(occ1_step5 - occ1)) print('occ1_step5', diff) # ================================================== # ===== step 6/ blur occlusion mask # ================================================== for iblur in range(0, BLUR_OCC): occ0 = blur(occ0) occ1 = blur(occ1) pickle.dump(occ0, open('occ0.step6.data', 'wb')) # save your intermediate result pickle.dump(occ1, open('occ1.step6.data', 'wb')) # save your intermediate result # ===== score occ0 = pickle.load(open('occ0.step6.data', 'rb')) # load your intermediate result occ1 = pickle.load(open('occ1.step6.data', 'rb')) # load your intermediate result occ0_step6 = pickle.load(open('occ0.step6.sample', 'rb')) # load sample result occ1_step6 = pickle.load(open('occ1.step6.sample', 'rb')) # load sample result diff = np.sum(np.abs(occ0_step6 - occ0)) print('occ0_step6', diff) diff = np.sum(np.abs(occ1_step6 - occ1)) print('occ1_step6', diff) # ================================================== # ===== step 7/ forward-warp the flow to time t to get flow_t # ================================================== flow_t = interpflow(flow0, frame0, frame1, t) #flow_t = pickle.load(open('flow_t.step7.sample', 'rb')) # load sample result pickle.dump(flow_t, open('flow_t.step7.data', 'wb')) # save your intermediate result # ====== score flow_t = pickle.load(open('flow_t.step7.data', 'rb')) # load your intermediate result flow_t_step7 = pickle.load(open('flow_t.step7.sample', 'rb')) # load sample result #flow_t = flow_t_step7 diff = np.sum(np.abs(flow_t - flow_t_step7)) print('flow_t_step7', diff) # ================================================== # ===== step 8/ find holes in the estimated flow_t # ================================================== holes1 = find_holes(flow_t) pickle.dump(holes1, open('holes1.step8.data', 'wb')) # save your intermediate result # ====== score holes1 = pickle.load(open('holes1.step8.data', 'rb')) # load your intermediate result holes1_step8 = pickle.load(open('holes1.step8.sample', 'rb')) # load sample result diff = np.sum(np.abs(holes1 - holes1_step8)) print('holes1_step8', diff) # ===== fill in any hole in flow_t using an outside-in strategy flow_t = holefill(flow_t, holes1) pickle.dump(flow_t, open('flow_t.step8.data', 'wb')) # save your intermediate result # ====== score flow_t = pickle.load(open('flow_t.step8.data', 'rb')) # load your intermediate result flow_t_step8 = pickle.load(open('flow_t.step8.sample', 'rb')) # load sample result diff = np.sum(np.abs(flow_t - flow_t_step8)) print('flow_t_step8', diff) # ================================================== # ===== 9/ inverse-warp frame 0 and frame 1 to the target time t # ================================================== #frame_t = warpimages(flow_t, frame0, frame1, occ0_step6, occ1_step6, t) frame_t = warpimages(flow_t, frame0, frame1, occ0, occ1, t) pickle.dump(frame_t, open('frame_t.step9.data', 'wb')) # save your intermediate result # ====== score frame_t = pickle.load(open('frame_t.step9.data', 'rb')) # load your intermediate result frame_t_step9 = pickle.load(open('frame_t.step9.sample', 'rb')) # load sample result diff = np.sqrt( np.mean( np.square( frame_t.astype(np.float32) - frame_t_step9.astype(np.float32)))) print('frame_t', diff) return frame_t
def capture_dense_optical_flow(path_in, name_of_videos, grids, ROI, time_interval, X_BLOCK_SIZE, Y_BLOCK_SIZE, if_show=True, if_normal_ori=True): ''' Objective: capture optical flow points from video input: path_in: dir of training video data, e.g './data/train/videos' name_of_videos: read the videos in the directory iteratively '/data/train/videos/name_of_videos'. 'train001' grids: the corner for each grid created by "make_grid" function if_show: a boolin defines if to show the video with optical flow and grid Output: an n by 6 array [starting x, starting y, magnitude, orientation, grid of the flow, time interval of the flow] ''' #set the Farneback parameters fb_params = dict( pyr_scale=0.03, levels=1, winsize=4, iterations=3, poly_n=5, poly_sigma=1.2, flags=0) # The video feed is read in as a VideoCapture object cap = cv2.VideoCapture(join(path_in, name_of_videos + '.avi')) # ret = a boolean return value from getting the frame, first_frame = the first frame in the entire video sequence ret, frame = cap.read() # Converts frame to grayscale because we only need the luminance channel for detecting edges - less computationally expensive prev_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Creates an image filled with zero intensities with the same dimensions as the frame mask = np.zeros_like(frame) # Sets image saturation to maximum mask[..., 1] = 255 # calculate different thresholds for different rows of magnitude to discard numbers_y_grid = int(frame.shape[0] / Y_BLOCK_SIZE) decreasing_rate = 0.88 original_threshold = 1.2 motion_threshold = [original_threshold] for row in range(1, numbers_y_grid): motion_threshold.insert(0, motion_threshold[0] * decreasing_rate) if row % 2 == 0: decreasing_rate -= .07 # if needs to show the video, portrait the grids if if_show == True: color_grid = (0, 255, 0) grid_mask = np.zeros_like(frame) num_of_grids = grids.shape[1] for g in range(num_of_grids): grid_mask = cv2.line(grid_mask, (grids[0][g] + X_BLOCK_SIZE, grids[1][g]), (grids[0][g], grids[1][g]), color_grid, 1) grid_mask = cv2.line(grid_mask, (grids[0][g], grids[1][g] + Y_BLOCK_SIZE), (grids[0][g], grids[1][g]), color_grid, 1) #for counting which frame is being processing for each video frame_count = 0 # 20 frames as a set, so record the which set the flows belong to set_of_frames = 1 flow_points = [] while cap.isOpened(): # ret = a boolean return value from getting the frame, frame = the current frame being projected in the video # 這個.read()一次就換下一張video 的frame ret, frame = cap.read() if not ret: break # Converts each frame to grayscale - we previously only converted the first frame to grayscale gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Calculates dense optical flow by Farneback method # https://docs.opencv.org/3.0-beta/modules/video/doc/motion_analysis_and_object_tracking.html#calcopticalflowfarneback flow = cv2.calcOpticalFlowFarneback(prev_gray, gray, None, **fb_params) # Computes the magnitude and angle of the 2D vectors magnitude, angle = cv2.cartToPolar(flow[..., 0], flow[..., 1], angleInDegrees=0) # to discard the stationary objects, and use different threshold for different rows for row in range(numbers_y_grid - 1): # for row in range(int(mask[..., 2].shape[0]/Y_BLOCK_SIZE)): thr_idx = magnitude[Y_BLOCK_SIZE * row:Y_BLOCK_SIZE * (row + 1), :] < motion_threshold[row] magnitude[Y_BLOCK_SIZE * row:Y_BLOCK_SIZE * (row + 1), :][thr_idx] = 0 # if row < 2: # magnitude[Y_BLOCK_SIZE*row : Y_BLOCK_SIZE*(row+1), :] = magnitude[Y_BLOCK_SIZE*row : Y_BLOCK_SIZE*(row+1), :] * median_ratio[row] thr_idx = magnitude[Y_BLOCK_SIZE * (row + 1):, :] < motion_threshold[ -1] # -1 here same as index row+1 magnitude[Y_BLOCK_SIZE * (row + 1):, :][thr_idx] = 0 # magnitude = np.array([m if m > motion_threshold else 0 for m in magnitude.reshape(-1)]).reshape(angle.shape[0], angle.shape[1]) # Sets image hue according to the optical flow direction mask[..., 0] = angle * 180 / np.pi / 2 # Sets image value according to the optical flow magnitude (normalized) mask[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX) # Converts HSV to RGB (BGR) color representation rgb = cv2.cvtColor(mask, cv2.COLOR_HSV2BGR) # Opens a new window and displays the output frame # Updates previous frame prev_gray = gray.copy() for roi in ROI: for y in range(grids[1][roi - 1], grids[1][roi - 1] + Y_BLOCK_SIZE): for x in range(grids[0][roi - 1], grids[0][roi - 1] + X_BLOCK_SIZE): if motion_threshold[0] < magnitude[y][x] < 15: if if_normal_ori: orientation = categorize_by_bin( mask[..., 0][y][x], 0, 180, 20) else: orientation = categorize_to_four(angle[y][x]) flow_points.append([ x, y, magnitude[y][x], orientation, roi, set_of_frames ]) ''' for y in range(magnitude.shape[0]): for x in range(magnitude.shape[1]): if motion_threshold[0] < magnitude[y][x] < 15: if if_normal_ori: orientation = categorize_by_bin(mask[..., 0][y][x], 0, 180, 20) else: orientation = categorize_to_four(angle[y][x]) which_grid = categorize_flow_points(x, y, grids, X_BLOCK_SIZE, Y_BLOCK_SIZE) flow_points.append([x, y, magnitude[y][x], orientation, which_grid, set_of_frames]) ''' frame_count += 1 if (frame_count + 1) % time_interval == 0: set_of_frames += 1 if if_show == True: # Opens a new window and displays the input frame cv2.imshow("input", frame) # Overlays the optical flow tracks on the original frame output = cv2.add(rgb, grid_mask) # Opens a new window and displays the output frame cv2.imshow("dense optical flow" + name_of_videos, output) # Frames are read by intervals of 10 milliseconds. The programs breaks out of the while loop when the user presses the 'q' key if cv2.waitKey(10) & 0xFF == ord('q'): break # The following frees up resources and closes all windows cap.release() cv2.destroyAllWindows() return np.array(flow_points)
def extractFrame(self, frame): frame = self.scale_frame(frame, self.percentage) #each frame to grayscale gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if self.prev_gray is None: self.prev_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if self.hsv_mask is None: self.hsv_mask = np.zeros_like(frame) self.hsv_mask[..., 1] = 255 #calculates optical flow/2D flow vector flow = cv2.calcOpticalFlowFarneback(self.prev_gray, gray, None, 0.5, 3, 15, 3, 5, 1.2, 0) #compute magnitude + angle of vector magnitude, angle = cv2.cartToPolar(flow[..., 0], flow[..., 1]) #set image hue according to optical flow direction (angle) self.hsv_mask[..., 0] = angle * 180 / np.pi / 2 #set image value according to optical flow magnitude (normalized) self.hsv_mask[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX) #convert to rgb rgb = cv2.cvtColor(self.hsv_mask, cv2.COLOR_HSV2BGR) #grayscale image h, s, v1 = cv2.split(rgb) rgb = v1 img = rgb _, thresh = cv2.threshold(img, 3, 255, cv2.THRESH_BINARY) #thresh = self.dilateErodeFrame(thresh, 10, 1) # Detect the contours in the image currentCentroids, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # max_kernel_size = 15 # result_array = np.zeros(shape=(max_kernel_size,max_kernel_size)) # for x in range(max_kernel_size): # for y in range(max_kernel_size): # morphed_thresh = cv2.erode(thresh, np.ones((x,x),np.uint8), 1) # morphed_thresh = cv2.erode(morphed_thresh, np.ones((y,y),np.uint8), 1) # currentCentroids, _ = cv2.findContours(morphed_thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE) # amountOfCentroids = len(currentCentroids) # result_array[x,y] = amountOfCentroids # if(amountOfCentroids <= self.MAXIMUM_TRACKABLE_OBJECTS and amountOfCentroids > 0): # for contour in currentCentroids: # img_x,img_y,w,h = cv2.boundingRect(contour) # cv2.rectangle(thresh, (img_x,img_y),(img_x+w,img_y+h), (255,255,0), 1) # print('contours found: ' + str(len(currentCentroids))) # print(result_array) # Draw all the contours img = cv2.drawContours(img, currentCentroids, -1, (0, 255, 0), 1) labelDictionary = {} cv2.imshow("Farneback Optical Flow", thresh) maxContourLength = 0 # Iterate through all the contours for contour in currentCentroids: if cv2.contourArea(contour) > maxContourLength: maxContourLength = cv2.contourArea(contour) # Find bounding rectangles x, y, w, h = cv2.boundingRect(contour) # Draw the rectangle print('found contour!') currentBoundingBox = [x, (x + w), y, (y + h)] #scale it back to the original frame size scaledBack_currentBoundingBox = list( map(self.scaleBack, currentBoundingBox)) labelDictionary = {'baer': scaledBack_currentBoundingBox} cv2.imshow("image", img) self.prev_gray = gray return labelDictionary
import numpy as np from PIL import Image import init_auv as auv im_filename = '*.png' imdir = '/Users/Mackeprang/Dropbox (Personlig)/Master Thesis/Pictures/20181005_084733.9640_Mission_1' filenames = auv.imagesFilePath(imdir) images = [] prev_frame = None hsv = np.zeros_like(cv2.imread(filenames[0])) hsv[...,1] = 255 for i,frame in enumerate(filenames): if auv.image_broken(frame): continue img = cv2.imread(frame) gray = auv.preprocess_image(img,size=None) if prev_frame is None: prev_frame = gray continue flow = cv2.calcOpticalFlowFarneback(prev_frame, gray, None, 0.5, 3, 30, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv[..., 0] = ang * 180 / np.pi / 2 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) cv2.imshow("Match", bgr) key = cv2.waitKey() if key & 0xFF == ord('q'): break prev_frame = gray.copy() cv2.destroyAllWindows()
def main(**kwargs): method = [ 'BM1', 'coarse2fine', 'DenseCV', 'HS', 'TVL', 'LK', 'PWCNet', 'BM2' ] sel_method = 0 pyr = True show_legend = False gt_paths = [ curr_dir + "/datasets/of_pred/noc_000045_10.png", curr_dir + "/datasets/of_pred/noc_000157_10.png" ] select_image = 0 #0: image 1, 2: image 2 im = cv2.imread(curr_dir + "/datasets/results/LKflow_000157_10.png", cv2.IMREAD_UNCHANGED) if select_image == 0: im1 = cv2.imread(curr_dir + "/datasets/of_pred/000045_10.png", cv2.IMREAD_UNCHANGED) im2 = cv2.imread(curr_dir + "/datasets/of_pred/000045_11.png", cv2.IMREAD_UNCHANGED) elif select_image == 1: im1 = cv2.imread(curr_dir + "/datasets/of_pred/000157_10.png", cv2.IMREAD_UNCHANGED) im2 = cv2.imread(curr_dir + "/datasets/of_pred/000157_11.png", cv2.IMREAD_UNCHANGED) gt = cv2.imread(gt_paths[select_image], cv2.IMREAD_UNCHANGED) #flow_im,valid_flow = decode_optical_flow(im) flow_gt, val_gt_flow = decode_optical_flow(gt) #Como llamar al Block Matching if sel_method == 1: flow, im_warped = coarse2fine_flow(im1, im2) elif sel_method == 2: flow = cv2.calcOpticalFlowFarneback(im1, im2, None, 0.5, 3, 15, 3, 5, 1.2, 0) elif sel_method == 3: if pyr == True: flow = opticalFlowHSPyr(im1, im2) else: flow = opticalFlowHS(im1, im2) elif sel_method == 4: if pyr == True: flow = tvl1_simple(im1, im2) else: flow = opticalFlowTVL1Pyr(im1, im2) elif sel_method == 5: if pyr == True: flow = opticalFlowLK(im1, im2) else: flow = opticalFlowLKPyr(im1, im2) elif sel_method == 0: flow = bm1.obtain_dense_mov(im1, im2, **kwargs) elif sel_method == 6: gpu_devices = ['/device:CPU:0'] controller = '/device:CPU:0' ckpt_path = '/Users/sergi/mcv-m6-2020-team5/src/opflows/tfoptflow/models/pwcnet-lg-6-2-multisteps-chairsthingsmix/pwcnet.ckpt-595000' nn_opts = deepcopy(_DEFAULT_PWCNET_TEST_OPTIONS) nn_opts['verbose'] = True nn_opts['ckpt_path'] = ckpt_path nn_opts['batch_size'] = 1 nn_opts['gpu_devices'] = gpu_devices nn_opts['controller'] = controller nn_opts['use_dense_cx'] = True nn_opts['use_res_cx'] = True nn_opts['pyr_lvls'] = 6 nn_opts['flow_pred_lvl'] = 2 nn_opts['adapt_info'] = (1, 376, 1241, 2) nn = ModelPWCNet(mode='test', options=nn_opts) nn.print_config() img_pairs = [] img_pairs.append((cv2.cvtColor(im1, cv2.COLOR_GRAY2RGB), cv2.cvtColor(im2, cv2.COLOR_GRAY2RGB))) print(np.asarray(img_pairs).shape) flow = nn.predict_from_img_pairs(img_pairs, batch_size=1, verbose=False) flow = np.asarray(flow) flow = np.squeeze(flow, axis=0) elif sel_method == 7: block_match2 = bm2.EBMA_searcher(15, 15) im_warped, flow = block_match2.run(im1, im2) flow = block_match2.get_original_size(flow, im1.shape[:2]) else: print("El método seleccionado no es válido.") if type(flow) is tuple: movsx, movsy, reliab = flow flow = np.stack((movsx, movsy), axis=2) color_plot = colorflow_white(flow) # cv2.imwrite("pyflow.png",color_plot) cv2.imshow("color_plot", color_plot) cv2.waitKey(1) if (show_legend): flow_legend = get_plot_legend(256, 256) color_flow_legend = colorflow_white(flow_legend) # im_empty = np.zeros((256,256)) # legend_arrow = arrow_flow(flow_legend.astype("float")/8,im_empty, filter_zero=False) cv2.imshow("color wheel", color_flow_legend) cv2.waitKey(1) # arrow_flow(flow,im1) ##metrics msen, pepn = flowmetrics.flowmetrics(flow, flow_gt, val_gt_flow) # print('MSEN') # print(msen) # print('PEPN') # print(pepn) return msen, pepn, flow, color_plot
while (capture.isOpened()): if cv2.waitKey(1)==32: decise_v+=1 while(decise_v%2==1): #print('pause') if cv2.waitKey(1)==32: decise_v+=1 stime=time.time() ret, frame =capture.read() imgC=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY) imgC=cv2.GaussianBlur(imgC,(5,5),0.5) flow=cv2.calcOpticalFlowFarneback(imgP,imgC,None,**params) definePass(frame,flow,TH) imgP=imgC.copy() max_x,max_y,t = frame.shape image_hsv=cv2.cvtColor(frame,cv2.COLOR_BGR2HSV) #hist = cv2.calcHist([hsv], [0, 1], None, [180, 256], [0, 180, 0, 256]) #hist2 = cv2.calcHist([hsv], [1, 2], None, [256, 256], [0, 256, 0, 256]) #mask_white=cv2.inRange(hsv,lower_white,upper_white) image_mask1=cv2.inRange(image_hsv, line_low1, line_up1) image_mask2=cv2.inRange(image_hsv, line_low2, line_up2) image_mask3=cv2.inRange(image_hsv, line_low3, line_up3) image_mask=image_mask1|image_mask2|image_mask3
show_hsv = False show_glitch = False cur_glitch = prev.copy() while True: ret, img = cam.read() # 读取视频的下一帧作为光流输入的当前帧 if ret == True: # 判断视频是否结束 if cv2.waitKey(10) == 27: break gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback( prevgray, gray, None, pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.5, flags=cv2.OPTFLOW_LK_GET_MIN_EIGENVALS) # Farnback光流法 prevgray = gray # 计算完光流后,将当前帧存储为下一次计算的前一帧 residual = me.LinearRegression(flow) cv2.imshow('flow', draw_flow(gray, flow)) cv2.imshow("residual", residual) # cv2.imshow("mod flow", Mod_flow(flow)) if show_hsv: cv2.imshow('flow HSV', draw_hsv(flow)) if show_glitch:
cap = cv2.VideoCapture(0) # Get first frame ret, first_frame = cap.read() previous_gray = cv2.cvtColor(first_frame, cv2.COLOR_BGR2GRAY) hsv = np.zeros_like(first_frame) hsv[..., 1] = 255 while True: # Read of video file ret, frame2 = cap.read() next = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY) # Computes the dense optical flow using the Gunnar Farneback’s algorithm flow = cv2.calcOpticalFlowFarneback(previous_gray, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) # use flow to calculate the magnitude (speed) and angle of motion # use these values to calculate the color to reflect speed and angle magnitude, angle = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv[..., 0] = angle * (180 / (np.pi / 2)) hsv[..., 2] = cv2.normalize(magnitude, None, 0, 255, cv2.NORM_MINMAX) final = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) # Show our demo of Dense Optical Flow cv2.imshow('Dense Optical Flow', final) if cv2.waitKey(1) == 13: #13 is the Enter Key break # Store current image as previous image previous_gray = next
def readData(Filename, data_shape): data_1 = [] data_2 = [] pic = [] pic_x = [] pic_y = [] for filename in glob.glob(Filename + '/image*.jpg'): pic.append(filename) pic.sort() a = 0 for i in range(len(pic) - 1): prev = cv2.imread(pic[i]) prev = cv2.resize(prev, (320, 240)) prev = cv2.cvtColor(prev, cv2.COLOR_RGB2GRAY) #prev = np.multiply(prev, 1/255.0) #print prev[156][0:100] cur = cv2.imread(pic[i + 1]) cur = cv2.resize(cur, (320, 240)) cur = cv2.cvtColor(cur, cv2.COLOR_RGB2GRAY) #cur = np.multiply(cur, 1/255.0) #print cur.shape #print cur[156][0:100] flow = cv2.calcOpticalFlowFarneback(prev, cur, 0.702, 5, 10, 2, 7, 1.5, cv2.OPTFLOW_FARNEBACK_GAUSSIAN) #flow = np.array(flow) #flow = np.multiply(flow, 255) #flow = cv2.resize(flow, (data_shape[2], data_shape[1]/10)) #array_bound = np.ones((data_shape[2], data_shape[1]/10, 2), dtype=int)*20 # array_bound = np.ones((256, 256, 2), dtype=int)*20 # flow_img = 255*((flow)+array_bound)/(2*20) # flow_img = np.uint8(flow_img) # flow_img = cv2.resize(flow_img, (data_shape[2], data_shape[1]/10)) # #aaaa += 1 # #flow = np.multiply(flow_img, 1/255.0) # # flow_1 = flow_img.transpose((2,0,1)) # cv2.imwrite('./flow/flow-'+str(a).zfill(4)+'.jpg',flow_1[0,...]) # flow_1 = np.multiply(flow_1, 1/255.0) # flow_1 = flow_1.tolist() # pic_x.append(flow_1[0]) # pic_y.append(flow_1[1]) # a += 1 flow_x = flow[..., 0] flow_y = flow[..., 1] flow_x = cv2.resize(flow_x, (data_shape[2], data_shape[1] / 10)) flow_y = cv2.resize(flow_y, (data_shape[2], data_shape[1] / 10)) array_bound = np.ones( (data_shape[2], data_shape[1] / 10), dtype=int) * 20 flow_x_img = 255 * ((flow_x) + array_bound) / (2 * 20) flow_y_img = 255 * ((flow_y) + array_bound) / (2 * 20) cv2.imwrite('./flow/flow-' + str(a).zfill(4) + '.jpg', flow_x_img) a += 1 flow_x_img = np.multiply(flow_x_img, 1 / 255.0) flow_y_img = np.multiply(flow_y_img, 1 / 255.0) flow_x_list = flow_x_img.tolist() flow_y_list = flow_y_img.tolist() pic_x.append(flow_x_list) pic_y.append(flow_y_list) for j in range(len(pic_x) - LEN_SEQ): data_1_1 = [] for i in range(LEN_SEQ): idx = j + i data_1_1.append([pic_x[idx], pic_y[idx]]) data_2.append(0) data_1.append(data_1_1) # le = len(pic_x)/LEN_SEQ # for j in range(2): # data_1_1 = [] # for i in range(LEN_SEQ): # ret = random.randint(i*le, (i+1)*le-1) # data_1_1.append([pic_x[ret], pic_y[ret]]) # data_2.append(0) # data_1.append(data_1_1) return (data_1, data_2)
def main(): with tf.Session() as sess: model_cfg, model_outputs = posenet.load_model(args.model, sess) output_stride = model_cfg['output_stride'] if args.file is not None: cap = cv2.VideoCapture(args.file) else: cap = cv2.VideoCapture(args.cam_id) cap.set(3, args.cam_width) cap.set(4, args.cam_height) video_dict = np.load('../atharva_old/stair_name_dict.npy',allow_pickle=True).item() frame_list = [] path_list = [] for video_name in video_dict: k = video_name[-12:-2] # video_path = '.' + video_name[:-1] video_path = '../atharva_old/' + video_name[2:-1] for j in range(video_dict[video_name]): path_list.append(video_path + str(j) + '.png') # print(video_path + str(j)) break # remove this for labelling # print(path_list) frame_c = 0 frame_n = len(path_list) # hasFrame, frame = cap.read() ret,frame = img_read(path_list,frame_c) frame_c += 1 if not ret: print('no frame') print(path_list[frame_c]) exit() # print(frame.shape) start = time.time() frame_count = 0 prvs = cv2.resize(frame,(224,224)) prvs = cv2.cvtColor(prvs,cv2.COLOR_BGR2GRAY) hsv = np.zeros((224,224,3),dtype=np.uint8) hsv[...,1] = 255 #intensity c = 0 while True: c += 1 flag = 0 t = time.time() # hasFrame, frame = cap.read() ret,frame = img_read(path_list,frame_c) frame_c += 1 if not ret: print('no frame') break next = cv2.resize(frame,(224,224)) # print(next.shape) next = cv2.cvtColor(next,cv2.COLOR_BGR2GRAY) prvs = cv2.medianBlur(prvs,5) next = cv2.medianBlur(next,5) # print(prvs.shape,next.shape) flow = cv2.calcOpticalFlowFarneback(prvs,next, None, 0.5,3,7,4,7,5, 0) mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1]) mag = (mag>1.4)*mag hsv[...,0] = ang*180/np.pi/2 #hue, colour hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX) #brightness up_mask = make_mask(hsv[...,0],130,145) #purple down_mask = make_mask(hsv[...,0],35,75) #green left_mask = make_mask(hsv[...,0],165,179) | make_mask(hsv[...,0],1,20)#red right_mask = make_mask(hsv[...,0],80,100) #blue hsv_up = apply_mask(hsv,up_mask) hsv_down = apply_mask(hsv,down_mask) hsv_left = apply_mask(hsv,left_mask) hsv_right = apply_mask(hsv,right_mask) #input_image, display_image, output_scale = posenet.read_cap( # cap, scale_factor=args.scale_factor, output_stride=output_stride) f= path_list[frame_c] input_image, draw_image, output_scale = posenet.read_imgfile( f, scale_factor=args.scale_factor, output_stride=output_stride) heatmaps_result, offsets_result, displacement_fwd_result, displacement_bwd_result = sess.run( model_outputs, feed_dict={'image:0':input_image} ) pose_scores, keypoint_scores, keypoint_coords = posenet.decode_multi.decode_multiple_poses( heatmaps_result.squeeze(axis=0), offsets_result.squeeze(axis=0), displacement_fwd_result.squeeze(axis=0), displacement_bwd_result.squeeze(axis=0), output_stride=output_stride, max_pose_detections=1, min_pose_score=0.15) keypoint_coords *= output_scale #what the hell is this #for i in range(len(pose_scores)): for pts in keypoint_coords: dist1 = [0] dist2 = [0] # print(pts.shape) for i in range(len(mag)): for j in range(len(mag[0])): if mag[i,j] > 10: # pass # dist1.append(dist_from_line(j,i,pts[7,:],pts[9,:])) # left hand # dist2.append(dist_from_line(j,i,pts[8,:],pts[10,:]))# right hand dist1.append(dist_from_pt(j,i,pts[9,:])) # left wrist dist2.append(dist_from_pt(j,i,pts[10,:])) # right wrist # if True: thresh = 140 up_thresh = 34 down_thresh = 16 left_thresh = 24 right_thresh = 28 if (np.mean(dist1) < thresh or np.mean(dist2)<thresh) and (np.mean(dist1) >0 and np.mean(dist2)>0): # if True: #original # print('please print') # print(np.mean(hsv_right[...,0])) if np.mean(hsv_up[...,0])>up_thresh and np.mean(mag)>0.07: # print(np.mean(hsv_up[...,0])) print('UP',c) flag = 1 elif np.mean(hsv_down[...,0])>down_thresh and np.mean(mag)>0.07: # print(np.mean(hsv_down[...,0])) print('DOWN',c) flag = 1 elif np.mean(hsv_left[...,0])>left_thresh and np.mean(mag)>0.08: # print(np.mean(hsv_left[...,0])) print('LEFT',c) flag = 1 elif np.mean(hsv_right[...,0])>right_thresh and np.mean(mag)>0.08: # print(np.mean(hsv_right[...,0])) print('RIGHT',c) flag = 1 #modified # if np.mean(hsv_up[...,0])>38 and np.mean(mag)>0.08: # print('UP',np.mean(hsv_up[...,0])) # flag = 1 # if np.mean(hsv_down[...,0])>16.5 and np.mean(mag)>0.08: # print('DOWN',np.mean(hsv_down[...,0])) # flag = 1 # if np.mean(hsv_left[...,0])>24 and np.mean(mag)>0.08: # print('LEFT',c) # flag = 1 # if np.mean(hsv_right[...,0])>28 and np.mean(mag)>0.08: # print('RIGHT',c) # flag = 1 # TODO this isn't particularly fast, use GL for drawing and display someday... overlay_image = posenet.draw_skel_and_kp( draw_image, pose_scores, keypoint_scores, keypoint_coords, min_pose_score=0.15, min_part_score=0.1) bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR) bgr = cv2.medianBlur(bgr,5) cv2.imshow('flow',bgr) cv2.imshow('posenet', overlay_image) prvs = next frame_count += 1 if cv2.waitKey(1) & 0xFF == ord('q'): exit() print('Average FPS: ', frame_count / (time.time() - start))
# ret, old_frame = cap.read() old_gray = cv2.cvtColor(old_frame, cv2.COLOR_BGR2GRAY) # p0 = cv2.goodFeaturesToTrack(old_gray, mask = None, **feature_params) # Create a mask image for drawing purposes # mask = np.zeros_like(old_frame) while(j < 500): print j j += 1 # frame = cv2.imread(video_path + '/img/' + img_seq[j], 1) frame = cv2.imread('/home/tiago/Desktop/2.jpg', 1) # frame = cv2.resize(frame, (0,0), fx=0.25, fy=0.25) frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # calculate optical flow # p1, st, err = cv2.calcOpticalFlowPyrLK(old_gray, frame_gray, p0, None, **lk_params) start = time.time() flow = cv2.calcOpticalFlowFarneback(old_gray, frame_gray, None, 0.5, 3, 15, 3, 5, 1.2, 0) print 'took', round(time.time() - start, 2), 'secs to compute OF' mean = np.mean(flow[:,:, 1 ]) print 'mean', mean cv2.imshow('1', flow[:, :,1]) cv2.waitKey(-1) break # Select good points # try: # good_new = p1[st==1] # good_old = p0[st==1] # # draw the tracks # for i,(new,old) in enumerate(zip(good_new,good_old)): # a,b = new.ravel() # c,d = old.ravel() # mask = cv2.line(mask, (a,b),(c,d), color[i].tolist(), 2)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--log_dir", type=str) parser.add_argument("--images_path", type=str) parser.add_argument("--images_ext", type=str, default='png') parser.add_argument("--labels_path", type=str, default='') parser.add_argument("--labels_ext", type=str, default='png') parser.add_argument("--labels_col", type=str, default='green') parser.add_argument("--seg_paths", type=str_to_list, default=[]) parser.add_argument("--seg_ext", type=str, default='png') parser.add_argument("--seg_root_dir", type=str, default='') parser.add_argument("--seg_labels", type=str_to_list, default=[]) parser.add_argument( "--seg_cols", type=str_to_list, default=['blue', 'forest_green', 'magenta', 'cyan', 'red']) parser.add_argument("--out_path", type=str, default='') parser.add_argument("--out_ext", type=str, default='jpg') parser.add_argument("--out_size", type=str, default='1920x1080') parser.add_argument("--fps", type=float, default=30) parser.add_argument("--codec", type=str, default='H264') parser.add_argument("--save_path", type=str, default='') parser.add_argument("--n_classes", type=int) parser.add_argument("--save_stitched", type=int, default=0) parser.add_argument("--load_ice_conc_diff", type=int, default=0) parser.add_argument("--start_id", type=int, default=0) parser.add_argument("--end_id", type=int, default=-1) parser.add_argument("--show_img", type=int, default=0) parser.add_argument("--stitch", type=int, default=0) parser.add_argument("--stitch_seg", type=int, default=1) parser.add_argument("--plot_changed_seg_count", type=int, default=0) parser.add_argument("--normalize_labels", type=int, default=0) parser.add_argument("--selective_mode", type=int, default=0) parser.add_argument("--ice_type", type=int, default=0, help='0: combined, 1: anchor, 2: frazil') parser.add_argument("--enable_plotting", type=int, default=1, help='enable_plotting') args = parser.parse_args() images_path = args.images_path images_ext = args.images_ext labels_path = args.labels_path labels_ext = args.labels_ext labels_col = args.labels_col seg_paths = args.seg_paths seg_root_dir = args.seg_root_dir seg_ext = args.seg_ext out_path = args.out_path out_ext = args.out_ext out_size = args.out_size fps = args.fps codec = args.codec # save_path = args.save_path n_classes = args.n_classes end_id = args.end_id start_id = args.start_id show_img = args.show_img stitch = args.stitch stitch_seg = args.stitch_seg save_stitched = args.save_stitched normalize_labels = args.normalize_labels selective_mode = args.selective_mode seg_labels = args.seg_labels seg_cols = args.seg_cols ice_type = args.ice_type plot_changed_seg_count = args.plot_changed_seg_count load_ice_conc_diff = args.load_ice_conc_diff enable_plotting = args.enable_plotting ice_types = { 0: 'Ice', 1: 'Anchor Ice', 2: 'Frazil Ice', } loc = (5, 120) size = 8 thickness = 6 fgr_col = (255, 255, 255) bgr_col = (0, 0, 0) font_id = 0 video_exts = ['mp4', 'mkv', 'avi', 'mpg', 'mpeg', 'mjpg'] labels_col_rgb = col_bgr[labels_col] seg_cols_rgb = [col_bgr[seg_col] for seg_col in seg_cols] ice_type_str = ice_types[ice_type] print('ice_type_str: {}'.format(ice_type_str)) src_files, src_labels_list, total_frames = read_data( images_path, images_ext, labels_path, labels_ext) if end_id < start_id: end_id = total_frames - 1 if seg_paths: n_seg_paths = len(seg_paths) n_seg_labels = len(seg_labels) if n_seg_paths != n_seg_labels: raise IOError( 'Mismatch between n_seg_labels: {} and n_seg_paths: {}'.format( n_seg_labels, n_seg_paths)) if seg_root_dir: seg_paths = [ os.path.join(seg_root_dir, name) for name in seg_paths ] if not out_path: if labels_path: out_path = labels_path + '_conc' elif seg_paths: out_path = seg_paths[0] + '_conc' if not os.path.isdir(out_path): os.makedirs(out_path) # print('Saving results data to {}'.format(out_path)) # if not save_path: # save_path = os.path.join(os.path.dirname(images_path), 'ice_concentration') # if not os.path.isdir(save_path): # os.makedirs(save_path) # if stitch and save_stitched: # print('Saving ice_concentration plots to: {}'.format(save_path)) # log_fname = os.path.join(out_path, 'vis_log_{:s}.txt'.format(getDateTime())) # print('Saving log to: {}'.format(log_fname)) if selective_mode: label_diff = int(255.0 / n_classes) else: label_diff = int(255.0 / (n_classes - 1)) print('label_diff: {}'.format(label_diff)) n_frames = end_id - start_id + 1 print_diff = int(n_frames * 0.01) labels_img = None n_cols = len(seg_cols_rgb) plot_y_label = '{} concentration (%)'.format(ice_type_str) plot_x_label = 'distance in pixels from left edge' dists = {} for _label in seg_labels: dists[_label] = { # 'bhattacharyya': [], 'euclidean': [], 'mae': [], 'mse': [], # 'frobenius': [], } plot_title = '{} concentration'.format(ice_type_str) out_size = tuple([int(x) for x in out_size.split('x')]) write_to_video = out_ext in video_exts out_width, out_height = out_size out_seq_name = os.path.basename(out_path) if enable_plotting: if write_to_video: stitched_seq_path = os.path.join( out_path, '{}.{}'.format(out_seq_name, out_ext)) print('Writing {}x{} output video to: {}'.format( out_width, out_height, stitched_seq_path)) save_dir = os.path.dirname(stitched_seq_path) fourcc = cv2.VideoWriter_fourcc(*codec) video_out = cv2.VideoWriter(stitched_seq_path, fourcc, fps, out_size) else: stitched_seq_path = os.path.join(out_path, out_seq_name) print('Writing {}x{} output images of type {} to: {}'.format( out_width, out_height, out_ext, stitched_seq_path)) save_dir = stitched_seq_path if save_dir and not os.path.isdir(save_dir): os.makedirs(save_dir) prev_seg_img = {} prev_conc_data_y = {} changed_seg_count = {} ice_concentration_diff = {} if load_ice_conc_diff: for seg_id in seg_labels: ice_concentration_diff[seg_id] = np.loadtxt(os.path.join( out_path, '{}_ice_concentration_diff.txt'.format(seg_id)), dtype=np.float64) _pause = 0 mae_data_y = [] for seg_id, _ in enumerate(seg_paths): mae_data_y.append([]) for img_id in range(start_id, end_id + 1): start_t = time.time() # img_fname = '{:s}_{:d}.{:s}'.format(fname_templ, img_id + 1, img_ext) img_fname = src_files[img_id] img_fname_no_ext = os.path.splitext(img_fname)[0] src_img_fname = os.path.join(images_path, img_fname) src_img = imread(src_img_fname) if src_img is None: raise SystemError('Source image could not be read from: {}'.format( src_img_fname)) try: src_height, src_width = src_img.shape[:2] except ValueError as e: print('src_img_fname: {}'.format(src_img_fname)) print('src_img: {}'.format(src_img)) print('src_img.shape: {}'.format(src_img.shape)) print('error: {}'.format(e)) sys.exit(1) conc_data_x = np.asarray(range(src_width), dtype=np.float64) plot_data_x = conc_data_x plot_data_y = [] plot_cols = [] plot_labels = [] stitched_img = src_img if labels_path: labels_img_fname = os.path.join( labels_path, img_fname_no_ext + '.{}'.format(labels_ext)) labels_img_orig = imread(labels_img_fname) if labels_img_orig is None: raise SystemError( 'Labels image could not be read from: {}'.format( labels_img_fname)) labels_height, labels_width = labels_img_orig.shape[:2] if labels_height != src_height or labels_width != src_width: raise AssertionError( 'Mismatch between dimensions of source: {} and label: {}'. format((src_height, src_width), (seg_height, seg_width))) if len(labels_img_orig.shape) == 3: labels_img_orig = np.squeeze(labels_img_orig[:, :, 0]) if show_img: cv2.imshow('labels_img_orig', labels_img_orig) if normalize_labels: labels_img = (labels_img_orig.astype(np.float64) / label_diff).astype(np.uint8) else: labels_img = np.copy(labels_img_orig) if len(labels_img.shape) == 3: labels_img = labels_img[:, :, 0].squeeze() conc_data_y = np.zeros((labels_width, ), dtype=np.float64) for i in range(labels_width): curr_pix = np.squeeze(labels_img[:, i]) if ice_type == 0: ice_pix = curr_pix[curr_pix != 0] else: ice_pix = curr_pix[curr_pix == ice_type] conc_data_y[i] = (len(ice_pix) / float(src_height)) * 100.0 conc_data = np.zeros((labels_width, 2), dtype=np.float64) conc_data[:, 0] = conc_data_x conc_data[:, 1] = conc_data_y plot_data_y.append(conc_data_y) plot_cols.append(labels_col_rgb) gt_dict = { conc_data_x[i]: conc_data_y[i] for i in range(labels_width) } if not normalize_labels: labels_img_orig = (labels_img_orig.astype(np.float64) * label_diff).astype(np.uint8) if len(labels_img_orig.shape) == 2: labels_img_orig = np.stack( (labels_img_orig, labels_img_orig, labels_img_orig), axis=2) stitched_img = np.concatenate((stitched_img, labels_img_orig), axis=1) plot_labels.append('GT') # gt_cl, _ = eval.extract_classes(labels_img_orig) # print('gt_cl: {}'.format(gt_cl)) mean_seg_counts = {} seg_count_data_y = [] curr_mae_data_y = [] mean_conc_diff = {} conc_diff_data_y = [] seg_img_disp_list = [] for seg_id, seg_path in enumerate(seg_paths): seg_img_fname = os.path.join( seg_path, img_fname_no_ext + '.{}'.format(seg_ext)) seg_img_orig = imread(seg_img_fname) seg_col = seg_cols_rgb[seg_id % n_cols] _label = seg_labels[seg_id] if seg_img_orig is None: raise SystemError( 'Seg image could not be read from: {}'.format( seg_img_fname)) seg_height, seg_width = seg_img_orig.shape[:2] if seg_height != src_height or seg_width != src_width: raise AssertionError( 'Mismatch between dimensions of source: {} and seg: {}'. format((src_height, src_width), (seg_height, seg_width))) if len(seg_img_orig.shape) == 3: seg_img_orig = np.squeeze(seg_img_orig[:, :, 0]) if seg_img_orig.max() > n_classes - 1: seg_img = (seg_img_orig.astype(np.float64) / label_diff).astype(np.uint8) seg_img_disp = seg_img_orig else: seg_img = seg_img_orig seg_img_disp = (seg_img_orig.astype(np.float64) * label_diff).astype(np.uint8) if len(seg_img_disp.shape) == 2: seg_img_disp = np.stack( (seg_img_disp, seg_img_disp, seg_img_disp), axis=2) ann_fmt = (font_id, loc[0], loc[1], size, thickness) + fgr_col + bgr_col put_text_with_background(seg_img_disp, seg_labels[seg_id], fmt=ann_fmt) seg_img_disp_list.append(seg_img_disp) # eval_cl, _ = eval.extract_classes(seg_img) # print('eval_cl: {}'.format(eval_cl)) if show_img: cv2.imshow('seg_img_orig', seg_img_orig) if len(seg_img.shape) == 3: seg_img = seg_img[:, :, 0].squeeze() conc_data_y = np.zeros((seg_width, ), dtype=np.float64) for i in range(seg_width): curr_pix = np.squeeze(seg_img[:, i]) if ice_type == 0: ice_pix = curr_pix[curr_pix != 0] else: ice_pix = curr_pix[curr_pix == ice_type] conc_data_y[i] = (len(ice_pix) / float(src_height)) * 100.0 plot_cols.append(seg_col) plot_data_y.append(conc_data_y) if labels_path: seg_dict = { conc_data_x[i]: conc_data_y[i] for i in range(seg_width) } # dists['bhattacharyya'].append(bhattacharyya(gt_dict, seg_dict)) dists[_label]['euclidean'].append(euclidean(gt_dict, seg_dict)) dists[_label]['mse'].append(mse(gt_dict, seg_dict)) dists[_label]['mae'].append(mae(gt_dict, seg_dict)) # dists['frobenius'].append(np.linalg.norm(conc_data_y - plot_data_y[0])) curr_mae_data_y.append(dists[_label]['mae'][-1]) else: if img_id > 0: if plot_changed_seg_count: flow = cv2.calcOpticalFlowFarneback( prev_seg_img[_label], seg_img, None, 0.5, 3, 15, 3, 5, 1.2, 0) print('flow: {}'.format(flow.shape)) # # Obtain the flow magnitude and direction angle # mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) # hsvImg = np.zeros((2160, 3840, 3), dtype=np.uint8) # hsvImg[..., 1] = 255 # # Update the color image # hsvImg[..., 0] = 0.5 * ang * 180 / np.pi # hsvImg[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) # rgbImg = cv2.cvtColor(hsvImg, cv2.COLOR_HSV2BGR) # rgbImg = resizeAR(rgbImg, width=out_width, height=out_height) # # Display the resulting frame # cv2.imshow('dense optical flow', rgbImg) # k = cv2.waitKey(0) curr_x, curr_y = (prev_x + flow[..., 0]).astype( np.int32), (prev_y + flow[..., 1]).astype(np.int32) seg_img_flow = seg_img[curr_y, curr_x] changed_seg_count[_label].append( np.count_nonzero( np.not_equal(seg_img, prev_seg_img[_label]))) seg_count_data_y.append(changed_seg_count[_label]) mean_seg_counts[_label] = np.mean( changed_seg_count[_label]) else: ice_concentration_diff[_label].append( np.mean( np.abs(conc_data_y - prev_conc_data_y[_label]))) conc_diff_data_y.append(ice_concentration_diff[_label]) mean_conc_diff[_label] = np.mean( ice_concentration_diff[_label]) else: if plot_changed_seg_count: prev_x, prev_y = np.meshgrid(range(seg_width), range(seg_height), sparse=False, indexing='xy') changed_seg_count[_label] = [] else: ice_concentration_diff[_label] = [] prev_seg_img[_label] = seg_img prev_conc_data_y[_label] = conc_data_y # conc_data = np.concatenate([conc_data_x, conc_data_y], axis=1) if labels_path: for i, k in enumerate(curr_mae_data_y): mae_data_y[i].append(k) n_test_images = img_id + 1 mae_data_X = np.asarray(range(1, n_test_images + 1), dtype=np.float64) print('') # print('mae_data_X:\n {}'.format(pformat(mae_data_X))) # print('mae_data_y:\n {}'.format(pformat(np.array(mae_data_y).transpose()))) if img_id == end_id: mae_data_y_arr = np.array(mae_data_y).transpose() print('mae_data_y:\n {}'.format( tabulate(mae_data_y_arr, headers=seg_labels, tablefmt='plain'))) pd.DataFrame(data=mae_data_y_arr, columns=seg_labels).to_clipboard(excel=True) mae_img = getPlotImage(mae_data_X, mae_data_y, plot_cols, 'MAE', seg_labels, 'frame', 'MAE') cv2.imshow('mae_img', mae_img) conc_diff_img = resize_ar(mae_img, seg_width, src_height, bkg_col=255) else: if img_id > 0: n_test_images = img_id seg_count_data_X = np.asarray(range(1, n_test_images + 1), dtype=np.float64) if plot_changed_seg_count: seg_count_img = getPlotImage(seg_count_data_X, seg_count_data_y, plot_cols, 'Count', seg_labels, 'frame', 'Changed Label Count') cv2.imshow('seg_count_img', seg_count_img) else: # print('seg_count_data_X:\n {}'.format(pformat(seg_count_data_X))) # print('conc_diff_data_y:\n {}'.format(pformat(conc_diff_data_y))) conc_diff_img = getPlotImage( seg_count_data_X, conc_diff_data_y, plot_cols, 'Mean concentration difference between consecutive frames' .format(ice_type_str), seg_labels, 'frame', 'Concentration Difference (%)') # cv2.imshow('conc_diff_img', conc_diff_img) conc_diff_img = resize_ar(conc_diff_img, seg_width, src_height, bkg_col=255) else: conc_diff_img = np.zeros((src_height, seg_width, 3), dtype=np.uint8) plot_labels += seg_labels if enable_plotting: plot_img = getPlotImage(plot_data_x, plot_data_y, plot_cols, plot_title, plot_labels, plot_x_label, plot_y_label, legend=0 # ylim=(0, 100) ) plot_img = resize_ar(plot_img, seg_width, src_height, bkg_col=255) # plt.plot(conc_data_x, conc_data_y) # plt.show() # conc_data_fname = os.path.join(out_path, img_fname_no_ext + '.txt') # np.savetxt(conc_data_fname, conc_data, fmt='%.6f') ann_fmt = (font_id, loc[0], loc[1], size, thickness) + labels_col_rgb + bgr_col put_text_with_background(src_img, 'frame {}'.format(img_id + 1), fmt=ann_fmt) if n_seg_paths == 1: print('seg_img_disp: {}'.format(seg_img_disp.shape)) print('plot_img: {}'.format(plot_img.shape)) stitched_seg_img = np.concatenate((seg_img_disp, plot_img), axis=1) print('stitched_seg_img: {}'.format(stitched_seg_img.shape)) print('stitched_img: {}'.format(stitched_img.shape)) stitched_img = np.concatenate((stitched_img, stitched_seg_img), axis=0 if labels_path else 1) elif n_seg_paths == 2: stitched_img = np.concatenate(( np.concatenate((src_img, conc_diff_img), axis=1), np.concatenate(seg_img_disp_list, axis=1), ), axis=0) elif n_seg_paths == 3: stitched_img = np.concatenate(( np.concatenate((src_img, plot_img, conc_diff_img), axis=1), np.concatenate(seg_img_disp_list, axis=1), ), axis=0) stitched_img = resize_ar(stitched_img, width=out_width, height=out_height) # print('dists: {}'.format(dists)) if write_to_video: video_out.write(stitched_img) else: stacked_img_path = os.path.join( stitched_seq_path, '{}.{}'.format(img_fname_no_ext, out_ext)) cv2.imwrite(stacked_img_path, stitched_img) cv2.imshow('stitched_img', stitched_img) k = cv2.waitKey(1 - _pause) if k == 27: break elif k == 32: _pause = 1 - _pause end_t = time.time() sys.stdout.write('\rDone {:d}/{:d} frames. fps: {}'.format( img_id + 1 - start_id, n_frames, 1.0 / (end_t - start_t))) sys.stdout.flush() print() if enable_plotting and write_to_video: video_out.release() if labels_path: median_dists = {} mean_dists = {} mae_data_y = [] for _label in seg_labels: _dists = dists[_label] mae_data_y.append(_dists['mae']) mean_dists[_label] = {k: np.mean(_dists[k]) for k in _dists} median_dists[_label] = {k: np.median(_dists[k]) for k in _dists} print('mean_dists:\n{}'.format(pformat(mean_dists))) print('median_dists:\n{}'.format(pformat(median_dists))) n_test_images = len(mae_data_y[0]) mae_data_x = np.asarray(range(1, n_test_images + 1), dtype=np.float64) mae_img = getPlotImage(mae_data_x, mae_data_y, plot_cols, 'MAE', seg_labels, 'test image', 'Mean Absolute Error') # plt.show() cv2.imshow('MAE', mae_img) k = cv2.waitKey(0) else: mean_seg_counts = {} median_seg_counts = {} seg_count_data_y = [] mean_conc_diff = {} median_conc_diff = {} conc_diff_data_y = [] for seg_id in ice_concentration_diff: if plot_changed_seg_count: seg_count_data_y.append(changed_seg_count[seg_id]) mean_seg_counts[seg_id] = np.mean(changed_seg_count[seg_id]) median_seg_counts[seg_id] = np.median( changed_seg_count[seg_id]) else: _ice_concentration_diff = ice_concentration_diff[seg_id] n_test_images = len(_ice_concentration_diff) conc_diff_data_y.append(_ice_concentration_diff) mean_conc_diff[seg_id] = np.mean(_ice_concentration_diff) median_conc_diff[seg_id] = np.median(_ice_concentration_diff) np.savetxt(os.path.join( out_path, '{}_ice_concentration_diff.txt'.format(seg_id)), _ice_concentration_diff, fmt='%8.4f', delimiter='\t') if plot_changed_seg_count: print('mean_seg_counts:\n{}'.format(pformat(mean_seg_counts))) print('median_seg_counts:\n{}'.format(pformat(median_seg_counts))) else: print('mean_conc_diff:') for seg_id in mean_conc_diff: print('{}\t{}'.format(seg_id, mean_conc_diff[seg_id])) print('median_conc_diff:') for seg_id in mean_conc_diff: print('{}\t{}'.format(seg_id, median_conc_diff[seg_id]))
if Path(sys.argv[1]).is_file(): file_name = sys.argv[1] cap = cv.VideoCapture(file_name) # Capture video from camera ret, frame1 = cap.read() prvs = cv.cvtColor(frame1, cv.COLOR_BGR2GRAY) hsv = np.zeros_like(frame1) hsv[..., 1] = 255 while True: ret, frame2 = cap.read() if not ret: break next = cv.cvtColor(frame2, cv.COLOR_BGR2GRAY) # prev, next, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags flow = cv.calcOpticalFlowFarneback(prvs, next, None, 0.5, 2, 15, 7, 5, 1.1, 0) mag, ang = cv.cartToPolar(flow[..., 0], flow[..., 1]) vert = ang % (np.pi / 2.0) vert = vert < (np.pi / 16.) hsv[..., 0] = ang * 180 / np.pi nmag = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX) nmag[vert] = 0.0 hsv[..., 2] = cv.normalize(mag, None, 0, 255, cv.NORM_MINMAX) bgr = cv.cvtColor(hsv, cv.COLOR_HSV2BGR) cv.imshow('frame2', bgr) k = cv.waitKey(5) & 0xff if k == 27: break elif k == ord('s'):
def extract_flow_angles(vidFile, hist_bins, mag_thresh, density=False, crop=240): ''' Extract optical flow maps from video vidFile for all the frames and put the angles with >mag_threshold in different bins. The bins vector is the feature representation for the stroke. Use only the strokes given by list of tuples frame_indx. Parameters: ------ vidFile: str complete path to a video start: int starting frame number end: int ending frame number hist_bins: 1d np array bin divisions (boundary values). Used np.linspace(0, 2*PI, 11) for 10 bins mag_thresh: int minimum size of the magnitude vectors that are considered (no. of pixels shifted in consecutive frames of OF) ''' cap = cv2.VideoCapture(vidFile) if not cap.isOpened(): print("Capture object not opened. Aborting !!") sys.exit(0) ret = True start = 0 end = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) stroke_features = [] prvs, next_ = None, None m, n = start, end #print("stroke {} ".format((m, n))) sum_norm_mag_ang = np.zeros( (len(hist_bins) - 1)) # for optical flow maxFrames - 1 size frameNo = m while ret: # and frameNo < n: # cap.set(cv2.CAP_PROP_POS_FRAMES, frameNo) ret, frame1 = cap.read() if not ret: # print("Frame not read. Aborting !!") break # resize if frame1.shape[0] < crop or frame1.shape[1] < crop: frame1 = cv2.resize(frame1, (crop, crop)) if (frameNo - m) == 0: # first frame condition # resize and then convert to grayscale if crop is not None: frame1 = center_crop(frame1, crop) prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) #prvs = scale_and_crop(prvs, scale) frameNo += 1 continue if crop is not None: frame1 = center_crop(frame1, crop) # resize and then convert to grayscale next_ = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs, next_, None, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) #print("Mag > 5 = {}".format(np.sum(mag>THRESH))) pixAboveThresh = np.sum(mag > mag_thresh) #use weights=mag[mag>THRESH] to be weighted with magnitudes #returns a tuple of (histogram, bin_boundaries) ang_hist = np.histogram(ang[mag > mag_thresh], bins=hist_bins, density=density) stroke_features.append(ang_hist[0]) #sum_norm_mag_ang +=ang_hist[0] # if not pixAboveThresh==0: # sum_norm_mag_ang[frameNo-m-1] = np.sum(mag[mag > THRESH])/pixAboveThresh # sum_norm_mag_ang[(maxFrames-1)+frameNo-m-1] = np.sum(ang[mag > THRESH])/pixAboveThresh frameNo += 1 prvs = next_ #stroke_features.append(sum_norm_mag_ang/(n-m+1)) cap.release() #cv2.destroyAllWindows() stroke_features = np.array(stroke_features) #Normalize row - wise #stroke_features = stroke_features/(1+stroke_features.sum(axis=1)[:, None]) return stroke_features
def runIterations(self, testAgent=False, doUpdate=False, iterations=1000): self.env.seed() observation = self.env.reset() observation = rescale(observation, self.zoom) obs2 = np.copy(observation) # all_rewards = [] reseting = 0 otp_flow = 1 if otp_flow: obs2[0:, 0:, 2] = obs2[0:, 0:, 2] * 0 #pass for t in range(iterations): if self.skip_frame_timer == self.skip_frames: if testAgent == False: if doUpdate: self.agent.update(self) t_before_action = time.time() action = self.agent.getNextAction(obs2) self.times["get_action"] += time.time() - t_before_action elif testAgent == True: self.env.render() action = self.agent.getBestAction(obs2) self.skip_frame_timer = 0 self.skip_frame_timer += 1 self.lastAction = action prev_state = np.copy(obs2) observation, reward, done, info = self.env.step(action) all_rewards.append(reward) #print(reward) observation = rescale(observation, self.zoom) obs2 = np.copy(observation) #plt.imshow(obs2) #plt.show() ####OPTICAL FLOW if reseting == 0: gray = rgb2gray(obs2) of_y = cv2.calcOpticalFlowFarneback(rgb2gray(prev_state * 255), gray * 255, None, 0.5, 3, 5, 3, 5, 1.2, 0)[0:, 0:, 1] #of = cv2.calcOpticalFlowFarneback(rgb2gray(prev_state)*255, # gray*255,None,0.5, 3, 5, 3, 5, 1.2, 0) obs2[0:, 0:, 2] = of_y / 10 #obs2[0:,0:,2] = drawShadow(gray,of) #obs2[0:,0:,0] = obs2[0:,0:,0]+ of_y*500 #obs2[0:,0:,1] = obs2[0:,0:,1]+ of_y*500 #obs2[0:,0:,2] = obs2[0:,0:,2]+ of_y*500 else: obs2[0:, 0:, 2] = obs2[0:, 0:, 2] * 0 #pass reseting = 0 r = reward #### log rewards all_rewards.append(r) #### r = np.clip(reward, -1, 1) if testAgent == False: if self.skip_frame_timer == 1: self.experienceData.addData(prev_state, action, obs2, r, done) if done: self.skip_frame_timer = self.skip_frames self.env.seed() observation = self.env.reset() observation = rescale(observation, self.zoom) obs2 = np.copy(observation) if otp_flow: obs2[0:, 0:, 2] = obs2[0:, 0:, 2] * 0 reseting = 1 if t == self.episode_maxLength: # never print("episode max length reached") #self.env.close() if testAgent == False: if self.agent.exploreChance > self.agent.exploration_final_eps: if doUpdate: self.agent.exploreChance *= 0.8 return all_rewards if testAgent == True: return all_rewards
def extract_flow_grid(vidFile, grid_size, crop): ''' Extract optical flow maps from video vidFile starting from start frame number to end frame no. The grid based features are flattened and appended. Parameters: ------ vidFile: str complete path to a video grid_size: int grid size for sampling at intersection points of 2D flow. Returns: ------ np.array 2D with N x (360/G * 640/G) where G is grid size ''' cap = cv2.VideoCapture(vidFile) if not cap.isOpened(): print("Capture object not opened. Aborting !!") sys.exit(0) ret = True start = 0 end = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) stroke_features = [] prvs, next_ = None, None m, n = start, end #print("stroke {} ".format((m, n))) #sum_norm_mag_ang = np.zeros((len(hist_bins)-1)) # for optical flow maxFrames - 1 size frameNo = m while ret: # and frameNo <= n: # cap.set(cv2.CAP_PROP_POS_FRAMES, frameNo) ret, frame1 = cap.read() if not ret: # print("Frame not read. Aborting !!") break # resize if frame1.shape[0] < crop or frame1.shape[1] < crop: frame1 = cv2.resize(frame1, (crop, crop)) if (frameNo - m) == 0: # first frame condition # resize and then convert to grayscale #cv2.imwrite(os.path.join(flow_numpy_path, str(frameNo)+".png"), frame1) if crop is not None: frame1 = center_crop(frame1, crop) prvs = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) #prvs = scale_and_crop(prvs, scale) frameNo += 1 continue if crop is not None: frame1 = center_crop(frame1, crop) next_ = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY) flow = cv2.calcOpticalFlowFarneback(prvs, next_, None, 0.5, 3, 15, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) # stack sliced arrays along the first axis (2, 12, 16) sliced_flow = np.stack(( mag[::grid_size, ::grid_size], \ ang[::grid_size, ::grid_size]), axis=0) # stroke_features.append(sliced_flow[1, ...].ravel()) # Only angles #feature = np.array(feature) stroke_features.append(sliced_flow.ravel()) # Both magnitude and angle frameNo += 1 prvs = next_ cap.release() #cv2.destroyAllWindows() stroke_features = np.array(stroke_features) #Normalize row - wise #stroke_features = stroke_features/(1+stroke_features.sum(axis=1)[:, None]) return stroke_features
####----New idea --- ## Once I get the mask of ROI, lets get the optical flow of the mask in each frame and from there we can catagorise activities # once we get the optical flow we can create another bounding box on the original frame with TAG : Picking / Not Picking clone = copy.copy(frame) curr_frame = copy.copy(clone) curr_frame = cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY) curr_frame = imutils.resize(curr_frame, width=WIDTH) #vel_x = None #vel_y = None #visual = None print ('clone_frame', np.shape(clone)) print ('curr_frame', np.shape(curr_frame)) flow = cv2.calcOpticalFlowFarneback(prev_frame, curr_frame, flow=None, pyr_scale=0.5, levels=3, winsize=15, iterations=3, poly_n=5, poly_sigma=1.5, flags=0) visual, vel_x, vel_y = draw_flow(curr_frame, flow) #vel_x_list = [] #vel_y_list = [] #vel_y_list = vel_y_list.append(vel_y) #vel_x_list = vel_x_list.append(vel_x) #print ('vel_x', vel_x) cv2.imshow('visual', visual) prev_frame = curr_frame ### Idea ENDS-------------- # show the output image cv2.imshow("Output", clone) cv2.waitKey(10000)
def crop_and_get_of(video_path, thresh, resize): optical_flows = [] # Open vide and read first frame cap = cv2.VideoCapture(video_path) ret = None current_frame = None frame = 0 while (current_frame is None): if (debug): print('None') ret, current_frame = cap.read() frame += 1 total_frame = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) # Assign current to prev and resize prev previous_frame = current_frame previous_frame = cv2.resize(previous_frame, resize) net = get_obj_detection_model() x_min, x_max, y_min, y_max, w, h = get_person_bounding_box( previous_frame, net) (h, w) = previous_frame.shape[:2] box_w = x_max - x_min box_h = y_max - y_min inc_x_min, inc_x_max, inc_y_min, inc_y_max = (max( 0, int(x_min - 0.5 * box_w)), min( w, int(x_max + 0.5 * box_w)), max( 0, int(y_min - 0.5 * box_h)), min(h, int(y_max + 0.5 * box_h))) of_x_min, of_x_max, of_y_min, of_y_max = inc_x_min, inc_x_max, inc_y_min, inc_y_max # Get bounding box coordinates if (debug): print("x_min " + str(x_min)) print("x_max " + str(x_max)) print("x_max - x_min " + str(x_max - x_min)) print("y_min " + str(y_min)) print("y_max " + str(y_max)) print("y_max - y_min " + str(y_max - y_min)) cv2_imshow(previous_frame[y_min:y_max, x_min:x_max]) hsv = zeros_like(previous_frame[of_y_min:of_y_max, of_x_min:of_x_max]) hsv[..., 1] = 255 of_acc = None a = [] width = cap.get(cv2.CAP_PROP_FRAME_WIDTH) height = cap.get(cv2.CAP_PROP_FRAME_HEIGHT) rec = False while (cap.isOpened()): frame += 1 print(str(frame) + '/' + str(total_frame)) current_frame = cv2.resize(current_frame, resize) current_frame_gray = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY) previous_frame_gray = cv2.cvtColor(previous_frame, cv2.COLOR_BGR2GRAY) # NEW MOTION DETECTION blurred_current = cv2.GaussianBlur(current_frame_gray, (11, 11), 0) blurred_prev = cv2.GaussianBlur(previous_frame_gray, (11, 11), 0) blurred_frame_diff = cv2.absdiff( blurred_current[of_y_min:of_y_max, of_x_min:of_x_max], blurred_prev[of_y_min:of_y_max, of_x_min:of_x_max]) thr = cv2.threshold(blurred_frame_diff, 25, 255, cv2.THRESH_BINARY)[1] thr = cv2.dilate(thr, None, iterations=1) cnts = cv2.findContours(thr, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) cnts = imutils.grab_contours(cnts) sum_area = 0 for c in cnts: sum_area += cv2.contourArea(c) sum_area2 = sum_area / (of_x_max - of_x_min) / (of_y_max - of_y_min) if (sum_area2 > thresh2): if (rec is False): rec = True start = frame bad = 0 bad += 1 cv2.fastNlMeansDenoising(previous_frame_gray, previous_frame_gray, 20, 5, 21) cv2.fastNlMeansDenoising(current_frame_gray, current_frame_gray, 20, 5, 21) if (bad < 18): ret, current_frame = cap.read() if (current_frame is None): break continue flow = cv2.calcOpticalFlowFarneback( previous_frame_gray[of_y_min:of_y_max, of_x_min:of_x_max], current_frame_gray[of_y_min:of_y_max, of_x_min:of_x_max], None, 0.5, 3, 4, 3, 5, 1.2, 0) mag, ang = cv2.cartToPolar(flow[..., 0], flow[..., 1]) hsv[..., 0] = ang * 180 / pi / 2 hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) if of_acc is None: of_acc = bgr else: of_acc += bgr else: if of_acc is not None: r = True optical_flows.append(of_acc) #break rec = False of_acc = None previous_frame = current_frame.copy() ret, current_frame = cap.read() if (current_frame is None): break cap.release() cv2.destroyAllWindows() return optical_flows, of_x_min, of_x_max, of_y_min, of_y_max, inc_x_min, inc_x_max, inc_y_min, inc_y_max, w, h
#cv2.putText(draw_roi,"Original Screen", (0,35), cv2.FONT_HERSHEY_PLAIN, 2,(255,255,255),3) #Set Frame Speed as 60 per second cap.set(cv2.CAP_PROP_FPS, int(60)) font = cv2.FONT_HERSHEY_SIMPLEX status_cap, frame = cap.read() frame = cv2.resize(frame, (0, 0), None, 0.5, 0.5) if not status_cap: break gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if init_flow: opt_flow = cv2.calcOpticalFlowFarneback( prev_frame, gray, None, 0.5, 5, 13, 10, 5, 1.1, cv2.OPTFLOW_FARNEBACK_GAUSSIAN) init_flow = False else: opt_flow = cv2.calcOpticalFlowFarneback( prev_frame, gray, opt_flow, 0.5, 5, 13, 10, 5, 1.1, cv2.OPTFLOW_USE_INITIAL_FLOW) prev_frame = np.copy(gray) img = display_flow(frame, opt_flow) if cv2.waitKey(1) & 0xFF == ord('q'): cv2.destroyAllWindows() break # Expand dimensions since the model expects images to have shape: [1, None, None, 3]