def __init__(self, min_accuracy, min_blend_area, kernel_fill=20, dist_threshold=15000, history=400): self.min_accuracy = max (min_accuracy, 0.7) self.min_blend_area = min_blend_area self.kernel_clean = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(4,4)) self.kernel_fill = np.ones((kernel_fill,kernel_fill),np.uint8) self.dist_threshold = dist_threshold self.history = history # read https://docs.opencv.org/3.3.0/d2/d55/group__bgsegm.html#gae561c9701970d0e6b35ec12bae149814 try: self.fgbg = cv2.bgsegm.createBackgroundSubtractorMOG(history=self.history, nmixtures=5, backgroundRatio=0.7, noiseSigma=0) except AttributeError as error: print ('It looks like your OpenCV version does not include bgsegm. Switching to createBackgroundSubtractorMOG2') self.fgbg = cv2.createBackgroundSubtractorMOG2(detectShadows=False, history=self.history) #self.fgbg = cv2.bgsegm.createBackgroundSubtractorGMG(decisionThreshold=0.98, initializationFrames=10) #self.fgbg = cv2.createBackgroundSubtractorMOG2(detectShadows=False, history=self.history) #self.fgbg=cv2.bgsegm.createBackgroundSubtractorGSOC(noiseRemovalThresholdFacBG=0.01, noiseRemovalThresholdFacFG=0.0001) #self.fgbg=cv2.bgsegm.createBackgroundSubtractorCNT(minPixelStability = 5, useHistory = True, maxPixelStability = 5 *60,isParallel = True) #self.fgbg=cv2.createBackgroundSubtractorKNN(detectShadows=False, history=self.history, dist2Threshold = self.dist_threshold) #fgbg=cv2.bgsegm.createBackgroundSubtractorLSBP() utils.success_print('Background subtraction initialized')
def __init__(self, configPath=None, weightPath=None, labelsPath=None, darknetLib=None, kernel_fill=3): if g.args['gpu']: utils.success_print('Using GPU model for YOLO') utils.success_print( 'If you run out of memory, please tweak yolo.cfg') self.m = yolo.SimpleYolo(configPath=configPath, weightPath=weightPath, darknetLib=darknetLib, labelsPath=labelsPath, useGPU=True) else: utils.success_print('Using CPU/OpenCV model for YOLO') self.net = cv2.dnn.readNetFromDarknet(configPath, weightPath) self.labels = open(labelsPath).read().strip().split("\n") np.random.seed(42) self.colors = np.random.randint(0, 255, size=(len(self.labels), 3), dtype="uint8") self.kernel_fill = np.ones((kernel_fill, kernel_fill), np.uint8) utils.success_print('YOLO initialized')
def __init__(self, configPath=None, weightPath=None, labelsPath=None, kernel_fill=3): if g.args['gpu'] and not g.args['use_opencv_dnn_cuda']: utils.success_print('Using Darknet GPU model for YOLO') utils.success_print( 'If you run out of memory, please tweak yolo.cfg') if not g.args['use_opencv_dnn_cuda']: self.m = yolo.SimpleYolo(configPath=configPath, weightPath=weightPath, darknetLib=g.args['darknet_lib'], labelsPath=labelsPath, useGPU=True) else: utils.success_print('Using OpenCV model for YOLO') utils.success_print( 'If you run out of memory, please tweak yolo.cfg') self.net = cv2.dnn.readNetFromDarknet(configPath, weightPath) self.labels = open(labelsPath).read().strip().split("\n") np.random.seed(42) self.colors = np.random.randint(0, 255, size=(len(self.labels), 3), dtype="uint8") self.kernel_fill = np.ones((kernel_fill, kernel_fill), np.uint8) if g.args['use_opencv_dnn_cuda'] and g.args['gpu']: (maj, minor, patch) = cv2.__version__.split('.') min_ver = int(maj + minor) if min_ver < 42: utils.fail_print('Not setting CUDA backend for OpenCV DNN') utils.dim_print( 'You are using OpenCV version {} which does not support CUDA for DNNs. A minimum of 4.2 is required. See https://www.pyimagesearch.com/2020/02/03/how-to-use-opencvs-dnn-module-with-nvidia-gpus-cuda-and-cudnn/ on how to compile and install openCV 4.2' .format(cv2.__version__)) else: utils.success_print( 'Setting CUDA backend for OpenCV. If you did not set your CUDA_ARCH_BIN correctly during OpenCV compilation, you will get errors during detection related to invalid device/make_policy' ) self.net.setPreferableBackend(cv2.dnn.DNN_BACKEND_CUDA) self.net.setPreferableTarget(cv2.dnn.DNN_TARGET_CUDA) utils.success_print('YOLO initialized')
def search_video(input_file=None, out_file=None, eid = None, mid = None): utils.dim_print ('Analyzing: {}'.format(input_file)) vid = cv2.VideoCapture(input_file) orig_fps = max(1, (g.args['fps'] or int(vid.get(cv2.CAP_PROP_FPS)))) frame_found = False # if any match found, this will be true out = None det_type = 'found' if g.args['present'] else 'missing' set_frames = { 'eventid': eid, 'monitorid': mid, 'type': det_type, 'frames':[] } # if we want to write frames to a new video, # make sure it uses the same FPS as the input video and is of the same size if g.args['write']: width = int(vid.get(3)) height = int(vid.get(4)) if g.args['resize']: resize = g.args['resize'] width = int (width * resize) height = int (height * resize) fourcc = cv2.VideoWriter_fourcc(*'mp4v') h,t = os.path.split(input_file) h = h or '.' dt = datetime.now().strftime("%m_%d_%Y_%H_%M_%S") if not out_file: out_file = h+'/analyzed-'+dt+'-'+t out = cv2.VideoWriter(out_file, fourcc, orig_fps, (width,height)) print ('If frames are matched, will write to output video: {}'.format(out_file)) # get metadata from the input video. There are times this may be off # fps_skip is set to 1/2 of FPS. So if you analyze a video with 10FPS, we will skip every 5 frames during analysis # basically, I think 2 fps for analysis is sufficient. You can override this if g.args['skipframes']: fps_skip = g.args['skipframes'] else: fps_skip = max(1, int(vid.get(cv2.CAP_PROP_FPS)/2)) total_frames = int(vid.get(cv2.CAP_PROP_FRAME_COUNT)) start_time = time.time() utils.dim_print ('fps={}, skipping {} frames, total frames={}'.format(orig_fps, fps_skip, total_frames)) utils.dim_print ('threshold={}, search type=if {}'.format(g.args['threshold'], det_type)) frame_cnt = 0 bar = tqdm(total=total_frames) # now loop through the input video while True: succ, frame = vid.read() if not succ: break frame_cnt = frame_cnt + 1 bar.update(1) if frame_cnt % fps_skip: # skip frames based on our skip frames count. We don't really need to process every frame continue if g.args['resize']: resize = g.args['resize'] rh, rw, rl = frame.shape frame = cv2.resize(frame, (int(rw*resize), int(rh*resize))) frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) if g.args['display']: cv2.imshow('frame', frame_gray) cv2.imshow('find', g.template) if cv2.waitKey(1) & 0xFF == ord('q'): exit(1) tl,br, minv, maxv = find_in_frame(frame_gray, g.template) #print (maxv) if maxv >= g.args['threshold'] and g.args['present']: # if we want to record frames where the object is present set_frames['frames'].append ({'time': int(frame_cnt/orig_fps), 'frame':frame_cnt, 'location':(tl,br), 'accuracy':'{:.2%}'.format(maxv)}) #matched.append('{}s, Frame: {}, at:{},{} (accuracy:{:.2%})'.format(int(frame_cnt/orig_fps),frame_cnt, tl, br, maxv)) cv2.rectangle(frame, tl, br, (255,0,0), 2) if g.args['write']: # put a box around the object, write to video text = '{}s, Frame: {}'.format(int(frame_cnt/orig_fps), frame_cnt) (tw, th) = cv2.getTextSize(text, cv2.FONT_HERSHEY_PLAIN, fontScale=1.5, thickness=1)[0] cv2.rectangle(frame, (width-tw-5,height-th-5), (width,height), (0,0,0), cv2.FILLED) cv2.putText(frame, text, (width-tw-2, height-2), cv2.FONT_HERSHEY_PLAIN, fontScale=1.5, color=(255,255,255), thickness=1) out.write(frame) frame_found = True if not g.args['all']: break if maxv < g.args['threshold'] and not g.args['present']: # if we want to record frames where the object is absent set_frames['frames'].append ({ 'time': int(frame_cnt/orig_fps), 'frame':frame_cnt, 'location':None, 'accuracy':'{:.2%}'.format(maxv)}) #missing.append('{}s, Frame: {} (accuracy:{:.2%})'.format(int(frame_cnt/orig_fps),frame_cnt, maxv)) if g.args['write']: text = 'MISSING: {}s, Frame: {}'.format(int(frame_cnt/orig_fps), frame_cnt) (tw, th) = cv2.getTextSize(text, cv2.FONT_HERSHEY_PLAIN, fontScale=1.5, thickness=1)[0] cv2.rectangle(frame, (width-tw-5,height-th-5), (width,height), (0,0,255), cv2.FILLED) cv2.putText(frame, text, (width-tw-2, height-2), cv2.FONT_HERSHEY_PLAIN, fontScale=1.5, color=(255,255,255), thickness=1) out.write(frame) frame_found = True if not g.args['all']: break frame_cnt = frame_cnt+1 # all done end_time = time.time() bar.close() # dump matches if frame_found: if g.args['present']: utils.success_print ('Match found in {} frames, starting at {}s, with initial accuracy of {}'.format(len(set_frames['frames']),set_frames['frames'][0]['time'], set_frames['frames'][0]['accuracy'])) g.json_out.append(set_frames) # for match in matched: # print (match) else: utils.success_print ('Object missing in {} frames, starting at {}s'.format(len(set_frames['frames']),set_frames['frames'][0]['time'])) g.json_out.append(set_frames) # for miss in missing: # print (miss) else: print ('No matches found') if g.args['write']: if frame_found: utils.success_print ('Video of frames written to {}'.format(out_file)) else: os.remove(out_file) # blank file, no frames try: if remove_downloaded: os.remove(g.args['input']) # input was a remote file that was downloaded, so remove local download except: pass print ('\nTime: {:.2}s'.format(end_time-start_time)) bar.close() vid.release() if out: out.release() return frame_found
def blend_video(input_file=None, out_file=None, eid=None, mid=None, starttime=None, delay=0): global det, det2 create_blend = False blend_frame_written_count = 0 set_frames = { 'eventid': eid, 'monitorid': mid, 'type': 'object', 'frames': [] } print('Blending: {}'.format(utils.secure_string(input_file))) vid = FVS.FileVideoStream(input_file) time.sleep(1) #vid = cv2.VideoCapture(input_file) cvobj = vid.get_stream_object() if not cvobj.isOpened(): raise ValueError('Error reading video {}'.format( utils.secure_string(input_file))) total_frames_vid = int(cvobj.get(cv2.CAP_PROP_FRAME_COUNT)) vid.start() if not g.orig_fps: orig_fps = max(1, (g.args['fps'] or int(cvobj.get(cv2.CAP_PROP_FPS)))) g.orig_fps = orig_fps else: orig_fps = g.orig_fps width = int(cvobj.get(3)) height = int(cvobj.get(4)) if g.args['resize']: resize = g.args['resize'] #print (width,height, resize) width = int(width * resize) height = int(height * resize) total_frames_vid_blend = 0 if os.path.isfile(blend_filename): vid_blend = FVS.FileVideoStream(blend_filename) time.sleep(1) cvobj_blend = vid_blend.get_stream_object() total_frames_vid_blend = int(cvobj_blend.get(cv2.CAP_PROP_FRAME_COUNT)) vid_blend.start() #vid_blend = cv2.VideoCapture(blend_filename) #utils.dim_print('Video blend {}'.format(vid_blend)) else: vid_blend = None cvobj_blend = None print('blend file will be created in this iteration') create_blend = True fourcc = cv2.VideoWriter_fourcc(*'mp4v') outf = cv2.VideoWriter('new-blended-temp.mp4', fourcc, orig_fps, (width, height)) utils.bold_print('Output video will be {}*{}@{}fps'.format( width, height, orig_fps)) if g.args['skipframes']: fps_skip = g.args['skipframes'] else: fps_skip = max(1, int(cvobj.get(cv2.CAP_PROP_FPS) / 2)) if vid_blend: utils.dim_print('frames in new video: {} vs blend: {}'.format( total_frames_vid, total_frames_vid_blend)) start_time = time.time() utils.dim_print('fps={}, skipping {} frames'.format(orig_fps, fps_skip)) utils.dim_print('delay for new video is {}s'.format(delay)) bar_new_video = tqdm(total=total_frames_vid, desc='New video', miniters=10) bar_blend_video = tqdm(total=total_frames_vid_blend, desc='Blend', miniters=10) is_trailing = False blend_frames_read = 0 # first wait for delay seconds # will only come in if blend video exists, as in first iter it is 0 # However, if blend wasn't created (no relevant frames), ignore delay if delay and not create_blend: frame_cnt = 0 bar_new_video.set_description('waiting for {}s'.format(delay)) prev_good_frame_b = None a = 0 b = 0 while True: if vid_blend and vid_blend.more(): frame_b = vid_blend.read() if frame_b is None: succ_b = False else: succ_b = True a = a + 1 #print ('delay read: {}'.format(a)) blend_frames_read = blend_frames_read + 1 prev_good_frame_b = frame_b else: succ_b = False vid_blend = None # If we have reached the end of blend, but have a good last frame # lets use it if not succ_b and prev_good_frame_b is not None: frame_b = prev_good_frame_b succ_b = True if not succ_b and not prev_good_frame_b: break frame_cnt = frame_cnt + 1 bar_blend_video.update(1) outf.write(frame_b) frame_dummy = np.zeros_like(frame_b) if g.args['display']: x = 320 y = 240 r_frame_b = cv2.resize(frame_b, (x, y)) r_frame_dummy = cv2.resize(frame_dummy, (x, y)) h1 = np.hstack((r_frame_dummy, r_frame_dummy)) h2 = np.hstack((r_frame_dummy, r_frame_b)) f = np.vstack((h1, h2)) cv2.imshow('display', f) if g.args['interactive']: key = cv2.waitKey(0) else: key = cv2.waitKey(1) if key & 0xFF == ord('q'): exit(1) if key & 0xFF == ord('c'): g.args['interactive'] = False blend_frame_written_count = blend_frame_written_count + 1 b = b + 1 #print ('delay write: {}'.format(b)) if (delay * orig_fps < frame_cnt): # if (frame_cnt/orig_fps > delay): #utils.dim_print('wait over') # print ('DELAY={} ORIGFPS={} FRAMECNT={}'.format(delay, orig_fps, frame_cnt)) break # now read new video along with blend bar_new_video.set_description('New video') frame_cnt = 0 while True: if vid.more(): frame = vid.read() if frame is None: succ = False else: succ = True else: frame = None succ = False #succ, frame = vid.read() frame_cnt = frame_cnt + 1 bar_new_video.update(1) if frame_cnt % fps_skip: continue if succ and g.args['resize']: resize = g.args['resize'] rh, rw, rl = frame.shape frame = cv2.resize(frame, (int(rw * resize), int(rh * resize))) succ_b = False if vid_blend: if vid_blend.more(): frame_b = vid_blend.read() if frame_b is None: succ_b = False else: succ_b = True bar_blend_video.update(1) blend_frames_read = blend_frames_read + 1 if not succ and not succ_b: bar_blend_video.write('both videos are done') break elif succ and succ_b: analyze = True relevant = False # may change on analysis #print ("succ and succ_b") elif succ and not succ_b: # print ('blend over') frame_b = frame.copy() analyze = True relevant = False # may change on analysis #print ("succ and not succ_b") elif not succ and succ_b: merged_frame = frame_b frame = frame_b boxed_frame = np.zeros_like(frame_b) txh, txw, _ = frame_b.shape frame_mask = np.zeros((txh, txw), dtype=np.uint8) foreground_a = np.zeros_like(frame_b) analyze = False relevant = True #print ("not succ and succ_b") if analyze: # only if both blend and new were read if g.args['balanceintensity']: intensity = np.mean(frame) intensity_b = np.mean(frame_b) if intensity > intensity_b: # new frame is brighter frame_b = utils.hist_match(frame_b, frame) else: # blend is brighter frame = utils.hist_match(frame, frame_b) #h1, w1 = frame.shape[:2] #hm, wm = frame_b.shape[:2] #print ("{}*{} frame == {}*{} frame_b".format(h1,w1,hm,wm)) merged_frame, foreground_a, frame_mask, relevant, boxed_frame = det.detect( frame, frame_b, frame_cnt, orig_fps, starttime, set_frames) #print ('RELEVANT={}'.format(relevant)) if relevant and g.args['detection_type'] == 'mixed': bar_new_video.set_description('YOLO running') #utils.dim_print('Adding YOLO, found relevance in backgroud motion') merged_frame, foreground_a, frame_mask, relevant, boxed_frame = det2.detect( frame, frame_b, frame_cnt, orig_fps, starttime, set_frames) #print ('YOLO RELEVANT={}'.format(relevant)) bar_new_video.set_description('New video') if relevant: is_trailing = True trail_frames = 0 if g.args['display']: x = 320 y = 240 r_frame_b = cv2.resize(frame_b, (x, y)) r_frame = cv2.resize(boxed_frame, (x, y)) r_fga = cv2.resize(foreground_a, (x, y)) r_frame_mask = cv2.resize(frame_mask, (x, y)) r_frame_mask = cv2.cvtColor(r_frame_mask, cv2.COLOR_GRAY2BGR) r_merged_frame = cv2.resize(merged_frame, (x, y)) h1 = np.hstack((r_frame, r_frame_mask)) h2 = np.hstack((r_fga, r_merged_frame)) f = np.vstack((h1, h2)) cv2.imshow('display', f) #cv2.imshow('merged_frame',cv2.resize(merged_frame, (640,480))) #cv2.imshow('frame_mask',cv2.resize(frame_mask, (640,480))) #cv2.imshow('frame_mask',frame_mask) if g.args['interactive']: key = cv2.waitKey(0) else: key = cv2.waitKey(1) if key & 0xFF == ord('q'): exit(1) if key & 0xFF == ord('c'): g.args['interactive'] = False # if we read a blend frame, merged frame will always be written # if we don't have a blend frame, then we write new frame only if its relevant # assuming we want relevant frames if relevant or not g.args['relevantonly'] or succ_b: #print ("WRITING") outf.write(merged_frame) blend_frame_written_count = blend_frame_written_count + 1 elif is_trailing: trail_frames = trail_frames + 1 if trail_frames > g.args['trailframes']: start_trailing = False else: bar_new_video.set_description('Trailing frame') # bar_new_video.write('trail frame: {}'.format(trail_frames)) outf.write(merged_frame) blend_frame_written_count = blend_frame_written_count + 1 else: #print ('irrelevant frame {}'.format(frame_cnt)) pass bar_blend_video.close() bar_new_video.close() vid.stop() outf.release() if vid_blend: vid_blend.stop() print('\n') #input("Press Enter to continue...") if create_blend and blend_frame_written_count == 0: utils.fail_print( 'No relevant frames found, blend file not created. Will try next iteration' ) os.remove('new-blended-temp.mp4') else: rel = 'relevant ' if g.args['relevantonly'] else '' utils.success_print( '{} total {}frames written to blend file ({} read)'.format( blend_frame_written_count, rel, blend_frames_read)) if blend_frame_written_count: try: os.remove(blend_filename) except: pass os.rename('new-blended-temp.mp4', blend_filename) utils.success_print( 'Blended file updated in {}'.format(blend_filename)) else: utils.success_print( 'No frames written this round, not updating blend file') g.json_out.append(set_frames) return False
def annotate_video(input_file=None, eid = None, mid = None, starttime=None): global det, det2 set_frames = { 'eventid': eid, 'monitorid': mid, 'type': 'object', 'frames':[] } print ('annotating: {}'.format(utils.secure_string(input_file))) #vid = cv2.VideoCapture(input_file) vid = FVS.FileVideoStream(input_file) time.sleep(1) cvobj = vid.get_stream_object() vid.start() if not cvobj.isOpened(): raise ValueError('Error reading video {}'.format(utils.secure_string(input_file))) if not g.orig_fps: orig_fps = max(1, (g.args['fps'] or int(cvobj.get(cv2.CAP_PROP_FPS)))) g.orig_fps = orig_fps else: orig_fps = g.orig_fps width = int(cvobj.get(3)) height = int(cvobj.get(4)) if g.args['resize']: resize = g.args['resize'] # print (width,height, resize) width = int(width * resize) height = int(height * resize) fourcc = cv2.VideoWriter_fourcc(*'mp4v') outf = cv2.VideoWriter(annotate_filename, fourcc, orig_fps, (width,height), True) utils.bold_print('Output video will be {}px*{}px @ {}fps'.format(width, height, orig_fps)) if g.args['skipframes']: fps_skip = g.args['skipframes'] else: fps_skip = max(1,int(cvobj.get(cv2.CAP_PROP_FPS)/2)) total_frames = int(cvobj.get(cv2.CAP_PROP_FRAME_COUNT)) start_time = time.time() utils.dim_print ('fps={}, skipping {} frames'.format(orig_fps, fps_skip)) bar_annotate_video = tqdm (total=total_frames, desc='annotating') frame_cnt = 0 while True: if vid.more(): frame = vid.read() if frame is None: succ = False else: succ = True else: frame = None succ = False #succ, frame = vid.read() if not succ: break frame_cnt = frame_cnt + 1 if not frame_cnt % 10: bar_annotate_video.update(10) if frame_cnt % fps_skip: continue if succ and g.args['resize']: resize = g.args['resize'] rh, rw, rl = frame.shape frame = cv2.resize(frame, (int(rw*resize), int(rh*resize))) frame_b = frame.copy() merged_frame, foreground_a, frame_mask, relevant, boxed_frame = det.detect(frame, frame_b, frame_cnt, orig_fps, starttime, set_frames) if relevant and g.args['detection_type'] == 'mixed': bar_annotate_video.set_description('YOLO running') #utils.dim_print('Adding YOLO, found relevance in backgroud motion') merged_frame, foreground_a, frame_mask, relevant, boxed_frame = det2.detect(frame, frame_b, frame_cnt, orig_fps, starttime, set_frames) bar_annotate_video.set_description('annotating') if g.args['display']: x = 320 y = 240 r_frame_b = cv2.resize (frame_b, (x, y)) r_frame = cv2.resize (boxed_frame, (x,y)) r_fga = cv2.resize (foreground_a, (x,y)) r_frame_mask = cv2.resize (frame_mask, (x, y)) r_frame_mask = cv2.cvtColor(r_frame_mask, cv2.COLOR_GRAY2BGR) r_merged_frame = cv2.resize (merged_frame, (x, y)) h1 = np.hstack((r_frame, r_frame_mask)) h2 = np.hstack((r_fga, r_merged_frame)) f = np.vstack((h1,h2)) cv2.imshow('display', f) #cv2.imshow('merged_frame',cv2.resize(merged_frame, (640,480))) #cv2.imshow('frame_mask',cv2.resize(frame_mask, (640,480))) #cv2.imshow('frame_mask',frame_mask) if g.args['interactive']: key = cv2.waitKey(0) else: key = cv2.waitKey(1) if key& 0xFF == ord('q'): exit(1) if key& 0xFF == ord('c'): g.args['interactive']=False if relevant or not g.args['relevantonly']: #print ("WRITING FRAME") outf.write (merged_frame) else: #print ('irrelevant frame {}'.format(frame_cnt)) pass bar_annotate_video.close() vid.stop() outf.release() utils.success_print('annotated file updated in {}'.format(annotate_filename)) g.json_out.append(set_frames) return False