def _find_target_coordinates(self, img): map = self._find_blobs(img, self._score_targets) bin = np.zeros_like(map) # as soon as we have three objects, we stop contours = [] for t in range(0, 255,1): cv2.threshold(map, t, 255,cv2.THRESH_BINARY ,bin) if CV_VERSION == 3: _, contours, h = cv2.findContours(bin,cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) else: contours, h = cv2.findContours(bin, cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) if len(contours) <3: raise EthoscopeException("There should be three targets. Only %i objects have been found" % (len(contours)), img) if len(contours) == 3: break target_diams = [cv2.boundingRect(c)[2] for c in contours] print(target_diams) mean_diam = np.mean(target_diams) mean_sd = np.std(target_diams) if mean_sd/mean_diam > 0.15: #0.10: # Increase the variation threshold raise EthoscopeException("Too much variation in the diameter of the targets. Something must be wrong since all target should have the same size", img) src_points = [] for c in contours: moms = cv2.moments(c) x , y = moms["m10"]/moms["m00"], moms["m01"]/moms["m00"] src_points.append((x,y)) a ,b, c = src_points pairs = [(a,b), (b,c), (a,c)] dists = [self._points_distance(*p) for p in pairs] # that is the AC pair hypo_vertices = pairs[np.argmax(dists)] # this is B : the only point not in (a,c) for sp in src_points: if not sp in hypo_vertices: break sorted_b = sp dist = 0 for sp in src_points: if sorted_b is sp: continue # b-c is the largest distance, so we can infer what point is c if self._points_distance(sp, sorted_b) > dist: dist = self._points_distance(sp, sorted_b) sorted_c = sp # the remaining point is a sorted_a = [sp for sp in src_points if not sp is sorted_b and not sp is sorted_c][0] sorted_src_pts = np.array([sorted_a, sorted_b, sorted_c], dtype=np.float32) return sorted_src_pts
def _find_target_coordinates(self, img, blob_function): cv2.imwrite( os.path.join(os.environ["HOME"], "target_dection_find_target_coordinates.png"), img) map = blob_function(img) cv2.imwrite( os.path.join(os.environ["HOME"], "target_dection_find_target_coordinates_after.png"), img) if debug: thresh = cv2.threshold(map, 1, 255, cv2.THRESH_BINARY)[1] cv2.imshow("map", thresh) cv2.waitKey(0) bin = np.zeros_like(map) # as soon as we have three objects, we stop contours = [] for t in range(0, 255, 1): cv2.threshold(map, t, 255, cv2.THRESH_BINARY, bin) if CV_VERSION == 3: _, contours, h = cv2.findContours(bin, cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) else: contours, h = cv2.findContours(bin, cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) if debug: cv2.imshow('bin', bin) cv2.waitKey(0) if len(contours) < 3: raise EthoscopeException( f"There should be three targets. Only {len(contours)} objects have been found", img) #raise EthoscopeException(f"There should be three targets. Only {len(contours)} objects have been found", np.stack((img, self._orig), axis=1)) if len(contours) == 3: break target_diams = [cv2.boundingRect(c)[2] for c in contours] mean_diam = np.mean(target_diams) mean_sd = np.std(target_diams) if mean_sd / mean_diam > 0.10: raise EthoscopeException( "Too much variation in the diameter of the targets. Something must be wrong since all target should have the same size", img) src_points = [] for c in contours: moms = cv2.moments(c) x, y = moms["m10"] / moms["m00"], moms["m01"] / moms["m00"] src_points.append((x, y)) sorted_src_pts = self._sort_src_pts(src_points) return sorted_src_pts
def __init__(self, device=0, target_fps=5, target_resolution=(960,720), *args, **kwargs): """ class to acquire stream from a video for linux compatible device (v4l2). :param device: The index of the device, or its path. :type device: int or str :param target_fps: the desired number of frames par second (FPS) :type target_fps: int :param target_fps: the desired resolution (W x H) :param target_resolution: (int,int) :param args: additional arguments :param kwargs: additional keyword arguments """ self.canbepickled = False self.capture = cv2.VideoCapture(device) self._warm_up() w, h = target_resolution if w <0 or h <0: self.capture.set(CAP_PROP_FRAME_WIDTH, 99999) self.capture.set(CAP_PROP_FRAME_HEIGHT, 99999) else: self.capture.set(CAP_PROP_FRAME_WIDTH, w) self.capture.set(CAP_PROP_FRAME_HEIGHT, h) if not isinstance(target_fps, int): raise EthoscopeException("FPS must be an integer number") if target_fps < 2: raise EthoscopeException("FPS must be at least 2") self.capture.set(CAP_PROP_FPS, target_fps) self._target_fps = float(target_fps) _, im = self.capture.read() # preallocate image buffer => faster if im is None: raise EthoscopeException("Error whist retrieving video frame. Got None instead. Camera not plugged?") self._frame = im assert(len(im.shape) >1) self._resolution = (im.shape[1], im.shape[0]) if self._resolution != target_resolution: if w > 0 and h > 0: logging.warning('Target resolution "%s" could NOT be achieved. Effective resolution is "%s"' % (target_resolution, self._resolution )) else: logging.info('Maximal effective resolution is "%s"' % str(self._resolution)) super(V4L2Camera, self).__init__(*args, **kwargs) self._start_time = time.time()
def __init__(self, target_fps=20, target_resolution=(1280, 960), *args, **kwargs): """ Class to acquire frames from the raspberry pi camera asynchronously. At the moment, frames are only greyscale images. :param target_fps: the desired number of frames par second (FPS) :type target_fps: int :param target_fps: the desired resolution (W x H) :param target_resolution: (int,int) :param args: additional arguments :param kwargs: additional keyword arguments """ logging.info("Initialising camera") print("Initialising camera") self.canbepickled = True #cv2.videocapture object cannot be serialized, hence cannot be picked w,h = target_resolution if not isinstance(target_fps, int): raise EthoscopeException("FPS must be an integer number") self._args = args self._kwargs = kwargs self._queue = multiprocessing.Queue(maxsize=1) self._stop_queue = multiprocessing.JoinableQueue(maxsize=1) self._p = self._frame_grabber_class(target_fps,target_resolution,self._queue,self._stop_queue, *args, **kwargs) self._p.daemon = True self._p.start() try: im = self._queue.get(timeout=10) except Exception as e: logging.error("Could not get any frame from the camera") self._stop_queue.cancel_join_thread() self._queue.cancel_join_thread() logging.warning("Stopping stop queue") self._stop_queue.close() logging.warning("Stopping queue") self._queue.close() logging.warning("Joining process") # we kill the frame grabber if it does not reply within 10s self._p.join(10) logging.warning("Process joined") raise e self._frame = cv2.cvtColor(im,cv2.COLOR_GRAY2BGR) if len(im.shape) < 2: raise EthoscopeException("The camera image is corrupted (less that 2 dimensions)") self._resolution = (im.shape[1], im.shape[0]) if self._resolution != target_resolution: if w > 0 and h > 0: logging.warning('Target resolution "%s" could NOT be achieved. Effective resolution is "%s"' % (target_resolution, self._resolution )) else: logging.info('Maximal effective resolution is "%s"' % str(self._resolution)) super(OurPiCameraAsync, self).__init__(*args, **kwargs) self._start_time = time.time() logging.info("Camera initialised")
def __init__(self, target_fps=20, target_resolution=(1280, 960), *args, **kwargs): """ Class to acquire frames from the raspberry pi camera asynchronously. At the moment, frames are greyscale images. :param target_fps: the desired number of frames par second (FPS) :type target_fps: int :param target_fps: the desired resolution (W x H) :param target_resolution: (int,int) :param args: additional arguments :param kwargs: additional keyword arguments """ logging.info("Initialising camera") w, h = target_resolution if not isinstance(target_fps, int): raise EthoscopeException("FPS must be an integer number") self._queue = multiprocessing.Queue(maxsize=1) self._stop_queue = multiprocessing.JoinableQueue(maxsize=1) self._p = PiFrameGrabber(target_fps, target_resolution, self._queue, self._stop_queue) self._p.daemon = True self._p.start() im = self._queue.get(timeout=10) self._frame = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) if len(im.shape) < 2: raise EthoscopeException( "The camera image is corrupted (less that 2 dimensions)") self._resolution = (im.shape[1], im.shape[0]) if self._resolution != target_resolution: if w > 0 and h > 0: logging.warning( 'Target resolution "%s" could NOT be achieved. Effective resolution is "%s"' % (target_resolution, self._resolution)) else: logging.info('Maximal effective resolution is "%s"' % str(self._resolution)) super(OurPiCameraAsync, self).__init__(*args, **kwargs) self._start_time = time.time() logging.info("Camera initialised")
def __init__(self, target_fps=10, target_resolution=(960, 720), *args, **kwargs): logging.info("Initialising camera") w, h = target_resolution self.capture = PiCamera() self.capture.resolution = target_resolution if not isinstance(target_fps, int): raise EthoscopeException("FPS must be an integer number") self.capture.framerate = target_fps self._raw_capture = PiRGBArray(self.capture, size=target_resolution) self._target_fps = float(target_fps) self._warm_up() self._cap_it = self._frame_iter() im = next(self._cap_it) if im is None: raise EthoscopeException( "Error whist retrieving video frame. Got None instead. Camera not plugged?" ) self._frame = im if len(im.shape) < 2: raise EthoscopeException( "The camera image is corrupted (less that 2 dimensions)") self._resolution = (im.shape[1], im.shape[0]) if self._resolution != target_resolution: if w > 0 and h > 0: logging.warning( 'Target resolution "%s" could NOT be achieved. Effective resolution is "%s"' % (target_resolution, self._resolution)) else: logging.info('Maximal effective resolution is "%s"' % str(self._resolution)) super(OurPiCamera, self).__init__(*args, **kwargs) self._start_time = time.time() logging.info("Camera initialised")
def __iter__(self): """ Iterate thought consecutive frames of this camera. :return: the time (in ms) and a frame (numpy array). :rtype: (int, :class:`~numpy.ndarray`) """ at_least_one_frame = False while True: if self.is_last_frame() or not self.is_opened(): if not at_least_one_frame: raise EthoscopeException( "Camera could not read the first frame") break t, out = self._next_time_image() if out is None: break t_ms = int(1000 * t) at_least_one_frame = True if (self._frame_idx % self._drop_each) == 0: yield t_ms, out if self._max_duration is not None and t > self._max_duration: break
def _next_image(self): try: g = self._queue.get(timeout=30) cv2.cvtColor(g,cv2.COLOR_GRAY2BGR,self._frame) return self._frame except Exception as e: raise EthoscopeException("Could not get frame from camera\n%s", traceback.format_exc(e))
def _next_image(self): try: g = self._queue.get(timeout=30) self._frame = imread(g) return self._frame except Exception as e: raise EthoscopeException("Could not get frame from camera\n%s", str(e))
def __init__(self, path, use_wall_clock=False, *args, **kwargs): """ Class to acquire frames from a video file. :param path: the path of the video file :type path: str :param use_wall_clock: whether to use the real time from the machine (True) or from the video file (False).\ The former can be useful for prototyping. :type use_wall_clock: bool :param args: additional arguments. :param kwargs: additional keyword arguments. """ #print "path", path self._frame_idx = 0 self._path = path self._use_wall_clock = use_wall_clock if not (isinstance(path, str) or isinstance(path, unicode)): raise EthoscopeException("path to video must be a string") if not os.path.exists(path): raise EthoscopeException("'%s' does not exist. No such file" % path) self.canbepickled = False #cv2.videocapture object cannot be serialized, hence cannot be picked self.capture = cv2.VideoCapture(path) w = self.capture.get(CAP_PROP_FRAME_WIDTH) h = self.capture.get(CAP_PROP_FRAME_HEIGHT) self._total_n_frames = self.capture.get(CAP_PROP_FRAME_COUNT) self._fps = self.capture.get(CAP_PROP_FPS) self._fourcc = self.capture.get(CAP_PROP_FOURCC) if self._total_n_frames == 0.: self._has_end_of_file = False else: self._has_end_of_file = True self._resolution = (int(w), int(h)) super(MovieVirtualCamera, self).__init__(*args, **kwargs) # emulates v4l2 (real time camera) from video file if self._use_wall_clock: self._start_time = time.time() else: self._start_time = 0
def _find_target_coordinates(self, img, blob_function): map = blob_function(img) _, map_bin = cv2.threshold(map, 1, 255, cv2.THRESH_BINARY) bin = np.zeros_like(map) # as soon as we have three objects, we stop contours = [] for t in range(0, 255,1): cv2.threshold(map, t, 255,cv2.THRESH_BINARY, bin) if CV_VERSION == 3: _, contours, h = cv2.findContours(bin,cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) else: contours, h = cv2.findContours(bin, cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) if len(contours) <3: raise EthoscopeException("There should be three targets. Only %i objects have been found" % (len(contours)), img) if len(contours) == 3: break target_diams = [cv2.boundingRect(c)[2] for c in contours] mean_diam = np.mean(target_diams) mean_sd = np.std(target_diams) if mean_sd/mean_diam > 0.10: raise EthoscopeException("Too much variation in the diameter of the targets. Something must be wrong since all target should have the same size", img) src_points = [] for c in contours: moms = cv2.moments(c) x , y = moms["m10"]/moms["m00"], moms["m01"]/moms["m00"] src_points.append((x,y)) # map_bin_dots1 = place_dots(map_bin.copy(), src_points, color = 0) sorted_src_pts = self._sort_src_pts(src_points) map_bin_dots = place_dots(map_bin.copy(), sorted_src_pts, color = 0) # cv2.imwrite(os.path.join(os.environ["HOME"], "target_detection_segmented_dots2.png" ), map_bin_dots) return sorted_src_pts
def __init__(self, target_fps=10, target_resolution=(960, 720), *args, **kwargs): logging.info("Initialising camera") w, h = target_resolution if not isinstance(target_fps, int): raise EthoscopeException("FPS must be an integer number") self._queue = multiprocessing.Queue(maxsize=2) self._stop_queue = multiprocessing.JoinableQueue(maxsize=1) self._p = PiFrameGrabber(target_fps, target_resolution, self._queue, self._stop_queue) self._p.daemon = True self._p.start() im = self._queue.get(timeout=5) self._frame = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) if len(im.shape) < 2: raise EthoscopeException( "The camera image is corrupted (less that 2 dimensions)") self._resolution = (im.shape[1], im.shape[0]) if self._resolution != target_resolution: if w > 0 and h > 0: logging.warning( 'Target resolution "%s" could NOT be achieved. Effective resolution is "%s"' % (target_resolution, self._resolution)) else: logging.info('Maximal effective resolution is "%s"' % str(self._resolution)) super(OurPiCameraAsync, self).__init__(*args, **kwargs) self._start_time = time.time() logging.info("Camera initialised")
def apply(self,img): """ Cut an image where the ROI is defined. :param img: An image. Typically either one or three channels `uint8`. :type img: :class:`~numpy.ndarray` :return: a tuple containing the resulting cropped image and the associated mask (both have the same dimension). :rtype: (:class:`~numpy.ndarray`, :class:`~numpy.ndarray`) """ x,y,w,h = self._rectangle try: out = img[y : y + h, x : x +w] except: raise EthoscopeException("Error whilst slicing region of interest %s" % str(self.get_feature_dict()), img) if out.shape[0:2] != self._mask.shape: raise EthoscopeException("Error whilst slicing region of interest. Possibly, the region out of the image: %s" % str(self.get_feature_dict()), img ) return out, self._mask
def _sort_src_pts(self, src_points): a, b, c = src_points #if debug: # for pt in src_points: # pt = tuple((int(e) for e in pt)) # thresh = cv2.circle(thresh, pt, 5, 0, -1) pairs = [(a,b), (b,c), (a,c)] dists = [self._points_distance(*p) for p in pairs] for d in dists: logging.info(d) if d < self._expected_min_target_dist: img = self._img if len(self._img.shape) == 2 else cv2.cvtColor(self._img, cv2.COLOR_BGR2GRAY) img = place_dots(self._img, src_points) output_path = os.path.join(os.environ["HOME"], "target_detection_fail_output.png") logging.info(f"Saving failed detection to {output_path}") cv2.imwrite(output_path, img) raise EthoscopeException(f"""The detected targets are too close. If you think this is correct, please decrease the value of _expected_min_target_dist to something that fits your setup. It is set to {self._expected_min_target_dist}""") # that is the AC pair hypo_vertices = pairs[np.argmax(dists)] # this is B : the only point not in (a,c) for sp in src_points: if not sp in hypo_vertices: break sorted_b = sp dist = 0 for sp in src_points: if sorted_b is sp: continue # b-c is the largest distance, so we can infer what point is c if self._points_distance(sp, sorted_b) > dist: dist = self._points_distance(sp, sorted_b) sorted_c = sp # the remaining point is a sorted_a = [sp for sp in src_points if not sp is sorted_b and not sp is sorted_c][0] sorted_src_pts = np.array([sorted_a, sorted_b, sorted_c], dtype=np.float32) self._sorted_src_pts = sorted_src_pts logging.warning("Sorted detection") logging.warning(sorted_src_pts) return sorted_src_pts
def _split_rois(self, bin, orig): bin_orig = bin.copy() if CV_VERSION == 3: _, contours, _ = cv2.findContours(bin, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) else: contours, _ = cv2.findContours(bin, cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) if debug: cv2.imshow("bin", bin) cv2.waitKey(0) while np.count_nonzero(bin) > 0: # bin = cv2.morphologyEx(bin, cv2.MORPH_TOPHAT, (9, 9)) bin = cv2.erode(bin, np.ones((1, 10))) if CV_VERSION == 3: _, contours, _ = cv2.findContours(bin, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) else: contours, _ = cv2.findContours(bin, cv2.RETR_EXTERNAL, CHAIN_APPROX_SIMPLE) if debug: cv2.imshow("eroded_bin", bin) cv2.waitKey(0) if len(contours) == 20: break if np.count_nonzero(bin) == 0: # TODO Adapt to any number of ROIs cv2.imwrite( os.path.join(os.environ["HOME"], 'failed_roi_builder_binary.png'), bin_orig) cv2.imwrite( os.path.join(os.environ["HOME"], 'failed_roi_builder_orig'), orig) raise EthoscopeException( 'I could not find 20 ROIs. Please try again or change the lighting conditions', np.stack((bin_orig, cv2.cvtColor(orig, cv2.COLOR_BGR2GRAY)), axis=1)) return contours
def __init__(self, mask_path): """ Class to build rois from greyscale image file. Each continuous region is used as a ROI. The greyscale value inside the ROI determines it's value. IMAGE HERE """ if not os.path.exists(mask_path): raise EthoscopeException("'%s' does not exist. No such file" % mask_path) self._mask = cv2.imread(mask_path, IMG_READ_FLAG_GREY) self._rois = [] self._rois_bounding_box = None super(ImgMaskROIBuilder, self).__init__()
def find_target_coordinates(self, camera): logging.info( "Checking targets in resulting video are visible and video is suitable for offline analysis" ) # to analyzs the same frames offline from ethoscope.hardware.input.cameras import MovieVirtualCamera from ethoscope.roi_builders.target_roi_builder import FSLSleepMonitorWithTargetROIBuilder from ethoscope.drawers.drawers import DefaultDrawer from ethoscope.core.tracking_unit import TrackingUnit from ethoscope.trackers.adaptive_bg_tracker import AdaptiveBGModel as tracker_class i = 0 try: # Log camera status and take shot target_detection_path = "/tmp/target_detection_{}.png" camera.framerate = 2 time.sleep(1) camera = configure_camera(camera, mode="target_detection") n = 0 roi_builder = FSLSleepMonitorWithTargetROIBuilder() drawer = DefaultDrawer() target_coord_file = self._video_prefix + "targets.pickle" rois_file = self._video_prefix + "rois.pickle" failure = True while failure and n < 5: try: camera = configure_camera(camera, mode="target_detection") report_camera(camera) camera.capture(target_detection_path.format(n)) time.sleep(1) img = cv2.imread(target_detection_path.format(n)) rois = roi_builder.build(img) logging.info("Annotating frame for human supervision") unit_trackers = [ TrackingUnit(tracker_class, r, None) for r in rois ] annotated = drawer.draw(img, tracking_units=unit_trackers, positions=None) tmp_dir = os.path.dirname(self._img_path) annotated_path = os.path.join(tmp_dir, "last_img_annotated.jpg") logging.info(f"Saving annotated frame to {annotated_path}") cv2.imwrite(annotated_path, annotated) logging.info("Saving pickle files") with open(target_coord_file, "wb") as fh: pickle.dump(roi_builder._sorted_src_pts, fh) with open(rois_file, "wb") as fh: pickle.dump(rois, fh) failure = False except Exception as e: logging.info(e) failure = True # not needed, but for readability n += 1 if failure: logging.info("Failure") raise EthoscopeException( "I tried taking 5 shots and none is good! Check them at /tmp/target_detection_{i}.png" ) else: logging.info("Success") return True except Exception as e: logging.error("Error on starting video recording process:" + traceback.format_exc())
def get_hull(self, img=None, t=None): if self.live_tracking and (img is None or t is None): raise EthoscopeException("Invalid input to get_hull") self._buff_fg_backup = np.copy(self._buff_fg) if self.live_tracking: self._check_prop_fg_pix() if CV_VERSION == 3: _, contours, _ = cv2.findContours(self._buff_fg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) else: contours, _ = cv2.findContours(self._buff_fg, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) contours = [cv2.approxPolyDP(c, 1.2, True) for c in contours] is_ambiguous = False if not contours: self._bg_model.increase_learning_rate() raise NoPositionError elif len(contours) > 1: if not self.live_tracking: hull = sorted(contours, key=cv2.contourArea, reverse=True)[0] return hull, False if not self.fg_model.is_ready: raise NoPositionError # hulls = [cv2.convexHull( c) for c in contours] hulls = contours #hulls = merge_blobs(hulls) hulls = [h for h in hulls if h.shape[0] >= 3] if len(hulls) < 1: raise NoPositionError elif len(hulls) > 1: is_ambiguous = True cluster_features = [self.fg_model.compute_features(img, h) for h in hulls] all_distances = [self.fg_model.distance(cf, t) for cf in cluster_features] good_clust = np.argmin(all_distances) hull = hulls[good_clust] distance = all_distances[good_clust] else: hull = contours[0] if not self.live_tracking: return hull, False if hull.shape[0] < 3: self._bg_model.increase_learning_rate() raise NoPositionError features = self.fg_model.compute_features(img, hull) distance = self.fg_model.distance(features, t) if distance > self._max_m_log_lik and self.live_tracking: self._bg_model.increase_learning_rate() raise NoPositionError return hull, is_ambiguous