コード例 #1
0
class FakeCam:
    def __init__(
        self,
        fps: int,
        width: int,
        height: int,
        codec: str,
        scale_factor: float,
        no_background: bool,
        background_blur: int,
        background_keep_aspect: bool,
        use_foreground: bool,
        hologram: bool,
        tiling: bool,
        bodypix_url: str,
        socket: str,
        background_image: str,
        foreground_image: str,
        foreground_mask_image: str,
        webcam_path: str,
        v4l2loopback_path: str,
        use_akvcam: bool,
        use_opencv: bool,
    ) -> None:
        self.no_background = no_background
        self.use_foreground = use_foreground
        self.hologram = hologram
        self.tiling = tiling
        self.background_blur = background_blur
        self.background_keep_aspect = background_keep_aspect
        self.background_image = background_image
        self.foreground_image = foreground_image
        self.foreground_mask_image = foreground_mask_image
        self.scale_factor = scale_factor
        self.real_cam = RealCam(webcam_path, width, height, fps, codec)
        # In case the real webcam does not support the requested mode.
        self.width = self.real_cam.get_frame_width()
        self.height = self.real_cam.get_frame_height()
        self.use_akvcam = use_akvcam
        self.use_opencv = use_opencv
        if not use_akvcam:
            self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path,
                                                    self.width, self.height)
        else:
            self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width,
                                            self.height)
        self.foreground_mask = None
        self.inverted_foreground_mask = None
        self.session = requests.Session()
        if use_opencv == None:
            if bodypix_url.startswith('/'):
                print("Looks like you want to use a unix socket")
                # self.session = requests_unixsocket.Session()
                self.bodypix_url = "http+unix:/" + bodypix_url
                self.socket = bodypix_url
                requests_unixsocket.monkeypatch()
        else:
            self.bodypix_url = bodypix_url
            self.socket = ""
        #If the session can't connect, set the session to None so that _get_mask will fall back
        #on to openCV
        # self.session = requests.Session()
        self.images: Dict[str, Any] = {}
        self.image_lock = asyncio.Lock()

    async def _get_mask(self, frame, session):
        frame = cv2.resize(frame, (0, 0),
                           fx=self.scale_factor,
                           fy=self.scale_factor)
        _, data = cv2.imencode(".png", frame)
        #print("Posting to " + self.bodypix_url)
        if self.use_opencv is False:
            async with session.post(
                    url=self.bodypix_url,
                    data=data.tobytes(),
                    headers={"Content-Type": "application/octet-stream"}) as r:
                mask = np.frombuffer(await r.read(), dtype=np.uint8)
                mask = mask.reshape((frame.shape[0], frame.shape[1]))
                mask = cv2.resize(mask, (0, 0),
                                  fx=1 / self.scale_factor,
                                  fy=1 / self.scale_factor,
                                  interpolation=cv2.INTER_NEAREST)
                mask = cv2.dilate(mask,
                                  np.ones((10, 10), np.uint8),
                                  iterations=1)
                mask = cv2.blur(mask.astype(float), (30, 30))
        else:
            backSub = self.real_cam.get_backsub()
            #Blur the frame (just like the backSub)
            frame = cv2.GaussianBlur(frame, (15, 15), 0)
            #frame = cv2.blur(frame,(5,5),0)
            mask = backSub.apply(frame, 0, 0)
            mask = mask.reshape((frame.shape[0], frame.shape[1]))
            _, mask = cv2.threshold(mask, 90, 255,
                                    cv2.THRESH_BINARY + cv2.THRESH_OTSU)

            contours, hierarchy = cv2.findContours(
                mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2:]
            try:
                contour = max(contours, key=cv2.contourArea)
                mask = np.zeros((frame.shape[0], frame.shape[1]))
                cv2.drawContours(mask, [contour],
                                 -1, (255, 255, 255),
                                 thickness=-1)
            except:
                pass
            #mask = cv2.GaussianBlur(mask,(15,15),0)
            #mask = cv2.blur(mask,(15,15),0)

            mask = cv2.resize(mask, (0, 0),
                              fx=1 / self.scale_factor,
                              fy=1 / self.scale_factor,
                              interpolation=cv2.INTER_NEAREST)
            mask = -mask

            #If the mask lets more than 40% of the image through, let nothing through
            #if((numpy.sum(mask)/np.size(mask))> .4):
            #    mask = 0
            #If I were to do contour detection, it would be here:
            #else:
            #    pass
        return mask

    def shift_image(self, img, dx, dy):
        img = np.roll(img, dy, axis=0)
        img = np.roll(img, dx, axis=1)
        if dy > 0:
            img[:dy, :] = 0
        elif dy < 0:
            img[dy:, :] = 0
        if dx > 0:
            img[:, :dx] = 0
        elif dx < 0:
            img[:, dx:] = 0
        return img

    """Rescale image to dimensions self.width, self.height. If keep_aspect is True
then scale & crop the image so that its pixels retain their aspect ratio."""

    def resize_image(self, img, keep_aspect):
        if self.width == 0 or self.height == 0:
            raise RuntimeError("Camera dimensions error w={} h={}".format(
                self.width, self.height))
        if keep_aspect:
            imgheight, imgwidth, = img.shape[:2]
            scale = max(self.width / imgwidth, self.height / imgheight)
            newimgwidth, newimgheight = int(np.floor(self.width / scale)), int(
                np.floor(self.height / scale))
            ix0 = int(np.floor(0.5 * imgwidth - 0.5 * newimgwidth))
            iy0 = int(np.floor(0.5 * imgheight - 0.5 * newimgheight))
            img = cv2.resize(
                img[iy0:iy0 + newimgheight, ix0:ix0 + newimgwidth, :],
                (self.width, self.height))
        else:
            img = cv2.resize(img, (self.width, self.height))
        return img

    async def load_images(self):
        async with self.image_lock:
            self.images: Dict[str, Any] = {}

            background = cv2.imread(self.background_image)
            if background is not None:
                if not self.tiling:
                    background = self.resize_image(background,
                                                   self.background_keep_aspect)
                else:
                    sizey, sizex = background.shape[0], background.shape[1]
                    if sizex > self.width and sizey > self.height:
                        background = cv2.resize(background,
                                                (self.width, self.height))
                    else:
                        repx = (self.width - 1) // sizex + 1
                        repy = (self.height - 1) // sizey + 1
                        background = np.tile(background, (repy, repx, 1))
                        background = background[0:self.height, 0:self.width]
                background = itertools.repeat(background)
            else:
                background_video = cv2.VideoCapture(self.background_image)
                if not background_video.isOpened():
                    raise RuntimeError("Couldn't open video '{}'".format(
                        self.background_image))
                self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
                # Initiate current fps to background video fps
                self.current_fps = self.bg_video_fps

                def read_frame():
                    ret, frame = background_video.read()
                    if not ret:
                        background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
                        ret, frame = background_video.read()
                        assert ret, 'cannot read frame %r' % self.background_image
                    return self.resize_image(frame,
                                             self.background_keep_aspect)

                def next_frame():
                    while True:
                        advrate = self.bg_video_fps / self.current_fps  # Number of frames we need to advance background movie. Fractional.
                        if advrate < 1:
                            # Number of frames<1 so to avoid movie freezing randomly choose whether to advance by one frame with correct probability.
                            self.bg_video_adv_rate = 1 if np.random.uniform(
                            ) < advrate else 0
                        else:
                            # Just round to nearest number of frames when >=1.
                            self.bg_video_adv_rate = round(advrate)
                        for i in range(self.bg_video_adv_rate):
                            frame = read_frame()
                        yield frame

                background = next_frame()

            self.images["background"] = background

            if self.use_foreground and self.foreground_image is not None:
                foreground = cv2.imread(self.foreground_image)
                self.images["foreground"] = cv2.resize(
                    foreground, (self.width, self.height))
                foreground_mask = cv2.imread(self.foreground_mask_image)
                foreground_mask = cv2.normalize(foreground_mask,
                                                None,
                                                alpha=0,
                                                beta=1,
                                                norm_type=cv2.NORM_MINMAX,
                                                dtype=cv2.CV_32F)
                foreground_mask = cv2.resize(foreground_mask,
                                             (self.width, self.height))
                self.images["foreground_mask"] = cv2.cvtColor(
                    foreground_mask, cv2.COLOR_BGR2GRAY)
                self.images["inverted_foreground_mask"] = 1 - self.images[
                    "foreground_mask"]

    def hologram_effect(self, img):
        # add a blue tint
        holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
        # add a halftone effect
        bandLength, bandGap = 3, 4
        for y in range(holo.shape[0]):
            if y % (bandLength + bandGap) < bandLength:
                holo[y, :, :] = holo[y, :, :] * np.random.uniform(0.1, 0.3)
        # add some ghosting
        holo_blur = cv2.addWeighted(holo, 0.2,
                                    self.shift_image(holo.copy(), 5, 5), 0.8,
                                    0)
        holo_blur = cv2.addWeighted(holo_blur, 0.4,
                                    self.shift_image(holo.copy(), -5, -5), 0.6,
                                    0)
        # combine with the original color, oversaturated
        out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
        return out

    async def mask_frame(self, session, frame):
        # fetch the mask with retries (the app needs to warmup and we're lazy)
        # e v e n t u a l l y c o n s i s t e n t
        mask = None
        while mask is None:
            try:
                mask = await self._get_mask(frame, session)
            except Exception as e:
                print(f"Mask request failed, retrying: {e}")
                traceback.print_exc()

        foreground_frame = background_frame = frame
        if self.hologram:
            foreground_frame = self.hologram_effect(foreground_frame)

        background_frame = cv2.blur(
            frame, (self.background_blur, self.background_blur),
            cv2.BORDER_DEFAULT)

        # composite the foreground and background
        async with self.image_lock:
            if self.no_background is False:
                background_frame = next(self.images["background"])
            for c in range(frame.shape[2]):
                frame[:, :,
                      c] = foreground_frame[:, :,
                                            c] * mask + background_frame[:, :,
                                                                         c] * (
                                                                             1
                                                                             -
                                                                             mask
                                                                         )

            if self.use_foreground and self.foreground_image is not None:
                for c in range(frame.shape[2]):
                    frame[:, :, c] = (frame[:, :, c] *
                                      self.images["inverted_foreground_mask"] +
                                      self.images["foreground"][:, :, c] *
                                      self.images["foreground_mask"])

        return frame

    def put_frame(self, frame):
        self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

    def stop(self):
        self.real_cam.stop()
        if self.use_akvcam:
            self.fake_cam.__del__()

    async def run(self):
        await self.load_images()
        self.real_cam.start()
        if self.socket != "":
            conn = aiohttp.UnixConnector(path=self.socket)
        else:
            conn = None
        async with aiohttp.ClientSession(connector=conn) as session:
            t0 = time.monotonic()
            print_fps_period = 1
            frame_count = 0
            while True:
                frame = self.real_cam.read()
                if frame is None:
                    await asyncio.sleep(0.1)
                    continue
                frame = await self.mask_frame(session, frame)
                self.put_frame(frame)
                frame_count += 1
                td = time.monotonic() - t0
                if td > print_fps_period:
                    self.current_fps = frame_count / td
                    print("FPS: {:6.2f}".format(self.current_fps), end="\r")
                    frame_count = 0
                    t0 = time.monotonic()
コード例 #2
0
class FakeCam:
    def __init__(self, fps: int, width: int, height: int, codec: str,
                 no_background: bool, background_blur: int,
                 background_keep_aspect: bool, use_foreground: bool,
                 hologram: bool, tiling: bool, background_image: str,
                 foreground_image: str, foreground_mask_image: str,
                 webcam_path: str, v4l2loopback_path: str,
                 use_akvcam: bool) -> None:
        self.no_background = no_background
        self.use_foreground = use_foreground
        self.hologram = hologram
        self.tiling = tiling
        self.background_blur = background_blur
        self.background_keep_aspect = background_keep_aspect
        self.background_image = background_image
        self.foreground_image = foreground_image
        self.foreground_mask_image = foreground_mask_image
        self.real_cam = RealCam(webcam_path, width, height, fps, codec)
        # In case the real webcam does not support the requested mode.
        self.width = self.real_cam.get_frame_width()
        self.height = self.real_cam.get_frame_height()
        self.use_akvcam = use_akvcam
        if not use_akvcam:
            self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path,
                                                    self.width, self.height)
        else:
            self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width,
                                            self.height)
        self.foreground_mask = None
        self.inverted_foreground_mask = None
        self.images: Dict[str, Any] = {}
        self.image_lock = asyncio.Lock()
        self.bodypix_model = load_model(
            download_model(BodyPixModelPaths.MOBILENET_FLOAT_50_STRIDE_16))

    def _get_mask(self, frame):
        res = self.bodypix_model.predict_single(frame)
        mask = res.get_mask(threshold=0.60)
        mask = np.frombuffer(mask, dtype=np.uint32)
        mask = mask.reshape((frame.shape[0], frame.shape[1]))
        mask = cv2.dilate(mask.astype(float),
                          np.ones((10, 10), np.uint8),
                          iterations=1)
        mask = cv2.blur(mask.astype(float), (30, 30))
        return mask

    def shift_image(self, img, dx, dy):
        img = np.roll(img, dy, axis=0)
        img = np.roll(img, dx, axis=1)
        if dy > 0:
            img[:dy, :] = 0
        elif dy < 0:
            img[dy:, :] = 0
        if dx > 0:
            img[:, :dx] = 0
        elif dx < 0:
            img[:, dx:] = 0
        return img

    """Rescale image to dimensions self.width, self.height. If keep_aspect is True
then scale & crop the image so that its pixels retain their aspect ratio."""

    def resize_image(self, img, keep_aspect):
        if self.width == 0 or self.height == 0:
            raise RuntimeError("Camera dimensions error w={} h={}".format(
                self.width, self.height))
        if keep_aspect:
            imgheight, imgwidth, = img.shape[:2]
            scale = max(self.width / imgwidth, self.height / imgheight)
            newimgwidth, newimgheight = int(np.floor(self.width / scale)), int(
                np.floor(self.height / scale))
            ix0 = int(np.floor(0.5 * imgwidth - 0.5 * newimgwidth))
            iy0 = int(np.floor(0.5 * imgheight - 0.5 * newimgheight))
            img = cv2.resize(
                img[iy0:iy0 + newimgheight, ix0:ix0 + newimgwidth, :],
                (self.width, self.height))
        else:
            img = cv2.resize(img, (self.width, self.height))
        return img

    async def load_images(self):
        async with self.image_lock:
            self.images: Dict[str, Any] = {}

            background = cv2.imread(self.background_image)
            if background is not None:
                if not self.tiling:
                    background = self.resize_image(background,
                                                   self.background_keep_aspect)
                else:
                    sizey, sizex = background.shape[0], background.shape[1]
                    if sizex > self.width and sizey > self.height:
                        background = cv2.resize(background,
                                                (self.width, self.height))
                    else:
                        repx = (self.width - 1) // sizex + 1
                        repy = (self.height - 1) // sizey + 1
                        background = np.tile(background, (repy, repx, 1))
                        background = background[0:self.height, 0:self.width]
                background = itertools.repeat(background)
            else:
                background_video = cv2.VideoCapture(self.background_image)
                if not background_video.isOpened():
                    raise RuntimeError("Couldn't open video '{}'".format(
                        self.background_image))
                self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
                # Initiate current fps to background video fps
                self.current_fps = self.bg_video_fps

                def read_frame():
                    ret, frame = background_video.read()
                    if not ret:
                        background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
                        ret, frame = background_video.read()
                        assert ret, 'cannot read frame %r' % self.background_image
                    return self.resize_image(frame,
                                             self.background_keep_aspect)

                def next_frame():
                    while True:
                        advrate = self.bg_video_fps / self.current_fps  # Number of frames we need to advance background movie. Fractional.
                        if advrate < 1:
                            # Number of frames<1 so to avoid movie freezing randomly choose whether to advance by one frame with correct probability.
                            self.bg_video_adv_rate = 1 if np.random.uniform(
                            ) < advrate else 0
                        else:
                            # Just round to nearest number of frames when >=1.
                            self.bg_video_adv_rate = round(advrate)
                        for i in range(self.bg_video_adv_rate):
                            frame = read_frame()
                        yield frame

                background = next_frame()

            self.images["background"] = background

            if self.use_foreground and self.foreground_image is not None:
                foreground = cv2.imread(self.foreground_image)
                self.images["foreground"] = cv2.resize(
                    foreground, (self.width, self.height))
                foreground_mask = cv2.imread(self.foreground_mask_image)
                foreground_mask = cv2.normalize(foreground_mask,
                                                None,
                                                alpha=0,
                                                beta=1,
                                                norm_type=cv2.NORM_MINMAX,
                                                dtype=cv2.CV_32F)
                foreground_mask = cv2.resize(foreground_mask,
                                             (self.width, self.height))
                self.images["foreground_mask"] = cv2.cvtColor(
                    foreground_mask, cv2.COLOR_BGR2GRAY)
                self.images["inverted_foreground_mask"] = 1 - self.images[
                    "foreground_mask"]

    def hologram_effect(self, img):
        # add a blue tint
        holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
        # add a halftone effect
        bandLength, bandGap = 3, 4
        for y in range(holo.shape[0]):
            if y % (bandLength + bandGap) < bandLength:
                holo[y, :, :] = holo[y, :, :] * np.random.uniform(0.1, 0.3)
        # add some ghosting
        holo_blur = cv2.addWeighted(holo, 0.2,
                                    self.shift_image(holo.copy(), 5, 5), 0.8,
                                    0)
        holo_blur = cv2.addWeighted(holo_blur, 0.4,
                                    self.shift_image(holo.copy(), -5, -5), 0.6,
                                    0)
        # combine with the original color, oversaturated
        out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
        return out

    async def mask_frame(self, frame):
        # fetch the mask with retries (the app needs to warmup and we're lazy)
        # e v e n t u a l l y c o n s i s t e n t
        mask = None
        while mask is None:
            try:
                mask = self._get_mask(frame)
            except Exception as e:
                print(f"Mask request failed, retrying: {e}")
                traceback.print_exc()

        foreground_frame = background_frame = frame
        if self.hologram:
            foreground_frame = self.hologram_effect(foreground_frame)

        background_frame = cv2.blur(
            frame, (self.background_blur, self.background_blur),
            cv2.BORDER_DEFAULT)

        # composite the foreground and background
        async with self.image_lock:
            if self.no_background is False:
                background_frame = next(self.images["background"])
            for c in range(frame.shape[2]):
                frame[:, :,
                      c] = foreground_frame[:, :,
                                            c] * mask + background_frame[:, :,
                                                                         c] * (
                                                                             1
                                                                             -
                                                                             mask
                                                                         )

            if self.use_foreground and self.foreground_image is not None:
                for c in range(frame.shape[2]):
                    frame[:, :, c] = (frame[:, :, c] *
                                      self.images["inverted_foreground_mask"] +
                                      self.images["foreground"][:, :, c] *
                                      self.images["foreground_mask"])

        return frame

    def put_frame(self, frame):
        self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

    def stop(self):
        self.real_cam.stop()
        if self.use_akvcam:
            self.fake_cam.__del__()

    async def run(self):
        await self.load_images()
        self.real_cam.start()

        t0 = time.monotonic()
        print_fps_period = 1
        frame_count = 0
        while True:
            frame = self.real_cam.read()
            if frame is None:
                await asyncio.sleep(0.1)
                continue
            frame = await self.mask_frame(frame)
            self.put_frame(frame)
            frame_count += 1
            td = time.monotonic() - t0
            if td > print_fps_period:
                self.current_fps = frame_count / td
                print("FPS: {:6.2f}".format(self.current_fps), end="\r")
                frame_count = 0
                t0 = time.monotonic()
コード例 #3
0
class FakeCam:
    def __init__(
        self,
        fps: int,
        width: int,
        height: int,
        scale_factor: float,
        use_foreground: bool,
        hologram: bool,
        tiling: bool,
        bodypix_url: str,
        background_image: str,
        foreground_image: str,
        foreground_mask_image: str,
        webcam_path: str,
        v4l2loopback_path: str,
        use_akvcam: bool
    ) -> None:
        self.use_foreground = use_foreground
        self.hologram = hologram
        self.tiling = tiling
        self.background_image = background_image
        self.foreground_image = foreground_image
        self.foreground_mask_image = foreground_mask_image
        self.scale_factor = scale_factor
        self.bodypix_url = bodypix_url
        self.real_cam = RealCam(webcam_path, width, height, fps)
        # In case the real webcam does not support the requested mode.
        self.width = self.real_cam.get_frame_width()
        self.height = self.real_cam.get_frame_height()
        self.use_akvcam = use_akvcam
        if not use_akvcam:
            self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path, self.width, self.height)
        else:
            self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width, self.height)
        self.foreground_mask = None
        self.inverted_foreground_mask = None
        self.session = requests.Session()
        self.images: Dict[str, Any] = {}
        self.image_lock = asyncio.Lock()

    async def _get_mask(self, frame, session):
        frame = cv2.resize(frame, (0, 0), fx=self.scale_factor,
                           fy=self.scale_factor)
        _, data = cv2.imencode(".png", frame)
        async with session.post(
            url=self.bodypix_url, data=data.tostring(),
            headers={"Content-Type": "application/octet-stream"}
        ) as r:
            mask = np.frombuffer(await r.read(), dtype=np.uint8)
            mask = mask.reshape((frame.shape[0], frame.shape[1]))
            mask = cv2.resize(
                mask, (0, 0), fx=1 / self.scale_factor,
                fy=1 / self.scale_factor, interpolation=cv2.INTER_NEAREST
            )
            mask = cv2.dilate(mask, np.ones((10, 10), np.uint8), iterations=1)
            mask = cv2.blur(mask.astype(float), (30, 30))
            return mask

    def shift_image(self, img, dx, dy):
        img = np.roll(img, dy, axis=0)
        img = np.roll(img, dx, axis=1)
        if dy > 0:
            img[:dy, :] = 0
        elif dy < 0:
            img[dy:, :] = 0
        if dx > 0:
            img[:, :dx] = 0
        elif dx < 0:
            img[:, dx:] = 0
        return img

    async def load_images(self):
        async with self.image_lock:
            self.images: Dict[str, Any] = {}

            background = cv2.imread(self.background_image)
            if background is not None:
                if not self.tiling:
                    background = cv2.resize(background, (self.width, self.height))
                else:
                    sizey, sizex = background.shape[0], background.shape[1]
                    if sizex > self.width and sizey > self.height:
                        background = cv2.resize(background, (self.width, self.height))
                    else:
                        repx = (self.width - 1) // sizex + 1
                        repy = (self.height - 1) // sizey + 1
                        background = np.tile(background,(repy, repx, 1))
                        background = background[0:self.height, 0:self.width]
                background = itertools.repeat(background)
            else:
                background_video = cv2.VideoCapture(self.background_image)
                self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
                # Initiate current fps to background video fps
                self.current_fps = self.bg_video_fps
                def read_frame():
                        ret, frame = background_video.read()
                        if not ret:
                            background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
                            ret, frame = background_video.read()
                            assert ret, 'cannot read frame %r' % self.background_image
                        frame = cv2.resize(frame, (self.width, self.height))
                        return frame
                def next_frame():
                    while True:
                        self.bg_video_adv_rate = round(self.bg_video_fps/self.current_fps)
                        for i in range(self.bg_video_adv_rate):
                            frame = read_frame();
                        yield frame
                background = next_frame()

            self.images["background"] = background

            if self.use_foreground and self.foreground_image is not None:
                foreground = cv2.imread(self.foreground_image)
                self.images["foreground"] = cv2.resize(foreground,
                                                       (self.width, self.height))
                foreground_mask = cv2.imread(self.foreground_mask_image)
                foreground_mask = cv2.normalize(
                    foreground_mask, None, alpha=0, beta=1,
                    norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
                foreground_mask = cv2.resize(foreground_mask,
                                             (self.width, self.height))
                self.images["foreground_mask"] = cv2.cvtColor(
                    foreground_mask, cv2.COLOR_BGR2GRAY)
                self.images["inverted_foreground_mask"] = 1 - self.images["foreground_mask"]

    def hologram_effect(self, img):
        # add a blue tint
        holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
        # add a halftone effect
        bandLength, bandGap = 2, 3
        for y in range(holo.shape[0]):
            if y % (bandLength+bandGap) < bandLength:
                holo[y,:,:] = holo[y,:,:] * np.random.uniform(0.1, 0.3)
        # add some ghosting
        holo_blur = cv2.addWeighted(holo, 0.2, self.shift_image(holo.copy(), 5, 5), 0.8, 0)
        holo_blur = cv2.addWeighted(holo_blur, 0.4, self.shift_image(holo.copy(), -5, -5), 0.6, 0)
        # combine with the original color, oversaturated
        out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
        return out 

    
    async def mask_frame(self, session, frame):
        # fetch the mask with retries (the app needs to warmup and we're lazy)
        # e v e n t u a l l y c o n s i s t e n t
        mask = None
        while mask is None:
            try:
                mask = await self._get_mask(frame, session)
            except Exception as e:
                print(f"Mask request failed, retrying: {e}")
                traceback.print_exc()     
      
        if self.hologram: 
            frame = self.hologram_effect(frame)

        # composite the foreground and background
        async with self.image_lock:
            background = next(self.images["background"])
            for c in range(frame.shape[2]):
                frame[:, :, c] = frame[:, :, c] * mask + background[:, :, c] * (1 - mask)

            if self.use_foreground and self.foreground_image is not None:
                for c in range(frame.shape[2]):
                    frame[:, :, c] = (
                        frame[:, :, c] * self.images["inverted_foreground_mask"]
                        + self.images["foreground"][:, :, c] * self.images["foreground_mask"]
                        )

        return frame

    def put_frame(self, frame):
        self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

    def stop(self):
        self.real_cam.stop()
        if self.use_akvcam:
            self.fake_cam.__del__()

    async def run(self):
        await self.load_images()
        self.real_cam.start()
        async with aiohttp.ClientSession() as session:
            t0 = time.monotonic()
            print_fps_period = 1
            frame_count = 0
            while True:
                frame = self.real_cam.read()
                if frame is None:
                    await asyncio.sleep(0.1)
                    continue
                await self.mask_frame(session, frame)
                self.put_frame(frame)
                frame_count += 1
                td = time.monotonic() - t0
                if td > print_fps_period:
                    self.current_fps = frame_count / td
                    print("FPS: {:6.2f}".format(self.current_fps), end="\r")
                    frame_count = 0
                    t0 = time.monotonic()
コード例 #4
0
class FakeCam:
    def __init__(
        self,
        fps: int,
        width: int,
        height: int,
        codec: str,
        no_background: bool,
        background_blur: int,
        background_keep_aspect: bool,
        use_foreground: bool,
        hologram: bool,
        tiling: bool,
        socket: str,
        background_image: str,
        foreground_image: str,
        foreground_mask_image: str,
        webcam_path: str,
        v4l2loopback_path: str,
        use_akvcam: bool
    ) -> None:
        self.no_background = no_background
        self.use_foreground = use_foreground
        self.hologram = hologram
        self.tiling = tiling
        self.background_blur = background_blur
        self.background_keep_aspect = background_keep_aspect
        self.background_image = background_image
        self.foreground_image = foreground_image
        self.foreground_mask_image = foreground_mask_image
        self.real_cam = RealCam(webcam_path, width, height, fps, codec)
        # In case the real webcam does not support the requested mode.
        self.width = self.real_cam.get_frame_width()
        self.height = self.real_cam.get_frame_height()
        self.use_akvcam = use_akvcam
        if not use_akvcam:
            self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path, self.width, self.height)
        else:
            self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width, self.height)
        self.foreground_mask = None
        self.inverted_foreground_mask = None
        self.images: Dict[str, Any] = {}
        self.classifier = mp.solutions.selfie_segmentation.SelfieSegmentation(model_selection=1)

    def shift_image(self, img, dx, dy):
        img = np.roll(img, dy, axis=0)
        img = np.roll(img, dx, axis=1)
        if dy > 0:
            img[:dy, :] = 0
        elif dy < 0:
            img[dy:, :] = 0
        if dx > 0:
            img[:, :dx] = 0
        elif dx < 0:
            img[:, dx:] = 0
        return img

    """Rescale image to dimensions self.width, self.height. If keep_aspect is True
then scale & crop the image so that its pixels retain their aspect ratio."""
    def resize_image(self, img, keep_aspect):
        if self.width==0 or self.height==0:
            raise RuntimeError("Camera dimensions error w={} h={}".format(self.width, self.height))
        if keep_aspect:
            imgheight, imgwidth,=img.shape[:2]
            scale=max(self.width/imgwidth, self.height/imgheight)
            newimgwidth, newimgheight=int(np.floor(self.width/scale)), int(np.floor(self.height/scale))
            ix0=int(np.floor(0.5*imgwidth-0.5*newimgwidth))
            iy0=int(np.floor(0.5*imgheight-0.5*newimgheight))
            img = cv2.resize(img[iy0:iy0+newimgheight, ix0:ix0+newimgwidth, :], (self.width, self.height))
        else:
            img = cv2.resize(img, (self.width, self.height))
        return img

    def load_images(self):
        self.images: Dict[str, Any] = {}

        background = cv2.imread(self.background_image)
        if background is not None:
            if not self.tiling:
                background = self.resize_image(background, self.background_keep_aspect)
            else:
                sizey, sizex = background.shape[0], background.shape[1]
                if sizex > self.width and sizey > self.height:
                    background = cv2.resize(background, (self.width, self.height))
                else:
                    repx = (self.width - 1) // sizex + 1
                    repy = (self.height - 1) // sizey + 1
                    background = np.tile(background,(repy, repx, 1))
                    background = background[0:self.height, 0:self.width]
            background = itertools.repeat(background)
        else:
            background_video = cv2.VideoCapture(self.background_image)
            if not background_video.isOpened():
                raise RuntimeError("Couldn't open video '{}'".format(self.background_image))
            self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
            # Initiate current fps to background video fps
            self.current_fps = self.bg_video_fps
            def read_frame():
                    ret, frame = background_video.read()
                    if not ret:
                        background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
                        ret, frame = background_video.read()
                        assert ret, 'cannot read frame %r' % self.background_image
                    return self.resize_image(frame, self.background_keep_aspect)
            def next_frame():
                while True:
                    advrate=self.bg_video_fps/self.current_fps # Number of frames we need to advance background movie. Fractional.
                    if advrate<1:
                        # Number of frames<1 so to avoid movie freezing randomly choose whether to advance by one frame with correct probability.
                        self.bg_video_adv_rate=1 if np.random.uniform()<advrate else 0
                    else:
                        # Just round to nearest number of frames when >=1.
                        self.bg_video_adv_rate = round(advrate)
                    for i in range(self.bg_video_adv_rate):
                        frame = read_frame();
                    yield frame
            background = next_frame()

        self.images["background"] = background

        if self.use_foreground and self.foreground_image is not None:
            foreground = cv2.imread(self.foreground_image)
            self.images["foreground"] = cv2.resize(foreground,
                                                    (self.width, self.height))
            foreground_mask = cv2.imread(self.foreground_mask_image)
            foreground_mask = cv2.normalize(
                foreground_mask, None, alpha=0, beta=1,
                norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
            foreground_mask = cv2.resize(foreground_mask,
                                            (self.width, self.height))
            self.images["foreground_mask"] = cv2.cvtColor(
                foreground_mask, cv2.COLOR_BGR2GRAY)
            self.images["inverted_foreground_mask"] = 1 - self.images["foreground_mask"]

    def hologram_effect(self, img):
        # add a blue tint
        holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
        # add a halftone effect
        bandLength, bandGap = 3, 4
        for y in range(holo.shape[0]):
            if y % (bandLength+bandGap) < bandLength:
                holo[y,:,:] = holo[y,:,:] * np.random.uniform(0.1, 0.3)
        # add some ghosting
        holo_blur = cv2.addWeighted(holo, 0.2, self.shift_image(holo.copy(), 5, 5), 0.8, 0)
        holo_blur = cv2.addWeighted(holo_blur, 0.4, self.shift_image(holo.copy(), -5, -5), 0.6, 0)
        # combine with the original color, oversaturated
        out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
        return out


    def compose_frame(self, frame):
        frame.flags.writeable = False
        mask =  self.classifier.process(frame).segmentation_mask

        if self.hologram:
            foreground_frame = self.hologram_effect(foreground_frame)

        # Get background image
        if self.no_background is False:
            background_frame = next(self.images["background"])
        else:
            background_frame = cv2.blur(frame, (self.background_blur, self.background_blur), cv2.BORDER_DEFAULT)
        frame.flags.writeable = True

        # Replace background
        for c in range(frame.shape[2]):
            frame[:, :, c] = frame[:, :, c] * mask + background_frame[:, :, c] * (1 - mask)

        # Add foreground if needed
        if self.use_foreground and self.foreground_image is not None:
            for c in range(frame.shape[2]):
                frame[:, :, c] = (
                    frame[:, :, c] * self.images["inverted_foreground_mask"] +
                    self.images["foreground"][:, :, c] * self.images["foreground_mask"]
                    )

        return frame

    def put_frame(self, frame):
        self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))

    def stop(self):
        self.real_cam.stop()
        if self.use_akvcam:
            self.fake_cam.__del__()

    def run(self):
        self.load_images()
        self.real_cam.start()
        t0 = time.monotonic()
        print_fps_period = 1
        frame_count = 0
        while True:
            frame = self.real_cam.read()
            if frame is None:
                time.sleep(0.1)
                continue
            frame = self.compose_frame(frame)
            self.put_frame(frame)
            frame_count += 1
            td = time.monotonic() - t0
            if td > print_fps_period:
                self.current_fps = frame_count / td
                print("FPS: {:6.2f}".format(self.current_fps), end="\r")
                frame_count = 0
                t0 = time.monotonic()