def __init__(self, fps: int, width: int, height: int, codec: str,
no_background: bool, background_blur: int,
background_keep_aspect: bool, use_foreground: bool,
hologram: bool, tiling: bool, background_image: str,
foreground_image: str, foreground_mask_image: str,
webcam_path: str, v4l2loopback_path: str,
use_akvcam: bool) -> None:
self.no_background = no_background
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_blur = background_blur
self.background_keep_aspect = background_keep_aspect
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.real_cam = RealCam(webcam_path, width, height, fps, codec)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path,
self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width,
self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.images: Dict[str, Any] = {}
self.image_lock = asyncio.Lock()
self.bodypix_model = load_model(
download_model(BodyPixModelPaths.MOBILENET_FLOAT_50_STRIDE_16))
def __init__(
self,
fps: int,
width: int,
height: int,
codec: str,
scale_factor: float,
no_background: bool,
background_blur: int,
background_keep_aspect: bool,
use_foreground: bool,
hologram: bool,
tiling: bool,
bodypix_url: str,
socket: str,
background_image: str,
foreground_image: str,
foreground_mask_image: str,
webcam_path: str,
v4l2loopback_path: str,
use_akvcam: bool,
) -> None:
self.no_background = no_background
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_blur = background_blur
self.background_keep_aspect = background_keep_aspect
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.scale_factor = scale_factor
self.real_cam = RealCam(webcam_path, width, height, fps, codec)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path,
self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width,
self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.session = requests.Session()
if bodypix_url.startswith("/"):
print("Looks like you want to use a unix socket")
# self.session = requests_unixsocket.Session()
self.bodypix_url = "http+unix:/" + bodypix_url
self.socket = bodypix_url
requests_unixsocket.monkeypatch()
else:
self.bodypix_url = bodypix_url
self.socket = ""
# self.session = requests.Session()
self.images: Dict[str, Any] = {}
self.image_lock = asyncio.Lock()
def __init__(
self,
fps: int,
width: int,
height: int,
codec: str,
no_background: bool,
background_blur: int,
background_keep_aspect: bool,
use_foreground: bool,
hologram: bool,
tiling: bool,
socket: str,
background_image: str,
foreground_image: str,
foreground_mask_image: str,
webcam_path: str,
v4l2loopback_path: str,
use_akvcam: bool
) -> None:
self.no_background = no_background
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_blur = background_blur
self.background_keep_aspect = background_keep_aspect
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.real_cam = RealCam(webcam_path, width, height, fps, codec)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path, self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width, self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.images: Dict[str, Any] = {}
self.classifier = mp.solutions.selfie_segmentation.SelfieSegmentation(model_selection=1)
def __init__(
self,
fps: int,
width: int,
height: int,
scale_factor: float,
use_foreground: bool,
hologram: bool,
tiling: bool,
bodypix_url: str,
background_image: str,
foreground_image: str,
foreground_mask_image: str,
webcam_path: str,
v4l2loopback_path: str,
use_akvcam: bool
) -> None:
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.scale_factor = scale_factor
self.bodypix_url = bodypix_url
self.real_cam = RealCam(webcam_path, width, height, fps)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path, self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width, self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.session = requests.Session()
self.images: Dict[str, Any] = {}
self.image_lock = asyncio.Lock()
class FakeCam:
def __init__(
self,
fps: int,
width: int,
height: int,
codec: str,
scale_factor: float,
no_background: bool,
background_blur: int,
background_keep_aspect: bool,
use_foreground: bool,
hologram: bool,
tiling: bool,
bodypix_url: str,
socket: str,
background_image: str,
foreground_image: str,
foreground_mask_image: str,
webcam_path: str,
v4l2loopback_path: str,
use_akvcam: bool,
use_opencv: bool,
) -> None:
self.no_background = no_background
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_blur = background_blur
self.background_keep_aspect = background_keep_aspect
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.scale_factor = scale_factor
self.real_cam = RealCam(webcam_path, width, height, fps, codec)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
self.use_opencv = use_opencv
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path,
self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width,
self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.session = requests.Session()
if use_opencv == None:
if bodypix_url.startswith('/'):
print("Looks like you want to use a unix socket")
# self.session = requests_unixsocket.Session()
self.bodypix_url = "http+unix:/" + bodypix_url
self.socket = bodypix_url
requests_unixsocket.monkeypatch()
else:
self.bodypix_url = bodypix_url
self.socket = ""
#If the session can't connect, set the session to None so that _get_mask will fall back
#on to openCV
# self.session = requests.Session()
self.images: Dict[str, Any] = {}
self.image_lock = asyncio.Lock()
async def _get_mask(self, frame, session):
frame = cv2.resize(frame, (0, 0),
fx=self.scale_factor,
fy=self.scale_factor)
_, data = cv2.imencode(".png", frame)
#print("Posting to " + self.bodypix_url)
if self.use_opencv is False:
async with session.post(
url=self.bodypix_url,
data=data.tobytes(),
headers={"Content-Type": "application/octet-stream"}) as r:
mask = np.frombuffer(await r.read(), dtype=np.uint8)
mask = mask.reshape((frame.shape[0], frame.shape[1]))
mask = cv2.resize(mask, (0, 0),
fx=1 / self.scale_factor,
fy=1 / self.scale_factor,
interpolation=cv2.INTER_NEAREST)
mask = cv2.dilate(mask,
np.ones((10, 10), np.uint8),
iterations=1)
mask = cv2.blur(mask.astype(float), (30, 30))
else:
backSub = self.real_cam.get_backsub()
#Blur the frame (just like the backSub)
frame = cv2.GaussianBlur(frame, (15, 15), 0)
#frame = cv2.blur(frame,(5,5),0)
mask = backSub.apply(frame, 0, 0)
mask = mask.reshape((frame.shape[0], frame.shape[1]))
_, mask = cv2.threshold(mask, 90, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)
contours, hierarchy = cv2.findContours(
mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2:]
try:
contour = max(contours, key=cv2.contourArea)
mask = np.zeros((frame.shape[0], frame.shape[1]))
cv2.drawContours(mask, [contour],
-1, (255, 255, 255),
thickness=-1)
except:
pass
#mask = cv2.GaussianBlur(mask,(15,15),0)
#mask = cv2.blur(mask,(15,15),0)
mask = cv2.resize(mask, (0, 0),
fx=1 / self.scale_factor,
fy=1 / self.scale_factor,
interpolation=cv2.INTER_NEAREST)
mask = -mask
#If the mask lets more than 40% of the image through, let nothing through
#if((numpy.sum(mask)/np.size(mask))> .4):
# mask = 0
#If I were to do contour detection, it would be here:
#else:
# pass
return mask
def shift_image(self, img, dx, dy):
img = np.roll(img, dy, axis=0)
img = np.roll(img, dx, axis=1)
if dy > 0:
img[:dy, :] = 0
elif dy < 0:
img[dy:, :] = 0
if dx > 0:
img[:, :dx] = 0
elif dx < 0:
img[:, dx:] = 0
return img
"""Rescale image to dimensions self.width, self.height. If keep_aspect is True
then scale & crop the image so that its pixels retain their aspect ratio."""
def resize_image(self, img, keep_aspect):
if self.width == 0 or self.height == 0:
raise RuntimeError("Camera dimensions error w={} h={}".format(
self.width, self.height))
if keep_aspect:
imgheight, imgwidth, = img.shape[:2]
scale = max(self.width / imgwidth, self.height / imgheight)
newimgwidth, newimgheight = int(np.floor(self.width / scale)), int(
np.floor(self.height / scale))
ix0 = int(np.floor(0.5 * imgwidth - 0.5 * newimgwidth))
iy0 = int(np.floor(0.5 * imgheight - 0.5 * newimgheight))
img = cv2.resize(
img[iy0:iy0 + newimgheight, ix0:ix0 + newimgwidth, :],
(self.width, self.height))
else:
img = cv2.resize(img, (self.width, self.height))
return img
async def load_images(self):
async with self.image_lock:
self.images: Dict[str, Any] = {}
background = cv2.imread(self.background_image)
if background is not None:
if not self.tiling:
background = self.resize_image(background,
self.background_keep_aspect)
else:
sizey, sizex = background.shape[0], background.shape[1]
if sizex > self.width and sizey > self.height:
background = cv2.resize(background,
(self.width, self.height))
else:
repx = (self.width - 1) // sizex + 1
repy = (self.height - 1) // sizey + 1
background = np.tile(background, (repy, repx, 1))
background = background[0:self.height, 0:self.width]
background = itertools.repeat(background)
else:
background_video = cv2.VideoCapture(self.background_image)
if not background_video.isOpened():
raise RuntimeError("Couldn't open video '{}'".format(
self.background_image))
self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
# Initiate current fps to background video fps
self.current_fps = self.bg_video_fps
def read_frame():
ret, frame = background_video.read()
if not ret:
background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
ret, frame = background_video.read()
assert ret, 'cannot read frame %r' % self.background_image
return self.resize_image(frame,
self.background_keep_aspect)
def next_frame():
while True:
advrate = self.bg_video_fps / self.current_fps # Number of frames we need to advance background movie. Fractional.
if advrate < 1:
# Number of frames<1 so to avoid movie freezing randomly choose whether to advance by one frame with correct probability.
self.bg_video_adv_rate = 1 if np.random.uniform(
) < advrate else 0
else:
# Just round to nearest number of frames when >=1.
self.bg_video_adv_rate = round(advrate)
for i in range(self.bg_video_adv_rate):
frame = read_frame()
yield frame
background = next_frame()
self.images["background"] = background
if self.use_foreground and self.foreground_image is not None:
foreground = cv2.imread(self.foreground_image)
self.images["foreground"] = cv2.resize(
foreground, (self.width, self.height))
foreground_mask = cv2.imread(self.foreground_mask_image)
foreground_mask = cv2.normalize(foreground_mask,
None,
alpha=0,
beta=1,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_32F)
foreground_mask = cv2.resize(foreground_mask,
(self.width, self.height))
self.images["foreground_mask"] = cv2.cvtColor(
foreground_mask, cv2.COLOR_BGR2GRAY)
self.images["inverted_foreground_mask"] = 1 - self.images[
"foreground_mask"]
def hologram_effect(self, img):
# add a blue tint
holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
# add a halftone effect
bandLength, bandGap = 3, 4
for y in range(holo.shape[0]):
if y % (bandLength + bandGap) < bandLength:
holo[y, :, :] = holo[y, :, :] * np.random.uniform(0.1, 0.3)
# add some ghosting
holo_blur = cv2.addWeighted(holo, 0.2,
self.shift_image(holo.copy(), 5, 5), 0.8,
0)
holo_blur = cv2.addWeighted(holo_blur, 0.4,
self.shift_image(holo.copy(), -5, -5), 0.6,
0)
# combine with the original color, oversaturated
out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
return out
async def mask_frame(self, session, frame):
# fetch the mask with retries (the app needs to warmup and we're lazy)
# e v e n t u a l l y c o n s i s t e n t
mask = None
while mask is None:
try:
mask = await self._get_mask(frame, session)
except Exception as e:
print(f"Mask request failed, retrying: {e}")
traceback.print_exc()
foreground_frame = background_frame = frame
if self.hologram:
foreground_frame = self.hologram_effect(foreground_frame)
background_frame = cv2.blur(
frame, (self.background_blur, self.background_blur),
cv2.BORDER_DEFAULT)
# composite the foreground and background
async with self.image_lock:
if self.no_background is False:
background_frame = next(self.images["background"])
for c in range(frame.shape[2]):
frame[:, :,
c] = foreground_frame[:, :,
c] * mask + background_frame[:, :,
c] * (
1
-
mask
)
if self.use_foreground and self.foreground_image is not None:
for c in range(frame.shape[2]):
frame[:, :, c] = (frame[:, :, c] *
self.images["inverted_foreground_mask"] +
self.images["foreground"][:, :, c] *
self.images["foreground_mask"])
return frame
def put_frame(self, frame):
self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
def stop(self):
self.real_cam.stop()
if self.use_akvcam:
self.fake_cam.__del__()
async def run(self):
await self.load_images()
self.real_cam.start()
if self.socket != "":
conn = aiohttp.UnixConnector(path=self.socket)
else:
conn = None
async with aiohttp.ClientSession(connector=conn) as session:
t0 = time.monotonic()
print_fps_period = 1
frame_count = 0
while True:
frame = self.real_cam.read()
if frame is None:
await asyncio.sleep(0.1)
continue
frame = await self.mask_frame(session, frame)
self.put_frame(frame)
frame_count += 1
td = time.monotonic() - t0
if td > print_fps_period:
self.current_fps = frame_count / td
print("FPS: {:6.2f}".format(self.current_fps), end="\r")
frame_count = 0
t0 = time.monotonic()
class FakeCam:
def __init__(
self,
fps: int,
width: int,
height: int,
scale_factor: float,
use_foreground: bool,
hologram: bool,
tiling: bool,
bodypix_url: str,
background_image: str,
foreground_image: str,
foreground_mask_image: str,
webcam_path: str,
v4l2loopback_path: str,
use_akvcam: bool
) -> None:
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.scale_factor = scale_factor
self.bodypix_url = bodypix_url
self.real_cam = RealCam(webcam_path, width, height, fps)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path, self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width, self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.session = requests.Session()
self.images: Dict[str, Any] = {}
self.image_lock = asyncio.Lock()
async def _get_mask(self, frame, session):
frame = cv2.resize(frame, (0, 0), fx=self.scale_factor,
fy=self.scale_factor)
_, data = cv2.imencode(".png", frame)
async with session.post(
url=self.bodypix_url, data=data.tostring(),
headers={"Content-Type": "application/octet-stream"}
) as r:
mask = np.frombuffer(await r.read(), dtype=np.uint8)
mask = mask.reshape((frame.shape[0], frame.shape[1]))
mask = cv2.resize(
mask, (0, 0), fx=1 / self.scale_factor,
fy=1 / self.scale_factor, interpolation=cv2.INTER_NEAREST
)
mask = cv2.dilate(mask, np.ones((10, 10), np.uint8), iterations=1)
mask = cv2.blur(mask.astype(float), (30, 30))
return mask
def shift_image(self, img, dx, dy):
img = np.roll(img, dy, axis=0)
img = np.roll(img, dx, axis=1)
if dy > 0:
img[:dy, :] = 0
elif dy < 0:
img[dy:, :] = 0
if dx > 0:
img[:, :dx] = 0
elif dx < 0:
img[:, dx:] = 0
return img
async def load_images(self):
async with self.image_lock:
self.images: Dict[str, Any] = {}
background = cv2.imread(self.background_image)
if background is not None:
if not self.tiling:
background = cv2.resize(background, (self.width, self.height))
else:
sizey, sizex = background.shape[0], background.shape[1]
if sizex > self.width and sizey > self.height:
background = cv2.resize(background, (self.width, self.height))
else:
repx = (self.width - 1) // sizex + 1
repy = (self.height - 1) // sizey + 1
background = np.tile(background,(repy, repx, 1))
background = background[0:self.height, 0:self.width]
background = itertools.repeat(background)
else:
background_video = cv2.VideoCapture(self.background_image)
self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
# Initiate current fps to background video fps
self.current_fps = self.bg_video_fps
def read_frame():
ret, frame = background_video.read()
if not ret:
background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
ret, frame = background_video.read()
assert ret, 'cannot read frame %r' % self.background_image
frame = cv2.resize(frame, (self.width, self.height))
return frame
def next_frame():
while True:
self.bg_video_adv_rate = round(self.bg_video_fps/self.current_fps)
for i in range(self.bg_video_adv_rate):
frame = read_frame();
yield frame
background = next_frame()
self.images["background"] = background
if self.use_foreground and self.foreground_image is not None:
foreground = cv2.imread(self.foreground_image)
self.images["foreground"] = cv2.resize(foreground,
(self.width, self.height))
foreground_mask = cv2.imread(self.foreground_mask_image)
foreground_mask = cv2.normalize(
foreground_mask, None, alpha=0, beta=1,
norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
foreground_mask = cv2.resize(foreground_mask,
(self.width, self.height))
self.images["foreground_mask"] = cv2.cvtColor(
foreground_mask, cv2.COLOR_BGR2GRAY)
self.images["inverted_foreground_mask"] = 1 - self.images["foreground_mask"]
def hologram_effect(self, img):
# add a blue tint
holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
# add a halftone effect
bandLength, bandGap = 2, 3
for y in range(holo.shape[0]):
if y % (bandLength+bandGap) < bandLength:
holo[y,:,:] = holo[y,:,:] * np.random.uniform(0.1, 0.3)
# add some ghosting
holo_blur = cv2.addWeighted(holo, 0.2, self.shift_image(holo.copy(), 5, 5), 0.8, 0)
holo_blur = cv2.addWeighted(holo_blur, 0.4, self.shift_image(holo.copy(), -5, -5), 0.6, 0)
# combine with the original color, oversaturated
out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
return out
async def mask_frame(self, session, frame):
# fetch the mask with retries (the app needs to warmup and we're lazy)
# e v e n t u a l l y c o n s i s t e n t
mask = None
while mask is None:
try:
mask = await self._get_mask(frame, session)
except Exception as e:
print(f"Mask request failed, retrying: {e}")
traceback.print_exc()
if self.hologram:
frame = self.hologram_effect(frame)
# composite the foreground and background
async with self.image_lock:
background = next(self.images["background"])
for c in range(frame.shape[2]):
frame[:, :, c] = frame[:, :, c] * mask + background[:, :, c] * (1 - mask)
if self.use_foreground and self.foreground_image is not None:
for c in range(frame.shape[2]):
frame[:, :, c] = (
frame[:, :, c] * self.images["inverted_foreground_mask"]
+ self.images["foreground"][:, :, c] * self.images["foreground_mask"]
)
return frame
def put_frame(self, frame):
self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
def stop(self):
self.real_cam.stop()
if self.use_akvcam:
self.fake_cam.__del__()
async def run(self):
await self.load_images()
self.real_cam.start()
async with aiohttp.ClientSession() as session:
t0 = time.monotonic()
print_fps_period = 1
frame_count = 0
while True:
frame = self.real_cam.read()
if frame is None:
await asyncio.sleep(0.1)
continue
await self.mask_frame(session, frame)
self.put_frame(frame)
frame_count += 1
td = time.monotonic() - t0
if td > print_fps_period:
self.current_fps = frame_count / td
print("FPS: {:6.2f}".format(self.current_fps), end="\r")
frame_count = 0
t0 = time.monotonic()
class FakeCam:
def __init__(self, fps: int, width: int, height: int, codec: str,
no_background: bool, background_blur: int,
background_keep_aspect: bool, use_foreground: bool,
hologram: bool, tiling: bool, background_image: str,
foreground_image: str, foreground_mask_image: str,
webcam_path: str, v4l2loopback_path: str,
use_akvcam: bool) -> None:
self.no_background = no_background
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_blur = background_blur
self.background_keep_aspect = background_keep_aspect
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.real_cam = RealCam(webcam_path, width, height, fps, codec)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path,
self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width,
self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.images: Dict[str, Any] = {}
self.image_lock = asyncio.Lock()
self.bodypix_model = load_model(
download_model(BodyPixModelPaths.MOBILENET_FLOAT_50_STRIDE_16))
def _get_mask(self, frame):
res = self.bodypix_model.predict_single(frame)
mask = res.get_mask(threshold=0.60)
mask = np.frombuffer(mask, dtype=np.uint32)
mask = mask.reshape((frame.shape[0], frame.shape[1]))
mask = cv2.dilate(mask.astype(float),
np.ones((10, 10), np.uint8),
iterations=1)
mask = cv2.blur(mask.astype(float), (30, 30))
return mask
def shift_image(self, img, dx, dy):
img = np.roll(img, dy, axis=0)
img = np.roll(img, dx, axis=1)
if dy > 0:
img[:dy, :] = 0
elif dy < 0:
img[dy:, :] = 0
if dx > 0:
img[:, :dx] = 0
elif dx < 0:
img[:, dx:] = 0
return img
"""Rescale image to dimensions self.width, self.height. If keep_aspect is True
then scale & crop the image so that its pixels retain their aspect ratio."""
def resize_image(self, img, keep_aspect):
if self.width == 0 or self.height == 0:
raise RuntimeError("Camera dimensions error w={} h={}".format(
self.width, self.height))
if keep_aspect:
imgheight, imgwidth, = img.shape[:2]
scale = max(self.width / imgwidth, self.height / imgheight)
newimgwidth, newimgheight = int(np.floor(self.width / scale)), int(
np.floor(self.height / scale))
ix0 = int(np.floor(0.5 * imgwidth - 0.5 * newimgwidth))
iy0 = int(np.floor(0.5 * imgheight - 0.5 * newimgheight))
img = cv2.resize(
img[iy0:iy0 + newimgheight, ix0:ix0 + newimgwidth, :],
(self.width, self.height))
else:
img = cv2.resize(img, (self.width, self.height))
return img
async def load_images(self):
async with self.image_lock:
self.images: Dict[str, Any] = {}
background = cv2.imread(self.background_image)
if background is not None:
if not self.tiling:
background = self.resize_image(background,
self.background_keep_aspect)
else:
sizey, sizex = background.shape[0], background.shape[1]
if sizex > self.width and sizey > self.height:
background = cv2.resize(background,
(self.width, self.height))
else:
repx = (self.width - 1) // sizex + 1
repy = (self.height - 1) // sizey + 1
background = np.tile(background, (repy, repx, 1))
background = background[0:self.height, 0:self.width]
background = itertools.repeat(background)
else:
background_video = cv2.VideoCapture(self.background_image)
if not background_video.isOpened():
raise RuntimeError("Couldn't open video '{}'".format(
self.background_image))
self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
# Initiate current fps to background video fps
self.current_fps = self.bg_video_fps
def read_frame():
ret, frame = background_video.read()
if not ret:
background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
ret, frame = background_video.read()
assert ret, 'cannot read frame %r' % self.background_image
return self.resize_image(frame,
self.background_keep_aspect)
def next_frame():
while True:
advrate = self.bg_video_fps / self.current_fps # Number of frames we need to advance background movie. Fractional.
if advrate < 1:
# Number of frames<1 so to avoid movie freezing randomly choose whether to advance by one frame with correct probability.
self.bg_video_adv_rate = 1 if np.random.uniform(
) < advrate else 0
else:
# Just round to nearest number of frames when >=1.
self.bg_video_adv_rate = round(advrate)
for i in range(self.bg_video_adv_rate):
frame = read_frame()
yield frame
background = next_frame()
self.images["background"] = background
if self.use_foreground and self.foreground_image is not None:
foreground = cv2.imread(self.foreground_image)
self.images["foreground"] = cv2.resize(
foreground, (self.width, self.height))
foreground_mask = cv2.imread(self.foreground_mask_image)
foreground_mask = cv2.normalize(foreground_mask,
None,
alpha=0,
beta=1,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_32F)
foreground_mask = cv2.resize(foreground_mask,
(self.width, self.height))
self.images["foreground_mask"] = cv2.cvtColor(
foreground_mask, cv2.COLOR_BGR2GRAY)
self.images["inverted_foreground_mask"] = 1 - self.images[
"foreground_mask"]
def hologram_effect(self, img):
# add a blue tint
holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
# add a halftone effect
bandLength, bandGap = 3, 4
for y in range(holo.shape[0]):
if y % (bandLength + bandGap) < bandLength:
holo[y, :, :] = holo[y, :, :] * np.random.uniform(0.1, 0.3)
# add some ghosting
holo_blur = cv2.addWeighted(holo, 0.2,
self.shift_image(holo.copy(), 5, 5), 0.8,
0)
holo_blur = cv2.addWeighted(holo_blur, 0.4,
self.shift_image(holo.copy(), -5, -5), 0.6,
0)
# combine with the original color, oversaturated
out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
return out
async def mask_frame(self, frame):
# fetch the mask with retries (the app needs to warmup and we're lazy)
# e v e n t u a l l y c o n s i s t e n t
mask = None
while mask is None:
try:
mask = self._get_mask(frame)
except Exception as e:
print(f"Mask request failed, retrying: {e}")
traceback.print_exc()
foreground_frame = background_frame = frame
if self.hologram:
foreground_frame = self.hologram_effect(foreground_frame)
background_frame = cv2.blur(
frame, (self.background_blur, self.background_blur),
cv2.BORDER_DEFAULT)
# composite the foreground and background
async with self.image_lock:
if self.no_background is False:
background_frame = next(self.images["background"])
for c in range(frame.shape[2]):
frame[:, :,
c] = foreground_frame[:, :,
c] * mask + background_frame[:, :,
c] * (
1
-
mask
)
if self.use_foreground and self.foreground_image is not None:
for c in range(frame.shape[2]):
frame[:, :, c] = (frame[:, :, c] *
self.images["inverted_foreground_mask"] +
self.images["foreground"][:, :, c] *
self.images["foreground_mask"])
return frame
def put_frame(self, frame):
self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
def stop(self):
self.real_cam.stop()
if self.use_akvcam:
self.fake_cam.__del__()
async def run(self):
await self.load_images()
self.real_cam.start()
t0 = time.monotonic()
print_fps_period = 1
frame_count = 0
while True:
frame = self.real_cam.read()
if frame is None:
await asyncio.sleep(0.1)
continue
frame = await self.mask_frame(frame)
self.put_frame(frame)
frame_count += 1
td = time.monotonic() - t0
if td > print_fps_period:
self.current_fps = frame_count / td
print("FPS: {:6.2f}".format(self.current_fps), end="\r")
frame_count = 0
t0 = time.monotonic()
class FakeCam:
def __init__(
self,
fps: int,
width: int,
height: int,
codec: str,
no_background: bool,
background_blur: int,
background_keep_aspect: bool,
use_foreground: bool,
hologram: bool,
tiling: bool,
socket: str,
background_image: str,
foreground_image: str,
foreground_mask_image: str,
webcam_path: str,
v4l2loopback_path: str,
use_akvcam: bool
) -> None:
self.no_background = no_background
self.use_foreground = use_foreground
self.hologram = hologram
self.tiling = tiling
self.background_blur = background_blur
self.background_keep_aspect = background_keep_aspect
self.background_image = background_image
self.foreground_image = foreground_image
self.foreground_mask_image = foreground_mask_image
self.real_cam = RealCam(webcam_path, width, height, fps, codec)
# In case the real webcam does not support the requested mode.
self.width = self.real_cam.get_frame_width()
self.height = self.real_cam.get_frame_height()
self.use_akvcam = use_akvcam
if not use_akvcam:
self.fake_cam = pyfakewebcam.FakeWebcam(v4l2loopback_path, self.width, self.height)
else:
self.fake_cam = AkvCameraWriter(v4l2loopback_path, self.width, self.height)
self.foreground_mask = None
self.inverted_foreground_mask = None
self.images: Dict[str, Any] = {}
self.classifier = mp.solutions.selfie_segmentation.SelfieSegmentation(model_selection=1)
def shift_image(self, img, dx, dy):
img = np.roll(img, dy, axis=0)
img = np.roll(img, dx, axis=1)
if dy > 0:
img[:dy, :] = 0
elif dy < 0:
img[dy:, :] = 0
if dx > 0:
img[:, :dx] = 0
elif dx < 0:
img[:, dx:] = 0
return img
"""Rescale image to dimensions self.width, self.height. If keep_aspect is True
then scale & crop the image so that its pixels retain their aspect ratio."""
def resize_image(self, img, keep_aspect):
if self.width==0 or self.height==0:
raise RuntimeError("Camera dimensions error w={} h={}".format(self.width, self.height))
if keep_aspect:
imgheight, imgwidth,=img.shape[:2]
scale=max(self.width/imgwidth, self.height/imgheight)
newimgwidth, newimgheight=int(np.floor(self.width/scale)), int(np.floor(self.height/scale))
ix0=int(np.floor(0.5*imgwidth-0.5*newimgwidth))
iy0=int(np.floor(0.5*imgheight-0.5*newimgheight))
img = cv2.resize(img[iy0:iy0+newimgheight, ix0:ix0+newimgwidth, :], (self.width, self.height))
else:
img = cv2.resize(img, (self.width, self.height))
return img
def load_images(self):
self.images: Dict[str, Any] = {}
background = cv2.imread(self.background_image)
if background is not None:
if not self.tiling:
background = self.resize_image(background, self.background_keep_aspect)
else:
sizey, sizex = background.shape[0], background.shape[1]
if sizex > self.width and sizey > self.height:
background = cv2.resize(background, (self.width, self.height))
else:
repx = (self.width - 1) // sizex + 1
repy = (self.height - 1) // sizey + 1
background = np.tile(background,(repy, repx, 1))
background = background[0:self.height, 0:self.width]
background = itertools.repeat(background)
else:
background_video = cv2.VideoCapture(self.background_image)
if not background_video.isOpened():
raise RuntimeError("Couldn't open video '{}'".format(self.background_image))
self.bg_video_fps = background_video.get(cv2.CAP_PROP_FPS)
# Initiate current fps to background video fps
self.current_fps = self.bg_video_fps
def read_frame():
ret, frame = background_video.read()
if not ret:
background_video.set(cv2.CAP_PROP_POS_FRAMES, 0)
ret, frame = background_video.read()
assert ret, 'cannot read frame %r' % self.background_image
return self.resize_image(frame, self.background_keep_aspect)
def next_frame():
while True:
advrate=self.bg_video_fps/self.current_fps # Number of frames we need to advance background movie. Fractional.
if advrate<1:
# Number of frames<1 so to avoid movie freezing randomly choose whether to advance by one frame with correct probability.
self.bg_video_adv_rate=1 if np.random.uniform()<advrate else 0
else:
# Just round to nearest number of frames when >=1.
self.bg_video_adv_rate = round(advrate)
for i in range(self.bg_video_adv_rate):
frame = read_frame();
yield frame
background = next_frame()
self.images["background"] = background
if self.use_foreground and self.foreground_image is not None:
foreground = cv2.imread(self.foreground_image)
self.images["foreground"] = cv2.resize(foreground,
(self.width, self.height))
foreground_mask = cv2.imread(self.foreground_mask_image)
foreground_mask = cv2.normalize(
foreground_mask, None, alpha=0, beta=1,
norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F)
foreground_mask = cv2.resize(foreground_mask,
(self.width, self.height))
self.images["foreground_mask"] = cv2.cvtColor(
foreground_mask, cv2.COLOR_BGR2GRAY)
self.images["inverted_foreground_mask"] = 1 - self.images["foreground_mask"]
def hologram_effect(self, img):
# add a blue tint
holo = cv2.applyColorMap(img, cv2.COLORMAP_WINTER)
# add a halftone effect
bandLength, bandGap = 3, 4
for y in range(holo.shape[0]):
if y % (bandLength+bandGap) < bandLength:
holo[y,:,:] = holo[y,:,:] * np.random.uniform(0.1, 0.3)
# add some ghosting
holo_blur = cv2.addWeighted(holo, 0.2, self.shift_image(holo.copy(), 5, 5), 0.8, 0)
holo_blur = cv2.addWeighted(holo_blur, 0.4, self.shift_image(holo.copy(), -5, -5), 0.6, 0)
# combine with the original color, oversaturated
out = cv2.addWeighted(img, 0.5, holo_blur, 0.6, 0)
return out
def compose_frame(self, frame):
frame.flags.writeable = False
mask = self.classifier.process(frame).segmentation_mask
if self.hologram:
foreground_frame = self.hologram_effect(foreground_frame)
# Get background image
if self.no_background is False:
background_frame = next(self.images["background"])
else:
background_frame = cv2.blur(frame, (self.background_blur, self.background_blur), cv2.BORDER_DEFAULT)
frame.flags.writeable = True
# Replace background
for c in range(frame.shape[2]):
frame[:, :, c] = frame[:, :, c] * mask + background_frame[:, :, c] * (1 - mask)
# Add foreground if needed
if self.use_foreground and self.foreground_image is not None:
for c in range(frame.shape[2]):
frame[:, :, c] = (
frame[:, :, c] * self.images["inverted_foreground_mask"] +
self.images["foreground"][:, :, c] * self.images["foreground_mask"]
)
return frame
def put_frame(self, frame):
self.fake_cam.schedule_frame(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
def stop(self):
self.real_cam.stop()
if self.use_akvcam:
self.fake_cam.__del__()
def run(self):
self.load_images()
self.real_cam.start()
t0 = time.monotonic()
print_fps_period = 1
frame_count = 0
while True:
frame = self.real_cam.read()
if frame is None:
time.sleep(0.1)
continue
frame = self.compose_frame(frame)
self.put_frame(frame)
frame_count += 1
td = time.monotonic() - t0
if td > print_fps_period:
self.current_fps = frame_count / td
print("FPS: {:6.2f}".format(self.current_fps), end="\r")
frame_count = 0
t0 = time.monotonic()