def mainloop():
global config, masks, layers, config_mtime
config, config_mtime_new = load_config(config_mtime, config)
if config_mtime != config_mtime_new:
config['width'] = width
config['height'] = height
layers = [] # Allow filters to run their destructors
layers = reload_layers(config)
config_mtime = config_mtime_new
if static_image is not None:
success, frame = True, static_image
else:
success, frame = cap.read()
if not success:
print("Error getting a webcam image!")
sys.exit(1)
# BGR to RGB
frame = frame[..., ::-1]
frame = frame.astype(np.float)
input_height, input_width = frame.shape[:2]
internal_resolution = config.get("internal_resolution", 0.5)
target_height, target_width = to_input_resolution_height_and_width(
internal_resolution, output_stride, input_height, input_width)
padT, padB, padL, padR = calc_padding(frame, target_height, target_width)
resized_frame = tf.image.resize_with_pad(
frame,
target_height,
target_width,
method=tf.image.ResizeMethod.BILINEAR)
resized_height, resized_width = resized_frame.shape[:2]
# Preprocessing
if model_type == "mobilenet":
resized_frame = np.divide(resized_frame, 127.5)
resized_frame = np.subtract(resized_frame, 1.0)
elif model_type == "resnet50":
m = np.array([-123.15, -115.90, -103.06])
resized_frame = np.add(resized_frame, m)
else:
assert (False)
sample_image = resized_frame[tf.newaxis, ...]
results = sess.run(output_tensor_names,
feed_dict={input_tensor: sample_image})
if model_type == "mobilenet":
segment_logits = results[1]
part_heatmaps = results[2]
heatmaps = results[4]
else:
segment_logits = results[6]
part_heatmaps = results[5]
heatmaps = results[2]
scaled_segment_scores = scale_and_crop_to_input_tensor_shape(
segment_logits, input_height, input_width, padT, padB, padL, padR,
True)
scaled_part_heatmap_scores = scale_and_crop_to_input_tensor_shape(
part_heatmaps, input_height, input_width, padT, padB, padL, padR, True)
scaled_heatmap_scores = scale_and_crop_to_input_tensor_shape(
heatmaps, input_height, input_width, padT, padB, padL, padR, True)
mask = to_mask_tensor(scaled_segment_scores,
config.get("segmentation_threshold", 0.75))
mask = np.reshape(mask, mask.shape[:2])
part_masks = to_mask_tensor(scaled_part_heatmap_scores, 0.999)
part_masks = np.array(part_masks)
heatmap_masks = to_mask_tensor(scaled_heatmap_scores, 0.99)
heatmap_masks = np.array(heatmap_masks)
# Average over the last N masks to reduce flickering
# (at the cost of seeing afterimages)
num_average_masks = max(1, config.get("average_masks", 3))
masks.insert(0, mask)
masks = masks[:num_average_masks]
mask = np.mean(masks, axis=0)
mask = (mask * 255).astype(np.uint8)
dilate_value = config.get("dilate", 0)
erode_value = config.get("erode", 0)
blur_value = config.get("blur", 0)
if dilate_value:
mask = cv2.dilate(mask,
np.ones((dilate_value, dilate_value), np.uint8),
iterations=1)
if erode_value:
mask = cv2.erode(mask,
np.ones((erode_value, erode_value), np.uint8),
iterations=1)
if blur_value:
mask = cv2.blur(mask, (blur_value, blur_value))
frame = np.append(frame, np.expand_dims(mask, axis=2), axis=2)
input_frame = frame.copy()
frame = np.zeros(input_frame.shape)
for layer_type, layer_filters in layers:
# Initialize the layer frame
layer_frame = np.zeros(frame.shape) # transparent black
if layer_type == "foreground":
layer_frame = input_frame.copy()
elif layer_type == "input":
layer_frame = input_frame.copy()
# make the frame opaque
layer_frame[:, :, 3] = 255 * np.ones(input_frame.shape[:2])
elif layer_type == "previous":
layer_frame = frame.copy()
# make the frame opaque
layer_frame[:, :, 3] = 255 * np.ones(input_frame.shape[:2])
elif layer_type == "empty":
pass
layer_frame = filters.apply_filters(layer_frame, mask, part_masks,
heatmap_masks, layer_filters)
if layer_frame.shape[2] == 4:
transparency = layer_frame[:, :, 3] / 255.0
transparency = np.expand_dims(transparency, axis=2)
frame[:,:,:3] = frame[:,:,:3] * \
(1.0 - transparency) + layer_frame[:,:,:3] * transparency
else:
frame[:, :, :3] = layer_frame[:, :, :3].copy()
# Remove alpha channel
frame = frame[:, :, :3]
if config.get("debug_show_mask") is not None:
mask_id = int(config.get("debug_show_mask", None))
if mask_id > -1 and mask_id < 24:
mask = part_masks[:, :, mask_id] * 255.0
frame[:, :, 0] = mask
frame[:, :, 1] = mask
frame[:, :, 2] = mask
elif config.get("debug_show_heatmap") is not None:
heatmap_id = int(config.get("debug_show_heatmap", None))
if heatmap_id > -1 and heatmap_id < 17:
mask = heatmap_masks[:, :, heatmap_id] * 255.0
frame[:, :, 0] = mask
frame[:, :, 1] = mask
frame[:, :, 2] = mask
frame = frame.astype(np.uint8)
fakewebcam.schedule_frame(frame)
def mainloop():
global config, masks, replacement_bgs, overlays
config = load_config(config)
success, frame = cap.read()
if not success:
print("Error getting a webcam image!")
sys.exit(1)
if config.get("flip_horizontal"):
frame = cv2.flip(frame, 1)
if config.get("flip_vertical"):
frame = cv2.flip(frame, 0)
image_filters = get_imagefilters(config.get("background_filters", []))
image_name = config.get("background_image", "background.jpg")
replacement_bgs = load_images(replacement_bgs, image_name,
height, width, "replacement_bgs",
config.get("background_interpolation_method"),
image_filters)
frame = frame[...,::-1]
if replacement_bgs is None:
if len(image_filters) == 0:
fakewebcam.schedule_frame(frame)
return
replacement_bg = np.copy(frame)
for image_filter in image_filters:
try:
replacement_bg = image_filter(replacement_bg)
except TypeError:
# caused by a wrong number of arguments in the config
pass
replacement_bgs = [replacement_bg]
input_height, input_width = frame.shape[:2]
target_height, target_width = to_input_resolution_height_and_width(
internal_resolution, output_stride, input_height, input_width)
padT, padB, padL, padR = calc_padding(frame, target_height, target_width)
resized_frame = tf.image.resize_with_pad(frame, target_height, target_width,
method=tf.image.ResizeMethod.BILINEAR)
resized_height, resized_width = resized_frame.shape[:2]
# Preprocessing for resnet
#m = np.array([-123.15, -115.90, -103.06])
#resized_frame = np.add(resized_frame, m)
# Preprocessing for mobilenet
resized_frame = np.divide(resized_frame, 127.5)
resized_frame = np.subtract(resized_frame, 1.0)
sample_image = resized_frame[tf.newaxis, ...]
results = sess.run(output_tensor_names,
feed_dict={input_tensor: sample_image})
segments = np.squeeze(results[1], 0)
segment_logits = results[1]
scaled_segment_scores = scale_and_crop_to_input_tensor_shape(
segment_logits, input_height, input_width,
padT, padB, padL, padR, True
)
mask = to_mask_tensor(scaled_segment_scores,
config["segmentation_threshold"])
mask = tf.dtypes.cast(mask, tf.int32)
mask = np.reshape(mask, mask.shape[:2])
# Average over the last N masks to reduce flickering
# (at the cost of seeing afterimages)
masks.insert(0, mask)
num_average_masks = max(1, config.get("average_masks", 3))
masks = masks[:num_average_masks]
mask = np.mean(masks, axis=0)
mask *= 255
if config["dilate"]:
mask = cv2.dilate(mask, np.ones((config["dilate"], config["dilate"]), np.uint8), iterations=1)
if config["erode"]:
mask = cv2.erode(mask, np.ones((config["erode"], config["erode"]), np.uint8), iterations=1)
if config["blur"]:
mask = cv2.blur(mask, (config["blur"], config["blur"]))
mask /= 255.
mask_inv = 1.0 - mask
# Filter the foreground
image_filters = get_imagefilters(config.get("foreground_filters", []))
for image_filter in image_filters:
try:
frame = image_filter(frame)
except TypeError:
# caused by a wrong number of arguments in the config
pass
replacement_bgs_idx = config.get("replacement_bgs_idx", 0)
for c in range(3):
frame[:,:,c] = frame[:,:,c] * mask + \
replacement_bgs[replacement_bgs_idx][:,:,c] * mask_inv
if time.time() - config.get("last_frame_bg", 0) > 1.0 / config.get("background_fps", 1):
config["replacement_bgs_idx"] = (replacement_bgs_idx + 1) % len(replacement_bgs)
config["last_frame_bg"] = time.time()
# Filter the result
image_filters = get_imagefilters(config.get("result_filters", []))
for image_filter in image_filters:
try:
frame = image_filter(frame)
except TypeError:
# caused by a wrong number of arguments in the config
pass
overlays_idx = config.get("overlays_idx", 0)
overlays = load_images(overlays, config.get("overlay_image", ""), height, width,
"overlays", get_imagefilters(config.get("overlay_filters", [])))
if overlays:
overlay = overlays[overlays_idx]
assert(overlay.shape[2] == 4) # The image has an alpha channel
for c in range(3):
frame[:,:,c] = frame[:,:,c] * (1.0 - overlay[:,:,3] / 255.0) + \
overlay[:,:,c] * (overlay[:,:,3] / 255.0)
if time.time() - config.get("last_frame_overlay", 0) > 1.0 / config.get("overlay_fps", 1):
config["overlays_idx"] = (overlays_idx + 1) % len(overlays)
config["last_frame_overlay"] = time.time()
if config.get("debug_show_mask", False):
frame[:,:,0] = mask * 255
frame[:,:,1] = mask * 255
frame[:,:,2] = mask * 255
fakewebcam.schedule_frame(frame)
last_frame_time = time.time()