def recv(self, frame: av.VideoFrame) -> av.VideoFrame:
img = frame.to_ndarray(format="bgr24")
# perform edge detection
img = cv2.cvtColor(cv2.Canny(img, 100, 200), cv2.COLOR_GRAY2BGR)
return av.VideoFrame.from_ndarray(img, format="bgr24")
def transform(self, frame: av.VideoFrame) -> np.ndarray:
global model
# Target area where the hand gestures should be.
img = frame.to_ndarray(format="bgr24")
cv2.rectangle(img, (self.x_, 0), (CROP_SIZE + self.x_, CROP_SIZE),
(0, 255, 0), 3)
#cv2.rectangle(img, (100, 5), (100, 400), (0, 200, 0), 3)
# Preprocessing the frame before input to the model.
cropped_image = img[0:CROP_SIZE, 0:CROP_SIZE]
resized_frame = cv2.resize(cropped_image, (IMAGE_SIZE, IMAGE_SIZE))
reshaped_frame = (np.array(resized_frame)).reshape(
(1, IMAGE_SIZE, IMAGE_SIZE, 3))
frame_for_model = data_generator.standardize(
np.float64(reshaped_frame))
# Predicting the frame.
prediction = np.array(model.predict(frame_for_model))
self.predicted_class = classes[
prediction.argmax()] # Selecting the max confidence index.
self.predict()
cv2.putText(img, self.word, (10, 450), 1, 2, (255, 255, 0), 2,
cv2.LINE_AA)
return img
def transform(self, frame: av.VideoFrame) -> np.ndarray:
in_image = frame.to_ndarray(format="bgr24")
out_image = in_image[:, ::-1, :] # Simple flipping for example.
with self.frame_lock:
self.in_image = in_image
self.out_image = out_image
return in_image
def transform(self, frame: av.VideoFrame) -> np.ndarray:
image = frame.to_ndarray(format="bgr24")
blob = cv2.dnn.blobFromImage(
cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
)
self._net.setInput(blob)
detections = self._net.forward()
annotated_image, labels = self._annotate_image(image, detections)
# TODO: Show labels
return annotated_image
def transform(self, frame: av.VideoFrame) -> np.ndarray:
image = frame.to_ndarray(format="bgr24")
blob = cv2.dnn.blobFromImage(cv2.resize(image, (300, 300)),
0.007843, (300, 300), 127.5)
self._net.setInput(blob)
detections = self._net.forward()
annotated_image, result = self._annotate_image(image, detections)
# NOTE: This `transform` method is called in another thread,
# so it must be thread-safe.
self.result_queue.put(result)
return annotated_image
def callback(frame: av.VideoFrame) -> av.VideoFrame:
image = frame.to_ndarray(format="bgr24")
blob = cv2.dnn.blobFromImage(
cv2.resize(image, (300, 300)), 0.007843, (300, 300), 127.5
)
net.setInput(blob)
detections = net.forward()
annotated_image, result = _annotate_image(image, detections)
# NOTE: This `recv` method is called in another thread,
# so it must be thread-safe.
result_queue.put(result) # TODO:
return av.VideoFrame.from_ndarray(annotated_image, format="bgr24")
def transform(self, frame: av.VideoFrame) -> np.ndarray:
img = frame.to_ndarray(format="bgr24")
img = cv2.flip(img, 1)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
gray_crop = gray[50:250, 400:600]
blur = cv2.GaussianBlur(gray_crop, (5, 5), 2)
th3 = cv2.adaptiveThreshold(blur, 255,
cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
cv2.THRESH_BINARY_INV, 11, 2)
ret, res = cv2.threshold(th3, 70, 255,
cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
self.signPrediction(res)
cv2.rectangle(img, (400, 50), (600, 250), (0, 0, 255), 5)
cv2.putText(img, self.current_symbol, (380, 90),
cv2.FONT_HERSHEY_SIMPLEX, 1.9, (255, 0, 0), 2)
cv2.putText(img, "Word->" + self.word, (10, 290),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2)
cv2.putText(img, "Sentence->" + self.sentence, (10, 390),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
return img
def transform(self, frame: av.VideoFrame) -> av.VideoFrame:
img = frame.to_ndarray(format="bgr24")
if self.type == "noop":
pass
elif self.type == "cartoon":
# prepare color
img_color = cv2.pyrDown(cv2.pyrDown(img))
for _ in range(6):
img_color = cv2.bilateralFilter(img_color, 9, 9, 7)
img_color = cv2.pyrUp(cv2.pyrUp(img_color))
# prepare edges
img_edges = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
img_edges = cv2.adaptiveThreshold(
cv2.medianBlur(img_edges, 7),
255,
cv2.ADAPTIVE_THRESH_MEAN_C,
cv2.THRESH_BINARY,
9,
2,
)
img_edges = cv2.cvtColor(img_edges, cv2.COLOR_GRAY2RGB)
# combine color and edges
img = cv2.bitwise_and(img_color, img_edges)
elif self.type == "edges":
# perform edge detection
img = cv2.cvtColor(cv2.Canny(img, 100, 200),
cv2.COLOR_GRAY2BGR)
elif self.type == "rotate":
# rotate image
rows, cols, _ = img.shape
M = cv2.getRotationMatrix2D((cols / 2, rows / 2),
frame.time * 45, 1)
img = cv2.warpAffine(img, M, (cols, rows))
return img
def transform(self, frame: av.VideoFrame) -> np.ndarray:
start = time.time()
image = frame.to_ndarray(format="bgr24")
# Resize frame of video to 1/4 size for faster face recognition processing
small_frame = cv2.resize(image, (0, 0), fx=0.25, fy=0.25)
face_locations = face_recognition.face_locations(small_frame)
self.cur_frame += 1
print(self.cur_frame)
if self.cur_frame % 10 == 0:
face_encodings = face_recognition.face_encodings(small_frame, face_locations)
self.face_names = []
for face_encoding in face_encodings:
# See if the face is a match for the known face(s)
matches = face_recognition.compare_faces(self.known_face_encodings, face_encoding)
name = "Unknown"
# Or instead, use the known face with the smallest distance to the new face
face_distances = face_recognition.face_distance(self.known_face_encodings, face_encoding)
best_match_index = np.argmin(face_distances)
if matches[best_match_index]:
name = self.known_face_names[best_match_index]
self.face_names.append(name)
annotated_image, result = self._annotate_image(image, face_locations, self.face_names)
self.result_queue.put(result)
end = time.time()
print("Transform time: ", str(end - start))
return annotated_image
def test_ndarray_rgb_align(self):
for format in ['rgb24', 'bgr24']:
frame = VideoFrame(318, 238, format=format)
array = frame.to_ndarray()
self.assertEqual(array.shape, (238, 318, 3))
def transform(self, frame: av.VideoFrame) -> av.VideoFrame:
img = frame.to_ndarray(format="bgr24")
img = cv2.flip(img, 1)
if self.type == "Live Stream":
pass
elif self.type == "Half Slide - Horizontal":
if (self.first_frame == False):
self.first_frame = True
else:
img = half_slide(img, self.last_frame, "horizontal")
self.last_frame = img
elif self.type == "Half Slide - Vertical":
if (self.first_frame == False):
self.first_frame = True
else:
img = half_slide(img, self.last_frame, "vertical")
self.last_frame = img
elif self.type == "Line Freeze - Horizontal":
if self.x == 0:
self.final_array = np.zeros(
(img.shape[0], img.shape[1], 3), dtype=np.uint8)
elif (self.x >= img.shape[1]):
self.x = 0
time.sleep(2)
else:
img = line_freeze(img, self.final_array, self.x,
"horizontal")
self.final_array = img
self.x += 1
elif self.type == "Line Freeze - Vertical":
if self.y == 0:
self.final_array = np.zeros(
(img.shape[0], img.shape[1], 3), dtype=np.uint8)
elif (self.y >= img.shape[0]):
self.y = 0
time.sleep(2)
else:
img = line_freeze(img, self.final_array, self.y,
"vertical")
self.final_array = img
self.y += 1
elif self.type == "Thug Life":
img = overlay_thuglife(img, file_checker_return[1])[0]
elif self.type == "Time Freeze - Ssim":
if self.init_frame == False:
self.start_time = time.time()
out_params = time_freeze(img, self.bg, [
self.stitched_img, self.prev_num, self.first_snap,
self.init_frame, self.start_time
], "ssim")
img, self.stitched_img, self.bg, self.prev_num, self.first_snap, self.init_frame = out_params
elif self.type == "Time Freeze - Rcnn":
if self.init_frame == False:
self.start_time = time.time()
out_params = time_freeze(img, self.bg, [
self.stitched_img, self.prev_num, self.first_snap,
self.init_frame, self.start_time
], "rcnn")
img, self.stitched_img, self.bg, self.prev_num, self.first_snap, self.init_frame = out_params
elif self.type == "Heart Eyes":
img = overlay_heart_eyes(img, file_checker_return[1])[0]
elif self.type == "Moustaches":
img = overlay_moustache(img, file_checker_return[1], 1)[0]
elif self.type == "Face Blur":
img = blur_all_faces(img, file_checker_return[1])[0]
elif self.type == "Devil-ie":
img = horns_nd_fangs_overlay(img, file_checker_return[1])[0]
return img
def test_ndarray_rgba_align(self):
for format in ['argb', 'rgba', 'abgr', 'bgra']:
frame = VideoFrame(318, 238, format=format)
array = frame.to_ndarray()
self.assertEqual(array.shape, (238, 318, 4))
def transform(self, frame: av.VideoFrame) -> np.ndarray:
img = frame.to_ndarray(format="bgr24")
cv2.rectangle(img, (self.x_, 0), (CROP_SIZE + self.x_, CROP_SIZE),
(0, 255, 0), 3)
return img
def test_ndarray_yuv420p_align(self):
frame = VideoFrame(318, 238, format='yuv420p')
array = frame.to_ndarray()
self.assertEqual(array.shape, (357, 318))
def test_ndarray_gray_align(self):
for format in ['gray', 'gray8']:
frame = VideoFrame(318, 238, format=format)
array = frame.to_ndarray()
self.assertEqual(array.shape, (238, 318))
def recv(self, frame: av.VideoFrame) -> av.VideoFrame:
image = frame.to_ndarray(format="bgr24")
image = self.gen_pred(image)
return av.VideoFrame.from_ndarray(image, format="bgr24")
def test_ndarray_yuyv422_align(self):
frame = VideoFrame(318, 238, format='yuyv422')
array = frame.to_ndarray()
self.assertEqual(array.shape, (238, 318, 2))
def transform(self, frame: av.VideoFrame) -> np.ndarray:
probability_minimum = 0.7
threshold = 0.3
bounding_boxes = []
confidences = []
class_numbers = []
frame = frame.to_ndarray(format="bgr24")
small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
h, w = small_frame.shape[:2]
if self.frame_cur == 0 or self.frame_cur % 5 == 0:
blob = cv2.dnn.blobFromImage(small_frame, 1 / 255.0, (416, 416),
swapRB=True, crop=False)
self.network.setInput(blob)
output_from_network = self.network.forward(self.layers_names_output)
for result in output_from_network:
for detected_objects in result:
scores = detected_objects[5:]
class_current = np.argmax(scores)
confidence_current = scores[class_current]
if confidence_current > probability_minimum:
box_current = detected_objects[0:4] * np.array([w, h, w, h])
x_center, y_center, box_width, box_height = box_current
x_min = int(x_center - (box_width / 2))
y_min = int(y_center - (box_height / 2))
bounding_boxes.append([x_min, y_min,
int(box_width), int(box_height)])
confidences.append(float(confidence_current))
class_numbers.append(class_current)
self.bounding_boxes = bounding_boxes
self.confidences = confidences
self.class_numbers = class_numbers
self.results = cv2.dnn.NMSBoxes(bounding_boxes, confidences, probability_minimum, threshold)
if len(self.results) > 0:
for i in self.results.flatten():
x_min, y_min = 4 * bounding_boxes[i][0], 4 * bounding_boxes[i][1]
box_width, box_height = 4 * bounding_boxes[i][2], 4 * bounding_boxes[i][3]
colour_box_current = self.colours[class_numbers[i]].tolist()
cv2.rectangle(frame, (x_min, y_min),
(x_min + box_width, y_min + box_height),
colour_box_current, 2)
text_box_current = '{}: {:.4f}'.format(self.labels[int(class_numbers[i])],
confidences[i])
cv2.putText(frame, text_box_current, (x_min, y_min - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, colour_box_current, 2)
else:
if len(self.results) > 0:
for i in self.results.flatten():
x_min, y_min = 4 * self.bounding_boxes[i][0], 4 * self.bounding_boxes[i][1]
box_width, box_height = 4 * self.bounding_boxes[i][2], 4 * self.bounding_boxes[i][3]
colour_box_current = self.colours[self.class_numbers[i]].tolist()
cv2.rectangle(frame, (x_min, y_min),
(x_min + box_width, y_min + box_height),
colour_box_current, 2)
text_box_current = '{}: {:.4f}'.format(self.labels[int(self.class_numbers[i])],
self.confidences[i])
cv2.putText(frame, text_box_current, (x_min, y_min - 5),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, colour_box_current, 2)
self.frame_cur += 1
return frame
def transform(self, frame: av.VideoFrame) -> np.ndarray:
return frame.to_ndarray(format="bgr24")
def test_basic_to_ndarray(self):
frame = VideoFrame(640, 480, 'rgb24')
array = frame.to_ndarray()
self.assertEqual(array.shape, (480, 640, 3))
def transform(self, frame: av.VideoFrame) -> av.VideoFrame:
img = frame.to_ndarray(format="bgr24") ## PIL ?
return self.assembly.forwardFrame(
img, soundOn=(self.type == "sound_warnings"))
