def __init__(self,
capture,
in_file_name,
preview_window_manager=None,
should_mirror_preview=False):
super(VideoProc, self).__init__()
# initialize variables
self.vm = VideoManager(capture, in_file_name)
self.in_file_name = in_file_name
self.preview_window_manager = preview_window_manager
self._out_filename = None
self._capture = capture
self.channel = 0
self.stopped = True
self.build_out_filename()
self._in_frame = None
self._out_frame = None
self.out_width = 480
self.out_height = 270
# for numbering frames
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.bottom_left_text = 10, self.out_height - 10
self.font_scale = 1
self.font_color = (255, 128, 128)
self.thickness = 2
self.counter = 0
self.build_out_filename()
def test_can_extract_audio(self):
manager = VideoManager(
os.path.join(os.path.dirname(__file__), "input", "test1.mp4"),
)
_, audio = tempfile.mkstemp(".wav")
extracted_audio = manager.extract_audio(audio)
self.assertEqual(audio, extracted_audio)
self.assertTrue(os.path.exists(extracted_audio))
os.remove(audio)
def test_can_transcribe_long_speech(self):
manager = VideoManager(
os.path.join(os.path.dirname(__file__), "input", "test1.mp4"), )
_, audio = tempfile.mkstemp(".wav")
manager.extract_audio(audio)
audio_manager = AudioManager(audio)
transcriber = Transcriber(audio_manager)
transcribe_result = list(transcriber.transcribe())
self.assertGreaterEqual(len(transcribe_result), 1)
def test_can_extract_thumbnail(self):
manager = VideoManager(
os.path.join(os.path.dirname(__file__), "input", "test2.mp4"),
)
_, thumbnail = tempfile.mkstemp(".jpg")
extracted_thumbnail = manager.extract_thumbnail(thumbnail)
self.assertEqual(thumbnail, extracted_thumbnail)
self.assertTrue(os.path.exists(extracted_thumbnail))
os.remove(thumbnail)
def main() -> int:
video_manager = VideoManager(LOGGER)
video_manager.collect()
video_url = video_manager.choose_random_video_url()
if not video_url:
return -1
print(video_url)
conf = Util.read_config("conf.json")
if not conf or "google_api_key" not in conf:
return -1
call_webhook(conf, video_url)
return 0
def extractData(self):
start_time = time.process_time()
print("Initiating Data Retrieval Process")
print("Reading Video...")
stream = VideoManager(self.media_source, self.key)
frame = stream.getNextFrame()
current_frame_index = stream.getCurrentFrame()
if frame.all() == -1:
raise Exception("Video frames corrupted!")
retrieved_data = ""
time.sleep(5)
hot_frame = FrameManager(frame, self.key)
number = 1
print("Decrypting Data...")
for byte in self.data:
if hot_frame.full():
# Find the next empty frame. Exit with exception if not available
while True:
if not stream.full():
frame = stream.getNextFrame()
current_frame_index = stream.getCurrentFrame()
if frame.all() == -1:
raise Exception("Video frames corrupted!")
hot_frame = FrameManager(frame, self.key)
if hot_frame.getBlobCount() != 0:
break
else:
raise Exception("Data storage capacity exceeded!")
retrieved_data += chr(byte)
number += 1
print("Done!")
execution_time = round(time.process_time() - start_time, 3)
print("Completed in", execution_time, "seconds")
return retrieved_data
class Manager():
def __init__(self, input, output, max_frames, skip, show, detector):
self.video = VideoManager(input, output, max_frames, skip, show)
self.horse = None
self.detector = detector
def match(self, frame, boxes, scores):
# if no horse and no boxes: return
# if no horse and boxes: assign first box
# if horse but no boxes: track
# if horse and boxes: find closest box
logger.info(f'detected boxes: {boxes}')
if self.horse is None and len(boxes) == 0: return
elif self.horse is None and len(boxes) > 0:
max_index = np.argmax(scores)
self.horse = Horse(boxes[max_index], 1)
elif self.horse is not None and len(boxes) == 0:
self.horse.track(frame)
elif self.horse is not None and len(boxes) > 0:
box = find_closest_box(self.horse, boxes)
if box is None:
#self.horse.track() # old
self.horse.track(frame)
else:
self.horse.detect(box)
def initialize(self):
raw = self.video.read()
frame = raw.copy()
smooth = raw.copy()
boxes, scores = self.detector.detect_boxes(frame)
self.match(frame, boxes, scores)
if self.horse is not None:
self.horse.draw(frame)
self.horse.draw_smooth(smooth)
horses = [] if self.horse is None else [self.horse]
self.video.write(raw, frame, smooth, horses)
def update(self):
raw = self.video.read()
frame = raw.copy()
smooth = raw.copy()
if self.horse is not None:
self.horse.updated = False
self.horse.last_detected += 1
boxes, scores = self.detector.detect_boxes(frame)
self.match(frame, boxes, scores)
if self.horse is not None:
self.horse.update(frame, [])
self.horse.draw(frame)
self.horse.draw_smooth(smooth)
if self.horse is not None and self.horse.gone():
logger.info('horse is gone')
self.horse = None
horses = [] if self.horse is None else [self.horse]
self.video.write(raw, frame, smooth, horses)
def getFrameCount(self):
return self.video.getFrameCount()
def __init__(self,
good_pose,
bad_pose,
good_bboxes,
bad_bboxes,
good_pose_video=None,
bad_pose_video=None,
weights=None):
self.good_poses = npy_to_poses(good_pose)
self.bad_poses = npy_to_poses(bad_pose)
self.good_bboxes = (np.load(good_bboxes, allow_pickle=True))[0]
self.bad_bboxes = np.load(bad_bboxes, allow_pickle=True)[0]
self.weights = list(
self.DEFAULT_WEIGHTS.values()) if weights is None else weights
self.good_poses_video = None
self.bad_poses_video = None
if good_pose_video is not None:
self.good_poses_video = VideoManager()
self.good_poses_video.get_video(good_pose_video)
if bad_pose_video is not None:
self.bad_poses_video = VideoManager()
self.bad_poses_video.get_video(bad_pose_video)
def main():
logger = logging.getLogger()
logger.info("start processing %s", input_file)
manager = VideoManager(input_file)
audio = manager.extract_audio(settings.ROOT / "output/a.wav")
thumbnail = manager.extract_thumbnail(settings.ROOT / "output/a.jpg")
audio_manager = AudioManager(audio)
transcriber = Transcriber(audio_manager)
filter_sections = []
for transcribtion in transcriber.transcribe():
logger.debug("transcription: %s", transcribtion)
word, start, end = transcribtion
if detect_mature_word(word):
logger.debug("mature word: %s, %s", word, detect_mature_word(word))
audio_manager.apply_beep(start, end)
manager.apply_mask(start, end)
filter_sections.append({
"start_time": start,
"end_time": end,
"word": word
})
manager.apply_audio(audio_manager.save(settings.ROOT /
"output/a_beep.wav"))
manager.save(settings.ROOT / "output/a.mp4")
print(
json.dumps({
"thumbnail": str(thumbnail),
"filter_sections": filter_sections,
"filter_video": str(settings.ROOT / "output" / "a.mp4"),
}))
def __init__(self, input, output, max_frames, skip, show, detector):
self.video = VideoManager(input, output, max_frames, skip, show)
self.horse = None
self.detector = detector
# for r in range(height):
# for c in range(width):
# gx,gy = y_gradient[r,c], x_gradient[r,c]
# avg_green_change = (gx+gy)/2
# green_channel[r,c,0] = gx
# # green_channel[r,c,2] = gy
greens = green_channel[:, :, 1]
high_greens = np.zeros(greens.shape)
high_greens[np.where(greens > 200)] = 1
green_channel[np.nonzero(high_greens)] = [255, 255, 255]
green_channel[np.where(high_greens == 0)] = [0, 0, 0]
visualize.show_image(green_channel)
# green_channel[np.where(gradient==0)] = [255, 0, 0]
if __name__ == '__main__':
scene_dir = 'scenes/overhead'
videos = file_utils.getPaths(scene_dir)
first_video = videos[0]
manager = VideoManager(first_video)
first_frame = manager.getFrame(0)
field_lines = findFieldLines(first_frame)
for fl in field_lines:
fl.draw(first_frame)
hashmarks = findHashMarks(first_frame, field_lines)
hashmarks.draw(first_frame)
visualize.show_image(first_frame)
import paho.mqtt.client as mqtt
import time
import serial
from video_manager import VideoManager
import sys
import struct
videomanager = VideoManager("/dev/video0", 1920, 1080, 30, "192.168.10.236",
5000, "RGB", 2000000, True, 1280, 720)
videomanager.run()
videomanager_lwir = VideoManager("/dev/video1", 640, 480, 25, "192.168.10.236",
5001, "LWIR", 1000000, False, 640, 480)
videomanager_lwir.run()
last_message = 0.0
ser = serial.Serial(port='/dev/ttyTHS1', baudrate=38400)
ser.isOpen()
def on_connect(client, userdata, flags, rc):
print("Conencted with result code" + str(rc))
client.subscribe("gimbal/#")
def on_message(client, userdata, msg):
if (msg.topic == 'gimbal/cameras/rgb/zoom'):
class Comparator:
DEFAULT_WEIGHTS = {
"BOTTOM SPINE": 0,
"RIGHT HIP": 1,
"RIGHT KNEE": 1,
"RIGHT ANKLE": 1,
"LEFT HIP": 1,
"LEFT KNEE": 1,
"LEFT ANKLE": 1,
"MIDDLE SPINE": 0,
"TOP SPINE": 0,
"MIDDLE HEAD": 0,
"TOP HEAD": 0,
"LEFT SHOULDER": 1,
"LEFT ELBOW": 1,
"LEFT HAND": 1,
"RIGHT SHOULDER": 1,
"RIGHT ELBOW": 1,
"RIGHT HAND": 1
}
def __init__(self,
good_pose,
bad_pose,
good_bboxes,
bad_bboxes,
good_pose_video=None,
bad_pose_video=None,
weights=None):
self.good_poses = npy_to_poses(good_pose)
self.bad_poses = npy_to_poses(bad_pose)
self.good_bboxes = (np.load(good_bboxes, allow_pickle=True))[0]
self.bad_bboxes = np.load(bad_bboxes, allow_pickle=True)[0]
self.weights = list(
self.DEFAULT_WEIGHTS.values()) if weights is None else weights
self.good_poses_video = None
self.bad_poses_video = None
if good_pose_video is not None:
self.good_poses_video = VideoManager()
self.good_poses_video.get_video(good_pose_video)
if bad_pose_video is not None:
self.bad_poses_video = VideoManager()
self.bad_poses_video.get_video(bad_pose_video)
def compare_images(self,
good_pose,
bad_pose,
good_bbox,
bad_bbox,
bad_image,
ignore=(0, 7, 8, 9, 10)):
joint_groups = good_pose.JOINT_GROUPS
images = []
good_arr = good_pose.to_array()
good_x = good_arr[0]
good_y = good_arr[2]
good_x = normalize_to_interval(good_x, good_bbox[2], good_bbox[0])
good_y = normalize_to_interval(good_y, good_bbox[3], good_bbox[1])
bad_arr = bad_pose.to_array()
bad_x = bad_arr[0]
bad_y = bad_arr[2]
bad_x = normalize_to_interval(bad_x, bad_bbox[2], bad_bbox[0])
bad_y = normalize_to_interval(bad_y, bad_bbox[3], bad_bbox[1])
for group in joint_groups:
for limb in group:
if limb[0] in ignore or limb[1] in ignore:
continue
image = bad_image.copy()
cv2.line(
image, (int(bad_x[limb[0]]), int(bad_y[limb[0]])),
(int(good_x[limb[1]] - good_x[limb[0]] + bad_x[limb[0]]),
int(good_y[limb[1]] - good_y[limb[0]] + bad_y[limb[0]])),
(0, 255, 0), 15)
images.append(image)
return images
def compute_pose_distance(self, pose_1, pose_2):
if len(pose_1) != len(pose_2):
return float("inf")
distance = 0
for index in range(len(pose_1)):
distance += pose_1[index].euclidian_distance(
pose_2[index]) * self.weights[index]
return distance
def compare_poses(self, treshold=168, frameskip=1):
# self.good_poses[0].prepare_2d_plot()
# self.good_poses[0].plot()
sets = []
good_pose_indexes = list(range(len(self.good_poses)))
# print(good_pose_indexes)
for index_1 in range(0, len(self.bad_poses), frameskip):
min_value = treshold + 1
min_index = 0
for index_2 in good_pose_indexes:
distance = self.bad_poses[index_1].angle_similarity(
self.good_poses[index_2])
min_value, min_index = (
distance, index_2) if distance < min_value else (min_value,
min_index)
if min_value > treshold:
continue
good_pose_indexes.remove(min_index)
# self.bad_poses[index_1].compute_corrections(self.good_poses[min_index])
if self.good_poses_video is not None and self.bad_poses_video is not None:
# print(str(index_1), "+", str(min_index), "=", str(min_value))
if self.good_poses_video[
min_index] is not None and self.bad_poses_video[
index_1] is not None:
frame_1 = self.good_poses[min_index].put_on_image(
self.good_poses_video[min_index],
self.good_bboxes[min_index])
frame_2 = self.bad_poses[index_1].put_on_image(
self.bad_poses_video[index_1],
self.bad_bboxes[index_1])
# plot_frames(frame_1, frame_2)
bad_frames = self.compare_images(
self.good_poses[min_index], self.bad_poses[index_1],
self.good_bboxes[min_index], self.bad_bboxes[index_1],
self.bad_poses_video[index_1])
set = [(frame_1, bad_frames[index])
for index in range(len(bad_frames))]
set.insert(0, (frame_1, frame_2))
sets.append(set)
return sets
class VideoProc(QObject):
# signals
in_display = pyqtSignal(QPixmap)
out_display = pyqtSignal(QPixmap)
def __init__(self,
capture,
in_file_name,
preview_window_manager=None,
should_mirror_preview=False):
super(VideoProc, self).__init__()
# initialize variables
self.vm = VideoManager(capture, in_file_name)
self.in_file_name = in_file_name
self.preview_window_manager = preview_window_manager
self._out_filename = None
self._capture = capture
self.channel = 0
self.stopped = True
self.build_out_filename()
self._in_frame = None
self._out_frame = None
self.out_width = 480
self.out_height = 270
# for numbering frames
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.bottom_left_text = 10, self.out_height - 10
self.font_scale = 1
self.font_color = (255, 128, 128)
self.thickness = 2
self.counter = 0
self.build_out_filename()
def build_out_filename(self):
self._out_filename = self.in_file_name[:-4] + "_small.mp4"
print('built out_file_name:', self._out_filename)
def process_image(self):
# resize the image
if self.vm._in_frame:
self._in_frame = self.vm._in_frame
else:
self._in_frame = np.ones((1920, 1080, 3))
print('entered:', self.vm._entered_frame, 'in_frame: ',
type(self.vm._in_frame))
print('type frame: ', self.vm._in_frame)
print('type:', type(self._in_frame))
print(self._in_frame.shape)
print(self.out_width, self.out_height)
self._out_frame = np.zeros((self.out_width, self.out_height, 3))
self._out_frame = cv2.resize(src=self._in_frame,
dsize=(self.out_width, self.out_height),
dst=self._out_frame,
fx=0,
fy=0,
interpolation=cv2.INTER_AREA)
# number the frames
cv2.putText(img=self._out_frame,
text=str(self.counter),
org=self.bottom_left_text,
fontFace=self.font,
fontScale=self.font_scale,
color=self.font_color,
thickness=self.thickness)
self.counter += 1
@pyqtSlot()
def start_video(self):
print("Thread started")
self.stopped = False
# start writing output video
self.vm.start_writing_video(self._out_filename)
print('name sent to video manager:', self._out_filename, 'end_of_line')
print('video filename:', self.vm._video_filename)
# begin main loop
while self._capture.isOpened() and not self.stopped:
self.vm.enter_frame()
retrieved = self.vm.retrieve_frame()
print('retrieved: ', retrieved)
if retrieved:
self.vm.update_elapsed_time_frames()
print("entered frame", 'elapsed: ', self.vm._frames_elapsed,
'NR:', self.vm._frame_not_ready_count)
self.process_image()
self.vm.update_elapsed_time_frames()
# self.vm.exit_frame()
print('exited frame', self.vm._frames_elapsed)
self.counter += 1
@pyqtSlot()
def stop_video(self):
print("stopping in progress")
self.stopped = True
self.vm.stop_video()
class Manager():
def __init__(self, input, output, max_frames, skip, show, detector):
self.video = VideoManager(input, output, max_frames, skip, show)
self.horses = np.array([], dtype=Horse)
self.detector = detector
self.global_horse_number = 1
def spawnHorse(self, box):
horse = Horse(box, self.global_horse_number)
self.global_horse_number += 1
logger.info(f'spawn horse {horse.number}')
self.horses = np.append(self.horses, horse)
return horse
def removeHorse(self, horse):
logger.info(f'remove horse {horse.number}')
self.horses = self.horses[self.horses != horse]
def detect(self, frame):
boxes, scores = self.detector.detect_boxes(frame)
relevant_boxes = []
for index in range(len(boxes)):
# todo: find appropriate value for low score
if scores[index] > 0.3: relevant_boxes.append(boxes[index])
return np.array(relevant_boxes)
def match(self, frame, detected):
detected_horses = []
for index, box in enumerate(detected):
intersects = False
for horse in detected_horses:
if horse.intersect(box): intersects = True
if intersects: continue
lone_horses = list_diff(self.horses, detected_horses)
horse = find_closest_horse(lone_horses, box)
if horse is None:
horse = self.spawnHorse(box)
detected_horses.append(horse)
else:
horse.detect(box)
detected_horses.append(horse)
lone_horses = list_diff(self.horses, detected_horses)
for horse in lone_horses:
horse.track(frame)
def initialize(self):
raw = self.video.read()
frame = raw.copy()
smooth = raw.copy()
detected = self.detect(frame)
self.match(frame, detected)
for horse in self.horses:
horse.draw(frame)
horse.draw_smooth(smooth)
self.video.write(raw, frame, smooth, self.horses)
def update(self):
raw = self.video.read()
frame = raw.copy()
smooth = raw.copy()
for horse in self.horses:
horse.updated = False
horse.last_detected += 1
detected = self.detect(frame)
self.match(frame, detected)
for horse in self.horses:
if horse.gone():
self.removeHorse(horse)
continue
horse.update(frame, self.horses)
horse.draw(frame)
horse.draw_smooth(smooth)
self.video.write(raw, frame, smooth, self.horses)
def getFrameCount(self):
return self.video.getFrameCount()
def embedData(self):
start_time = time.process_time()
print("Encrypting Data...")
encrypted_data = self.crypto.mac(self.data)
print("Reading Video...")
stream = VideoManager(self.media_source, self.key)
frame = stream.getNextFrame()
current_frame_index = stream.getCurrentFrame()
if frame.all() == -1:
raise Exception("Video frames corrupted!")
hot_frame = FrameManager(frame, self.key)
number = 1
print("Embedding Data...")
for byte in encrypted_data:
if hot_frame.full():
# Find the next empty frame. Exit with exception if not available
while True:
if not stream.full():
frame = stream.getNextFrame()
current_frame_index = stream.getCurrentFrame()
if frame.all() == -1:
raise Exception("Video frames corrupted!")
hot_frame = FrameManager(frame, self.key)
if hot_frame.getBlobCount() != 0:
break
else:
raise Exception("Data storage capacity exceeded!")
hot_frame.save_frame("frame_before_" + str(number))
hot_frame.embed(byte)
stream.write_frame(current_frame_index, hot_frame.getFrame())
hot_frame.save_frame("frame_after_" + str(number))
number += 1
print("Generating Video...")
stream.generate_video()
print("Done!")
execution_time = round(time.process_time() - start_time, 3)
print("Completed in", execution_time, "seconds")
def __init__(self, input, output, max_frames, skip, show, detector):
self.video = VideoManager(input, output, max_frames, skip, show)
self.horses = np.array([], dtype=Horse)
self.detector = detector
self.global_horse_number = 1