def __init__(self, module_name, settings, pipe):
"""Initialize video capture, logger, and processing plugins.
Args:
module_name -- TODO
settings -- dict of settings for this object.
pipe -- TODO
"""
self.module_name = module_name
self.master_settings = settings['sensor/vision/control']
self.settings = settings[module_name]
self.processor = FrameProcessor()
self._com = Communicator(module_name)
# Nagging messages from the parent come from here. The process must
# respond quickly enough or be killed by the parent.
self._pipe = pipe
self._fresh_frame_available = Event()
self._got_im, self._im = None, None
self._init_stream()
# Individual stream images will be processed by modules stored here.
self._plugins = []
# Load, instantiate, and store the modules defined in the settings file.
for plugin_name in self.settings['plugins']:
# Name of module and the class should be the same.
module_obj = getattr(import_module(
'..'+plugin_name, package='plugins.subpkg'),
plugin_name)(self.processor, self.settings)
self._plugins += [module_obj]
# Configure the VideoCapture object.
self._vision_process()
class StreamProcessor(object):
"""Open a video stream and process frames. Report finds to the grapevine."""
def __init__(self, module_name, settings, pipe):
"""Initialize video capture, logger, and processing plugins.
Args:
module_name -- TODO
settings -- dict of settings for this object.
pipe -- TODO
"""
self.module_name = module_name
self.master_settings = settings['sensor/vision/control']
self.settings = settings[module_name]
self.processor = FrameProcessor()
self._com = Communicator(module_name)
# Nagging messages from the parent come from here. The process must
# respond quickly enough or be killed by the parent.
self._pipe = pipe
self._fresh_frame_available = Event()
self._got_im, self._im = None, None
self._init_stream()
# Individual stream images will be processed by modules stored here.
self._plugins = []
# Load, instantiate, and store the modules defined in the settings file.
for plugin_name in self.settings['plugins']:
# Name of module and the class should be the same.
module_obj = getattr(import_module(
'..'+plugin_name, package='plugins.subpkg'),
plugin_name)(self.processor, self.settings)
self._plugins += [module_obj]
# Configure the VideoCapture object.
self._vision_process()
def _init_stream(self):
"""Initialize the video stream."""
if self.settings['stream_type'] == 'device':
cam_path = os.path.realpath(self.settings['symlink'])
# Check that the symlink points to something that really exists.
if '/dev/video' not in cam_path:
raise Exception('Broken symlink {sym}'.format(
sym=self.settings['symlink']))
self._cap = cv2.VideoCapture(int(cam_path[-1]))
self._cap.set(cv.CV_CAP_PROP_FPS, self.settings['fps'])
self._frame_grabber_thread = Thread(target=self.refresh_frame)
self._frame_grabber_thread.daemon = True
self._frame_grabber_thread.start()
#self._frame_grabber_thread = None
elif self.settings['stream_type'] == 'recorded_video':
# The 'symlink' key didn't exist. This must be a test with recorded
# video.
self._cap = cv2.VideoCapture(self.settings['recorded_video'])
self._frame_grabber_thread = None
else:
raise Exception(
'No stream type specified for {module_name}'.format(
module_name=module_name))
def _get_frame(self):
"""Get the most recent frame from the video stream."""
if self._frame_grabber_thread is not None:
self._fresh_frame_available.wait()
return (self._got_im, self._im)
else:
return self._cap.read()
def refresh_frame(self):
"""Get and store latest frame from video stream as fast as possible."""
failed_count = 0
# Assuming 30fps, 150 fails in a row means 5s of no video.
while failed_count < self.master_settings['max_failed_frames']:
self._got_im, self._im = self._cap.read()
self._fresh_frame_available.set()
self._fresh_frame_available.clear()
if not self._got_im:
failed_count += 1
else:
failed_count = 0
raise Exception('Camera sucks')
def _vision_process(self):
"""Detect obstacle details from video stream. Report findings."""
while True:
start_time = time()
got_im, im = self._get_frame()
#print 'new frame'
if self._pipe.poll() is True:
# Send the message back to parent to prove this process
# isn't frozen.
self._pipe.send(self._pipe.recv())
if got_im:
# Check if parent sent a message over the pipe.
self.processor.load_im(im)
# self.processor.hacky_display() # FIXME this needs to go away.
packet = {}
for plugin in self._plugins:
plugin.process_image(packet)
self._com.publish_message(packet)
def main():
args = parse_args()
FOLDER = args.output_folder
FRAME_LIMIT = args.frame_limit
VIDEO_FILE = args.video_file
DRAW_BOXES = args.draw_boxes
SCORES_FILE = args.scores_file
DETECTION = args.detection
GAZE_ESTIMATION = not args.no_gaze_estimation
OUTPUT_TYPE = args.output_type
NO_SCORES = args.no_scores
if GAZE_ESTIMATION:
processor = FrameProcessor()
else:
processor = None
if DETECTION == "cv2":
detector = CVFaceDetector()
elif DETECTION == "cnn":
detector = MTCNNFaceDetector()
else:
raise Exception(f"detection method {DETECTION} is invalid, expected one of [cv2, cnn]")
df_faces = []
if NO_SCORES:
print(f">> Reading frames from {VIDEO_FILE}")
frames = [
cv2.imread(os.path.join(VIDEO_FILE, frame_file))
for frame_file in os.listdir(VIDEO_FILE)
]
h, w, _ = frames[0].shape
df = pd.DataFrame([{
'track_id': i // 128,
'frame_id': i,
'abs_frame_id': i,
'box_id': 0,
'x': 0,
'y': 0,
'x2': w,
'y2': h,
} for i in range(len(frames))])
else:
print(f">> Reading video from {VIDEO_FILE}")
df = pd.read_csv(SCORES_FILE, header=None, index_col=None, names=[
"track_id", "frame_id", "box_id", "x", "y", "x2", "y2"] + list(range(80)))
df["abs_frame_id"] = df.frame_id + df.track_id - 128
frames = collect_frames(VIDEO_FILE, FRAME_LIMIT)
if os.path.isdir(FOLDER):
shutil.rmtree(FOLDER)
print(">> Matching frames")
for i, frame in tqdm(enumerate(frames), total=len(frames)):
new_frame = match_faces_bodies(
frame,
df[df.abs_frame_id == i],
processor=processor,
detector=detector,
draw_boxes=DRAW_BOXES
)
frames[i], new_df_faces = new_frame
df_faces += new_df_faces
if not os.path.isdir(FOLDER):
print(f">> Creating folder {FOLDER}")
os.makedirs(FOLDER)
if OUTPUT_TYPE == "frames":
os.makedirs(os.path.join(FOLDER, "frames"))
save_video_to_files(frames, os.path.join(FOLDER, "frames"))
df_faces = pd.DataFrame(df_faces)
print(">> Saving box files")
box_ids = df_faces.box_id.unique()
for box_id in tqdm(box_ids, total=len(box_ids)):
with open(os.path.join(FOLDER, f"person{box_id}.txt"), "w") as f:
for _, frame in df_faces[df_faces.box_id == box_id].iterrows():
bbox = str([frame.x, frame.y, frame.x2, frame.y2]).replace(" ", "").strip("[]")
f.write(f"frame_{frame.frame_id}.png,{bbox}\n")
else:
output_video_file(os.path.join(FOLDER, "video.mp4"), frames)
agent = PG(run_name=arg.run_name,
input_shape=[160,160],
n_action=3,
learning_rate=arg.learning_rate,
save_path=arg.save_path,
record_io=False,
record=True,
gpu_fraction=0.9)
for i in range(arg.games_num):
replay_buffer = ReplayBuffer(input_shape=[160, 160], start_size=32, max_size=10000000)
memory = []
total_reward = 0
obs = env.reset()
frame_processor = FrameProcessor(obs)
obs, reward, done, _ = env.step(agent.random_action() + 1) # do random action at the first frame
total_reward += reward
# play one game
while not done:
input_frame = frame_processor.process(obs)
prob = agent.get_action_prob(input_frame)
action = np.random.choice(3, p=prob)
obs, reward, done, _ = env.step(action + 1)
if reward == 0:
replay_buffer.store_transition(input_frame, action)
else:
total_reward += reward
if reward == 1:
class StreamProcessor(object):
"""Open a video stream and process frames. Report finds to the grapevine."""
def __init__(self, module_name, settings, pipe):
"""Initialize video capture, logger, and processing plugins.
Args:
module_name -- TODO
settings -- dict of settings for this object.
pipe -- TODO
"""
self.module_name = module_name
self.master_settings = settings['sensor/vision/control']
self.settings = settings[module_name]
self.processor = FrameProcessor()
self._com = Communicator(module_name)
# Nagging messages from the parent come from here. The process must
# respond quickly enough or be killed by the parent.
self._pipe = pipe
self._fresh_frame_available = Event()
self._got_im, self._im = None, None
self._init_stream()
# Individual stream images will be processed by modules stored here.
self._plugins = []
# Load, instantiate, and store the modules defined in the settings file.
for plugin_name in self.settings['plugins']:
# Name of module and the class should be the same.
module_obj = getattr(
import_module('..' + plugin_name, package='plugins.subpkg'),
plugin_name)(self.processor, self.settings)
self._plugins += [module_obj]
# Configure the VideoCapture object.
self._vision_process()
def _init_stream(self):
"""Initialize the video stream."""
if self.settings['stream_type'] == 'device':
cam_path = os.path.realpath(self.settings['symlink'])
# Check that the symlink points to something that really exists.
if '/dev/video' not in cam_path:
raise Exception('Broken symlink {sym}'.format(
sym=self.settings['symlink']))
self._cap = cv2.VideoCapture(int(cam_path[-1]))
self._cap.set(cv.CV_CAP_PROP_FPS, self.settings['fps'])
self._frame_grabber_thread = Thread(target=self.refresh_frame)
self._frame_grabber_thread.daemon = True
self._frame_grabber_thread.start()
#self._frame_grabber_thread = None
elif self.settings['stream_type'] == 'recorded_video':
# The 'symlink' key didn't exist. This must be a test with recorded
# video.
self._cap = cv2.VideoCapture(self.settings['recorded_video'])
self._frame_grabber_thread = None
else:
raise Exception(
'No stream type specified for {module_name}'.format(
module_name=module_name))
def _get_frame(self):
"""Get the most recent frame from the video stream."""
if self._frame_grabber_thread is not None:
self._fresh_frame_available.wait()
return (self._got_im, self._im)
else:
return self._cap.read()
def refresh_frame(self):
"""Get and store latest frame from video stream as fast as possible."""
failed_count = 0
# Assuming 30fps, 150 fails in a row means 5s of no video.
while failed_count < self.master_settings['max_failed_frames']:
self._got_im, self._im = self._cap.read()
self._fresh_frame_available.set()
self._fresh_frame_available.clear()
if not self._got_im:
failed_count += 1
else:
failed_count = 0
raise Exception('Camera sucks')
def _vision_process(self):
"""Detect obstacle details from video stream. Report findings."""
while True:
start_time = time()
got_im, im = self._get_frame()
#print 'new frame'
if self._pipe.poll() is True:
# Send the message back to parent to prove this process
# isn't frozen.
self._pipe.send(self._pipe.recv())
if got_im:
# Check if parent sent a message over the pipe.
self.processor.load_im(im)
# self.processor.hacky_display() # FIXME this needs to go away.
packet = {}
for plugin in self._plugins:
plugin.process_image(packet)
self._com.publish_message(packet)
