def __setup_subscriber(self):
self.rc_data.delete(self.opt.sub_channel)
# Dummy set value
# expired_at = 2 # in seconds
# redis_set(self.rc_data, self.opt.sub_channel, True, expired_at)
redis_set(self.rc_data, self.opt.sub_channel, 1)
def run(self):
print("\nReading video:")
# while True:
while self.is_running:
try:
# t0_stream_setup = time.time()
self.cap = cv.VideoCapture(self.opt.source)
# t_stream_setup = time.time() - t0_stream_setup
# t_stream_setup_key = "stream-setup-" + str(self.opt.drone_id)
t_stream_setup_key = "stream-start-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_stream_setup_key, time.time())
# Latency:
# t_stream_setup_key = "stream-setup-" + str(self.opt.drone_id)
# redis_set(self.rc_latency, t_stream_setup_key, t_stream_setup)
# print('\nLatency [Stream Setup Time]: (%.5fs)' % (t_stream_setup))
if self.opt.enable_cv_out:
cv.namedWindow("Image", cv.WND_PROP_FULLSCREEN)
cv.resizeWindow("Image", 1366, 768) # Enter your size
self.__start_streaming()
except:
print(
"\nUnable to communicate with the Streaming. Restarting . . ."
)
# time.sleep(1) # Delay 1 second before trying again
# The following frees up resources and closes all windows
self.cap.release()
if self.opt.enable_cv_out:
cv.destroyAllWindows()
def __load_balancing(self, frame_id, ret, frame, save_path):
# Initially, send process into first worker
self.worker_id += 1
# self.worker_id = 1
stream_channel = common_settings["redis_config"][
"channel_prefix"] + str(self.worker_id)
# Check worker status first, please wait while still working
# print(">>>>>> __worker_status = ", self.__worker_status())
w = 0
wait_time = self.wait_time
if redis_get(self.rc_data, self.worker_id) is None:
print("This worker is OFF. Nothing to do")
print(
"TBD next time: Should skip this worker and move to the next worker instead"
)
self.__find_optimal_worker()
else:
# None = DISABLED; 1=Ready; 0=Busy
while self.__worker_status() == 0:
# print("\nWorker-%d is still processing other image, waiting (%ds) ..." % (self.worker_id, w))
# print("\nWorker-%d is still processing other image, waiting ... " % self.worker_id)
# time.sleep(0.005)
if not self.opt.disable_delay:
time.sleep(self.wait_time)
w += self.wait_time
# Send multi-process and set the worker as busy (value=False)
print("### Sending the work into [worker-#%d] @ `%s`" %
((self.worker_id), stream_channel))
Process(target=frame_producer,
args=(
self.rc,
frame_id,
ret,
frame,
save_path,
stream_channel,
self.rc_latency,
self.opt.drone_id,
)).start()
redis_set(self.rc_data, self.worker_id, 0)
self.__reset_worker()
def __iterate_frames(self, this_frame_id=None):
# Run inference
self.t0 = time.time()
# frame_id = 0
for path, img, im0s, vid_cap in self.dataset:
self.frame_id += 1
t = time.time()
# Get detections
ts_det = time.time()
img = torch.from_numpy(img).to(self.device)
if img.ndimension() == 3:
img = img.unsqueeze(0)
self.pred = self.model(img)[0]
# print('\n # Total Inference time: (%.3fs)' % (time.time() - ts_det))
t_inference = time.time() - ts_det
self.time_inference += t_inference
self.time_inference_list.append(t_inference)
# Latency: Inference
print('\nLatency [Inference] of frame-%s: (%.5fs)' %
(str(this_frame_id), t_inference))
t_inference_key = "inference-" + str(
self.opt.drone_id) + "-" + str(this_frame_id)
redis_set(self.rc_latency, t_inference_key, t_inference)
# Default: Disabled
if self.opt.half:
self.pred = self.pred.float()
# Apply NMS: Non-Maximum Suppression
ts_nms = time.time()
# to Removes detections with lower object confidence score than 'conf_thres'
self.pred = non_max_suppression(self.pred,
self.opt.conf_thres,
self.opt.iou_thres,
classes=self.opt.classes,
agnostic=self.opt.agnostic_nms)
# print('\n # Total Non-Maximum Suppression (NMS) time: (%.3fs)' % (time.time() - ts_nms))
t_nms = time.time() - ts_nms
self.time_nms += t_nms
self.time_nms_list.append(t_nms)
# Latency: NMS
print('\nLatency [NMS] of frame-%s: (%.5fs)' %
(str(this_frame_id), t_nms))
t_nms_key = "nms-" + str(
self.opt.drone_id) + "-" + str(this_frame_id)
redis_set(self.rc_latency, t_nms_key, t_nms)
# Apply Classifier: Default DISABLED
if self.classify:
self.pred = apply_classifier(self.pred, self.modelc, img, im0s)
# Process detections
'''
p = path
s = string for printing
im0 = image (matrix)
'''
for i, det in enumerate(self.pred): # detections per image
if self.webcam: # batch_size >= 1
p, self.str_output, im0 = path[i], '%g: ' % i, im0s[i]
else:
p, self.str_output, im0 = path, '', im0s
self.save_path = str(Path(self.out) / Path(p).name)
self.str_output += '%gx%g ' % img.shape[2:] # print string
if det is not None and len(det):
# Rescale boxes from img_size to im0 size
det[:, :4] = scale_coords(img.shape[2:], det[:, :4],
im0.shape).round()
ts_mbbox = time.time()
if self.mbbox_algorithm:
self.__mbbox_detection(det, im0) # modifying Mb-box
t_mbbox = time.time() - ts_mbbox
self.time_mbbox += t_mbbox
self.time_mbbox_list.append(t_mbbox)
# Latency: MB-Box
print('\nLatency [MB-Box Algorithm] of frame-%s: (%.5fs)' %
(str(this_frame_id), t_mbbox))
t_mbbox_key = "mbbox-" + str(
self.opt.drone_id) + "-" + str(this_frame_id)
redis_set(self.rc_latency, t_mbbox_key, t_mbbox)
ts_default = time.time()
if not self.opt.maximize_latency:
if self.default_algorithm:
self.__default_detection(det, im0)
t_default = time.time() - ts_default
self.time_default += t_default
self.time_default_list.append(t_default)
# Latency: Default Detection
print(
'\nLatency [Default Detection] of frame-%s: (%.5fs)' %
(str(this_frame_id), t_default))
t_default_key = "draw2bbox-" + str(
self.opt.drone_id) + "-" + str(this_frame_id)
redis_set(self.rc_latency, t_default_key, t_default)
# Print time (inference + NMS)
# print('%sDone. (%.3fs)' % (self.str_output, time.time() - t))
print(
'Done. (%.3fs) --> '
'Inference(%.3fs); NMS(%.3fs); MB-Box(%.3fs); Default-Bbox(%.3fs)'
% ((time.time() - t), t_inference, t_nms, t_mbbox,
t_default))
# Save end latency calculation
t_end_key = "end-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_end_key, time.time())
# Mark last processed frame
t_last_frame_key = "last-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_last_frame_key,
int(this_frame_id))
# latency: End-to-end Latency, each frame
t_sframe_key = "start-fi-" + str(
self.opt.drone_id
) # to calculate end2end latency each frame.
t_end2end_frame = redis_get(
self.rc_latency, t_end_key) - redis_get(
self.rc_latency, t_sframe_key)
t_e2e_frame_key = "end2end-frame-" + str(
self.opt.drone_id) + "-" + str(this_frame_id)
redis_set(self.rc_latency, t_e2e_frame_key,
t_end2end_frame)
print(
'\nLatency [End2end this frame] of frame-%s: (%.5fs)' %
(str(this_frame_id), t_end2end_frame))
# latency: End-to-end Latency TOTAL
t_start_key = "start-" + str(self.opt.drone_id)
t_end2end = redis_get(self.rc_latency,
t_end_key) - redis_get(
self.rc_latency, t_start_key)
t_e2e_key = "end2end-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_e2e_key, t_end2end)
print('\nLatency [This End2end] of frame-%s: (%.5fs)' %
(str(this_frame_id), t_end2end))
# Stream results
# if self.view_img:
# cv2.imshow(p, im0)
# if cv2.waitKey(1) == ord('q'): # q to quit
# raise StopIteration
self.__save_results(im0, vid_cap, self.frame_id)
def waiting_frames(self):
print("Starting YOLOv3 image detector")
t0 = time.time()
self.__load_weight()
t_load_weight = time.time() - t0
# print(".. Load `weight` in (%.3fs)" % t_load_weight)
# Latency: Load YOLOv3 Weight
t_weight_key = "weight-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_weight_key, t_load_weight)
print('\nLatency [Load `weight`]: (%.5fs)' % (t_load_weight))
t0 = time.time()
self.__eval_model()
t_load_eval_model = time.time() - t0
# print(".. Load function `eval_model` in (%.3fs)" % t_load_eval_model)
# Latency: Execute Evaluation Model
t_eval_model_key = "evalModel-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_eval_model_key, t_load_eval_model)
print('\nLatency [Load `Eval Model`]: (%.5fs)' % (t_load_eval_model))
self.__get_names_colors()
# Setup redis subscriber availability
self.__setup_subscriber()
# Waiting for get the image information
stream_ch = "stream_" + self.opt.sub_channel
print("\n### Waiting for get the image information @ Channel `%s`" %
stream_ch)
pubsub = self.rc.pubsub()
pubsub.subscribe([stream_ch])
for item in pubsub.listen():
# noinspection PyPackageRequirements
try:
t0_sub2frame = time.time()
# Start fetching information
fetch_data = json.loads(item['data'])
frame_id = fetch_data['frame_id']
# print('Streamer collects : ', fetch_data)
# Latency: subscribing frame
t_sub2frame = time.time() - t0_sub2frame
t_sub2frame_key = "sub2frame-" + str(
self.opt.drone_id) + "-" + str(frame_id)
redis_set(self.rc_latency, t_sub2frame_key, t_sub2frame)
print(
'\nLatency [Subscriber extracting information] of frame-%s: (%.5fs)'
% (str(frame_id), t_sub2frame))
self.source = fetch_data["img_path"]
# print('img_path : ', self.source)
# Load image from pub: get `img_path`
t0_load_img = time.time()
self.__set_data_loader()
t_load_img = time.time() - t0_load_img
# print(".. load image into variable in (%.3fs)" % (time.time() - t0))
t_load_img_key = "loadIMG-" + str(
self.opt.drone_id) + "-" + str(frame_id)
redis_set(self.rc_latency, t_load_img_key, t_load_img)
print(
'\nLatency [Loads image into YOLOv3 Worker] of frame-%s: (%.5fs)'
% (str(frame_id), t_load_img))
# # Check worker status first, please wait while still working
# is_worker_ready = redis_get(self.rc_data, self.opt.sub_channel)
# print(">>>>>> is_worker_ready = ", is_worker_ready)
# if is_worker_ready:
# print("\nWorker-%d is still running, waiting to finish ..." % self.opt.sub_channel)
# time.sleep(0.1)
# Start processing image
print("\nStart processing to get MB-Box.")
redis_set(self.rc_data, self.opt.sub_channel,
0) # set as `Busy`
# print("Set as busy Done.")
self.__iterate_frames(
frame_id) # Perform detection in each frame here
# print(" >>> Hasil MB-Box Img = ", self.mbbox_img)
# print("Process finished.")
frame_id = str(fetch_data["frame_id"])
prev_fid = str(self.opt.drone_id) + "-" + str(
(int(frame_id) - 1))
# # TBD: Start behavior detection in background process, in every 30 frames (FPS=30)
# behave_det = BehaviorDetection(self.opt, int(frame_id), self.detected_mbbox)
# if int(frame_id) % self.fps and self.detected_mbbox > 0:
# behave_det.run()
# Preparing to send the frames.
if self.mbbox_img is not None:
# output_path = self.opt.mbbox_output + "frame-%s.jpg" % frame_id
output_path = self.opt.mbbox_output + str(
self.opt.drone_id) + "/frame-%s.jpg" % frame_id
# Try writing synchronously: store ONLY when (frame-1) has been stored
# print(" ######## PREV_ID = ", redis_get(self.rc, prev_fid))
while redis_get(self.rc,
prev_fid) is None and int(frame_id) > 1:
# print(" ######## Waiting previous frame to be stored. PREV_ID = ", redis_get(self.rc, prev_fid))
# time.sleep(1)
pass
t0_save_mbbox_img = time.time()
cv2.imwrite(output_path, self.mbbox_img)
# print("Saving image in: ", output_path)
# print(".. MB-Box image is saved in (%.3fs)" % (time.time() - t0))
t_save_mbbox_img = time.time() - t0_save_mbbox_img
t_save_mbbox_img_key = "save2mmboxIMG-" + str(
self.opt.drone_id) + "-" + str(frame_id)
redis_set(self.rc_latency, t_save_mbbox_img_key,
t_save_mbbox_img)
print(
'\nLatency [Save MB-Box Image] of frame-%s: (%.5fs)' %
(str(frame_id), t_save_mbbox_img))
# Set that this frame has been successfully added
expired_at = 20 # in seconds
key = str(self.opt.drone_id) + "-" + frame_id
redis_set(self.rc, key, True, expired_at)
else:
print("This MB-Box is NONE. nothing to be saved yet.")
# self.__save_latency_to_csv()
# Restore availibility
redis_set(self.rc_data, self.opt.sub_channel,
1) # set as `Ready`
print("\n### This Worker-%s is ready to serve again. \n\n" %
self.opt.sub_channel)
# Set prev_fid as DONE
redis_set(self.rc_data, prev_fid, None) # set as `Ready`
# if this is latest frame: Set this frame as latest
# key = "drone_id-latest", value: "drone_id-frame_id"
# redis_set(self.rc_data, prev_fid, None) # set as `Ready`
except:
pass
def __start_streaming(self):
n = 0
frame_id = 0
received_frame_id = 0
# t_start = time.time()
# redis_set(self.rc_latency, "start", t_start)
# Save timestamp to start extracting video streaming.
t_start_key = "start-" + str(self.opt.drone_id)
redis_set(self.rc_latency, t_start_key, time.time())
# while (self.cap.isOpened()):
while (self.cap.isOpened()) and self.is_running:
received_frame_id += 1
t_sframe_key = "start-fi-" + str(
self.opt.drone_id) # to calculate end2end latency each frame.
redis_set(self.rc_latency, t_sframe_key, time.time())
n += 1
t0_frame = time.time()
# ret = a boolean return value from getting the frame, frame = the current frame being projected in the video
try:
ret, frame = self.cap.read()
# Latency: capture each frame
t_frame = time.time() - t0_frame
print('\nLatency [Reading stream frame] of frame-%d: (%.5fs)' %
(received_frame_id, t_frame))
t_frame_key = "frame-" + str(
self.opt.drone_id) + "-" + str(frame_id)
redis_set(self.rc_latency, t_frame_key, t_frame)
if n == self.opt.delay: # read every n-th frame
if ret:
# Start capturing here
if received_frame_id >= self.start_frame_id:
frame_id += 1
# Force stop after n frames
# if frame_id > int(self.max_frames):
if frame_id == (int(self.max_frames) + 1):
self.is_running = False
break
save_path = self.opt.output_folder + str(
self.opt.drone_id) + "/frame-%d.jpg" % frame_id
self.__load_balancing(frame_id, ret, frame,
save_path)
if self.opt.enable_cv_out:
cv.imshow("Image", frame)
else:
print("IMAGE is INVALID.")
print("I guess there is no more frame to show.")
break
n = 0
# time.sleep(0.01) # wait time
except:
print("No more frame to show.")
break
if cv.waitKey(10) & 0xFF == ord('q'):
break