def generalize_result(self, eng_input, eng_output):
# Pipeline returns None if any error happened
if eng_output is None:
eng_output = {}
# If pipeline generate multiple outputs simultaneously
#
# In this case, the format of engine output is
#
# {
# 'annotations': {...},
# 'bytes': '...'
# }
if all(key in eng_output.keys() for key in ['annotations', 'bytes']):
logger.debug('Pipeline output type: multiple')
logger.debug('eng_input type = {0}, len = {1}'.format(
type(eng_input), len(eng_input)))
# FIXME: Re-cap why eng_input is a list and only contains 1 item.
eng_input = eng_input[0]
try:
eng_input['annotations'] = eng_output['annotations']
logger.debug('output image type: {0}, len: {1}'.format(
type(eng_output['bytes']), len(eng_output['bytes'])))
pipeline_img = eng_output['bytes'][0]
retval, jpg_bytes = cv2.imencode('.jpg', pipeline_img)
eng_input['bytes'] = payload.stringify_jpg(jpg_bytes)
except Exception as e:
logger.critical(e)
else:
logger.debug('Pipeline output type: simple')
# FIXME: Workaround for spec incompatibility
# DLBox spec use 'image_blob', but BerryNet use 'bytes', so we have to
# do a convert here
if isinstance(eng_output, list):
inf_output = eng_output[0]
else:
inf_output = eng_output
if len(eng_input) > 1:
for i in range(len(eng_input)):
try:
retval, jpg_bytes = cv2.imencode('.jpg', inf_output)
eng_input[i]['bytes'] = payload.stringify_jpg(
jpg_bytes)
#eng_input[i].pop('bytes')
except Exception as e:
print(e)
else:
try:
eng_input, = eng_input
retval, jpg_bytes = cv2.imencode('.jpg', inf_output)
eng_input['bytes'] = payload.stringify_jpg(jpg_bytes)
#eng_input.pop('bytes')
except Exception as e:
print(e)
return eng_input
def draw_label(bgr_nparr, infres, class_color, save_image_path=None):
"""Draw bounding boxes on an image.
Args:
bgr_nparr: image data in numpy array format
infres: Inference results followed generic format specification.
class_color: Label color, a RGB tuple.
Returens:
Generalized result whose image data is drew w/ labels.
"""
left = 0
top = 0
for res in infres['annotations']:
imgHeight, imgWidth, _ = bgr_nparr.shape
thick = int((imgHeight + imgWidth) // 300)
# putText can not handle newline char yet,
# so we have to put multiple texts manually.
cv2.putText(
bgr_nparr,
'{0}: {1}'.format(res['label'], res['confidence']),
(left + 10, top + 20), # bottom-left corner of text
0, # fontFace
1e-3 * imgHeight, # fontScale
class_color,
thick // 3)
top += 20
infres['bytes'] = payload.stringify_jpg(cv2.imencode('.jpg', bgr_nparr)[1])
if save_image_path:
cv2.imwrite(save_image_path, bgr_nparr)
return infres
def draw_bb(bgr_nparr, infres, class_colors, labels):
"""Draw bounding boxes on an image.
Args:
bgr_nparr: image data in numpy array format
infres: Darkflow inference results
class_colors: Bounding box color candidates, list of RGB tuples.
Returens:
Generalized result whose image data is drew w/ bounding boxes.
"""
for res in infres['annotations']:
left = int(res['left'])
top = int(res['top'])
right = int(res['right'])
bottom = int(res['bottom'])
label = res['label']
color = class_colors[labels.index(label)]
confidence = res['confidence']
imgHeight, imgWidth, _ = bgr_nparr.shape
thick = int((imgHeight + imgWidth) // 300)
cv2.rectangle(bgr_nparr, (left, top), (right, bottom), color, thick)
cv2.putText(bgr_nparr, label, (left, top - 12), 0, 1e-3 * imgHeight,
color, thick // 3)
#cv2.imwrite('prediction.jpg', bgr_nparr)
infres['bytes'] = payload.stringify_jpg(cv2.imencode('.jpg', bgr_nparr)[1])
return infres
def generalize_result(self, eng_inputs, eng_outputs):
# Pipeline returns None if any error happened
if eng_outputs is None:
eng_outputs = [{}]
if len(eng_inputs) != len(eng_outputs):
logger.warning('Input length != output length: {} != {}'.format(
len(eng_inputs), len(eng_outputs)))
# We guarantee len of inputs will always be 1 (at least now), so
# it's safer to access eng_inputs by index than to eng_outputs
c_id = int(eng_inputs[0]['meta']['channel_id'])
eng_outputs = [eng_outputs[c_id]]
# FIXME: Workaround for spec incompatibility
# DLBox spec use 'image_blob', but BerryNet use 'bytes', so we have to
# do a convert here
for eng_in, eng_out in list(zip(eng_inputs, eng_outputs)):
if isinstance(eng_out, np.ndarray):
r, result_img = cv2.imencode('.jpg', eng_out)
eng_in['bytes'] = payload.stringify_jpg(result_img)
else:
try:
eng_in.update(eng_out)
except KeyError as e:
logger.exception('{} ({}): {}'.format(
e.__class__, e.__doc__, e))
eng_in['image_blob'] = eng_in.pop('bytes')
return eng_inputs
def main():
args = parse_args()
if args['debug']:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
comm_config = {
'subscribe': {},
'broker': {
'address': args['broker_ip'],
'port': args['broker_port']
}
}
comm = Communicator(comm_config, debug=True)
duration = lambda t: (datetime.now() - t).microseconds / 1000
if args['mode'] == 'stream':
counter = 0
# Check input stream source
if args['stream_src'].isdigit():
# source is a physically connected camera
stream_source = '/dev/video{}'.format(int(args['stream_src']))
capture = cv2.VideoCapture(int(args['stream_src']))
else:
# source is an IP camera
stream_source = args['stream_src']
capture = cv2.VideoCapture(args['stream_src'])
cam_fps = capture.get(cv2.CAP_PROP_FPS)
if cam_fps > 30 or cam_fps < 1:
logger.warn(
'Camera FPS is {} (>30 or <1). Set it to 30.'.format(cam_fps))
cam_fps = 30
out_fps = args['fps']
interval = int(cam_fps / out_fps)
# warmup
#t_warmup_start = time.time()
#t_warmup_now = time.time()
#warmup_counter = 0
#while t_warmup_now - t_warmup_start < 1:
# capture.read()
# warmup_counter += 1
# t_warmup_now = time.time()
logger.debug('===== VideoCapture Information =====')
logger.debug('Stream Source: {}'.format(stream_source))
logger.debug('Camera FPS: {}'.format(cam_fps))
logger.debug('Output FPS: {}'.format(out_fps))
logger.debug('Interval: {}'.format(interval))
#logger.debug('Warmup Counter: {}'.format(warmup_counter))
logger.debug('====================================')
while True:
status, im = capture.read()
if (status is False):
logger.warn('ERROR: Failure happened when reading frame')
# NOTE: Hard-coding rotation for AIKEA onboard camera.
# We will add parameter support in the future.
im = tinycv.rotate_ccw_opencv(im)
counter += 1
if counter == interval:
logger.debug('Drop frames: {}'.format(counter - 1))
counter = 0
# Open a window and display the ready-to-send frame.
# This is useful for development and debugging.
if args['display']:
cv2.imshow('Frame', im)
cv2.waitKey(1)
t = datetime.now()
#logger.debug('write frame to /tmp/output.jpg')
#cv2.imwrite('/tmp/output.jpg', im)
retval, jpg_bytes = cv2.imencode('.jpg', im)
obj = {}
obj['timestamp'] = datetime.now().isoformat()
obj['bytes'] = payload.stringify_jpg(jpg_bytes)
obj['meta'] = {
'roi': [{
'top': 50,
#'left': 341,
#'bottom': 500,
#'right': 682,
#'left': 640,
#'bottom': 980,
#'right': 1280,
'left': 10,
'bottom': 600,
'right': 600,
'overlap_threshold': 0.5
}]
}
logger.debug('timestamp: {}'.format(obj['timestamp']))
logger.debug('bytes len: {}'.format(len(obj['bytes'])))
logger.debug('meta: {}'.format(obj['meta']))
mqtt_payload = payload.serialize_payload([obj])
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('send: {} ms'.format(duration(t)))
else:
pass
elif args['mode'] == 'file':
# Prepare MQTT payload
im = cv2.imread(args['filepath'])
retval, jpg_bytes = cv2.imencode('.jpg', im)
t = datetime.now()
obj = {}
obj['timestamp'] = datetime.now().isoformat()
obj['bytes'] = payload.stringify_jpg(jpg_bytes)
obj['meta'] = {
'roi': [{
'top': 50,
'left': 10,
'bottom': 600,
'right': 600,
'overlap_threshold': 0.5
}]
}
mqtt_payload = payload.serialize_payload([obj])
logger.debug('payload: {} ms'.format(duration(t)))
logger.debug('payload size: {}'.format(len(mqtt_payload)))
# Client publishes payload
t = datetime.now()
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('mqtt.publish: {} ms'.format(duration(t)))
logger.debug('publish at {}'.format(datetime.now().isoformat()))
else:
logger.error('User assigned unknown mode {}'.format(args['mode']))