def check_param_keys(self):
"""
Check if any default key is missing in the self.param.
"""
default_keys = [
"model_file", "model_description", "label_file", "device"
]
for _key in default_keys:
if _key not in self.param.keys():
logger.error("%s missing in self.param" % _key)
self.terminate_flag = True
else:
continue
logger.debug("keys of self.param are checked")
def main():
args = parse_args()
comm_config = {
'subscribe': {},
'broker': {
'address': args['broker_ip'],
'port': args['broker_port']
}
}
comm = Communicator(comm_config, debug=True)
duration = lambda t: (datetime.now() - t).microseconds / 1000
if args['mode'] == 'stream':
counter = 0
capture = cv2.VideoCapture(0)
while True:
status, im = capture.read()
if (status is False):
logger.warn('ERROR: Failure happened when reading frame')
t = datetime.now()
retval, jpg_bytes = cv2.imencode('.jpg', im)
mqtt_payload = payload.serialize_jpg(jpg_bytes)
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('send: {} ms'.format(duration(t)))
time.sleep(1.0 / args['fps'])
elif args['mode'] == 'file':
# Prepare MQTT payload
im = cv2.imread(args['filepath'])
retval, jpg_bytes = cv2.imencode('.jpg', im)
t = datetime.now()
mqtt_payload = payload.serialize_jpg(jpg_bytes)
logger.debug('payload: {} ms'.format(duration(t)))
logger.debug('payload size: {}'.format(len(mqtt_payload)))
# Client publishes payload
t = datetime.now()
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('mqtt.publish: {} ms'.format(duration(t)))
logger.debug('publish at {}'.format(datetime.now().isoformat()))
else:
logger.error('User assigned unknown mode {}'.format(args['mode']))
def main_process(self):
if len(self.input_data) == 0:
logger.error('no input_data found')
self.terminate_flag = True
logger.debug('self.input_data len: {}'.format(len(self.input_data)))
for img_array in self.input_data:
orig_h, orig_w = img_array.shape[:-1]
img_array = self.process_input(img_array)
inf_results = self.inference(img_array)
det_results = self.process_output(inf_results)
# why this code looks so redundant here is beacause that
# this script is modified from berrynet openvino_engine.py,
# and I follow the principle that make least change of original
# code. the annotations structure in process_output is original
# defined in berrynet, and in order to use the lab_tools, we
# must re-define the structure
annotations = [[
det_result['label'], det_result['confidence'],
[
det_result['top'], det_result['bottom'],
det_result['left'], det_result['right']
]
] for det_result in det_results]
# orders, labinfo = self.create_labinfo(inf_results)
res = lab_tools.output_pred_detection(
# input_path=self.orig_input_path,
input_path='',
annotations=annotations,
img_size=(orig_h, orig_w),
# labinfo=labinfo
labinfo={})
self.results.append(res)
logger.debug('self.results: {}'.format(self.results))
def main():
args = parse_args()
if args['debug']:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
comm_config = {
'subscribe': {},
'broker': {
'address': args['broker_ip'],
'port': args['broker_port']
}
}
comm = Communicator(comm_config, debug=True)
duration = lambda t: (datetime.now() - t).microseconds / 1000
if args['mode'] == 'stream':
counter = 0
# Check input stream source
if args['stream_src'].isdigit():
# source is a physically connected camera
stream_source = '/dev/video{}'.format(int(args['stream_src']))
capture = cv2.VideoCapture(int(args['stream_src']))
else:
# source is an IP camera
stream_source = args['stream_src']
capture = cv2.VideoCapture(args['stream_src'])
cam_fps = capture.get(cv2.CAP_PROP_FPS)
if cam_fps > 30 or cam_fps < 1:
logger.warn(
'Camera FPS is {} (>30 or <1). Set it to 30.'.format(cam_fps))
cam_fps = 30
out_fps = args['fps']
interval = int(cam_fps / out_fps)
# warmup
#t_warmup_start = time.time()
#t_warmup_now = time.time()
#warmup_counter = 0
#while t_warmup_now - t_warmup_start < 1:
# capture.read()
# warmup_counter += 1
# t_warmup_now = time.time()
logger.debug('===== VideoCapture Information =====')
logger.debug('Stream Source: {}'.format(stream_source))
logger.debug('Camera FPS: {}'.format(cam_fps))
logger.debug('Output FPS: {}'.format(out_fps))
logger.debug('Interval: {}'.format(interval))
#logger.debug('Warmup Counter: {}'.format(warmup_counter))
logger.debug('====================================')
while True:
status, im = capture.read()
if (status is False):
logger.warn('ERROR: Failure happened when reading frame')
# NOTE: Hard-coding rotation for AIKEA onboard camera.
# We will add parameter support in the future.
im = tinycv.rotate_ccw_opencv(im)
counter += 1
if counter == interval:
logger.debug('Drop frames: {}'.format(counter - 1))
counter = 0
# Open a window and display the ready-to-send frame.
# This is useful for development and debugging.
if args['display']:
cv2.imshow('Frame', im)
cv2.waitKey(1)
t = datetime.now()
#logger.debug('write frame to /tmp/output.jpg')
#cv2.imwrite('/tmp/output.jpg', im)
retval, jpg_bytes = cv2.imencode('.jpg', im)
obj = {}
obj['timestamp'] = datetime.now().isoformat()
obj['bytes'] = payload.stringify_jpg(jpg_bytes)
obj['meta'] = {
'roi': [{
'top': 50,
#'left': 341,
#'bottom': 500,
#'right': 682,
#'left': 640,
#'bottom': 980,
#'right': 1280,
'left': 10,
'bottom': 600,
'right': 600,
'overlap_threshold': 0.5
}]
}
logger.debug('timestamp: {}'.format(obj['timestamp']))
logger.debug('bytes len: {}'.format(len(obj['bytes'])))
logger.debug('meta: {}'.format(obj['meta']))
mqtt_payload = payload.serialize_payload([obj])
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('send: {} ms'.format(duration(t)))
else:
pass
elif args['mode'] == 'file':
# Prepare MQTT payload
im = cv2.imread(args['filepath'])
retval, jpg_bytes = cv2.imencode('.jpg', im)
t = datetime.now()
obj = {}
obj['timestamp'] = datetime.now().isoformat()
obj['bytes'] = payload.stringify_jpg(jpg_bytes)
obj['meta'] = {
'roi': [{
'top': 50,
'left': 10,
'bottom': 600,
'right': 600,
'overlap_threshold': 0.5
}]
}
mqtt_payload = payload.serialize_payload([obj])
logger.debug('payload: {} ms'.format(duration(t)))
logger.debug('payload size: {}'.format(len(mqtt_payload)))
# Client publishes payload
t = datetime.now()
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('mqtt.publish: {} ms'.format(duration(t)))
logger.debug('publish at {}'.format(datetime.now().isoformat()))
else:
logger.error('User assigned unknown mode {}'.format(args['mode']))
def __init__(self, model, device='CPU', labels=None, top_k=3):
"""
Args:
model: Path to an .xml file with a trained model.
device: Specify the target device to infer on; CPU, GPU, FPGA
or MYRIAD is acceptable. Sample will look for a suitable
plugin for device specified (CPU by default)
labels: Labels mapping file
top_k: Number of top results
"""
super(OpenVINOClassifierEngine, self).__init__()
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
if labels:
with open(labels, 'r') as f:
# Allow label name with spaces. To use onlyh the 1st word,
# uncomment another labels_map implementation below.
self.labels_map = [l.strip() for l in f.readlines()]
#self.labels_map = [x.split(sep=' ', maxsplit=1)[-1].strip()
# for x in f]
else:
self.labels_map = None
self.top_k = top_k
# Plugin initialization for specified device and
# load extensions library if specified
#
# Note: MKLDNN CPU-targeted custom layer support is not included
# because we do not use it yet.
self.plugin = IEPlugin(device=device, plugin_dirs=None)
# Read IR
logger.debug('Loading network files:'
'\n\txml: {0}\n\tbin: {1}'.format(model_xml, model_bin))
net = IENetwork.from_ir(model=model_xml, weights=model_bin)
if self.plugin.device == "CPU":
supported_layers = self.plugin.get_supported_layers(net)
not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
logger.error("Following layers are not supported by the plugin for specified device {}:\n {}".format(self.plugin.device, ', '.join(not_supported_layers)))
sys.exit(1)
assert len(net.inputs.keys()) == 1, "Sample supports only single input topologies"
assert len(net.outputs) == 1, "Sample supports only single output topologies"
# input_blob and and out_blob are the layer names in string format.
logger.debug("Preparing input blobs")
self.input_blob = next(iter(net.inputs))
self.out_blob = next(iter(net.outputs))
net.batch_size = 1
self.n, self.c, self.h, self.w = net.inputs[self.input_blob].shape
# Loading model to the plugin
logger.debug("Loading model to the plugin")
self.exec_net = self.plugin.load(network=net)
del net
def __init__(self, model, device='CPU', labels=None, threshold=0.3):
super(OpenVINODetectorEngine, self).__init__()
# Prepare model and labels
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
if labels:
with open(labels, 'r') as f:
self.labels_map = [x.strip() for x in f]
else:
self.labels_map = None
self.threshold = threshold
# Plugin initialization for specified device and
# load extensions library if specified
#
# Note: MKLDNN CPU-targeted custom layer support is not included
# because we do not use it yet.
if device == 'CPU':
plugin_dirs = '/opt/intel/openvino_2019.1.144/deployment_tools/inference_engine/lib/intel64'
self.plugin = IEPlugin(device=device, plugin_dirs=plugin_dirs)
self.plugin.add_cpu_extension(plugin_dirs + '/libcpu_extension_avx2.so')
self.plugin.add_cpu_extension(plugin_dirs + '/libcpu_extension_avx512.so')
self.plugin.add_cpu_extension(plugin_dirs + '/libcpu_extension_sse4.so')
logger.debug('plugin dirs: {}'.format(plugin_dirs))
else:
plugin_dirs = None
self.plugin = IEPlugin(device=device, plugin_dirs=plugin_dirs)
# Read IR
logger.debug('Loading network files:'
'\n\txml: {0}\n\tbin: {1}'.format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
if self.plugin.device == "CPU":
supported_layers = self.plugin.get_supported_layers(net)
not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
logger.error("Following layers are not supported by the plugin for specified device {}:\n {}".
format(self.plugin.device, ', '.join(not_supported_layers)))
logger.error("Please try to specify cpu extensions library path in demo's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(net.inputs.keys()) == 1, "Demo supports only single input topologies"
assert len(net.outputs) == 1, "Demo supports only single output topologies"
# input_blob and and out_blob are the layer names in string format.
logger.debug("Preparing input blobs")
self.input_blob = next(iter(net.inputs))
self.out_blob = next(iter(net.outputs))
self.n, self.c, self.h, self.w = net.inputs[self.input_blob].shape
# Loading model to the plugin
self.exec_net = self.plugin.load(network=net, num_requests=2)
del net
# Initialize engine mode: sync or async
#
# FIXME: async mode does not work currently.
# process_input needs to provide two input tensors for async.
self.is_async_mode = False
self.cur_request_id = 0
self.next_request_id = 1
def main():
args = parse_args()
if args['debug']:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
comm_config = {
'subscribe': {},
'broker': {
'address': args['broker_ip'],
'port': args['broker_port']
}
}
comm = Communicator(comm_config, debug=True)
duration = lambda t: (datetime.now() - t).microseconds / 1000
metadata = json.loads(args.get('meta', '{}'))
if args['mode'] == 'stream':
counter = 0
fail_counter = 0
# Check input stream source
if args['stream_src'].isdigit():
# source is a physically connected camera
stream_source = int(args['stream_src'])
else:
# source is an IP camera
stream_source = args['stream_src']
capture = cv2.VideoCapture(stream_source)
cam_fps = capture.get(cv2.CAP_PROP_FPS)
if cam_fps > 30 or cam_fps < 1:
logger.warn('Camera FPS is {} (>30 or <1). Set it to 30.'.format(cam_fps))
cam_fps = 30
out_fps = args['fps']
interval = int(cam_fps / out_fps)
# warmup
#t_warmup_start = time.time()
#t_warmup_now = time.time()
#warmup_counter = 0
#while t_warmup_now - t_warmup_start < 1:
# capture.read()
# warmup_counter += 1
# t_warmup_now = time.time()
logger.debug('===== VideoCapture Information =====')
if stream_source.isdigit():
stream_source_uri = '/dev/video{}'.format(stream_source)
else:
stream_source_uri = stream_source
logger.debug('Stream Source: {}'.format(stream_source_uri))
logger.debug('Camera FPS: {}'.format(cam_fps))
logger.debug('Output FPS: {}'.format(out_fps))
logger.debug('Interval: {}'.format(interval))
logger.debug('Send MQTT Topic: {}'.format(args['topic']))
#logger.debug('Warmup Counter: {}'.format(warmup_counter))
logger.debug('====================================')
while True:
status, im = capture.read()
# To verify whether the input source is alive, you should use the
# return value of capture.read(). It will not work by capturing
# exception of a capture instance, or by checking the return value
# of capture.isOpened().
#
# Two reasons:
# 1. If a dead stream is alive again, capture will not notify
# that input source is dead.
#
# 2. If you check capture.isOpened(), it will keep retruning
# True if a stream is dead afterward. So you can not use
# the capture return value (capture status) to determine
# whether a stream is alive or not.
if (status is True):
counter += 1
if counter == interval:
logger.debug('Drop frames: {}'.format(counter-1))
counter = 0
# Open a window and display the ready-to-send frame.
# This is useful for development and debugging.
if args['display']:
cv2.imshow('Frame', im)
cv2.waitKey(1)
t = datetime.now()
retval, jpg_bytes = cv2.imencode('.jpg', im)
mqtt_payload = payload.serialize_jpg(jpg_bytes, args['hash'], metadata)
comm.send(args['topic'], mqtt_payload)
logger.debug('send: {} ms'.format(duration(t)))
else:
pass
else:
fail_counter += 1
logger.critical('ERROR: Failure #{} happened when reading frame'.format(fail_counter))
# Re-create capture.
capture.release()
logger.critical('Re-create a capture and reconnect to {} after 5s'.format(stream_source))
time.sleep(5)
capture = cv2.VideoCapture(stream_source)
elif args['mode'] == 'file':
# Prepare MQTT payload
im = cv2.imread(args['filepath'])
retval, jpg_bytes = cv2.imencode('.jpg', im)
t = datetime.now()
mqtt_payload = payload.serialize_jpg(jpg_bytes, args['hash'], metadata)
logger.debug('payload: {} ms'.format(duration(t)))
logger.debug('payload size: {}'.format(len(mqtt_payload)))
# Client publishes payload
t = datetime.now()
comm.send(args['topic'], mqtt_payload)
logger.debug('mqtt.publish: {} ms'.format(duration(t)))
logger.debug('publish at {}'.format(datetime.now().isoformat()))
else:
logger.error('User assigned unknown mode {}'.format(args['mode']))
def main():
args = parse_args()
if args['debug']:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
comm_config = {
'subscribe': {},
'broker': {
'address': args['broker_ip'],
'port': args['broker_port']
}
}
comm = Communicator(comm_config, debug=True)
duration = lambda t: (datetime.now() - t).microseconds / 1000
if args['mode'] == 'stream':
counter = 0
# Check input stream source
if args['stream_src'].isdigit():
# source is a physically connected camera
stream_source = '/dev/video{}'.format(int(args['stream_src']))
capture = cv2.VideoCapture(int(args['stream_src']))
else:
# source is an IP camera
stream_source = args['stream_src']
capture = cv2.VideoCapture(args['stream_src'])
cam_fps = capture.get(cv2.CAP_PROP_FPS)
if cam_fps > 30 or cam_fps < 1:
logger.warn(
'Camera FPS is {} (>30 or <1). Set it to 30.'.format(cam_fps))
cam_fps = 30
out_fps = args['fps']
interval = int(cam_fps / out_fps)
# warmup
#t_warmup_start = time.time()
#t_warmup_now = time.time()
#warmup_counter = 0
#while t_warmup_now - t_warmup_start < 1:
# capture.read()
# warmup_counter += 1
# t_warmup_now = time.time()
logger.debug('===== VideoCapture Information =====')
logger.debug('Stream Source: {}'.format(stream_source))
logger.debug('Camera FPS: {}'.format(cam_fps))
logger.debug('Output FPS: {}'.format(out_fps))
logger.debug('Interval: {}'.format(interval))
#logger.debug('Warmup Counter: {}'.format(warmup_counter))
logger.debug('====================================')
while True:
status, im = capture.read()
if (status is False):
logger.warn('ERROR: Failure happened when reading frame')
counter += 1
if counter == interval:
logger.debug('Drop frames: {}'.format(counter - 1))
counter = 0
# Open a window and display the ready-to-send frame.
# This is useful for development and debugging.
if args['display']:
cv2.imshow('Frame', im)
cv2.waitKey(1)
t = datetime.now()
retval, jpg_bytes = cv2.imencode('.jpg', im)
mqtt_payload = payload.serialize_jpg(jpg_bytes)
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('send: {} ms'.format(duration(t)))
else:
pass
elif args['mode'] == 'file':
# Prepare MQTT payload
im = cv2.imread(args['filepath'])
retval, jpg_bytes = cv2.imencode('.jpg', im)
t = datetime.now()
mqtt_payload = payload.serialize_jpg(jpg_bytes)
logger.debug('payload: {} ms'.format(duration(t)))
logger.debug('payload size: {}'.format(len(mqtt_payload)))
# Client publishes payload
t = datetime.now()
comm.send('berrynet/data/rgbimage', mqtt_payload)
logger.debug('mqtt.publish: {} ms'.format(duration(t)))
logger.debug('publish at {}'.format(datetime.now().isoformat()))
else:
logger.error('User assigned unknown mode {}'.format(args['mode']))
def __init__(self, dyda_config_path='', debug=False):
""" __init__ of DetectorOpenVINO
Trainer Variables:
input_data: a list of image array
results: defined by lab_tools.output_pred_detection()
Arguments:
dyda_config_path -- Trainer config filepath
"""
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
# Setup dyda config
super(DetectorOpenVINO,
self).__init__(dyda_config_path=dyda_config_path)
self.set_param(self.class_name)
self.check_param_keys()
if "threshold" in self.param.keys():
self.threshold = self.param["threshold"]
else:
self.threshold = 0.3
# Setup DL model
model_xml = self.param['model_description']
model_bin = self.param['model_file']
with open(self.param['label_file'], 'r') as f:
self.labels_map = [x.strip() for x in f]
# Setup OpenVINO
#
# Plugin initialization for specified device and
# load extensions library if specified
#
# Note: MKLDNN CPU-targeted custom layer support is not included
# because we do not use it yet.
self.plugin = IEPlugin(device=self.param['device'],
plugin_dirs=self.param['plugin_dirs'])
if self.param['device'] == 'CPU':
for ext in self.param['cpu_extensions']:
logger.info('Add cpu extension: {}'.format(ext))
self.plugin.add_cpu_extension(ext)
logger.debug("Computation device: {}".format(self.param['device']))
# Read IR
logger.debug("Loading network files:\n\t{}\n\t{}".format(
model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
if self.plugin.device == "CPU":
supported_layers = self.plugin.get_supported_layers(net)
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
logger.error(
('Following layers are not supported '
'by the plugin for specified device {}:\n {}').format(
self.plugin.device, ', '.join(not_supported_layers)))
logger.error("Please try to specify cpu "
"extensions library path in demo's "
"command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
assert len(net.inputs.keys()) == 1, (
'Demo supports only single input topologies')
assert len(
net.outputs) == 1, ('Demo supports only single output topologies')
# input_blob and and out_blob are the layer names in string format.
logger.debug("Preparing input blobs")
self.input_blob = next(iter(net.inputs))
self.out_blob = next(iter(net.outputs))
self.n, self.c, self.h, self.w = net.inputs[self.input_blob].shape
# Loading model to the plugin
self.exec_net = self.plugin.load(network=net, num_requests=2)
del net
# Initialize engine mode: sync or async
#
# FIXME: async mode does not work currently.
# process_input needs to provide two input tensors for async.
self.is_async_mode = False
self.cur_request_id = 0
self.next_request_id = 1