def init():
"""Initialize model
Returns: model
"""
model_xml = "/usr/local/ev_sdk/model/openvino/yolov3.xml"
if not os.path.isfile(model_xml):
log.error(f'{model_xml} does not exist')
return None
model_bin = pathlib.Path(model_xml).with_suffix('.bin').as_posix()
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# Load Inference Engine
log.info('Loading Inference Engine')
global ie
ie = IECore()
global exec_net
exec_net = ie.load_network(network=net, device_name=device)
log.info('Device info:')
versions = ie.get_versions(device)
print("{}".format(device))
print("MKLDNNPlugin version ......... {}.{}".format(versions[device].major, versions[device].minor))
print("Build ........... {}".format(versions[device].build_number))
input_blob = next(iter(net.inputs))
n, c, h, w = net.inputs[input_blob].shape
global input_h, input_w, input_c, input_n
input_h, input_w, input_c, input_n = h, w, c, n
return net
def boot_myriad():
myriad = {}
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# ------------ Load Inference Engine and network files -------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("IR Engine info:")
log.info(" Available IR Devices: {}".format(ie.available_devices))
if 'MYRIAD' in ie.available_devices:
verinfo = ie.get_versions('MYRIAD')['MYRIAD']
log.info(" Myriad Plugin version: {}.{}".format(
verinfo.major, verinfo.minor))
log.info(" Myriad Build: {}".format(verinfo.build_number))
devices = ie.get_metric(metric_name='AVAILABLE_DEVICES',
device_name='MYRIAD')
if (len(devices) > 1):
ir_device = "MULTI:"
for device in devices:
ir_device += "MYRIAD." + device + ","
ir_device = ir_device[:-1]
else:
ir_device = "MYRIAD"
else:
ir_device = "CPU"
log.warn("!! MYRIAD DEVICE NOT FOUND. USING CPU !!")
log.info(" Selected Device(s): {}".format(ir_device))
log.info("Loading network files XML [{}] and BIN [{}]".format(
model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
net_input_layer_name = next(iter(net.inputs))
n, c, h, w = net.inputs[net_input_layer_name].shape
log.info(" Network shape: [{},{},{},{}]".format(n, c, h, w))
log.info("Loading labels file [{}]".format(args.labels.name))
labels_map = [x.strip() for x in args.labels]
log.info("Loading network to device... Please wait..")
exec_net = ie.load_network(network=net,
device_name=ir_device,
num_requests=4)
ir_num_requests = exec_net.get_metric('OPTIMAL_NUMBER_OF_INFER_REQUESTS')
log.info(" Optimal Number of Infer Requests: {}".format(ir_num_requests))
log.info("Done loading network")
myriad = {
'n': n,
'c': c,
'h': h,
'w': w,
'labels_map': labels_map,
'net': net,
'exec_net': exec_net,
'net_input_layer_name': net_input_layer_name,
}
return myriad
def test_get_version(device):
ie = IECore()
version = ie.get_versions(device)
assert isinstance(version, dict), "Returned version must be a dictionary"
assert device in version, "{} plugin version wasn't found in versions"
assert hasattr(version[device], "major"), "Returned version has no field 'major'"
assert hasattr(version[device], "minor"), "Returned version has no field 'minor'"
assert hasattr(version[device], "description"), "Returned version has no field 'description'"
assert hasattr(version[device], "build_number"), "Returned version has no field 'build_number'"
def __init__(self,
model_xml,
model_bin,
input_names,
output_names,
vocab_file,
device='MYRIAD'):
self.context = None
self.input_names = input_names
self.output_names = output_names
log.info("Loading vocab file:\t{}".format(vocab_file))
with open(vocab_file, "r", encoding="utf-8") as r:
self.vocab = {
t.rstrip("\n"): i
for i, t in enumerate(r.readlines())
}
log.info("{} tokens loaded".format(len(self.vocab)))
log.info("Initializing Inference Engine")
ie = IECore()
ie.set_config({'VPU_HW_STAGES_OPTIMIZATION': 'NO'}, "MYRIAD")
version = ie.get_versions(device)[device]
version_str = "{}.{}.{}".format(version.major, version.minor,
version.build_number)
log.info("Plugin version is {}".format(version_str))
# read IR
log.info("Loading network files:\n\t{}\n\t{}".format(
model_xml, model_bin))
ie_encoder = ie.read_network(model=model_xml, weights=model_bin)
# maximum number of tokens that can be processed by network at once
self.max_length = ie_encoder.input_info[
self.input_names[0]].input_data.shape[1]
# load model to the device
log.info("Loading model to the {}".format(device))
self.ie_encoder_exec = ie.load_network(network=ie_encoder,
device_name=device)
def intelNeuralStick_init():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(DEVICE_NAME)
print("{}{}".format(" " * 10, DEVICE_NAME))
print("{}Plugin version ......... {}.{}".format(
" " * 10, versions[DEVICE_NAME].major, versions[DEVICE_NAME].minor))
print("{}Build ........... {}".format(" " * 10,
versions[DEVICE_NAME].build_number))
# Read IR
objectDetection = intelNeuralStick_modelRead(MODEL_OBJECT_DETECTION,
DEVICE_NAME, ie)
faceDetection = intelNeuralStick_modelRead(MODEL_FACE_DETECTION,
DEVICE_NAME, ie)
genderAgeDetection = intelNeuralStick_modelRead(
MODEL_AGE_GENDER_PREDICTION, DEVICE_NAME, ie)
return objectDetection, faceDetection, genderAgeDetection
class LandmarkDetector:
'''
Class for the Face Detection Model.
'''
def __init__(
self,
model_name='models/intel/facial-landmarks-35-adas-0002/FP32-INT8/facial-landmarks-35-adas-0002',
device='CPU',
extensions=None):
self.model_name = model_name
self.model_w = model_name + '.bin'
self.model_s = model_name + '.xml'
self.device = device
self.extension = extensions
self.inference_results = None
self.pre_image = None
self.post_image = None
self.network = None
self.input_name = None
self.input_shape = None
self.output_shape = None
self.output_name = None
self.plugin = None
self.exec_network = None
def load_model(self):
self.plugin = IECore()
log.info("Attempting to load network for model:")
log.info(self.model_name)
self.network = self.plugin.read_network(model=self.model_s,
weights=self.model_w)
self.exec_network = self.plugin.load_network(self.network, self.device)
self.input_name = next(iter(self.network.inputs))
self.output_name = next(iter(self.network.outputs))
self.input_shape = self.network.inputs[self.input_name].shape
self.output_shape = self.network.outputs[self.output_name].shape
def predict(self, image):
self.pre_image = self.preprocess_input(image)
self.inference_results = self.exec_network.infer(
{self.input_name: self.pre_image})
landmarks = self.preprocess_output(self.inference_results, image)
return landmarks
def check_model(self):
self.plugin = IECore()
log.info("Checking model layers for model:")
log.info(self.model_name)
self.network = self.plugin.read_network(model=self.model_s,
weights=self.model_w)
# double check supported network layers
# code taken from Project-01 (ND131)
if "CPU" in self.device:
supported_layers = self.plugin.query_network(self.network, "CPU")
not_supported_layers = [
l for l in self.network.layers.keys()
if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(self.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
versions = self.plugin.get_versions(self.device)
log.info("{}{}".format(" " * 8, self.device))
log.info("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[self.device].major, versions[self.device].minor))
log.info("{}Build ........... {}".format(
" " * 8, versions[self.device].build_number))
def preprocess_input(self, image):
post_frame = cv2.resize(image,
(self.input_shape[3], self.input_shape[2]))
post_frame = post_frame.transpose((2, 0, 1))
post_frame = post_frame.reshape(1, *post_frame.shape)
input_width = post_frame.shape[1]
input_height = post_frame.shape[0]
log.debug("input width x height: %d x %d", self.input_shape[3],
self.input_shape[2])
return post_frame
def preprocess_output(self, outputs, image):
'''
We only care about the eye coordinates for this project, parse the eye's (X,Y) positions
and repackage for further processing
'''
width = image.shape[1]
height = image.shape[0]
outputs = outputs[self.output_name][0]
##Left Eye Position (X,Y)
eye_l_x = int(outputs[0] * width)
eye_l_y = int(outputs[1] * height)
##Right Eye Position (X,Y)
eye_r_x = int(outputs[2] * width)
eye_r_y = int(outputs[3] * height)
nose_x = int(outputs[4] * width)
nose_y = int(outputs[5] * height)
lip_l_x = int(outputs[6] * width)
lip_l_y = int(outputs[7] * height)
lip_r_x = int(outputs[8] * width)
lip_r_y = int(outputs[9] * height)
##used for later drawing
result = {
'eye_left': [eye_l_x, eye_l_y],
'eye_right': [eye_r_x, eye_r_y],
'nose': [nose_x, nose_y],
'lip_left': [lip_l_x, lip_l_y],
'lip_right': [lip_r_x, lip_r_y]
}
return result
class InferenceEngine:
def __init__(self, model_xml, device, stride):
self.device = device
self.stride = stride
model_bin = os.path.splitext(model_xml)[0] + '.bin'
log.info("Loading network files:\n\t{}\n\t{}".format(
model_xml, model_bin))
self.net = IENetwork(model=model_xml, weights=model_bin)
log.info("Loading Inference Engine")
self.ie = IECore()
log.info("Device info:")
versions = self.ie.get_versions(device)
log.info("{}{}".format(" " * 8, device))
log.info("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[device].major, versions[device].minor))
log.info("{}Build ........... {}".format(
" " * 8, versions[device].build_number))
self.input_blob = next(iter(self.net.inputs))
log.info(
f"Input blob: {self.input_blob} - shape: {self.net.inputs[self.input_blob].shape}"
)
for o in self.net.outputs.keys():
log.info(f"Output blob: {o} - shape: {self.net.outputs[o].shape}")
if o == "Mconv7_stage2_L2":
self.heatmaps_blob = "Mconv7_stage2_L2"
self.pafs_blob = "Mconv7_stage2_L1"
elif o == "heatmaps":
self.heatmaps_blob = "heatmaps"
self.pafs_blob = "pafs"
log.info(
f"Heatmaps blob: {self.heatmaps_blob} - PAFs blob: {self.pafs_blob}"
)
log.info("Loading model to the plugin")
self.exec_net = self.ie.load_network(network=self.net,
num_requests=1,
device_name=device)
def infer(self, img):
img = img[0:img.shape[0] - (img.shape[0] % self.stride),
0:img.shape[1] - (img.shape[1] % self.stride)]
n, c, h, w = self.net.inputs[self.input_blob].shape
if h != img.shape[0] or w != img.shape[1]:
log.info(f"Reshaping of network")
self.net.reshape(
{self.input_blob: (n, c, img.shape[0], img.shape[1])})
log.info(
f"Input blob: {self.input_blob} - new shape: {self.net.inputs[self.input_blob].shape}"
)
for o in self.net.outputs.keys():
log.info(
f"Output blob: {o} - new shape: {self.net.outputs[o].shape}"
)
self.exec_net = self.ie.load_network(network=self.net,
num_requests=1,
device_name=self.device)
img = np.transpose(img, (2, 0, 1))[None, ]
inference_result = self.exec_net.infer(inputs={self.input_blob: img})
inference_result = (inference_result[self.heatmaps_blob][0],
inference_result[self.pafs_blob][0])
return inference_result
class Classification:
def __init__(self, device: str, number_infer_requests, number_iterations, duration_seconds, api_type):
self.device = device
self.ie = IECore()
self.nireq = number_infer_requests
self.niter = number_iterations
self.duration_seconds = get_duration_seconds(duration_seconds, self.niter, self.device)
self.api_type = api_type
self.device_number_streams = {}
def __del__(self):
del self.ie
def add_extension(self, path_to_extension: str=None, path_to_cldnn_config: str=None):
if GPU_DEVICE_NAME in self.device:
if path_to_cldnn_config:
self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config}, GPU_DEVICE_NAME)
logger.info('GPU extensions is loaded {}'.format(path_to_cldnn_config))
if CPU_DEVICE_NAME in self.device or MYRIAD_DEVICE_NAME in self.device:
if path_to_extension:
self.ie.add_extension(extension_path=path_to_extension, device_name=CPU_DEVICE_NAME)
logger.info('CPU extensions is loaded {}'.format(path_to_extension))
def get_version_info(self) -> str:
logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format('', 'API version', get_version()))
version_string = 'Device info\n'
for device, version in self.ie.get_versions(self.device).items():
version_string += '{: <9}{}\n'.format('', device)
version_string += '{: <9}{:.<24}{} {}.{}\n'.format('', version.description, ' version', version.major,
version.minor)
version_string += '{: <9}{:.<24} {}\n'.format('', 'Build', version.build_number)
return version_string
@staticmethod
def reshape(ie_network: IENetwork, batch_size: int):
new_shapes = {}
for input_layer_name, input_layer in ie_network.inputs.items():
new_shapes[input_layer_name] = get_blob_shape(input_layer, batch_size)
if new_shapes:
logger.info('Resizing network to batch = {}'.format(batch_size))
ie_network.reshape(new_shapes)
def set_config(self, number_streams: int, api_type: str = 'async',
number_threads: int = None, infer_threads_pinning: int = None):
devices = parse_devices(self.device)
self.device_number_streams = parse_nstreams_value_per_device(devices, number_streams)
for device_name in self.device_number_streams.keys():
key = device_name + "_THROUGHPUT_STREAMS"
supported_config_keys = self.ie.get_metric(device_name, 'SUPPORTED_CONFIG_KEYS')
if key not in supported_config_keys:
raise Exception("Device " + device_name + " doesn't support config key '" + key + "'! " +
"Please specify -nstreams for correct devices in format <dev1>:<nstreams1>,<dev2>:<nstreams2>");
for device in devices:
if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys
# limit threading for CPU portion of inference
if number_threads:
self.ie.set_config({'CPU_THREADS_NUM': str(number_threads)}, device)
if MULTI_DEVICE_NAME in self.device and GPU_DEVICE_NAME in self.device:
self.ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME)
else:
# pin threads for CPU portion of inference
self.ie.set_config({'CPU_BIND_THREAD': infer_threads_pinning}, device)
# for CPU execution, more throughput-oriented execution via streams
# for pure CPU execution, more throughput-oriented execution via streams
if api_type == 'async':
cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'}
if device in self.device_number_streams.keys():
cpu_throughput['CPU_THROUGHPUT_STREAMS'] = str(self.device_number_streams.get(device))
self.ie.set_config(cpu_throughput, device)
self.device_number_streams[device] = self.ie.get_config(device, 'CPU_THROUGHPUT_STREAMS')
elif device == GPU_DEVICE_NAME:
if api_type == 'async':
gpu_throughput = {'GPU_THROUGHPUT_STREAMS': 'GPU_THROUGHPUT_AUTO'}
if device in self.device_number_streams.keys():
gpu_throughput['GPU_THROUGHPUT_STREAMS'] = str(self.device_number_streams.get(device))
self.ie.set_config(gpu_throughput, device)
self.device_number_streams[device] = self.ie.get_config(device, 'GPU_THROUGHPUT_STREAMS')
if MULTI_DEVICE_NAME in self.device and CPU_DEVICE_NAME in self.device:
# multi-device execution with the CPU+GPU performs best with GPU trottling hint,
# which releases another CPU thread (that is otherwise used by the GPU driver for active polling)
self.ie.set_config({'CLDNN_PLUGIN_THROTTLE': '1'}, device)
elif device == MYRIAD_DEVICE_NAME:
self.ie.set_config({'LOG_LEVEL': 'LOG_INFO'}, MYRIAD_DEVICE_NAME)
def read_network(self, path_to_model: str):
xml_filename = os.path.abspath(path_to_model)
head, tail = os.path.splitext(xml_filename)
bin_filename = os.path.abspath(head + BIN_EXTENSION)
ie_network = self.ie.read_network(xml_filename, bin_filename)
input_info = ie_network.inputs
if not input_info:
raise AttributeError('No inputs info is provided')
return ie_network
def load_network(self, ie_network: IENetwork, perf_counts: bool):
config = {'PERF_COUNT': ('YES' if perf_counts else 'NO')}
exe_network = self.ie.load_network(ie_network,
self.device,
config=config,
num_requests=1 if self.api_type == 'sync' else self.nireq or 0)
# Number of requests
self.nireq = len(exe_network.requests)
return exe_network
def infer(self, exe_network, batch_size, progress_bar=None):
progress_count = 0
infer_requests = exe_network.requests
# warming up - out of scope
if self.api_type == 'sync':
infer_requests[0].infer()
else:
infer_requests[0].async_infer()
status = exe_network.wait()
if status != StatusCode.OK:
raise Exception("Wait for all requests is failed with status code {}!".format(status))
out_blob = next(iter(exe_network.outputs))
start_time = datetime.utcnow()
exec_time = 0
iteration = 0
times = []
in_fly = set()
# Start inference & calculate performance
# to align number if iterations to guarantee that last infer requests are executed in the same conditions **/
#(self.duration_seconds and exec_time < self.duration_seconds) or \
while (self.niter and iteration < self.niter) or \
(len(in_fly) == iteration*self.nireq) or \
(self.api_type == 'async' and iteration % self.nireq):
if self.api_type == 'sync':
infer_requests[0].infer()
times.append(infer_requests[0].latency)
else:
infer_request_id = exe_network.get_idle_request_id()
if infer_request_id < 0:
status = exe_network.wait(num_requests=1)
if status != StatusCode.OK:
raise Exception("Wait for idle request failed!")
infer_request_id = exe_network.get_idle_request_id()
if infer_request_id < 0:
raise Exception("Invalid request id!")
if infer_request_id in in_fly:
times.append(infer_requests[infer_request_id].latency)
else:
in_fly.add(infer_request_id)
infer_requests[infer_request_id].async_infer()
#times.append(infer_requests[infer_request_id].latency)
iteration += 1
exec_time = (datetime.utcnow() - start_time).total_seconds()
if progress_bar:
if self.duration_seconds:
# calculate how many progress intervals are covered by current iteration.
# depends on the current iteration time and time of each progress interval.
# Previously covered progress intervals must be skipped.
progress_interval_time = self.duration_seconds / progress_bar.total_num
new_progress = int(exec_time / progress_interval_time - progress_count)
progress_bar.add_progress(new_progress)
progress_count += new_progress
elif self.niter:
progress_bar.add_progress(1)
# wait the latest inference executions
inference_output = []
status = exe_network.wait()
for infer_request in infer_requests:
output = infer_request.outputs[out_blob]
inference_output.append(output.tolist())
if status != StatusCode.OK:
raise Exception("Wait for all requests is failed with status code {}!".format(status))
total_duration_sec = (datetime.utcnow() - start_time).total_seconds()
for infer_request_id in in_fly:
times.append(infer_requests[infer_request_id].latency)
times.sort()
latency_ms = median(times)
fps = batch_size * 1000 / latency_ms if self.api_type == 'sync' else batch_size * iteration / total_duration_sec
if progress_bar:
progress_bar.finish()
return inference_output,fps, latency_ms, total_duration_sec, iteration
def setup(url):
global vocab
global ie_encoder
global input_names
global output_names
global model
global c_tokens_id
global ie_encoder_exec
global args
global c_tokens_se
global context
global COLOR_RED
global COLOR_RESET
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
args = build_argparser().parse_args()
if args.colors:
COLOR_RED = "\033[91m"
COLOR_RESET = "\033[0m"
else:
COLOR_RED = ""
COLOR_RESET = ""
# load vocabulary file for model
log.info("Loading vocab file:\t{}".format(args.vocab))
vocab = load_vocab_file(args.vocab)
log.info("{} tokens loaded".format(len(vocab)))
# get context as a string (as we might need it's length for the sequence reshape)
p = url
paragraphs = get_paragraphs([p])
context = '\n'.join(paragraphs)
log.info("Size: {} chars".format(len(context)))
log.info("Context: " + COLOR_RED + context + COLOR_RESET)
# encode context into token ids list
c_tokens_id, c_tokens_se = text_to_tokens(context.lower(), vocab)
log.info("Initializing Inference Engine")
ie = IECore()
version = ie.get_versions(args.device)[args.device]
version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number)
log.info("Plugin version is {}".format(version_str))
# read IR
model_xml = args.model
model_bin = model_xml.with_suffix(".bin")
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
ie_encoder = ie.read_network(model=model_xml, weights=model_bin)
if args.reshape:
# reshape the sequence length to the context + maximum question length (in tokens)
first_input_layer = next(iter(ie_encoder.inputs))
c = ie_encoder.inputs[first_input_layer].shape[1]
# find the closest multiple of 64, if it is smaller than current network's sequence length, let' use that
seq = min(c, int(np.ceil((len(c_tokens_id) + args.max_question_token_num) / 64) * 64))
if seq < c:
input_info = list(ie_encoder.inputs)
new_shapes = dict([])
for i in input_info:
n, c = ie_encoder.inputs[i].shape
new_shapes[i] = [n, seq]
log.info("Reshaped input {} from {} to the {}".format(i, ie_encoder.inputs[i].shape, new_shapes[i]))
log.info("Attempting to reshape the network to the modified inputs...")
try:
ie_encoder.reshape(new_shapes)
log.info("Successful!")
except RuntimeError:
log.error("Failed to reshape the network, please retry the demo without '-r' option")
sys.exit(-1)
else:
log.info("Skipping network reshaping,"
" as (context length + max question length) exceeds the current (input) network sequence length")
# check input and output names
input_names = list(i.strip() for i in args.input_names.split(','))
output_names = list(o.strip() for o in args.output_names.split(','))
if ie_encoder.inputs.keys() != set(input_names) or ie_encoder.outputs.keys() != set(output_names):
log.error("Input or Output names do not match")
log.error(" The demo expects input->output names: {}->{}. "
"Please use the --input_names and --output_names to specify the right names "
"(see actual values below)".format(input_names, output_names))
log.error(" Actual network input->output names: {}->{}".format(list(ie_encoder.inputs.keys()),
list(ie_encoder.outputs.keys())))
log.error(" Actual network input->output values: {}->{}".format(list(ie_encoder.inputs.values()),
list(ie_encoder.outputs.values())))
raise Exception("Unexpected network input or output names")
# load model to the device
log.info("Loading model to the {}".format(args.device))
ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device)
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Loading Inference Engine")
ie = IECore()
log.info("Initializing plugin for {} device...".format(args.device))
plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir)
model_xml = args.model #'face-detection-retail-0004.xml'
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = ie.read_network(model=model_xml, weights=model_bin)
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
log.info("Loading IR to the plugin...")
exec_net = plugin.load(network=net, num_requests=2)
if args.labels:
with open(args.labels, 'r') as f:
labels_map = [x.strip() for x in f]
else:
labels_map = None
n, c, h, w = net.inputs[input_blob].shape
del net
#s3client = boto3.resource('s3')
expiresIn = 5 * 24 * 3600 #expires recorded voice after 5 days
cur_request_id = 0
next_request_id = 1
render_time = 0
last_epoch = time.time()
cap = cv2.VideoCapture(0)
cap.set(3, 640)
cap.set(4, 480)
ret, frame = cap.read()
log.info("To close the application, press 'CTRL+C'")
while cap.isOpened():
ret, frame = cap.read()
if not ret:
break
initial_w = cap.get(3)
initial_h = cap.get(4)
# Main sync point:
# in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete
# in the regular mode we start the CURRENT request and immediately wait for it's completion
inf_start = time.time()
in_frame = cv2.resize(frame, (w, h))
in_frame = in_frame.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
in_frame = in_frame.reshape((n, c, h, w))
exec_net.start_async(request_id=cur_request_id,
inputs={input_blob: in_frame})
if exec_net.requests[cur_request_id].wait(-1) == 0:
inf_end = time.time()
det_time = inf_end - inf_start
# Parse detection results of the current request
res = exec_net.requests[cur_request_id].outputs[out_blob]
#log.info("Processing output blobs")
#print(res)
for obj in res[0][0]:
# Draw only objects when probability more than specified threshold
if obj[2] > args.prob_threshold:
xmin = int(obj[3] * initial_w)
ymin = int(obj[4] * initial_h)
xmax = int(obj[5] * initial_w)
ymax = int(obj[6] * initial_h)
class_id = int(obj[1])
# Draw box and label\class_id
#color = (min(class_id * 12.5, 255), min(class_id * 7, 255), min(class_id * 5, 255))
color = (232, 35, 244)
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2)
det_label = labels_map[class_id] if labels_map else str(
class_id)
cv2.putText(
frame,
det_label + ' ' + str(round(obj[2] * 100, 1)) + ' %',
(xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color,
1)
diff = (time.time() - last_epoch)
log.info("Object detected {:.1f} s".format(
round(obj[2] * 100, 1)))
if diff > 60:
last_epoch = time.time()
t = TempImage()
cv2.imwrite(t.path, frame)
#s3client.meta.client.upload_file(t.path, args.bucket, t.key )
#log.info("upload image to {}".format(signedUrl))
play_mp3('alert.wav')
t.cleanup()
inf_time_message = "Inference time: {:.3f} ms".format(det_time *
1000)
render_time_message = "OpenCV rendering time: {:.3f} ms".format(
render_time * 1000)
cv2.putText(frame, inf_time_message, (15, 15),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
cv2.putText(frame, render_time_message, (15, 30),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
render_start = time.time()
cv2.imshow("Detection Results", frame)
render_end = time.time()
render_time = render_end - render_start
key = cv2.waitKey(1)
if key == 27:
break
cv2.destroyAllWindows()
cap.release()
class facial_landmarks_detection:
'''
Class for the facial_landmarks_detection.
'''
def __init__(self, model, device='CPU', extensions=None):
'''
TODO: Use this to set your instance variables.
'''
self.model = model
self.device = device
self.extension = extensions
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
self.device_list = []
self.print_once = True
self.network_input_shape = None
self.filtered_result = [[]]
# ====== Load model files, verify model, and get the input shap parameters at start ======
log.info(
"<---------- class facial_landmark_detection model --------->")
self.load_model()
self.check_model()
self.network_input_shape = self.get_input_shape()
def load_model(self):
'''
TODO: You will need to complete this method.
This method is for loading the model to the device specified by the user.
If your model requires any Plugins, this is where you can load them.
'''
self.plugin = IECore()
log.info("------device avaibility--------")
for available_devices in self.plugin.available_devices: #Dont use device variable, conflicts.
self.device_list.append(available_devices)
log.info("Available device: " +
str(self.device_list)) #get name of available devices
log.info("---------Plugin version--------")
ver = self.plugin.get_versions("CPU")["CPU"] # get plugin info
log.info("descr: maj.min.num" + str(ver.description) + "." +
str(ver.major) + "." + str(ver.minor) + "." +
str(ver.build_number))
### Load IR files into their related class
model_xml = self.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Check if path is not given or model name doesnt ends with .xml
if model_xml is not None and model_xml.find('.xml') != -1:
f, s = model_xml.rsplit(
".", 1
) #check from last "." and "r"split only one element from last
model_bin = f + ".bin"
else:
log.error(
"Error! Model files are not found or invalid, check paths")
log.error("Program stopped")
sys.exit() #exit program no further execution
log.info("-------------Model path----------")
log.info("XML: " + str(model_xml))
log.info("bin: " + str(model_bin))
self.network = IENetwork(model=model_xml, weights=model_bin)
log.info("ModelFiles are successfully loaded into IENetwork")
return
def check_model(self):
'''
This function will check the input model for compatibility with hardware,
It will check for supported and unsuported layers if any.
No return required.
'''
### Check for supported layers ###
log.info("Checking for supported Network layers...")
# Query network will return all the layer, required all the time if device changes.
supported_layers = self.plugin.query_network(network=self.network,
device_name="CPU")
log.info("------Status of default Network layers--------")
log.info("No. of Layers in network: " + str(len(self.network.layers)))
log.info("No. of supported layers:" + str(len(supported_layers)))
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.info("Unsupported layers found:" + str(unsupported_layers))
log.info("CPU extension required and adding...")
#exit(1)
### Adding any necessary extensions ###
if self.extension and "CPU" in self.device:
self.plugin.add_extension(self.extension, self.device)
log.info("Checking for CPU extension compatibility...")
# Again Query network will return fresh list of supported layers.
supported_layers = self.plugin.query_network(
network=self.network, device_name="CPU")
log.info(
"------Status of Network layers with CPU Extension--------"
)
log.info("No. of Layers in network:" +
str(len(self.network.layers)))
log.info("No. of supported layers:" +
str(len(supported_layers)))
log.info("CPU extension added sucessfully!")
unsupported_layers = [
l for l in self.network.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
log.error("Unsupported layers found: " +
str((unsupported_layers)))
log.error("Error! Model not supported, Program stopped")
exit()
else:
log.error("Error! cpu extension not found")
log.error("Program stopped")
exit()
else:
log.info("All the layers are supported, No CPU extension required")
# This will enable all following four functions ref:intel doc. ie_api.IECore Class Reference
self.exec_network = self.plugin.load_network(self.network, "CPU")
print("IR successfully loaded into Inference Engine")
return
def get_input_shape(self):
'''
Get dimensions of inputs
'''
### Get the input layer informations ###
log.info("-----Accessing input layer information-----")
log.info('facial_landmarks_detection model Network input layers = ' +
str(list(self.network.inputs)))
log.info('facial_landmarks_detection Network input layers type: ' +
str(type(self.network.inputs)))
self.input_blob = next(iter(self.network.inputs)) #Origional
log.info("-------------------------------")
return self.network.inputs[self.input_blob].shape #Origional
def wait(self):
### Wait for the Async request to be complete. ###
status = self.exec_network.requests[0].wait(-1)
return status
def get_output(self):
### Extract and return the output results
self.output_blob = next(iter(self.network.outputs))
# First return the name of blob as dictionary and second output of first blob as Nd array
return self.exec_network.requests[
0].outputs, self.exec_network.requests[0].outputs[self.output_blob]
def preprocess_input(self, input_frames_raw, network_input_shape_height,
network_input_shape_width):
'''
Before feeding the data into the model for inference,
you might have to preprocess it. This function is where you can do that.
'''
p_frame = cv2.resize(
input_frames_raw,
(network_input_shape_height,
network_input_shape_width)) #Resize as per network input spec.
p_frame = p_frame.transpose((2, 0, 1)) #swap channel cxhxw
p_frame = p_frame.reshape(
1,
*p_frame.shape) #add one axis 1 to make 4D shape for network input
#print(p_frame.shape) #Debug output
return p_frame
def predict(self, input_frames_raw, input_frame_raw_height,
input_frame_raw_width):
'''
TODO: You will need to complete this method.
This method is meant for running predictions on the input image or video frame.
input: RBG image in jpg format or video frame.
'''
# pre-process origional frame to match network inputs.
p_frame = self.preprocess_input(input_frames_raw,
self.network_input_shape[3],
self.network_input_shape[2])
# Run Async inference
self.exec_network.start_async(
request_id=0, #Origional
inputs={self.input_blob: p_frame}) # run inference
# # Run sync inference
# self.exec_network.infer({self.input_blob: p_frame}) # run inference
# wait until result available
if self.wait() == 0:
### the results of the inference request ###
blob, result = self.get_output(
) # origioinal for single blob outputs
# Print available blob infirmation
blob_list = []
if self.print_once: # Print only Once
self.print_once = False
for name, output_ in blob.items(
): #Find the possible BLOBS for name,
blob_list.append(name)
log.info(
"The name of available blob of facial_landmarks_detection model is: "
+ str(blob_list))
return result
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### TODO: Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
self.count_input_layers = 0
self.first_input_layer = None
self.second_input_layer = None
def load_model(self, model, device="CPU", cpu_extension=None):
### TODO: Load the model ###
self.plugin = IECore()
print("------device avaibility--------")
for devices in self.plugin.available_devices: #Dont use device variable, conflicts.
print("Available device:", devices) #get name of available devices
print("---------Plugin version--------")
ver = self.plugin.get_versions("CPU")["CPU"] # get plugin info
print("{descr}: {maj}.{min}.{num}".format(descr=ver.description,
maj=ver.major,
min=ver.minor,
num=ver.build_number))
### Load IR files into their related class
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# Check if path is not given or model name doesnt ends with .xml
if model_xml is not None and model_xml.find('.xml') != -1:
f, s = model_xml.rsplit(
".", 1
) #check from last "." and "r"split only one element from last
model_bin = f + ".bin"
else:
print("Error! Model files are not found or invalid, check paths")
print("Program stopped")
sys.exit() #exit program no further execution
print("-------------Model path----------")
print("XML:", model_xml)
print("bin:", model_bin)
self.network = IENetwork(model=model_xml, weights=model_bin)
print("ModelFiles are successfully loaded into IENetwork")
### TODO: Check for supported layers ###
print("Checking for supported Network layers...")
# Query network will return all the layer, required all the time if device changes.
supported_layers = self.plugin.query_network(network=self.network,
device_name="CPU")
print("------Status of default Network layers--------")
print("No. of Layers in network", len(self.network.layers))
print("No. of supported layers:", len(supported_layers))
unsupported_layers = [
l for l in self.network.layers.keys() if l not in supported_layers
]
if len(unsupported_layers) != 0:
print("Unsupported layers found: {}".format(unsupported_layers))
print("CPU extension required and adding...")
#exit(1)
### TODO: Add any necessary extensions ###
#print(cpu_extension)
#print(device)
if cpu_extension and "CPU" in device:
self.plugin.add_extension(cpu_extension, device)
print("Checking for CPU extension compatibility...")
# Again Query network will return fresh list of supported layers.
supported_layers = self.plugin.query_network(
network=self.network, device_name="CPU")
print(
"------Status of Network layers with CPU Extension--------"
)
print("No. of Layers in network", len(self.network.layers))
print("No. of supported layers:", len(supported_layers))
unsupported_layers = [
l for l in self.network.layers.keys()
if l not in supported_layers
]
if len(unsupported_layers) != 0:
print("Unsupported layers found: {}".format(
unsupported_layers))
print("Error! Model not supported, Program stopped")
exit()
else:
print("Error! cpu extension not found")
print("Program stopped")
exit()
else:
print("All the layers are supported, No CPU extension required")
### TODO: Return the loaded inference plugin ###
# This will enable all following four functions ref:intel doc. ie_api.IECore Class Reference
self.exec_network = self.plugin.load_network(self.network, "CPU")
print("IR successfully loaded into Inference Engine")
### Note: You may need to update the function parameters. ###
return
def get_input_shape(self):
### TODO: Return the shape of the input layer ###
# Get the input layer
# To avoid unnecessory conversion in realtime in exec_net() and get_input_shape()
self.count_input_layers = len(list(self.network.inputs))
print("-----Accessing input layer information-----")
print('Network input layers = ' + str(list(self.network.inputs)))
print('Network input layers type: ', type(self.network.inputs))
# FasterRcnn model Fix for Openvino V2019R3
if self.count_input_layers > 1: # check if more than 1 input layers
# Model:TF-faster_rcnn_inception_v2_coco_2018_01_28
# Network input layers name = ['image_info', 'image_tensor']
# Access it by dictionary element
# To avoid unnecessory conversion in realtime in exec_net()
self.first_input_layer = list(self.network.inputs)[0]
self.second_input_layer = list(self.network.inputs)[1]
#print("Var:",self.first_input_layer,self.second_input_layer) #Debug
print("More than one input layers found")
print("### Warning!!! Manual data feed may require... ###")
print("Read respective model documentation for more info.")
self.input_blob = self.network.inputs[self.second_input_layer]
print("Manually selected input layer: ", self.second_input_layer)
print("Fixed data applied to other input layer: ",
self.first_input_layer, ":", [600, 1024, 1])
print("in function exec_net()")
print("-------------------------------")
return self.input_blob.shape
else: # If regular model
self.input_blob = next(iter(self.network.inputs)) #Origional
print("-------------------------------")
return self.network.inputs[self.input_blob].shape #Origional
def exec_net(self, image):
### TODO: Start an asynchronous request ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
# FasterRcnn model Fix for Openvino V2019R3
if self.count_input_layers > 1: # check if more than 1 input layers
# Model:TF-faster_rcnn_inception_v2_coco_2018_01_28
# Network input layers name = ['image_info', 'image_tensor']
# Access it by dictionary element
# Manually feed data according to model documentation
# inputs layes names={'image_info':[600,1024,1],'image_tensor': image})
# image_info:[HxWxS], height, width and scale factor usually 1
self.exec_network.start_async(request_id=0,
inputs={
self.first_input_layer:
[600, 1024, 1],
self.second_input_layer:
image
})
else: # If regular model
self.exec_network.start_async(
request_id=0, #Origional
inputs={self.input_blob: image})
return
def wait(self):
### TODO: Wait for the request to be complete. ###
### TODO: Return any necessary information ###
### Note: You may need to update the function parameters. ###
status = self.exec_network.requests[0].wait(-1)
return status
def get_output(self):
### TODO: Extract and return the output results
### Note: You may need to update the function parameters. ###
self.output_blob = next(iter(self.network.outputs))
# First return the name of blob as dictionary and second output of first blob as Nd array
return self.exec_network.requests[
0].outputs, self.exec_network.requests[0].outputs[self.output_blob]
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Loading Inference Engine")
ie = IECore()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_name = splitext(basename(model_xml))[0]
model_bin = model_xml[:-3] + 'bin'
log.info("Loading network files:\n\t{}\n".format(model_xml))
net = ie.read_network(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
tab = pd.DataFrame(data=d)
tab.to_csv(globalize(f"{args.tab_base}_{model_name}.csv", "C:/frames/"))
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Device info:")
device = 'CPU'
versions = ie.get_versions(device)
print("{}{}".format(" " * 8, device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[device].major, versions[device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[device].build_number))
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
infos = [*net.input_info]
print("inputs number: " + str(len(infos)))
print("input shape: " + str(net.input_info[infos[0]].input_data.shape))
print("input key: " + infos[0])
n, c, h, w = net.input_info[infos[0]].input_data.shape
log.info("Preparing input blobs")
out_blob = next(iter(net.outputs))
input_name = infos[0]
log.info("Batch size is {}".format(net.batch_size))
net.input_info[infos[0]].precision = 'U8'
log.info('Preparing output blobs')
output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))]
for output_key in net.outputs:
if net.layers[output_key].type == "DetectionOutput":
output_name, output_info = output_key, net.outputs[output_key]
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=device)
log.info("Creating infer request and starting inference")
# -----------------------------------------------------------------------------------------------------
if args.video is not None and args.input is not None:
raise RuntimeError('Either use video or images input')
if args.video is None and args.input is None:
raise RuntimeError('Need an input: video or images')
has_video = False
if args.video is not None:
has_video = True
log.info('Processing and ' + ('sav' if args.save else 'show') + 'ing ' +
('video' if has_video else 'images'))
if args.save:
if has_video:
outdir = dirname(args.video) + sep + 'results' + sep
else:
outdir = args.input + sep + 'results' + sep
makedirs(outdir, exist_ok=True)
if has_video:
cap = cv2.VideoCapture(args.video)
fps = cap.get(cv2.CAP_PROP_FPS)
video_length = cap.get(cv2.CAP_PROP_FRAME_COUNT)
ret, image = cap.read() # first frame just to read size (...)
size = (image.shape[1], image.shape[0])
if args.save:
writer = cv2.VideoWriter(outdir + basename(args.video),
cv2.VideoWriter_fourcc(*'mp4v'), fps,
size)
else:
filenames = sorted(glob.glob(args.input + '/*.jpg'), key=getmtime)
network_ratio = w / h
count = -1
x1, y1, ws, hs, paths, objects, probas = [], [], [], [], [], [], []
while True:
count += 1
if has_video:
ret, image = cap.read()
name = f'frame{count}'
if count % 100 == 0:
print('Progress: %.2f%%' % (100.0 * count / video_length),
end='\r',
flush=True)
if count % 1000 == 0:
log.info('Progress: %.2f%%' %
(100.0 * count / video_length))
if not ret:
break
else:
if count == len(filenames):
break
name = filenames[count]
image = cv2.imread(name)
output = []
ih, iw = image.shape[:-1]
input_ratio = iw / ih
if input_ratio < network_ratio:
new_h = int(floor(w / input_ratio))
new_w = w
scale_ratio = iw / w
off_h = int(floor((new_h - h) / 2))
off_w = 0
else:
new_h = h
new_w = int(floor(h * input_ratio))
scale_ratio = ih / h
off_h = 0
off_w = int(floor((new_w - w) / 2))
crop = cv2.resize(image, (new_w, new_h))
crop = crop[off_h:off_h + h, off_w:off_w + w, :]
images_hw = crop.shape[:-1]
data = {
input_name: crop.transpose((2, 0, 1))
} # Change data layout from HWC to CHW
res = exec_net.infer(inputs=data)
res = res[out_blob][0][0]
for number, proposal in enumerate(res):
if proposal[2] > 0:
ih, iw = images_hw
label = np.int(proposal[1])
confidence = proposal[2]
xmin = np.int(scale_ratio * (off_w + iw * proposal[3]))
ymin = np.int(scale_ratio * (off_h + ih * proposal[4]))
xmax = np.int(scale_ratio * (off_w + iw * proposal[5]))
ymax = np.int(scale_ratio * (off_h + ih * proposal[6]))
if confidence > args.confidence:
output.append((xmin, ymin, xmax, ymax, label,
basename(name), confidence))
"""if not args.save:
print("[{},{}] element, prob = {:.6} ({},{})-({},{})" \
.format(number, label, confidence, xmin, ymin, xmax, ymax))"""
for box in output:
if box[5] == 1:
cl = (255, 0, 0)
else:
cl = (0, 0, 255)
x, y, width, height = box[0], box[
1], box[2] - box[0], box[3] - box[1]
cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), cl, 2)
multiple_append([x1, y1, ws, hs, objects, paths, probas],
[x, y, width, height, box[4], box[5], box[6]])
if args.save:
if has_video:
writer.write(image)
else:
base = basename(name)
log.info(f'Write to {outdir + base}')
cv2.imwrite(outdir + 'detection_' + model_name + '_' + base,
image)
else:
if args.hide:
cv2.imshow('result', image)
cv2.waitKey(0)
d = {
'file': paths,
'object': objects,
'x1': x1,
'y1': y1,
'w': ws,
'h': hs,
'p': probas
}
tab = pd.DataFrame(data=d)
tab.to_csv(args.tab_base, index_label='#')
# -----------------------------------------------------------------------------------------------------
if has_video:
cap.release()
if args.save:
writer.release()
log.info("Execution successful\n")
def main():
log.basicConfig(format='[ %(levelname)s ] %(message)s',
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info('Loading Inference Engine')
ie = IECore()
# ---1. Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format ---
model = args.model
log.info(f'Loading network:')
log.info(f' {model}')
net = ie.read_network(model=model)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info('Device info:')
versions = ie.get_versions(args.device)
log.info(f' {args.device}')
log.info(
f' MKLDNNPlugin version ......... {versions[args.device].major}.{versions[args.device].minor}'
)
log.info(f' Build ........... {versions[args.device].build_number}')
# -----------------------------------------------------------------------------------------------------
# --------------------------- 3. Read and preprocess input --------------------------------------------
log.info(f'Inputs number: {len(net.input_info.keys())}')
assert len(net.input_info.keys()
) == 1, 'Sample supports clean SSD network with one input'
assert len(net.outputs.keys()
) == 1, 'Sample supports clean SSD network with one output'
input_name = list(net.input_info.keys())[0]
input_info = net.input_info[input_name]
supported_input_dims = 4 # Supported input layout - NHWC
log.info(f' Input name: {input_name}')
log.info(f' Input shape: {str(input_info.input_data.shape)}')
if len(input_info.input_data.layout) == supported_input_dims:
n, c, h, w = input_info.input_data.shape
assert n == 1, 'Sample supports topologies with one input image only'
else:
raise AssertionError('Sample supports input with NHWC shape only')
image = cv2.imread(args.input)
h_new, w_new = image.shape[:-1]
images = np.ndarray(shape=(n, c, h_new, w_new))
log.info('File was added: ')
log.info(f' {args.input}')
image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW
images[0] = image
log.info(
'Reshaping the network to the height and width of the input image')
net.reshape({input_name: [n, c, h_new, w_new]})
log.info(
f'Input shape after reshape: {str(net.input_info["data"].input_data.shape)}'
)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info('Preparing input blobs')
if len(input_info.layout) == supported_input_dims:
input_info.precision = 'U8'
data = {}
data[input_name] = images
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
func = ng.function_from_cnn(net)
ops = func.get_ordered_ops()
output_name, output_info = '', net.outputs[next(iter(net.outputs.keys()))]
output_ops = {op.friendly_name : op for op in ops \
if op.friendly_name in net.outputs and op.get_type_name() == 'DetectionOutput'}
if len(output_ops) == 1:
output_name, output_info = output_ops.popitem()
assert output_name != '', 'Can' 't find a DetectionOutput layer in the topology'
output_dims = output_info.shape
assert output_dims != 4, 'Incorrect output dimensions for SSD model'
assert output_dims[3] == 7, 'Output item should have 7 as a last dimension'
output_info.precision = 'FP32'
# -----------------------------------------------------------------------------------------------------
# --------------------------- Performing inference ----------------------------------------------------
log.info('Loading model to the device')
exec_net = ie.load_network(network=net, device_name=args.device)
log.info('Creating infer request and starting inference')
exec_result = exec_net.infer(inputs=data)
# -----------------------------------------------------------------------------------------------------
# --------------------------- Read and postprocess output ---------------------------------------------
log.info('Processing output blobs')
result = exec_result[output_name]
boxes = {}
detections = result[0][0] # [0][0] - location of detections in result blob
for number, proposal in enumerate(detections):
imid, label, confidence, coords = np.int(proposal[0]), np.int(
proposal[1]), proposal[2], proposal[3:]
if confidence > 0.5:
# correcting coordinates to actual image resolution
xmin, ymin, xmax, ymax = w_new * coords[0], h_new * coords[
1], w_new * coords[2], h_new * coords[3]
log.info(
f' [{number},{label}] element, prob = {confidence:.6f}, '
f'bbox = ({xmin:.3f},{ymin:.3f})-({xmax:.3f},{ymax:.3f}), batch id = {imid}'
)
if not imid in boxes.keys():
boxes[imid] = []
boxes[imid].append([xmin, ymin, xmax, ymax])
imid = 0 # as sample supports one input image only, imid in results will always be 0
tmp_image = cv2.imread(args.input)
for box in boxes[imid]:
# drawing bounding boxes on the output image
cv2.rectangle(tmp_image, (np.int(box[0]), np.int(box[1])),
(np.int(box[2]), np.int(box[3])),
color=(232, 35, 244),
thickness=2)
cv2.imwrite('out.bmp', tmp_image)
log.info('Image out.bmp created!')
# -----------------------------------------------------------------------------------------------------
log.info('Execution successful\n')
log.info(
'This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool'
)
return parser
#log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
args = build_argparser().parse_args()
#log.info("Loading Inference Engine")
ie = IECore()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
#log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = ie.read_network(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
#log.info("Device info:")
versions = ie.get_versions(args.device)
#print("{}{}".format(" " * 8, args.device))
#print("{}MKLDNNPlugin version ......... {}.{}".format(" " * 8, versions[args.device].major,
versions[args.device].minor))
#print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number))
# CPU를 device로 선택했을때
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
def main():
total_timer = Timer(name='total')
total_timer.tic()
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1
), "Sample supports topologies only with 1 input"
input_name = next(iter(net.inputs.keys()))
input_info = net.inputs[input_name]
input_info.precision = 'FP32'
log.info('input shape: {}'.format(input_info.shape))
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
assert (len(net.outputs.keys()) == 2
), "Sample supports topologies only with 2 output"
loc_out_name = "797"
class_out_name = "741"
assert (loc_out_name in net.outputs.keys()) and (class_out_name
in net.outputs.keys())
loc_out_info = net.outputs[loc_out_name]
class_out_info = net.outputs[class_out_name]
loc_out_info.precision = "FP32"
class_out_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------------
log.info("Loading model to the device")
# cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'}
ie.set_config({'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'},
args.device)
ie.set_config({'CPU_BIND_THREAD': 'YES'}, args.device)
exec_net = ie.load_network(network=net,
device_name=args.device,
num_requests=0)
infer_requests = exec_net.requests
request_queue = InferRequestsQueue(infer_requests)
log.info('nreqs: {}, nstream:{}'.format(
len(infer_requests),
ie.get_config(args.device, 'CPU_THROUGHPUT_STREAMS')))
# --------------------------- 3. Read and preprocess input --------------------------------------------
# -----------------------------------------------------------------------------------------------------
if not os.path.exists(args.result_dir):
os.makedirs(args.result_dir)
if args.voc_res_file and os.path.exists(args.voc_res_file):
os.remove(args.voc_res_file)
load_data_timer = Timer(name='load_data')
post_process_timer = Timer(name='post_process')
adapter = RetinaNetAdapter(input_shape=args.patch_size)
# --------------------------- Performing inference ----------------------------------------------------
result_all_images = defaultdict(list)
data_loader = DataLoader(args.image_dir, args.strides, args.patch_size)
while True:
load_data_timer.tic()
input_data = data_loader.next()
load_data_timer.toc()
if input_data == None:
break
infer_request = request_queue.get_idle_request()
if not infer_request:
raise Exception('No idle Infer Requests!')
if infer_request.cur_meta == None:
infer_request.start_async(input_name, input_data)
continue
# get result
post_process_timer.tic()
image_name = infer_request.cur_meta['image_name']
x = infer_request.cur_meta['x']
y = infer_request.cur_meta['y']
loc_out = infer_request.request.outputs[loc_out_name][0]
class_out = infer_request.request.outputs[class_out_name][0]
## start infer
infer_request.start_async(input_name, input_data)
## post-process
result = adapter.process(loc_out, class_out)
result, _ = nms(result, thresh=0.5, keep_top_k=100)
result[:, 0] += x
result[:, 1] += y
result[:, 2] += x
result[:, 3] += y
result_all_images[image_name].append(result)
post_process_timer.toc()
# wait the latest inference executions
request_queue.wait_all()
post_process_timer.tic()
for infer_request in request_queue.requests:
# get result
image_name = infer_request.cur_meta['image_name']
x = infer_request.cur_meta['x']
y = infer_request.cur_meta['y']
loc_out = infer_request.request.outputs[loc_out_name][0]
class_out = infer_request.request.outputs[class_out_name][0]
## post-process
result = adapter.process(loc_out, class_out)
result, _ = nms(result, thresh=0.5, keep_top_k=100)
result[:, 0] += x
result[:, 1] += y
result[:, 2] += x
result[:, 3] += y
result_all_images[image_name].append(result)
post_process_timer.toc()
post_process_timer.tic()
## process total image result
for image_name, result_per_image in result_all_images.items():
result_per_image = np.concatenate(result_per_image, axis=0)
nms_result, _ = nms(result_per_image, thresh=0.5)
voc_format = '{} {:.4f} {} {} {} {}'
pos_all = []
voc_all = []
for i in range(nms_result.shape[0]):
x = int(nms_result[i, 0])
y = int(nms_result[i, 1])
w = max(int(nms_result[i, 2] - nms_result[i, 0]), 1)
h = max(int(nms_result[i, 3] - nms_result[i, 1]), 1)
p = float(nms_result[i, 4])
pos = {'x': x, 'y': y, 'w': w, 'h': h, 'p': p}
pos_all.append(pos)
if args.voc_res_file:
xmin = x
ymin = y
xmax = int(nms_result[i, 2])
ymax = int(nms_result[i, 3])
voc_str = voc_format.format(
os.path.splitext(image_name)[0], p, xmin, ymin, xmax, ymax)
voc_all.append(voc_str)
file_name = os.path.splitext(image_name)[0] + '.json'
with open(os.path.join(args.result_dir, file_name), 'w') as f:
json.dump(pos_all, f)
if args.voc_res_file:
with open(args.voc_res_file, 'a') as f:
for voc_str in voc_all:
f.write(voc_str + '\n')
post_process_timer.toc()
total_timer.toc()
# -----------------------------------------------------------------------------------------------------
all_timers = []
# all_timers.extend([create_anchor_timer,
# read_img_timer,
# preprocess_timer,
# infer_timer,
# adapter_timer,
# patch_img_nms_timer,
# whole_img_nms_timer,
# add_offset_timer,
# write_result_timer,
# total_timer])
all_timers.extend([load_data_timer, post_process_timer, total_timer])
for timer in all_timers:
log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format(
timer.name, timer.avg * 1000, timer.total))
log.info('infer: {:2f}s'.format(request_queue.get_duration_in_seconds()))
log.info("Execution successful\n")
def main():
total_timer = Timer(name='total')
total_timer.tic()
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1
), "Sample supports topologies only with 1 input"
input_name = next(iter(net.inputs.keys()))
input_info = net.inputs[input_name]
# input_info.precision = 'FP32'
input_info.precision = 'U8'
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
assert (len(net.outputs.keys()) == 2
), "Sample supports topologies only with 2 output"
loc_out_name = args.loc_out_name
class_out_name = args.class_out_name
assert (loc_out_name in net.outputs.keys()) and (class_out_name
in net.outputs.keys())
loc_out_info = net.outputs[loc_out_name]
class_out_info = net.outputs[class_out_name]
loc_out_info.precision = "FP32"
class_out_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------------
log.info("Loading model to the device")
# ie.set_config({'CPU_THREADS_NUM': str(12)}, 'CPU')
exec_net = ie.load_network(network=net, device_name=args.device)
# # --------------------------- 3. Read and preprocess input --------------------------------------------
# # -----------------------------------------------------------------------------------------------------
if not os.path.exists(args.result_dir):
os.makedirs(args.result_dir)
if args.voc_res_file and os.path.exists(args.voc_res_file):
os.remove(args.voc_res_file)
# create_anchor_timer = Timer(name='create_anchor')
# read_img_timer = Timer(name='read_img')
# preprocess_timer = Timer(name='preprocess')
# infer_timer = Timer(name='infer')
# adapter_timer = Timer(name='adapter')
# patch_img_nms_timer = Timer(name='patch_img_nms')
# whole_img_nms_timer = Timer(name='whole_img_nms')
# add_offset_timer = Timer(name='add_offset')
# write_result_timer = Timer(name='write_result')
queue = Queue()
producer = Producer('Producer', queue, args)
consumer = Consumer('Consumer', queue, exec_net, input_name, args)
producer.start()
consumer.start()
producer.join()
consumer.join()
log.info('All threads finished!')
total_timer.toc()
# # -----------------------------------------------------------------------------------------------------
all_timers = []
# all_timers.extend([create_anchor_timer,
# read_img_timer,
# preprocess_timer,
# infer_timer,
# adapter_timer,
# patch_img_nms_timer,
# whole_img_nms_timer,
# add_offset_timer,
# write_result_timer,
# total_timer])
all_timers.extend([total_timer])
for timer in all_timers:
log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format(
timer.name, timer.avg * 1000, timer.total))
log.info("Execution successful\n")
class Benchmark:
def __init__(self, device: str, number_infer_requests, number_iterations,
duration_seconds, api_type):
self.device = device.upper()
self.ie = IECore()
self.nireq = number_infer_requests
self.niter = number_iterations
self.duration_seconds = get_duration_seconds(duration_seconds,
self.niter, self.device)
self.api_type = api_type
self.device_number_streams = {}
def __del__(self):
del self.ie
def add_extension(self,
path_to_extension: str = None,
path_to_cldnn_config: str = None):
if GPU_DEVICE_NAME in self.device:
if path_to_cldnn_config:
self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config},
GPU_DEVICE_NAME)
logger.info(
'GPU extensions is loaded {}'.format(path_to_cldnn_config))
if CPU_DEVICE_NAME in self.device or MYRIAD_DEVICE_NAME in self.device:
if path_to_extension:
self.ie.add_extension(extension_path=path_to_extension,
device_name=CPU_DEVICE_NAME)
logger.info(
'CPU extensions is loaded {}'.format(path_to_extension))
def get_version_info(self) -> str:
logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format(
'', 'API version', get_version()))
version_string = 'Device info\n'
for device, version in self.ie.get_versions(self.device).items():
version_string += '{: <9}{}\n'.format('', device)
version_string += '{: <9}{:.<24}{} {}.{}\n'.format(
'', version.description, ' version', version.major,
version.minor)
version_string += '{: <9}{:.<24} {}\n'.format(
'', 'Build', version.build_number)
return version_string
@staticmethod
def reshape(ie_network: IENetwork, batch_size: int):
new_shapes = {}
for input_layer_name, input_layer in ie_network.inputs.items():
shape = input_layer.shape
layout = input_layer.layout
try:
batch_index = layout.index('N')
except ValueError:
batch_index = 1 if layout == 'C' else -1
if batch_index != -1 and shape[batch_index] != batch_size:
shape[batch_index] = batch_size
new_shapes[input_layer_name] = shape
if new_shapes:
logger.info('Resizing network to batch = {}'.format(batch_size))
ie_network.reshape(new_shapes)
def set_config(self,
number_streams: int,
api_type: str = 'async',
number_threads: int = None,
infer_threads_pinning: int = None):
devices = parse_devices(self.device)
self.device_number_streams = parse_value_per_device(
devices, number_streams)
for device in devices:
if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys
# limit threading for CPU portion of inference
if number_threads:
self.ie.set_config(
{'CPU_THREADS_NUM': str(number_threads)}, device)
if MULTI_DEVICE_NAME in self.device and GPU_DEVICE_NAME in self.device:
self.ie.set_config({'CPU_BIND_THREAD': 'NO'},
CPU_DEVICE_NAME)
else:
# pin threads for CPU portion of inference
self.ie.set_config(
{'CPU_BIND_THREAD': infer_threads_pinning}, device)
# for CPU execution, more throughput-oriented execution via streams
# for pure CPU execution, more throughput-oriented execution via streams
if api_type == 'async':
cpu_throughput = {
'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'
}
if device in self.device_number_streams.keys():
cpu_throughput['CPU_THROUGHPUT_STREAMS'] = str(
self.device_number_streams.get(device))
self.ie.set_config(cpu_throughput, device)
self.device_number_streams[device] = self.ie.get_config(
device, 'CPU_THROUGHPUT_STREAMS')
elif device == GPU_DEVICE_NAME:
if api_type == 'async':
gpu_throughput = {
'GPU_THROUGHPUT_STREAMS': 'GPU_THROUGHPUT_AUTO'
}
if device in self.device_number_streams.keys():
gpu_throughput['GPU_THROUGHPUT_STREAMS'] = str(
self.device_number_streams.get(device))
self.ie.set_config(gpu_throughput, device)
self.device_number_streams[device] = self.ie.get_config(
device, 'GPU_THROUGHPUT_STREAMS')
if MULTI_DEVICE_NAME in self.device and CPU_DEVICE_NAME in self.device:
# multi-device execution with the CPU+GPU performs best with GPU trottling hint,
# which releases another CPU thread (that is otherwise used by the GPU driver for active polling)
self.ie.set_config({'CLDNN_PLUGIN_THROTTLE': '1'}, device)
elif device == MYRIAD_DEVICE_NAME:
self.ie.set_config({'LOG_LEVEL': 'LOG_INFO'},
MYRIAD_DEVICE_NAME)
def load_network(self,
ie_network: IENetwork,
perf_counts: bool,
number_infer_requests: int = None):
config = {'PERF_COUNT': ('YES' if perf_counts else 'NO')}
exe_network = self.ie.load_network(ie_network,
self.device,
config=config,
num_requests=number_infer_requests
or 0)
return exe_network
def infer(self, request_queue, requests_input_data, batch_size,
progress_bar):
progress_count = 0
# warming up - out of scope
infer_request = request_queue.get_idle_request()
if not infer_request:
raise Exception('No idle Infer Requests!')
if self.api_type == 'sync':
infer_request.infer(requests_input_data[infer_request.req_id])
else:
infer_request.start_async(
requests_input_data[infer_request.req_id])
request_queue.wait_all()
request_queue.reset_times()
start_time = datetime.now()
exec_time = (datetime.now() - start_time).total_seconds()
iteration = 0
# Start inference & calculate performance
# to align number if iterations to guarantee that last infer requests are executed in the same conditions **/
while (self.niter and iteration < self.niter) or \
(self.duration_seconds and exec_time < self.duration_seconds) or \
(self.api_type == 'async' and iteration % self.nireq):
infer_request = request_queue.get_idle_request()
if not infer_request:
raise Exception('No idle Infer Requests!')
if self.api_type == 'sync':
infer_request.infer(requests_input_data[infer_request.req_id])
else:
infer_request.start_async(
requests_input_data[infer_request.req_id])
iteration += 1
exec_time = (datetime.now() - start_time).total_seconds()
if self.duration_seconds:
# calculate how many progress intervals are covered by current iteration.
# depends on the current iteration time and time of each progress interval.
# Previously covered progress intervals must be skipped.
progress_interval_time = self.duration_seconds / progress_bar.total_num
new_progress = int(exec_time / progress_interval_time -
progress_count)
progress_bar.add_progress(new_progress)
progress_count += new_progress
elif self.niter:
progress_bar.add_progress(1)
# wait the latest inference executions
request_queue.wait_all()
total_duration_sec = request_queue.get_duration_in_seconds()
times = request_queue.times
times.sort()
latency_ms = median(times)
fps = batch_size * 1000 / latency_ms
if self.api_type == 'async':
fps = batch_size * iteration / total_duration_sec
progress_bar.finish()
return fps, latency_ms, total_duration_sec, iteration
def main(args=None):
try:
# ------------------------------ 1. Parsing and validating input arguments -------------------------------------
next_step()
if not args:
args = parse_args()
# ------------------------------ 2. Loading Inference Engine ---------------------------------------------------
next_step()
device_name = args.target_device.upper()
ie = IECore()
if CPU_DEVICE_NAME in device_name:
if args.path_to_extension:
ie.add_cpu_extension(extension_path=args.path_to_extension,
device_name=CPU_DEVICE_NAME)
if GPU_DEVICE_NAME in device_name:
if args.path_to_cldnn_config:
ie.set_config({'CONFIG_FILE': args.path_to_cldnn_config},
GPU_DEVICE_NAME)
logger.info("GPU extensions is loaded {}".format(
args.path_to_cldnn_config))
logger.info("InferenceEngine:\n{: <9}{}".format("", get_version()))
version_string = "Device is {}\n".format(device_name)
for device, version in ie.get_versions(device_name).items():
version_string += "{: <9}{}\n".format("", device)
version_string += "{: <9}{:.<24}{} {}.{}\n".format(
"", version.description, " version", version.major,
version.minor)
version_string += "{: <9}{:.<24} {}\n".format(
"", "Build", version.build_number)
logger.info(version_string)
# --------------------- 3. Read the Intermediate Representation of the network ---------------------------------
next_step()
xml_filename = os.path.abspath(args.path_to_model)
head, tail = os.path.splitext(xml_filename)
bin_filename = os.path.abspath(head + BIN_EXTENSION)
ie_network = IENetwork(xml_filename, bin_filename)
input_info = ie_network.inputs
if len(input_info) == 0:
raise AttributeError('No inputs info is provided')
# --------------------- 4. Resizing network to match image sizes and given batch -------------------------------
next_step()
batch_size = ie_network.batch_size
precision = ie_network.precision
if args.batch_size and args.batch_size != ie_network.batch_size:
new_shapes = {}
for key in input_info.keys():
shape = input_info[key].shape
layout = input_info[key].layout
batchIndex = -1
if ((layout == 'NCHW') or (layout == 'NCDHW')
or (layout == 'NHWC') or (layout == 'NDHWC')
or (layout == 'NC')):
batchIndex = 0
elif (layout == 'CN'):
batchIndex = 1
if ((batchIndex != -1)
and (shape[batchIndex] != args.batch_size)):
shape[batchIndex] = args.batch_size
new_shapes[key] = shape
if (len(new_shapes) > 0):
logger.info("Resizing network to batch = {}".format(
args.batch_size))
ie_network.reshape(new_shapes)
batch_size = args.batch_size
logger.info("Network batch size: {}, precision {}".format(
batch_size, precision))
# --------------------- 5. Configuring input of the model ------------------------------------------------------
next_step()
for key in input_info.keys():
if (isImage(input_info[key])):
# Set the precision of input data provided by the user
# Should be called before load of the network to the plugin
input_info[key].precision = 'U8'
# --------------------- 6. Setting device configuration --------------------------------------------------------
next_step()
devices = parseDevices(device_name)
device_nstreams = parseValuePerDevice(devices, args.number_streams)
for device in devices:
if device == CPU_DEVICE_NAME: ## CPU supports few special performance-oriented keys
## limit threading for CPU portion of inference
if args.number_threads:
ie.set_config(
{'CPU_THREADS_NUM': str(args.number_threads)}, device)
if MULTI_DEVICE_NAME in device_name and GPU_DEVICE_NAME in device_name:
ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME)
else:
# pin threads for CPU portion of inference
ie.set_config(
{'CPU_BIND_THREAD': args.infer_threads_pinning},
device)
## for CPU execution, more throughput-oriented execution via streams
# for pure CPU execution, more throughput-oriented execution via streams
if args.api_type == 'async':
ie.set_config(
{
'CPU_THROUGHPUT_STREAMS':
str(device_nstreams.get(device))
if device in device_nstreams.keys() else
'CPU_THROUGHPUT_AUTO'
}, device)
device_nstreams[device] = int(
ie.get_config(device, 'CPU_THROUGHPUT_STREAMS'))
elif device == GPU_DEVICE_NAME:
if args.api_type == 'async':
ie.set_config(
{
'GPU_THROUGHPUT_STREAMS':
str(device_nstreams.get(device))
if device in device_nstreams.keys() else
'GPU_THROUGHPUT_AUTO'
}, device)
device_nstreams[device] = int(
ie.get_config(device, 'GPU_THROUGHPUT_STREAMS'))
if MULTI_DEVICE_NAME in device_name and CPU_DEVICE_NAME in device_name:
## multi-device execution with the CPU+GPU performs best with GPU trottling hint,
## which releases another CPU thread (that is otherwise used by the GPU driver for active polling)
ie.set_config({'CLDNN_PLUGIN_THROTTLE': str(1)}, device)
elif device == MYRIAD_DEVICE_NAME:
ie.set_config(
{
'LOG_LEVEL': 'LOG_INFO',
'VPU_LOG_LEVEL': 'LOG_WARNING'
}, MYRIAD_DEVICE_NAME)
# --------------------- 7. Loading the model to the device -----------------------------------------------------
next_step()
config = {'PERF_COUNT': ('YES' if args.perf_counts else 'NO')}
exe_network = ie.load_network(ie_network,
device_name,
config=config,
num_requests=args.number_infer_requests
if args.number_infer_requests else 0)
# --------------------- 8. Setting optimal runtime parameters --------------------------------------------------
next_step()
## Number of requests
infer_requests = exe_network.requests
nireq = len(infer_requests)
## Iteration limit
niter = args.number_iterations
if niter and args.api_type == 'async':
niter = (int)((niter + nireq - 1) / nireq) * nireq
if (args.number_iterations != niter):
logger.warn(
"Number of iterations was aligned by request number "
"from {} to {} using number of requests {}".format(
args.number_iterations, niter, nireq))
## Time limit
duration_seconds = 0
if args.time:
## time limit
duration_seconds = args.time
elif not args.number_iterations:
## default time limit
duration_seconds = get_duration_in_secs(device)
# ------------------------------------ 8. Creating infer requests and filling input blobs ----------------------
next_step()
request_queue = InferRequestsQueue(infer_requests)
path_to_input = os.path.abspath(
args.path_to_input) if args.path_to_input else None
requests_input_data = getInputs(path_to_input, batch_size,
ie_network.inputs, infer_requests)
# ------------------------------------ 9. Measuring performance ------------------------------------------------
progress_count = 0
progress_bar_total_count = 10000
output_string = "Start inference {}ronously".format(args.api_type)
if (args.api_type == "async"):
if output_string != "":
output_string += ", "
output_string += str(nireq) + " inference requests"
device_ss = ''
for device, nstreams in device_nstreams.items():
if device_ss != '':
device_ss += ', '
device_ss += "{} streams for {}".format(str(nstreams), device)
if device_ss != '':
output_string += " using " + device_ss
output_string += ", limits: "
if niter:
if not duration_seconds:
progress_bar_total_count = niter
output_string += str(niter) + " iterations"
if duration_seconds:
if niter:
output_string += ", "
output_string += str(
getDurationInMilliseconds(duration_seconds)) + " ms duration"
next_step(output_string)
## warming up - out of scope
infer_request = request_queue.getIdleRequest()
if not infer_request:
raise Exception("No idle Infer Requests!")
if (args.api_type == 'sync'):
infer_request.infer(requests_input_data[infer_request.id])
else:
infer_request.startAsync(requests_input_data[infer_request.id])
request_queue.waitAll()
request_queue.resetTimes()
start_time = datetime.now()
exec_time = (datetime.now() - start_time).total_seconds()
iteration = 0
progress_bar = ProgressBar(progress_bar_total_count,
args.stream_output, args.progress)
## Start inference & calculate performance
## to align number if iterations to guarantee that last infer requests are executed in the same conditions **/
while ((niter and iteration < niter)
or (duration_seconds and exec_time < duration_seconds)
or (args.api_type == "async" and iteration % nireq != 0)):
infer_request = request_queue.getIdleRequest()
if not infer_request:
raise Exception("No idle Infer Requests!")
if (args.api_type == 'sync'):
infer_request.infer(requests_input_data[infer_request.id])
else:
infer_request.startAsync(requests_input_data[infer_request.id])
iteration += 1
exec_time = (datetime.now() - start_time).total_seconds()
if niter:
progress_bar.add_progress(1)
else:
## calculate how many progress intervals are covered by current iteration.
## depends on the current iteration time and time of each progress interval.
## Previously covered progress intervals must be skipped.
progress_interval_time = duration_seconds / progress_bar_total_count
new_progress = (int)(exec_time / progress_interval_time -
progress_count)
progress_bar.add_progress(new_progress)
progress_count += new_progress
## wait the latest inference executions
request_queue.waitAll()
total_duration_sec = request_queue.getDurationInSeconds()
times = request_queue.times
times.sort()
latency_ms = median(times)
fps = batch_size * 1000 / latency_ms if args.api_type == 'sync' else batch_size * iteration / total_duration_sec
progress_bar.finish()
# ------------------------------------ 10. Dumping statistics report -------------------------------------------
next_step()
if args.exec_graph_path:
try:
exec_graph_info = exe_network.get_exec_graph_info()
exec_graph_info.serialize(args.exec_graph_path)
logger.info("Executable graph is stored to {}".format(
args.exec_graph_path))
del exec_graph_info
except Exception as e:
logging.exception(e)
if args.perf_counts:
for ni in range(int(nireq)):
perf_counts = exe_network.requests[ni].get_perf_counts()
logger.info(
"Pefrormance counts for {}-th infer request".format(ni))
for layer, stats in perf_counts.items():
max_layer_name = 30
print("{:<30}{:<15}{:<30}{:<20}{:<20}{:<20}".format(
layer[:max_layer_name - 4] + '...' if
(len(layer) >= max_layer_name) else layer,
stats['status'],
'layerType: ' + str(stats['layer_type']),
'realTime: ' + str(stats['real_time']),
'cpu: ' + str(stats['cpu_time']),
'execType: ' + str(stats['exec_type'])))
print("Count: {} iterations".format(iteration))
print("Duration: {:.2f} ms".format(
getDurationInMilliseconds(total_duration_sec)))
if not MULTI_DEVICE_NAME in device_name:
print("Latency: {:.4f} ms".format(latency_ms))
print("Throughput: {:.2f} FPS".format(fps))
del exe_network
del ie
next_step.step_id = 0
except Exception as e:
logging.exception(e)
def main(args):
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
n, w, h, c = net.inputs[input_blob].shape
# net.inputs[input_blob].precision = 'U8'
# input_blob.precision = 'I8'
log.info('shape: [{},{},{},{}]'.format(n, w, h, c))
exec_net = ie.load_network(network=net, device_name=args.device)
handler = SignalHandler()
print('model loaded, starting detection loop', flush=True)
while not handler.interrupted:
sleep(0.25)
r = requests.get(args.motion_url, timeout=0.5)
if not r:
log.error(
'error getting motion from url, trying again after a minute')
sleep(60)
dat = np.frombuffer(r.content, dtype=np.int8)
dat = np.reshape(dat, [1, w, h, 4])
dat = dat[:, :, :, 0:2]
dat = dat.astype(np.float32)
res = exec_net.infer(inputs={input_blob: dat})
res = res[out_blob]
res = np.reshape(res, [])
if res > args.total_motion:
requests.post(args.post_url, data='"on"')
print('motion detected: {}'.format(res), flush=True)
sleep(600)
print('exited.')
class HandPoseOpenvino(HandPose):
def __init__(self,
pd_xml=PALM_DETECTION_OPENVINO_FP32,
lm_xml=LANDMARK_OPENVINO_FP32,
pd_device="CPU",
lm_device="CPU",
**kwargs):
"""
pd_xml : palm detection model xml filepath
pd_device : device on which palm detection model is run
lm_xml : landmark model xml filepath
lm_device : device on which landmark model is run
kwargs : args inherited from HandPose class
"""
from openvino.inference_engine import IENetwork, IECore
super().__init__(**kwargs)
log.info("Loading Inference Engine")
self.ie = IECore()
log.info("Device info:")
versions = self.ie.get_versions(pd_device)
log.info("{}{}".format(" " * 8, pd_device))
log.info("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[pd_device].major, versions[pd_device].minor))
log.info("{}Build ........... {}".format(
" " * 8, versions[pd_device].build_number))
# Palm detection model
self.pd_name = os.path.splitext(pd_xml)[0]
pd_bin = self.pd_name + '.bin'
log.info(
"Palm detection model - Loading network files:\n\t{}\n\t{}".format(
pd_xml, pd_bin))
self.pd_net = IENetwork(model=pd_xml, weights=pd_bin)
# [ INFO ] Input blob: input - shape: [1, 3, 256, 256]
# [ INFO ] Output blob: classificators - shape: [1, 2944, 1] : tensor scores
# [ INFO ] Output blob: regressors - shape: [1, 2944, 18] : tensor bboxes
self.pd_input_blob = next(iter(self.pd_net.inputs))
log.info(
f"Input blob: {self.pd_input_blob} - shape: {self.pd_net.inputs[self.pd_input_blob].shape}"
)
_, _, self.pd_h, self.pd_w = self.pd_net.inputs[
self.pd_input_blob].shape
for o in self.pd_net.outputs.keys():
log.info(
f"Output blob: {o} - shape: {self.pd_net.outputs[o].shape}")
self.pd_scores = "classificators"
self.pd_bboxes = "regressors"
log.info("Loading palm detection model to the plugin")
self.pd_exec_net = self.ie.load_network(network=self.pd_net,
num_requests=1,
device_name=pd_device)
# Landmarks model
if self.use_landmarks:
if lm_device != pd_device:
log.info("Device info:")
versions = self.ie.get_versions(pd_device)
log.info("{}{}".format(" " * 8, pd_device))
log.info("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[pd_device].major,
versions[pd_device].minor))
log.info("{}Build ........... {}".format(
" " * 8, versions[pd_device].build_number))
self.lm_name = os.path.splitext(lm_xml)[0]
lm_bin = self.lm_name + '.bin'
log.info(
"Landmark model - Loading network files:\n\t{}\n\t{}".format(
lm_xml, lm_bin))
self.lm_net = IENetwork(model=lm_xml, weights=lm_bin)
# [ INFO ] Input blob: input - shape: [1, 3, 256, 256]
# [ INFO ] Output blob: StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid/Sigmoid - shape: [1, 1, 1, 1]: score
# [ INFO ] Output blob: StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid_1/Sigmoid_1 - shape: [1, 1, 1, 1] : handedness
# [ INFO ] Output blob: StatefulPartitionedCall/functional_1/tf_op_layer_ld_21_3d/ld_21_3d - shape: [1, 63] : landmarks
self.lm_input_blob = next(iter(self.lm_net.inputs))
log.info(
f"Input blob: {self.lm_input_blob} - shape: {self.lm_net.inputs[self.lm_input_blob].shape}"
)
_, _, self.lm_h, self.lm_w = self.lm_net.inputs[
self.lm_input_blob].shape
for o in self.lm_net.outputs.keys():
log.info(
f"Output blob: {o} - shape: {self.lm_net.outputs[o].shape}"
)
self.lm_score = "StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid/Sigmoid"
self.lm_handedness = "StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid_1/Sigmoid_1"
self.lm_landmarks = "StatefulPartitionedCall/functional_1/tf_op_layer_ld_21_3d/ld_21_3d"
log.info("Loading landmark model to the plugin")
self.lm_exec_net = self.ie.load_network(network=self.lm_net,
num_requests=1,
device_name=lm_device)
def palm_detection_infer(self, img_nn):
"""
img_nn: normalized and resized image, can be directly fed into the palm detection model
"""
inference_result = self.pd_exec_net.infer(
inputs={self.pd_input_blob: img_nn})
scores = np.squeeze(inference_result[self.pd_scores])
bboxes = inference_result[self.pd_bboxes][0]
return scores, bboxes
def landmarks_regression_infer(self, img_nn):
"""
img_nn: normalized and resized image, can be directly fed into the landmarks model
Returns :
- lm_score : probability between 0 and 1 that img_nn contains a handArgumentParser
- lm handedness : probability between 0 and 1 that the hand is a right handArgumentParser
- lm_array : array of size 63 (=3x21) containing 21 (x, y, z) coordinates of 21 landmarks
"""
inference_result = self.lm_exec_net.infer(
inputs={self.lm_input_blob: img_nn})
lm_score = np.squeeze(inference_result[self.lm_score])
handedness = np.squeeze(inference_result[self.lm_handedness])
lm_array = np.squeeze(inference_result[self.lm_landmarks])
return lm_score, handedness, lm_array
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Loading Inference Engine")
ie = IECore()
# ---1. Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format ---
model = args.model
log.info(f"Loading network:\n\t{model}")
net = ie.read_network(model=model)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
# -----------------------------------------------------------------------------------------------------
# --------------------------- 3. Read and preprocess input --------------------------------------------
print("inputs number: " + str(len(net.input_info.keys())))
assert len(
net.input_info.keys()) == 1, 'Sample supports networks with one input'
for input_key in net.input_info:
print("input shape: " +
str(net.input_info[input_key].input_data.shape))
print("input key: " + input_key)
if len(net.input_info[input_key].input_data.layout) == 4:
n, c, h, w = net.input_info[input_key].input_data.shape
images = np.ndarray(shape=(n, c, h, w))
images_hw = []
for i in range(n):
image = cv2.imread(args.input)
ih, iw = image.shape[:-1]
images_hw.append((ih, iw))
log.info("File was added: ")
log.info(" {}".format(args.input))
if (ih, iw) != (h, w):
log.warning("Image {} is resized from {} to {}".format(
args.input, image.shape[:-1], (h, w)))
image = cv2.resize(image, (w, h))
image = image.transpose(
(2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.input_info.keys()) == 1 or len(net.input_info.keys())
== 2), "Sample supports topologies only with 1 or 2 inputs"
out_blob = next(iter(net.outputs))
input_name, input_info_name = "", ""
for input_key in net.input_info:
if len(net.input_info[input_key].layout) == 4:
input_name = input_key
net.input_info[input_key].precision = 'U8'
elif len(net.input_info[input_key].layout) == 2:
input_info_name = input_key
net.input_info[input_key].precision = 'FP32'
if net.input_info[input_key].input_data.shape[1] != 3 and net.input_info[input_key].input_data.shape[1] != 6 or \
net.input_info[input_key].input_data.shape[0] != 1:
log.error(
'Invalid input info. Should be 3 or 6 values length.')
data = {}
data[input_name] = images
if input_info_name != "":
infos = np.ndarray(shape=(n, c), dtype=float)
for i in range(n):
infos[i, 0] = h
infos[i, 1] = w
infos[i, 2] = 1.0
data[input_info_name] = infos
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
func = ng.function_from_cnn(net)
ops = func.get_ordered_ops()
output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))]
output_ops = {op.friendly_name : op for op in ops \
if op.friendly_name in net.outputs and op.get_type_name() == "DetectionOutput"}
if len(output_ops) != 0:
output_name, output_info = output_ops.popitem()
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# --------------------------- Performing inference ----------------------------------------------------
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=args.device)
log.info("Creating infer request and starting inference")
res = exec_net.infer(inputs=data)
# -----------------------------------------------------------------------------------------------------
# --------------------------- Read and postprocess output ---------------------------------------------
log.info("Processing output blobs")
res = res[out_blob]
boxes, classes = {}, {}
data = res[0][0]
for number, proposal in enumerate(data):
if proposal[2] > 0:
imid = np.int(proposal[0])
ih, iw = images_hw[imid]
label = np.int(proposal[1])
confidence = proposal[2]
xmin = np.int(iw * proposal[3])
ymin = np.int(ih * proposal[4])
xmax = np.int(iw * proposal[5])
ymax = np.int(ih * proposal[6])
print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}" \
.format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="")
if proposal[2] > 0.5:
print(" WILL BE PRINTED!")
if not imid in boxes.keys():
boxes[imid] = []
boxes[imid].append([xmin, ymin, xmax, ymax])
if not imid in classes.keys():
classes[imid] = []
classes[imid].append(label)
else:
print()
for imid in classes:
tmp_image = cv2.imread(args.input)
for box in boxes[imid]:
cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]),
(232, 35, 244), 2)
cv2.imwrite("out.bmp", tmp_image)
log.info("Image out.bmp created!")
# -----------------------------------------------------------------------------------------------------
log.info("Execution successful\n")
log.info(
"This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool"
)
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Loading Inference Engine")
ie = IECore()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = ie.read_network(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" "*8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(" "*8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" "*8, versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
if len(not_supported_layers) != 0:
log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
format(args.device, ', '.join(not_supported_layers)))
log.error("Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 3. Read and preprocess input --------------------------------------------
input_blob = next(iter(net.inputs))
n, c, h, w = net.inputs[input_blob].shape
images = np.ndarray(shape=(n, c, h, w))
images_hw = []
for i in range(n):
image = cv2.imread(args.input[i])
ih, iw = image.shape[:-1]
images_hw.append((ih, iw))
log.info("File was added: ")
log.info(" {}".format(args.input[i]))
if (ih, iw) != (h, w):
image = cv2.resize(image, (w, h))
log.warning("Image {} is resized from {} to {}".format(args.input[i], image.shape[:-1], (h, w)))
image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW
images[i] = image
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1 or len(net.inputs.keys()) == 2), "Sample supports topologies only with 1 or 2 inputs"
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
input_name, input_info_name = "", ""
for input_key in net.inputs:
if len(net.inputs[input_key].layout) == 4:
input_name = input_key
log.info("Batch size is {}".format(net.batch_size))
net.inputs[input_key].precision = 'U8'
elif len(net.inputs[input_key].layout) == 2:
input_info_name = input_key
net.inputs[input_key].precision = 'FP32'
if net.inputs[input_key].shape[1] != 3 and net.inputs[input_key].shape[1] != 6 or net.inputs[input_key].shape[0] != 1:
log.error('Invalid input info. Should be 3 or 6 values length.')
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))]
for output_key in net.outputs:
if net.layers[output_key].type == "DetectionOutput":
output_name, output_info = output_key, net.outputs[output_key]
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# --------------------------- Performing inference ----------------------------------------------------
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=args.device)
log.info("Creating infer request and starting inference")
res = exec_net.infer(inputs={input_blob: images})
# -----------------------------------------------------------------------------------------------------
# --------------------------- Read and postprocess output ---------------------------------------------
log.info("Processing output blobs")
res = res[out_blob]
boxes, classes = {}, {}
data = res[0][0]
for number, proposal in enumerate(data):
if proposal[2] > 0:
imid = np.int(proposal[0])
ih, iw = images_hw[imid]
label = np.int(proposal[1])
confidence = proposal[2]
xmin = np.int(iw * proposal[3])
ymin = np.int(ih * proposal[4])
xmax = np.int(iw * proposal[5])
ymax = np.int(ih * proposal[6])
print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}"\
.format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="")
if proposal[2] > 0.5:
print(" WILL BE PRINTED!")
if not imid in boxes.keys():
boxes[imid] = []
boxes[imid].append([xmin, ymin, xmax, ymax])
if not imid in classes.keys():
classes[imid] = []
classes[imid].append(label)
else:
print()
for imid in classes:
tmp_image = cv2.imread(args.input[imid])
for box in boxes[imid]:
cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]), (232, 35, 244), 2)
cv2.imshow('output', tmp_image)
if (cv2.waitKey(0) == 27):
break
cv2.destroyAllWindows()
cv2.imwrite("out.bmp", tmp_image)
log.info("Image out.bmp created!")
# -----------------------------------------------------------------------------------------------------
log.info("Execution successful\n")
log.info("This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool")
def main():
args = parse_arguments()
# --------------------------------- 1. Load Plugin for inference engine ---------------------------------
logger.info("Creating Inference Engine")
ie = IECore()
if 'CPU' in args.target_device:
if args.path_to_extension:
ie.add_extension(args.path_to_extension, "CPU")
if args.number_threads is not None:
ie.set_config({'CPU_THREADS_NUM': str(args.number_threads)}, "CPU")
elif 'GPU' in args.target_device:
if args.path_to_cldnn_config:
ie.set_config({'CONFIG_FILE': args.path_to_cldnn_config}, "GPU")
logger.info("GPU extensions is loaded {}".format(
args.path_to_cldnn_config))
else:
raise AttributeError(
"Device {} do not support of 3D convolution. "
"Please use CPU, GPU or HETERO:*CPU*, HETERO:*GPU*")
logger.info("Device is {}".format(args.target_device))
version = ie.get_versions(args.target_device)[args.target_device]
version_str = "{}.{}.{}".format(version.major, version.minor,
version.build_number)
logger.info("Plugin version is {}".format(version_str))
# --------------------- 2. Read IR Generated by ModelOptimizer (.xml and .bin files) ---------------------
xml_filename = os.path.abspath(args.path_to_model)
bin_filename = os.path.abspath(os.path.splitext(xml_filename)[0] + '.bin')
ie_network = IENetwork(xml_filename, bin_filename)
input_info = ie_network.inputs
if len(input_info) == 0:
raise AttributeError('No inputs info is provided')
elif len(input_info) != 1:
raise AttributeError("only one input layer network is supported")
input_name = next(iter(input_info))
out_name = next(iter(ie_network.outputs))
if args.shape:
logger.info("Reshape of network from {} to {}".format(
input_info[input_name].shape, args.shape))
ie_network.reshape({input_name: args.shape})
input_info = ie_network.inputs
# ---------------------------------------- 4. Preparing input data ----------------------------------------
logger.info("Preparing inputs")
if len(input_info[input_name].shape) != 5:
raise AttributeError(
"Incorrect shape {} for 3d convolution network".format(args.shape))
n, c, d, h, w = input_info[input_name].shape
ie_network.batch_size = n
if not os.path.exists(args.path_to_input_data):
raise AttributeError("Path to input data: '{}' does not exist".format(
args.path_to_input_data))
is_nifti_data = os.path.isdir(args.path_to_input_data)
if is_nifti_data:
series_name = find_series_name(args.path_to_input_data)
original_data, data_crop, affine, original_size, bbox = \
read_image(args.path_to_input_data, series_name=series_name, sizes=(h, w, d))
else:
if not (fnmatch(args.path_to_input_data, '*.tif')
or fnmatch(args.path_to_input_data, '*.tiff')):
raise AttributeError("Input file extension must have tiff format")
data_crop = np.zeros(shape=(n, c, d, h, w), dtype=np.float)
im_seq = ImageSequence.Iterator(Image.open(args.path_to_input_data))
for i, page in enumerate(im_seq):
im = np.array(page).reshape(h, w, c)
for channel in range(c):
data_crop[:, channel, i, :, :] = im[:, :, channel]
original_data = data_crop
original_size = original_data.shape[-3:]
test_im = {input_name: data_crop}
# ------------------------------------- 4. Loading model to the plugin -------------------------------------
logger.info("Loading model to the plugin")
executable_network = ie.load_network(network=ie_network,
device_name=args.target_device)
del ie_network
# ---------------------------------------------- 5. Do inference --------------------------------------------
logger.info("Start inference")
start_time = datetime.now()
res = executable_network.infer(test_im)
infer_time = datetime.now() - start_time
logger.info("Finish inference")
logger.info("Inference time is {}".format(infer_time))
# ---------------------------- 6. Processing of the received inference results ------------------------------
result = res[out_name]
batch, channels, out_d, out_h, out_w = result.shape
list_img = list()
list_seg_result = list()
logger.info("Processing of the received inference results is started")
start_time = datetime.now()
for batch, data in enumerate(result):
seg_result = np.zeros(shape=original_size, dtype=np.uint8)
if data.shape[1:] != original_size:
x = bbox[1] - bbox[0]
y = bbox[3] - bbox[2]
z = bbox[5] - bbox[4]
seg_result[bbox[0]:bbox[1], bbox[2]:bbox[3], bbox[4]:bbox[5]] = \
np.argmax(resample_np(data, (channels, x, y, z), 1), axis=0)
elif channels == 1:
reshaped_data = data.reshape(out_d, out_h, out_w)
mask = reshaped_data[:, :, :] > 0.5
reshaped_data[mask] = 1
seg_result = reshaped_data.astype(int)
else:
seg_result = np.argmax(data, axis=0).astype(int)
im = np.stack([
original_data[batch, 0, :, :, :], original_data[batch, 0, :, :, :],
original_data[batch, 0, :, :, :]
],
axis=3)
im = 255 * (im - im.min()) / (im.max() - im.min())
color_seg_frame = np.zeros(im.shape, dtype=np.uint8)
for idx, c in enumerate(CLASSES_COLOR_MAP):
color_seg_frame[seg_result[:, :, :] == idx, :] = np.array(
c, dtype=np.uint8)
mask = seg_result[:, :, :] > 0
im[mask] = color_seg_frame[mask]
for k in range(out_d):
if is_nifti_data:
list_img.append(
Image.fromarray(im[:, :, k, :].astype('uint8'), 'RGB'))
else:
list_img.append(
Image.fromarray(im[k, :, :, :].astype('uint8'), 'RGB'))
if args.output_nifti and is_nifti_data:
list_seg_result.append(seg_result)
result_processing_time = datetime.now() - start_time
logger.info("Processing of the received inference results is finished")
logger.info("Processing time is {}".format(result_processing_time))
# --------------------------------------------- 7. Save output -----------------------------------------------
tiff_output_name = os.path.join(args.path_to_output, 'output.tiff')
Image.new('RGB',
(data.shape[3], data.shape[2])).save(tiff_output_name,
append_images=list_img,
save_all=True)
logger.info("Result tiff file was saved to {}".format(tiff_output_name))
if args.output_nifti and is_nifti_data:
for seg_res in list_seg_result:
nii_filename = os.path.join(
args.path_to_output,
'output_{}.nii.gz'.format(list_seg_result.index(seg_res)))
nib.save(nib.Nifti1Image(seg_res, affine=affine), nii_filename)
logger.info(
"Result nifti file was saved to {}".format(nii_filename))
class Benchmark:
def __init__(self,
device: str,
number_infer_requests: int = None,
number_iterations: int = None,
duration_seconds: int = None,
api_type: str = 'async'):
self.device = device
self.ie = IECore()
self.nireq = number_infer_requests
self.niter = number_iterations
self.duration_seconds = get_duration_seconds(duration_seconds,
self.niter, self.device)
self.api_type = api_type
def __del__(self):
del self.ie
def add_extension(self,
path_to_extension: str = None,
path_to_cldnn_config: str = None):
if path_to_cldnn_config:
self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config},
GPU_DEVICE_NAME)
logger.info(
'GPU extensions is loaded {}'.format(path_to_cldnn_config))
if path_to_extension:
self.ie.add_extension(extension_path=path_to_extension,
device_name=CPU_DEVICE_NAME)
logger.info(
'CPU extensions is loaded {}'.format(path_to_extension))
def get_version_info(self) -> str:
logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format(
'', 'API version', get_version()))
version_string = 'Device info\n'
for device, version in self.ie.get_versions(self.device).items():
version_string += '{: <9}{}\n'.format('', device)
version_string += '{: <9}{:.<24}{} {}.{}\n'.format(
'', version.description, ' version', version.major,
version.minor)
version_string += '{: <9}{:.<24} {}\n'.format(
'', 'Build', version.build_number)
return version_string
def set_config(self, config={}):
for device in config.keys():
self.ie.set_config(config[device], device)
def read_network(self, path_to_model: str):
model_filename = os.path.abspath(path_to_model)
head, ext = os.path.splitext(model_filename)
weights_filename = os.path.abspath(
head + BIN_EXTENSION) if ext == XML_EXTENSION else ""
ie_network = self.ie.read_network(model_filename, weights_filename)
return ie_network
def load_network(self, ie_network: IENetwork, config={}):
exe_network = self.ie.load_network(
ie_network,
self.device,
config=config,
num_requests=1 if self.api_type == 'sync' else self.nireq or 0)
# Number of requests
self.nireq = len(exe_network.requests)
return exe_network
def import_network(self, path_to_file: str, config={}):
exe_network = self.ie.import_network(
model_file=path_to_file,
device_name=self.device,
config=config,
num_requests=1 if self.api_type == 'sync' else self.nireq or 0)
# Number of requests
self.nireq = len(exe_network.requests)
return exe_network
def first_infer(self, exe_network):
infer_request = exe_network.requests[0]
# warming up - out of scope
if self.api_type == 'sync':
infer_request.infer()
else:
infer_request.async_infer()
status = exe_network.wait()
if status != StatusCode.OK:
raise Exception(
"Wait for all requests is failed with status code {}!".
format(status))
return infer_request.latency
def infer(self, exe_network, batch_size, progress_bar=None):
progress_count = 0
infer_requests = exe_network.requests
start_time = datetime.utcnow()
exec_time = 0
iteration = 0
times = []
in_fly = set()
# Start inference & calculate performance
# to align number if iterations to guarantee that last infer requests are executed in the same conditions **/
while (self.niter and iteration < self.niter) or \
(self.duration_seconds and exec_time < self.duration_seconds) or \
(self.api_type == 'async' and iteration % self.nireq):
if self.api_type == 'sync':
infer_requests[0].infer()
times.append(infer_requests[0].latency)
else:
infer_request_id = exe_network.get_idle_request_id()
if infer_request_id < 0:
status = exe_network.wait(num_requests=1)
if status != StatusCode.OK:
raise Exception("Wait for idle request failed!")
infer_request_id = exe_network.get_idle_request_id()
if infer_request_id < 0:
raise Exception("Invalid request id!")
if infer_request_id in in_fly:
times.append(infer_requests[infer_request_id].latency)
else:
in_fly.add(infer_request_id)
infer_requests[infer_request_id].async_infer()
iteration += 1
exec_time = (datetime.utcnow() - start_time).total_seconds()
if progress_bar:
if self.duration_seconds:
# calculate how many progress intervals are covered by current iteration.
# depends on the current iteration time and time of each progress interval.
# Previously covered progress intervals must be skipped.
progress_interval_time = self.duration_seconds / progress_bar.total_num
new_progress = int(exec_time / progress_interval_time -
progress_count)
progress_bar.add_progress(new_progress)
progress_count += new_progress
elif self.niter:
progress_bar.add_progress(1)
# wait the latest inference executions
status = exe_network.wait()
if status != StatusCode.OK:
raise Exception(
"Wait for all requests is failed with status code {}!".format(
status))
total_duration_sec = (datetime.utcnow() - start_time).total_seconds()
for infer_request_id in in_fly:
times.append(infer_requests[infer_request_id].latency)
times.sort()
latency_ms = median(times)
fps = batch_size * 1000 / latency_ms if self.api_type == 'sync' else batch_size * iteration / total_duration_sec
if progress_bar:
progress_bar.finish()
return fps, latency_ms, total_duration_sec, iteration
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
args = build_argparser().parse_args()
if args.colors:
COLOR_RED = "\033[91m"
COLOR_RESET = "\033[0m"
else:
COLOR_RED = ""
COLOR_RESET = ""
# load vocabulary file for model
log.info("Loading vocab file:\t{}".format(args.vocab))
vocab = load_vocab_file(args.vocab)
log.info("{} tokens loaded".format(len(vocab)))
# get context as a string (as we might need it's length for the sequence reshape)
paragraphs = get_paragraphs(args.input)
context = '\n'.join(paragraphs)
log.info("Size: {} chars".format(len(context)))
log.info("Context: " + COLOR_RED + context + COLOR_RESET)
# encode context into token ids list
c_tokens_id, c_tokens_se = text_to_tokens(context.lower(), vocab)
log.info("Initializing Inference Engine")
ie = IECore()
version = ie.get_versions(args.device)[args.device]
version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number)
log.info("Plugin version is {}".format(version_str))
# read IR
model_xml = args.model
model_bin = model_xml.with_suffix(".bin")
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
ie_encoder = ie.read_network(model=model_xml, weights=model_bin)
if args.reshape:
# reshape the sequence length to the context + maximum question length (in tokens)
first_input_layer = next(iter(ie_encoder.input_info))
c = ie_encoder.input_info[first_input_layer].input_data.shape[1]
# find the closest multiple of 64, if it is smaller than current network's sequence length, let' use that
seq = min(c, int(np.ceil((len(c_tokens_id) + args.max_question_token_num) / 64) * 64))
if seq < c:
new_shapes = {}
for input_name, input_info in ie_encoder.input_info.items():
n, c = input_info.input_data.shape
new_shapes[input_name] = [n, seq]
log.info("Reshaped input {} from {} to the {}".format(
input_name, input_info.input_data.shape, new_shapes[input_name]))
log.info("Attempting to reshape the network to the modified inputs...")
try:
ie_encoder.reshape(new_shapes)
log.info("Successful!")
except RuntimeError:
log.error("Failed to reshape the network, please retry the demo without '-r' option")
sys.exit(-1)
else:
log.info("Skipping network reshaping,"
" as (context length + max question length) exceeds the current (input) network sequence length")
# check input and output names
input_names = [i.strip() for i in args.input_names.split(',')]
output_names = [o.strip() for o in args.output_names.split(',')]
if ie_encoder.input_info.keys() != set(input_names) or ie_encoder.outputs.keys() != set(output_names):
log.error("Input or Output names do not match")
log.error(" The demo expects input->output names: {}->{}. "
"Please use the --input_names and --output_names to specify the right names "
"(see actual values below)".format(input_names, output_names))
log.error(" Actual network input->output names: {}->{}".format(list(ie_encoder.input_info.keys()),
list(ie_encoder.outputs.keys())))
raise Exception("Unexpected network input or output names")
# load model to the device
log.info("Loading model to the {}".format(args.device))
ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device)
if args.questions:
def questions():
for question in args.questions:
log.info("Question: {}".format(question))
yield question
else:
def questions():
while True:
yield input('Type question (empty string to exit):')
# loop on user's or prepared questions
for question in questions():
if not question.strip():
break
q_tokens_id, _ = text_to_tokens(question.lower(), vocab)
# maximum number of tokens that can be processed by network at once
max_length = ie_encoder.input_info[input_names[0]].input_data.shape[1]
# calculate number of tokens for context in each inference request.
# reserve 3 positions for special tokens
# [CLS] q_tokens [SEP] c_tokens [SEP]
c_wnd_len = max_length - (len(q_tokens_id) + 3)
# token num between two neighbour context windows
# 1/2 means that context windows are overlapped by half
c_stride = c_wnd_len // 2
t0 = time.perf_counter()
t_count = 0
# array of answers from each window
answers = []
# init a window to iterate over context
c_s, c_e = 0, min(c_wnd_len, len(c_tokens_id))
# iterate while context window is not empty
while c_e > c_s:
# form the request
tok_cls = vocab['[CLS]']
tok_sep = vocab['[SEP]']
input_ids = [tok_cls] + q_tokens_id + [tok_sep] + c_tokens_id[c_s:c_e] + [tok_sep]
token_type_ids = [0] + [0] * len(q_tokens_id) + [0] + [1] * (c_e - c_s) + [0]
attention_mask = [1] * len(input_ids)
# pad the rest of the request
pad_len = max_length - len(input_ids)
input_ids += [0] * pad_len
token_type_ids += [0] * pad_len
attention_mask += [0] * pad_len
# create numpy inputs for IE
inputs = {
input_names[0]: np.array([input_ids], dtype=np.int32),
input_names[1]: np.array([attention_mask], dtype=np.int32),
input_names[2]: np.array([token_type_ids], dtype=np.int32),
}
if len(input_names)>3:
inputs[input_names[3]] = np.arange(len(input_ids), dtype=np.int32)[None, :]
t_start = time.perf_counter()
# infer by IE
res = ie_encoder_exec.infer(inputs=inputs)
t_end = time.perf_counter()
t_count += 1
log.info("Sequence of length {} is processed with {:0.2f} requests/sec ({:0.2} sec per request)".format(
max_length,
1 / (t_end - t_start),
t_end - t_start
))
# get start-end scores for context
def get_score(name):
out = np.exp(res[name].reshape((max_length,)))
return out / out.sum(axis=-1)
score_s = get_score(output_names[0])
score_e = get_score(output_names[1])
# get 'no-answer' score (not valid if model has been fine-tuned on squad1.x)
if args.model_squad_ver.split('.')[0] == '1':
score_na = 0
else:
score_na = score_s[0] * score_e[0]
# find product of all start-end combinations to find the best one
c_s_idx = len(q_tokens_id) + 2 # index of first context token in tensor
c_e_idx = max_length - (1 + pad_len) # index of last+1 context token in tensor
score_mat = np.matmul(
score_s[c_s_idx:c_e_idx].reshape((c_e - c_s, 1)),
score_e[c_s_idx:c_e_idx].reshape((1, c_e - c_s))
)
# reset candidates with end before start
score_mat = np.triu(score_mat)
# reset long candidates (>max_answer_token_num)
score_mat = np.tril(score_mat, args.max_answer_token_num - 1)
# find the best start-end pair
max_s, max_e = divmod(score_mat.flatten().argmax(), score_mat.shape[1])
max_score = score_mat[max_s, max_e] * (1 - score_na)
# convert to context text start-end index
max_s = c_tokens_se[c_s + max_s][0]
max_e = c_tokens_se[c_s + max_e][1]
# check that answers list does not have duplicates (because of context windows overlapping)
same = [i for i, a in enumerate(answers) if a[1] == max_s and a[2] == max_e]
if same:
assert len(same) == 1
# update existing answer record
a = answers[same[0]]
answers[same[0]] = (max(max_score, a[0]), max_s, max_e)
else:
# add new record
answers.append((max_score, max_s, max_e))
# check that context window reached the end
if c_e == len(c_tokens_id):
break
# move to next window position
c_s = min(c_s + c_stride, len(c_tokens_id))
c_e = min(c_s + c_wnd_len, len(c_tokens_id))
t1 = time.perf_counter()
log.info("The performance below is reported only for reference purposes, "
"please use the benchmark_app tool (part of the OpenVINO samples) for any actual measurements.")
log.info("{} requests of {} length were processed in {:0.2f}sec ({:0.2}sec per request)".format(
t_count,
max_length,
t1 - t0,
(t1 - t0) / t_count
))
# print top 3 results
answers = sorted(answers, key=lambda x: -x[0])
for score, s, e in answers[:3]:
log.info("---answer: {:0.2f} {}".format(score, context[s:e]))
c_s, c_e = find_sentence_range(context, s, e)
log.info(" " + context[c_s:s] + COLOR_RED + context[s:e] + COLOR_RESET + context[e:c_e])
class Model:
'''
Class for the Face Detection Model.
'''
def __init__(self, model_name, device_name, extensions=None):
'''
TODO: Use this to set your instance variables. model_name, device='CPU', extensions=None
'''
self.ie = IECore()
self.extension = extensions
self.model_structure = model_name + '.xml'
self.model_weights = model_name + '.bin'
_, self.model_name = self.path_leaf(model_name)
self.device = device_name
self.net = None
self.layers_map = None
self.exec_net = None
try:
t_start = time.perf_counter()
self.net = self.ie.read_network(model=self.model_structure,
weights=self.model_weights)
t_end = time.perf_counter()
log.info(
"model {} is processed with loading {:0.2} sec per request)".
format(self.model_name, t_end - t_start))
except Exception as e:
raise ValueError(
"Could not Initialise the network. Have you enterred the correct model path?",
e)
self.layers_map = self.ie.query_network(network=self.net,
device_name=device_name)
self.input_blob = list(self.net.inputs.keys())
self.output_blob = list(self.net.outputs.keys())
self.input_shape = None
self.output_shape = None
def load_model(self):
return self.net
def get_input_blob(self):
return self.input_blob
def get_output_blob(self):
return self.output_blob
def get_input_shape(self):
model_inputs_name = np.array(self.input_blob)
array_data_inputs = []
for i in range(len(model_inputs_name)):
array_data_inputs.append(
self.net.inputs[model_inputs_name[i]].shape)
return np.array(array_data_inputs)
def get_output_shape(self):
model_outputs_name = np.array(self.output_blob)
array_data_outputs = []
for i in range(len(model_outputs_name)):
array_data_outputs.append(
self.net.outputs[model_outputs_name[i]].shape)
return np.array(array_data_outputs)
def load_network(self):
return self.ie.load_network(self.net, self.device, num_requests=0)
def get_inference(self, input_data):
self.exec_net = self.load_network()
return self.exec_net.infer(inputs=input_data)
def get_infer_output(self, input_data):
self.get_inference(input_data)
return self.exec_net.requests[0].outputs
def get_supported_layer(self):
for pair in self.layers_map.items():
print("supported_layers: ", pair)
def get_unsupported_layer(self):
not_supported_layers = [
l for l in self.net.layers.keys() if l not in self.layers_map
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(self.device, ', '.join(not_supported_layers)))
else:
log.info(f"All layers are supported for {self.model_name} model!")
def get_model_name(self):
return self.model_structure
def get_openvino_version(self, device_name):
return self.ie.get_versions(device_name)
def get_model_name(self):
return self.model_name
def path_leaf(self, path):
tail, head = ntpath.split(path)
return tail, ntpath.basename(head)
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout)
args = build_argparser().parse_args()
# load vocabulary file for model
log.info("Loading vocab file:\t{}".format(args.vocab))
vocab = load_vocab_file(args.vocab)
log.info("{} tokens loaded".format(len(vocab)))
# get context as a string (as we might need it's length for the sequence reshape)
paragraphs = get_paragraphs(args.input)
context = '\n'.join(paragraphs)
log.info("Size: {} chars".format(len(context)))
sentences = re.split(sentence_splitter, context)
preprocessed_sentences = [text_to_tokens(sentence, vocab) for sentence in sentences]
max_sent_length = max([len(tokens) + 2 for tokens, _ in preprocessed_sentences])
log.info("Initializing Inference Engine")
ie = IECore()
version = ie.get_versions(args.device)[args.device]
version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number)
log.info("Plugin version is {}".format(version_str))
# read IR
model_xml = args.model
model_bin = model_xml.with_suffix(".bin")
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
ie_encoder = ie.read_network(model=model_xml, weights=model_bin)
# check input and output names
input_names = [i.strip() for i in args.input_names.split(',')]
if ie_encoder.input_info.keys() != set(input_names):
log.error("Input names do not match")
log.error(" The demo expects input names: {}. "
"Please use the --input_names to specify the right names "
"(see actual values below)".format(input_names))
log.error(" Actual network input names: {}".format(list(ie_encoder.input_info.keys())))
raise Exception("Unexpected network input names")
if len(ie_encoder.outputs) != 1:
log.log.error('Demo expects model with single output, while provided {}'.format(len(ie_encoder.outputs)))
raise Exception('Unexpected number of outputs')
output_names = list(ie_encoder.outputs)
max_length = ie_encoder.input_info[input_names[0]].input_data.shape[1]
if max_sent_length > max_length:
input_shapes = {
input_names[0]: [1, max_sent_length],
input_names[1]: [1, max_sent_length],
input_names[2]: [1, max_sent_length]
}
ie_encoder.reshape(input_shapes)
max_length = max_sent_length
# load model to the device
log.info("Loading model to the {}".format(args.device))
ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device)
# maximum number of tokens that can be processed by network at once
t0 = time.perf_counter()
t_count = 0
def get_score(name):
out = np.exp(res[name][0])
return out / out.sum(axis=-1, keepdims=True)
for sentence, (c_tokens_id, c_token_s_e) in zip(sentences, preprocessed_sentences):
# form the request
tok_cls = vocab['[CLS]']
tok_sep = vocab['[SEP]']
input_ids = [tok_cls] + c_tokens_id + [tok_sep]
token_type_ids = [0] * len(input_ids)
attention_mask = [1] * len(input_ids)
# pad the rest of the request
pad_len = max_length - len(input_ids)
input_ids += [0] * pad_len
token_type_ids += [0] * pad_len
attention_mask += [0] * pad_len
# create numpy inputs for IE
inputs = {
input_names[0]: np.array([input_ids], dtype=np.int32),
input_names[1]: np.array([attention_mask], dtype=np.int32),
input_names[2]: np.array([token_type_ids], dtype=np.int32),
}
if len(input_names)>3:
inputs[input_names[3]] = np.arange(len(input_ids), dtype=np.int32)[None, :]
t_start = time.perf_counter()
# infer by IE
res = ie_encoder_exec.infer(inputs=inputs)
t_end = time.perf_counter()
t_count += 1
log.info("Sequence of length {} is processed with {:0.2f} requests/sec ({:0.2} sec per request)".format(
max_length,
1 / (t_end - t_start),
t_end - t_start
))
score = get_score(output_names[0])
labels_idx = score.argmax(-1)
filtered_labels_idx = [
(idx, label_idx)
for idx, label_idx in enumerate(labels_idx)
if label_idx != 0 and 0 < idx < max_length - pad_len
]
if not filtered_labels_idx:
continue
log.info('Sentence: \n\t{}'.format(sentence))
visualized = set()
for idx, label_idx in filtered_labels_idx:
word_s, word_e = c_token_s_e[idx - 1]
if (word_s, word_e) in visualized:
continue
visualized.add((word_s, word_e))
word = sentence[word_s:word_e]
log.info('\n\tWord: {}\n\tConfidence: {}\n\tTag: {}'.format(word, score[idx][label_idx], label_to_tag[label_idx]))
t1 = time.perf_counter()
log.info("The performance below is reported only for reference purposes, "
"please use the benchmark_app tool (part of the OpenVINO samples) for any actual measurements.")
log.info("{} requests of {} length were processed in {:0.2f}sec ({:0.2}sec per request)".format(
t_count,
max_length,
t1 - t0,
(t1 - t0) / t_count
))
def main():
total_timer = Timer(name='total')
total_timer.tic()
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
# --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
# -----------------------------------------------------------------------------------------------------
# ------------- 2. Load Plugin for inference engine and extensions library if specified --------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("Device info:")
versions = ie.get_versions(args.device)
print("{}{}".format(" " * 8, args.device))
print("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[args.device].major, versions[args.device].minor))
print("{}Build ........... {}".format(" " * 8,
versions[args.device].build_number))
if args.cpu_extension and "CPU" in args.device:
ie.add_extension(args.cpu_extension, "CPU")
log.info("CPU extension loaded: {}".format(args.cpu_extension))
if "CPU" in args.device:
supported_layers = ie.query_network(net, "CPU")
not_supported_layers = [
l for l in net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
# -----------------------------------------------------------------------------------------------------
# --------------------------- 4. Configure input & output ---------------------------------------------
# --------------------------- Prepare input blobs -----------------------------------------------------
log.info("Preparing input blobs")
assert (len(net.inputs.keys()) == 1
), "Sample supports topologies only with 1 input"
input_name = next(iter(net.inputs.keys()))
input_info = net.inputs[input_name]
input_info.precision = 'FP32'
# --------------------------- Prepare output blobs ----------------------------------------------------
log.info('Preparing output blobs')
assert (len(net.outputs.keys()) == 2
), "Sample supports topologies only with 2 output"
loc_out_name = "797"
class_out_name = "741"
assert (loc_out_name in net.outputs.keys()) and (class_out_name
in net.outputs.keys())
loc_out_info = net.outputs[loc_out_name]
class_out_info = net.outputs[class_out_name]
loc_out_info.precision = "FP32"
class_out_info.precision = "FP32"
# -----------------------------------------------------------------------------------------------------
# -----------------------------------------------------------------------------------------------------
log.info("Loading model to the device")
exec_net = ie.load_network(network=net, device_name=args.device)
# --------------------------- 3. Read and preprocess input --------------------------------------------
# -----------------------------------------------------------------------------------------------------
if not os.path.exists(args.result_dir):
os.makedirs(args.result_dir)
if args.voc_res_file and os.path.exists(args.voc_res_file):
os.remove(args.voc_res_file)
create_anchor_timer = Timer(name='create_anchor')
read_img_timer = Timer(name='read_img')
preprocess_timer = Timer(name='preprocess')
infer_timer = Timer(name='infer')
adapter_timer = Timer(name='adapter')
patch_img_nms_timer = Timer(name='patch_img_nms')
whole_img_nms_timer = Timer(name='whole_img_nms')
add_offset_timer = Timer(name='add_offset')
write_result_timer = Timer(name='write_result')
create_anchor_timer.tic()
adapter = RetinaNetAdapter(input_shape=args.patch_size)
create_anchor_timer.toc()
image_names = os.listdir(args.image_dir)
log.info("image_nums: {}".format(len(image_names)))
for image_id, image_name in enumerate(image_names):
read_img_timer.tic()
image_path = os.path.join(args.image_dir, image_name)
img = cv2.imread(image_path).astype('float32')
read_img_timer.toc()
height, width, _ = img.shape
image_shape = (width, height)
strides = args.strides
patch_size = args.patch_size
x_num, y_num = calc_split_num(image_shape, patch_size, strides)
log.info("id:{}, name: {}, shape: ({},{}), x_num:{}, y_num:{}".format(
image_id, image_name, height, width, x_num, y_num))
preprocess_timer.tic()
img = img.transpose((2, 0, 1)) # Change data layout from HWC to CHW
preprocess_timer.toc()
result_all = []
for i in range(x_num):
for j in range(y_num):
x = strides[0] * i if i < x_num - 1 else image_shape[
0] - args.patch_size[0]
y = strides[1] * j if j < y_num - 1 else image_shape[
1] - args.patch_size[1]
# print('processing {} , x: {}, y: {}'.format(image_name, x, y))
preprocess_timer.tic()
crop_img = img[:, y:y + patch_size[1],
x:x + patch_size[0]].copy()
crop_img = crop_img[np.newaxis, :, :, :]
preprocess_timer.toc()
# --------------------------- Performing inference ----------------------------------------------------
infer_timer.tic()
res = exec_net.infer(inputs={input_name: crop_img})
loc_out = res[loc_out_name][0]
class_out = res[class_out_name][0]
infer_timer.toc()
adapter_timer.tic()
result = adapter.process(loc_out, class_out)
adapter_timer.toc()
patch_img_nms_timer.tic()
result, _ = nms(result, thresh=0.5, keep_top_k=100)
patch_img_nms_timer.toc()
# import pdb;pdb.set_trace()
add_offset_timer.tic()
result[:, 0] += x
result[:, 1] += y
result[:, 2] += x
result[:, 3] += y
result_all.append(result)
add_offset_timer.toc()
# import pdb;pdb.set_trace()
whole_img_nms_timer.tic()
result_all = np.concatenate(result_all, axis=0)
nms_result, _ = nms(result_all, thresh=0.5)
whole_img_nms_timer.toc()
write_result_timer.tic()
voc_format = '{} {:.4f} {} {} {} {}'
pos_all = []
voc_all = []
for i in range(nms_result.shape[0]):
x = int(nms_result[i, 0])
y = int(nms_result[i, 1])
w = max(int(nms_result[i, 2] - nms_result[i, 0]), 1)
h = max(int(nms_result[i, 3] - nms_result[i, 1]), 1)
p = float(nms_result[i, 4])
pos = {'x': x, 'y': y, 'w': w, 'h': h, 'p': p}
pos_all.append(pos)
if args.voc_res_file:
xmin = x
ymin = y
xmax = int(nms_result[i, 2])
ymax = int(nms_result[i, 3])
voc_str = voc_format.format(
os.path.splitext(image_name)[0], p, xmin, ymin, xmax, ymax)
voc_all.append(voc_str)
file_name = os.path.splitext(image_name)[0] + '.json'
with open(os.path.join(args.result_dir, file_name), 'w') as f:
json.dump(pos_all, f)
if args.voc_res_file:
with open(args.voc_res_file, 'a') as f:
for voc_str in voc_all:
f.write(voc_str + '\n')
write_result_timer.toc()
total_timer.toc()
# -----------------------------------------------------------------------------------------------------
all_timers = []
all_timers.extend([
create_anchor_timer, read_img_timer, preprocess_timer, infer_timer,
adapter_timer, patch_img_nms_timer, whole_img_nms_timer,
add_offset_timer, write_result_timer, total_timer
])
for timer in all_timers:
log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format(
timer.name, timer.avg * 1000, timer.total))
log.info("Execution successful\n")
class Network:
"""
Load and configure inference plugins for the specified target devices
and performs synchronous and asynchronous modes for the specified infer requests.
"""
def __init__(self):
### Initialize any class variables desired ###
self.plugin = None
self.network = None
self.input_blob = None
self.input_blob_name = None
self.output_blob = None
self.exec_network = None
self.infer_request = None
self.output_name = None
self.output_info = None
self.feed_dict = None
def load_model(self, model, device="CPU", cpu_extension=None):
### Load the model ###
model_xml = model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
### init inference engine plugin
self.plugin = IECore()
### Add any necessary extensions ###
log.info("attempting to add CPU extension!")
log.info(cpu_extension)
if cpu_extension and "CPU" in device:
plugin_dir = '/opt/intel/openvino/deployment_tools/inference_engine/lib/intel64'
self.plugin.add_extension(cpu_extension, device)
#plugin = IEPlugin(device=‘CPU', plugin_dirs=plugin_dir)
#cpu_extension_path = '/home/temp/intel/openvino/deployment_tools/inference_engine/lib/intel64/libMKLDNNPlugin.so'
log.info("Loading network files:\n\t{}\n\t{}".format(
model_xml, model_bin))
self.network = IENetwork(model=model_xml, weights=model_bin)
### Check for supported layers ###
if "CPU" in device:
supported_layers = self.plugin.query_network(self.network, "CPU")
not_supported_layers = [
l for l in self.network.layers.keys()
if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
"Following layers are not supported by the plugin for specified device {}:\n {}"
.format(args.device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
versions = self.plugin.get_versions(device)
log.info("{}{}".format(" " * 8, device))
log.info("{}MKLDNNPlugin version ......... {}.{}".format(
" " * 8, versions[device].major, versions[device].minor))
log.info("{}Build ........... {}".format(
" " * 8, versions[device].build_number))
### Return the loaded inference plugin ###
self.network = IENetwork(model=model_xml, weights=model_bin)
self.exec_network = self.plugin.load_network(self.network, device)
## grab the input layer ##
self.input_blob = self.network.inputs
##next(iter(self.network.inputs))
img_info_input_blob = None
self.feed_dict = {}
for blob_name in self.network.inputs:
log.info("blob_name: " + blob_name)
if len(self.network.inputs[blob_name].shape) == 4:
self.input_blob_name = blob_name
elif len(self.network.inputs[blob_name].shape) == 2:
img_info_input_blob = blob_name
else:
raise RuntimeError(
"Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported"
.format(len(net.inputs[blob_name].shape), blob_name))
self.output_blob = next(iter(self.network.outputs))
if img_info_input_blob:
log.info("set input info")
n, c, h, w = self.get_input_shape()
self.feed_dict[img_info_input_blob] = [h, w, 1]
self.output_name, self.output_info = "", self.network.outputs[next(
iter(self.network.outputs.keys()))]
for output_key in self.network.outputs:
if self.network.layers[output_key].type == "DetectionOutput":
self.output_name, self.output_info = output_key, self.network.outputs[
output_key]
if self.output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
log.info("load model done")
### Note: You may need to update the function parameters. ###
log.info('Preparing output blobs')
output_name, output_info = "", self.network.outputs[next(
iter(self.network.outputs.keys()))]
for output_key in self.network.outputs:
log.info("iteration: %s", output_key)
if self.network.layers[output_key].type == "DetectionOutput":
output_name, output_info = output_key, self.network.outputs[
output_key]
if output_name == "":
log.error("Can't find a DetectionOutput layer in the topology")
output_dims = output_info.shape
if len(output_dims) != 4:
log.error("Incorrect output dimensions for SSD model")
max_proposal_count, object_size = output_dims[2], output_dims[3]
log.info("Max Proposal Count is: %s", max_proposal_count)
log.info("Output Object Size: %s, %s", output_dims[2], output_dims[3])
if object_size != 7:
log.error("Output item should have 7 as a last dimension")
output_info.precision = "FP32"
return
def get_input_blob(self):
##return self.network.inputs[self.input_blob]
##input_blob = None
##return input_blob
##self.network.inputs[input_blob].shape
return self.input_blob
def get_input_blob_name(self):
##return self.network.inputs[self.input_blob]
##input_blob = None
##return input_blob
##self.network.inputs[input_blob].shape
return self.input_blob_name
def get_input_shape(self):
### Return the shape of the input layer ###
return self.network.inputs[self.input_blob_name].shape
def get_output_name(self):
return self.output_name
def get_output_info(self):
return self.output_info
def exec_net(self, image):
### Start an asynchronous request ###
### Return any necessary information ###
##log.info("Start Async Inference")
self.input_blob[self.input_blob_name] = image
self.feed_dict[self.input_blob_name] = image
self.exec_network.start_async(request_id=0, inputs=self.feed_dict)
return
def wait(self):
### Wait for the request to be complete. ###
### Return any necessary information ###
### Note: You may need to update the function parameters. ###
status = self.exec_network.requests[0].wait(-1)
return status
def get_output(self):
### Extract and return the output results
### Note: You may need to update the function parameters. ###
res = self.exec_network.requests[0].outputs[self.output_blob]
return res
