def _test(self, framework_model, ref_net, ie_device, precision, ir_version, temp_dir, infer_timeout=60,
enabled_transforms='', disabled_transforms='', **kwargs):
"""
:param enabled_transforms/disabled_transforms: string with idxs of transforms that should be enabled/disabled.
Example: "transform_1,transform_2"
"""
model_path = self.produce_model_path(framework_model=framework_model, save_path=temp_dir)
# TODO Pass environment variables via subprocess environment
os.environ['MO_ENABLED_TRANSFORMS'] = enabled_transforms
os.environ['MO_DISABLED_TRANSFORMS'] = disabled_transforms
mo_params = {self.input_model_key: model_path,
"output_dir": temp_dir,
"data_type": precision, "model_name": 'model'
}
if 'input_shapes' in kwargs and len(kwargs['input_shapes']):
input_shapes_str = []
for ishape in kwargs['input_shapes']:
input_shapes_str.append('[' + ','.join([str(i) for i in ishape]) + ']')
mo_params.update(dict(input_shape=','.join(input_shapes_str)))
if 'input_names' in kwargs and len(kwargs['input_names']):
mo_params.update(dict(input=','.join(kwargs['input_names'])))
exit_code, stderr = generate_ir(**mo_params)
del os.environ['MO_ENABLED_TRANSFORMS']
del os.environ['MO_DISABLED_TRANSFORMS']
assert not exit_code, ("IR generation failed with {} exit code: {}".format(exit_code, stderr))
path_to_xml = Path(temp_dir, 'model.xml')
path_to_bin = Path(temp_dir, 'model.bin')
ir = IREngine(path_to_xml, path_to_bin, precision=precision)
if ref_net is not None:
(flag, resp) = ir.compare(ref_net)
assert flag, '\n'.join(resp)
# Prepare feed dict
if 'kwargs_to_prepare_input' in kwargs and kwargs['kwargs_to_prepare_input']:
inputs_dict = self._prepare_input(ir.get_inputs(), kwargs['kwargs_to_prepare_input'])
else:
inputs_dict = self._prepare_input(ir.get_inputs())
# IE infer:
ie_engine = IEInfer(model=path_to_xml,
weights=path_to_bin,
device=ie_device)
infer_res = ie_engine.infer(input_data=inputs_dict, infer_timeout=infer_timeout)
if hasattr(self, 'skip_framework') and self.skip_framework:
warnings.warn('Framework is skipped')
return
# Framework infer:
fw_res = self.get_framework_results(inputs_dict=inputs_dict, model_path=model_path)
if len(fw_res) == len(infer_res) == 1:
# match output layers directly
mapping_dict = {next(iter(fw_res)): next(iter(infer_res))}
else:
# Load mapping file
mapping_dict = mapping_parser(path_to_xml.with_suffix('.mapping'))
if 'custom_eps' in kwargs and kwargs['custom_eps'] is not None:
custom_eps = kwargs['custom_eps']
else:
custom_eps = 1e-4
# Compare Ie results with Framework results
fw_eps = custom_eps if precision == 'FP32' else 5e-2
assert self.compare_ie_results_with_framework(infer_res=infer_res, framework_res=fw_res,
mapping_dict=mapping_dict, framework_eps=fw_eps), \
"Comparing with Framework failed: ie_res={}; framework_res={}.".format(infer_res, fw_res)
def _test(self,
framework_model,
ref_net,
ie_device,
precision,
ir_version,
temp_dir,
api_2,
use_new_frontend=False,
infer_timeout=60,
enabled_transforms='',
disabled_transforms='',
**kwargs):
"""
:param enabled_transforms/disabled_transforms: string with idxs of transforms that should be enabled/disabled.
Example: "transform_1,transform_2"
"""
model_path = self.produce_model_path(framework_model=framework_model,
save_path=temp_dir)
self.use_new_frontend = use_new_frontend
self.api_2 = api_2
# TODO Pass environment variables via subprocess environment
os.environ['MO_ENABLED_TRANSFORMS'] = enabled_transforms
os.environ['MO_DISABLED_TRANSFORMS'] = disabled_transforms
mo_params = {
self.input_model_key: model_path,
"output_dir": temp_dir,
"data_type": precision,
"model_name": 'model'
}
if 'input_shapes' in kwargs and len(kwargs['input_shapes']):
input_shapes_str = []
for ishape in kwargs['input_shapes']:
input_shapes_str.append('[' +
','.join([str(i)
for i in ishape]) + ']')
mo_params.update(dict(input_shape=','.join(input_shapes_str)))
if 'input_names' in kwargs and len(kwargs['input_names']):
mo_params.update(dict(input=','.join(kwargs['input_names'])))
if use_new_frontend:
mo_params["use_new_frontend"] = True
exit_code, stderr = generate_ir(**mo_params)
del os.environ['MO_ENABLED_TRANSFORMS']
del os.environ['MO_DISABLED_TRANSFORMS']
assert not exit_code, (
"IR generation failed with {} exit code: {}".format(
exit_code, stderr))
path_to_xml = Path(temp_dir, 'model.xml')
path_to_bin = Path(temp_dir, 'model.bin')
# TODO: need to update ref graphs or get rid of this comparison
# if ref_net is not None:
# ir = IREngine(path_to_xml, path_to_bin, precision=precision)
# (flag, resp) = ir.compare(ref_net)
# assert flag, '\n'.join(resp)
if api_2:
ie_engine = InferAPI20(model=path_to_xml,
weights=path_to_bin,
device=ie_device)
else:
ie_engine = IEInfer(model=path_to_xml,
weights=path_to_bin,
device=ie_device)
# Prepare feed dict
if 'kwargs_to_prepare_input' in kwargs and kwargs[
'kwargs_to_prepare_input']:
inputs_dict = self._prepare_input(
ie_engine.get_inputs_info(precision),
kwargs['kwargs_to_prepare_input'])
else:
inputs_dict = self._prepare_input(
ie_engine.get_inputs_info(precision))
# IE infer:
infer_res = ie_engine.infer(input_data=inputs_dict,
infer_timeout=infer_timeout)
if hasattr(self, 'skip_framework') and self.skip_framework:
warnings.warn('Framework is skipped')
return
# Framework infer:
fw_res = self.get_framework_results(inputs_dict=inputs_dict,
model_path=model_path)
if len(fw_res) == len(infer_res) == 1:
# match output layers directly
mapping_dict = {next(iter(fw_res)): next(iter(infer_res))}
else:
# Load mapping file
mapping_dict = mapping_parser(path_to_xml.with_suffix('.mapping'))
if 'custom_eps' in kwargs and kwargs['custom_eps'] is not None:
custom_eps = kwargs['custom_eps']
else:
custom_eps = 1e-4
# Compare Ie results with Framework results
fw_eps = custom_eps if precision == 'FP32' else 5e-2
assert self.compare_ie_results_with_framework(infer_res=infer_res, framework_res=fw_res,
mapping_dict=mapping_dict,
framework_eps=fw_eps), \
"Comparing with Framework failed: ie_res={}; framework_res={}.".format(infer_res,
fw_res)
if len(inputs_dict.keys()) > 1 or len(infer_res.keys()) > 1:
tree = ET.parse(path_to_xml)
# findall returns elements in document order, this order should be the same as
# order of inputs/outputs in original model
inputs_ie = [
child for child in tree.findall('.//layer[@type="Parameter"]')
]
outputs_ie = [
child for child in tree.findall('.//layer[@type="Result"]')
]
if 'input_names' in kwargs:
input_names = kwargs['input_names']
for i, input_name in enumerate(input_names):
assert inputs_ie[i].attrib['name'] == input_name, \
'Input order does not match framework order. Input with index {} is {}, ' \
'but expected {}'.format(i, inputs_ie[i].attrib['name'], input_name)
if 'output_names' in kwargs:
output_names = kwargs['output_names']
for i, output_name in enumerate(output_names):
output_name_ie = outputs_ie[i].attrib['name']
output_without_sink_port = re.sub(r'\/sink_port_.', '',
output_name_ie)
assert output_without_sink_port == output_name, \
'Output order does not match framework order. Output with index {} is {}, ' \
'but expected {}'.format(i, output_without_sink_port, output_name)