def _test(self, framework_model, ref_net, ie_device, precision, ir_version, temp_dir, infer_timeout=60, enabled_transforms='', disabled_transforms='', **kwargs): """ :param enabled_transforms/disabled_transforms: string with idxs of transforms that should be enabled/disabled. Example: "transform_1,transform_2" """ model_path = self.produce_model_path(framework_model=framework_model, save_path=temp_dir) # TODO Pass environment variables via subprocess environment os.environ['MO_ENABLED_TRANSFORMS'] = enabled_transforms os.environ['MO_DISABLED_TRANSFORMS'] = disabled_transforms mo_params = {self.input_model_key: model_path, "output_dir": temp_dir, "data_type": precision, "model_name": 'model' } if 'input_shapes' in kwargs and len(kwargs['input_shapes']): input_shapes_str = [] for ishape in kwargs['input_shapes']: input_shapes_str.append('[' + ','.join([str(i) for i in ishape]) + ']') mo_params.update(dict(input_shape=','.join(input_shapes_str))) if 'input_names' in kwargs and len(kwargs['input_names']): mo_params.update(dict(input=','.join(kwargs['input_names']))) exit_code, stderr = generate_ir(**mo_params) del os.environ['MO_ENABLED_TRANSFORMS'] del os.environ['MO_DISABLED_TRANSFORMS'] assert not exit_code, ("IR generation failed with {} exit code: {}".format(exit_code, stderr)) path_to_xml = Path(temp_dir, 'model.xml') path_to_bin = Path(temp_dir, 'model.bin') ir = IREngine(path_to_xml, path_to_bin, precision=precision) if ref_net is not None: (flag, resp) = ir.compare(ref_net) assert flag, '\n'.join(resp) # Prepare feed dict if 'kwargs_to_prepare_input' in kwargs and kwargs['kwargs_to_prepare_input']: inputs_dict = self._prepare_input(ir.get_inputs(), kwargs['kwargs_to_prepare_input']) else: inputs_dict = self._prepare_input(ir.get_inputs()) # IE infer: ie_engine = IEInfer(model=path_to_xml, weights=path_to_bin, device=ie_device) infer_res = ie_engine.infer(input_data=inputs_dict, infer_timeout=infer_timeout) if hasattr(self, 'skip_framework') and self.skip_framework: warnings.warn('Framework is skipped') return # Framework infer: fw_res = self.get_framework_results(inputs_dict=inputs_dict, model_path=model_path) if len(fw_res) == len(infer_res) == 1: # match output layers directly mapping_dict = {next(iter(fw_res)): next(iter(infer_res))} else: # Load mapping file mapping_dict = mapping_parser(path_to_xml.with_suffix('.mapping')) if 'custom_eps' in kwargs and kwargs['custom_eps'] is not None: custom_eps = kwargs['custom_eps'] else: custom_eps = 1e-4 # Compare Ie results with Framework results fw_eps = custom_eps if precision == 'FP32' else 5e-2 assert self.compare_ie_results_with_framework(infer_res=infer_res, framework_res=fw_res, mapping_dict=mapping_dict, framework_eps=fw_eps), \ "Comparing with Framework failed: ie_res={}; framework_res={}.".format(infer_res, fw_res)
def _test(self, framework_model, ref_net, ie_device, precision, ir_version, temp_dir, api_2, use_new_frontend=False, infer_timeout=60, enabled_transforms='', disabled_transforms='', **kwargs): """ :param enabled_transforms/disabled_transforms: string with idxs of transforms that should be enabled/disabled. Example: "transform_1,transform_2" """ model_path = self.produce_model_path(framework_model=framework_model, save_path=temp_dir) self.use_new_frontend = use_new_frontend self.api_2 = api_2 # TODO Pass environment variables via subprocess environment os.environ['MO_ENABLED_TRANSFORMS'] = enabled_transforms os.environ['MO_DISABLED_TRANSFORMS'] = disabled_transforms mo_params = { self.input_model_key: model_path, "output_dir": temp_dir, "data_type": precision, "model_name": 'model' } if 'input_shapes' in kwargs and len(kwargs['input_shapes']): input_shapes_str = [] for ishape in kwargs['input_shapes']: input_shapes_str.append('[' + ','.join([str(i) for i in ishape]) + ']') mo_params.update(dict(input_shape=','.join(input_shapes_str))) if 'input_names' in kwargs and len(kwargs['input_names']): mo_params.update(dict(input=','.join(kwargs['input_names']))) if use_new_frontend: mo_params["use_new_frontend"] = True exit_code, stderr = generate_ir(**mo_params) del os.environ['MO_ENABLED_TRANSFORMS'] del os.environ['MO_DISABLED_TRANSFORMS'] assert not exit_code, ( "IR generation failed with {} exit code: {}".format( exit_code, stderr)) path_to_xml = Path(temp_dir, 'model.xml') path_to_bin = Path(temp_dir, 'model.bin') # TODO: need to update ref graphs or get rid of this comparison # if ref_net is not None: # ir = IREngine(path_to_xml, path_to_bin, precision=precision) # (flag, resp) = ir.compare(ref_net) # assert flag, '\n'.join(resp) if api_2: ie_engine = InferAPI20(model=path_to_xml, weights=path_to_bin, device=ie_device) else: ie_engine = IEInfer(model=path_to_xml, weights=path_to_bin, device=ie_device) # Prepare feed dict if 'kwargs_to_prepare_input' in kwargs and kwargs[ 'kwargs_to_prepare_input']: inputs_dict = self._prepare_input( ie_engine.get_inputs_info(precision), kwargs['kwargs_to_prepare_input']) else: inputs_dict = self._prepare_input( ie_engine.get_inputs_info(precision)) # IE infer: infer_res = ie_engine.infer(input_data=inputs_dict, infer_timeout=infer_timeout) if hasattr(self, 'skip_framework') and self.skip_framework: warnings.warn('Framework is skipped') return # Framework infer: fw_res = self.get_framework_results(inputs_dict=inputs_dict, model_path=model_path) if len(fw_res) == len(infer_res) == 1: # match output layers directly mapping_dict = {next(iter(fw_res)): next(iter(infer_res))} else: # Load mapping file mapping_dict = mapping_parser(path_to_xml.with_suffix('.mapping')) if 'custom_eps' in kwargs and kwargs['custom_eps'] is not None: custom_eps = kwargs['custom_eps'] else: custom_eps = 1e-4 # Compare Ie results with Framework results fw_eps = custom_eps if precision == 'FP32' else 5e-2 assert self.compare_ie_results_with_framework(infer_res=infer_res, framework_res=fw_res, mapping_dict=mapping_dict, framework_eps=fw_eps), \ "Comparing with Framework failed: ie_res={}; framework_res={}.".format(infer_res, fw_res) if len(inputs_dict.keys()) > 1 or len(infer_res.keys()) > 1: tree = ET.parse(path_to_xml) # findall returns elements in document order, this order should be the same as # order of inputs/outputs in original model inputs_ie = [ child for child in tree.findall('.//layer[@type="Parameter"]') ] outputs_ie = [ child for child in tree.findall('.//layer[@type="Result"]') ] if 'input_names' in kwargs: input_names = kwargs['input_names'] for i, input_name in enumerate(input_names): assert inputs_ie[i].attrib['name'] == input_name, \ 'Input order does not match framework order. Input with index {} is {}, ' \ 'but expected {}'.format(i, inputs_ie[i].attrib['name'], input_name) if 'output_names' in kwargs: output_names = kwargs['output_names'] for i, output_name in enumerate(output_names): output_name_ie = outputs_ie[i].attrib['name'] output_without_sink_port = re.sub(r'\/sink_port_.', '', output_name_ie) assert output_without_sink_port == output_name, \ 'Output order does not match framework order. Output with index {} is {}, ' \ 'but expected {}'.format(i, output_without_sink_port, output_name)