def test_build_device_cpu(mocker):
mocker.patch("ie_serving.models.ir_engine.IEPlugin")
cpu_extension_mock = mocker.patch(
"ie_serving.models.ir_engine.IEPlugin.add_cpu_extension")
model_xml = 'model1.xml'
model_bin = 'model1.bin'
with pytest.raises(FileNotFoundError):
IrEngine.build(model_bin=model_bin, model_xml=model_xml)
cpu_extension_mock.assert_called_once_with()
def test_build_device_other(mocker):
mocker.patch("ie_serving.models.ir_engine.IEPlugin")
device_mocker = mocker.patch("ie_serving.models.ir_engine.DEVICE")
device_mocker.return_value = 'other'
cpu_extension_mock = mocker.patch(
"ie_serving.models.ir_engine.IEPlugin.add_cpu_extension")
model_xml = 'model1.xml'
model_bin = 'model1.bin'
with pytest.raises(FileNotFoundError):
IrEngine.build(model_bin=model_bin, model_xml=model_xml)
assert not cpu_extension_mock.assert_called_once_with()
def test_build_device_cpu(mocker):
mocker.patch("ie_serving.models.ir_engine.IEPlugin")
cpu_extension_mock = mocker.patch(
"ie_serving.models.ir_engine.IEPlugin.add_cpu_extension")
model_xml = 'model1.xml'
model_bin = 'model1.bin'
batch_size = None
mapping_config = 'mapping_config.json'
with pytest.raises(Exception):
IrEngine.build(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
batch_size=batch_size)
cpu_extension_mock.assert_called_once_with()
def get_fake_model():
mapping_config = 'mapping_config.json'
exec_net = None
net = MockedNet(
inputs={DEFAULT_INPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')},
outputs={DEFAULT_OUTPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')})
plugin = None
batching_info = BatchingInfo(None)
shape_info = ShapeInfo(None, net.inputs)
new_engines = {}
available_versions = [1, 2, 3]
for version in available_versions:
engine = IrEngine(model_name='test',
model_version=version,
mapping_config=mapping_config,
exec_net=exec_net,
net=net,
plugin=plugin,
batching_info=batching_info,
shape_info=shape_info)
new_engines.update({version: engine})
model_name = "test"
versions_statuses = {}
batch_size_param, shape_param = None, None
for version in available_versions:
versions_statuses[version] = ModelVersionStatus(model_name, version)
new_model = LocalModel(model_name=model_name,
model_directory='fake_path/model/',
available_versions=available_versions,
engines=new_engines,
batch_size_param=batch_size_param,
shape_param=shape_param,
version_policy_filter=lambda versions: versions[:],
versions_statuses=versions_statuses)
return new_model
def get_engine_for_version(cls, version_attributes):
engine = IrEngine.build(
model_bin=version_attributes['bin_file'],
model_xml=version_attributes['xml_file'],
mapping_config=version_attributes['mapping_config'],
batch_size=version_attributes['batch_size'])
return engine
def test_build_device_other(mocker):
mocker.patch("ie_serving.models.ir_engine.IEPlugin")
device_mocker = mocker.patch("ie_serving.models.ir_engine.DEVICE")
device_mocker.return_value = 'other'
cpu_extension_mock = mocker.patch(
"ie_serving.models.ir_engine.IEPlugin.add_cpu_extension")
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
batch_size = None
with pytest.raises(Exception):
IrEngine.build(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
batch_size=batch_size)
assert not cpu_extension_mock.assert_called_once_with()
def test_build_device_other(mocker):
mocker.patch("ie_serving.models.ir_engine.IECore")
cpu_extension_mock = mocker.patch(
"ie_serving.models.ir_engine.IECore.add_extension")
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
batch_size_param, shape_param = None, None
with pytest.raises(Exception):
IrEngine.build(model_name='test', model_version=1,
model_bin=model_bin, model_xml=model_xml,
mapping_config=mapping_config,
batch_size_param=batch_size_param,
shape_param=shape_param, num_ireq=1,
target_device='other', plugin_config=None)
assert not cpu_extension_mock.assert_called_once_with()
def test_init_class():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
net = None
batch_size = None
plugin = None
input_key = 'input'
inputs = {input_key: Layer('FP32', (1, 1), 'NCHW')}
outputs = {'output': Layer('FP32', (1, 1), 'NCHW')}
engine = IrEngine(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
exec_net=exec_net,
inputs=inputs,
outputs=outputs,
net=net,
plugin=plugin,
batch_size=batch_size)
assert model_xml == engine.model_xml
assert model_bin == engine.model_bin
assert exec_net == engine.exec_net
assert [input_key] == engine.input_tensor_names
assert inputs == engine.input_tensors
assert ['output'] == engine.output_tensor_names
assert {
'inputs': {
'input': 'input'
},
'outputs': {
'output': 'output'
}
} == engine.model_keys
assert [input_key] == engine.input_key_names
def test_init_class():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
exec_net = None
input_key = 'input'
inputs = {input_key: []}
outputs = ['output']
engine = IrEngine(model_bin=model_bin,
model_xml=model_xml,
exec_net=exec_net,
inputs=inputs,
outputs=outputs)
assert model_xml == engine.model_xml
assert model_bin == engine.model_bin
assert exec_net == engine.exec_net
assert [input_key] == engine.input_tensor_names
assert inputs == engine.input_tensors
assert outputs == engine.output_tensor_names
assert {
'inputs': {
'input': 'input'
},
'outputs': {
'output': 'output'
}
} == engine.model_keys
assert [input_key] == engine.input_key_names
def get_fake_model():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
net = None
plugin = None
batch_size = None
input_key = DEFAULT_INPUT_KEY
output_key = DEFAULT_OUTPUT_KEY
inputs = {input_key: Layer('FP32', [1, 1, 1], 'NCHW')}
outputs = {output_key: Layer('FP32', [1, 1, 1], 'NCHW')}
engine = IrEngine(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
exec_net=exec_net,
inputs=inputs,
outputs=outputs,
net=net,
plugin=plugin,
batch_size=batch_size)
new_engines = {1: engine, 2: engine, 3: engine}
available_versions = [1, 2, 3]
model_name = "test"
versions_statuses = {}
for version in available_versions:
versions_statuses[version] = ModelVersionStatus(model_name, version)
new_model = LocalModel(model_name=model_name,
model_directory='fake_path/model/',
available_versions=[1, 2, 3],
engines=new_engines,
batch_size=batch_size,
version_policy_filter=lambda versions: versions[:],
versions_statuses=versions_statuses)
return new_model
def get_fake_ir_engine():
mapping_config = 'mapping_config.json'
exec_net = MockedExecNet()
net = MockedNet(
inputs={DEFAULT_INPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')},
outputs={DEFAULT_OUTPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')})
core = None
batching_info = BatchingInfo(None)
shape_info = ShapeInfo(None, net.inputs)
requests_queue = queue.Queue()
free_ireq_index_queue = queue.Queue(maxsize=1)
free_ireq_index_queue.put(0)
engine = IrEngine(model_name='test',
model_version=1,
mapping_config=mapping_config,
exec_net=exec_net,
net=net,
core=core,
batching_info=batching_info,
shape_info=shape_info,
target_device="CPU",
free_ireq_index_queue=free_ireq_index_queue,
plugin_config=None,
num_ireq=1,
requests_queue=requests_queue)
return engine
def test_init_class():
mapping_config = 'mapping_config.json'
exec_net = None
net = MockedNet(inputs={'input': MockedIOInfo('FP32', [1, 1, 1], 'NCHW')},
outputs={'output': MockedIOInfo('FP32', [1, 1], 'NCHW')})
batching_info = BatchingInfo(None)
shape_info = ShapeInfo(None, net.inputs)
plugin = None
requests_queue = queue.Queue()
free_ireq_index_queue = queue.Queue(maxsize=1)
free_ireq_index_queue.put(0)
engine = IrEngine(model_name='test',
model_version=1,
mapping_config=mapping_config,
exec_net=exec_net,
net=net,
plugin=plugin,
batching_info=batching_info,
shape_info=shape_info,
num_ireq=1,
free_ireq_index_queue=free_ireq_index_queue,
requests_queue=requests_queue,
target_device='CPU',
plugin_config=None)
assert exec_net == engine.exec_net
assert list(net.inputs.keys()) == engine.input_tensor_names
assert list(net.outputs.keys()) == engine.output_tensor_names
assert engine.free_ireq_index_queue.qsize() == 1
def get_fake_model():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
net = None
plugin = None
batch_size = None
input_key = 'input'
inputs = {input_key: Layer('FP32', [1, 1, 1], 'NCHW')}
outputs = {'output': Layer('FP32', [1, 1, 1], 'NCHW')}
engine = IrEngine(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
exec_net=exec_net,
inputs=inputs,
outputs=outputs,
net=net,
plugin=plugin,
batch_size=batch_size)
new_engines = {1: engine, 2: engine, 3: engine}
new_model = LocalModel(model_name="test",
model_directory='fake_path/model/',
available_versions=[1, 2, 3],
engines=new_engines,
batch_size=batch_size)
return new_model
def get_engine_for_version(cls, version_attributes):
local_xml_file, local_bin_file, local_mapping_config = \
cls.create_local_mirror(version_attributes)
logger.info('Downloaded files from GCS')
engine = IrEngine.build(model_xml=local_xml_file,
model_bin=local_bin_file,
mapping_config=local_mapping_config)
cls.delete_local_mirror(
[local_xml_file, local_bin_file, local_mapping_config])
logger.info('Deleted temporary files')
return engine
def get_fake_ir_engine():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
input_key = 'input'
inputs = {input_key: []}
outputs = ['output']
engine = IrEngine(model_bin=model_bin, model_xml=model_xml,
mapping_config=mapping_config, exec_net=exec_net,
inputs=inputs, outputs=outputs)
return engine
def get_engine_for_version(cls, model_name, version_attributes):
version_attributes['xml_file'], version_attributes['bin_file'], \
version_attributes['mapping_config'] = cls.create_local_mirror(
version_attributes)
logger.info('Downloaded files from GCS')
engine_spec = cls._get_engine_spec(model_name, version_attributes)
engine = IrEngine.build(**engine_spec)
cls.delete_local_mirror([version_attributes['xml_file'],
version_attributes['bin_file'],
version_attributes['mapping_config']])
logger.info('Deleted temporary files')
return engine
def test_init_class():
mapping_config = 'mapping_config.json'
exec_net = None
net = MockedNet(inputs={'input': MockedIOInfo('FP32', [1, 1, 1], 'NCHW')},
outputs={'output': MockedIOInfo('FP32', [1, 1], 'NCHW')})
batching_info = BatchingInfo(None)
shape_info = ShapeInfo(None, net.inputs)
plugin = None
engine = IrEngine(model_name='test', model_version=1,
mapping_config=mapping_config,
exec_net=exec_net,
net=net, plugin=plugin, batching_info=batching_info,
shape_info=shape_info)
assert exec_net == engine.exec_net
assert list(net.inputs.keys()) == engine.input_tensor_names
assert list(net.outputs.keys()) == engine.output_tensor_names
def get_fake_model():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
input_key = 'input'
inputs = {input_key: [1, 1]}
outputs = ['test_output']
engine = IrEngine(model_bin=model_bin, model_xml=model_xml,
mapping_config=mapping_config, exec_net=exec_net,
inputs=inputs, outputs=outputs)
new_engines = {1: engine, 2: engine, 3: engine}
new_model = LocalModel(model_name="test",
model_directory='fake_path/model/',
available_versions=[1, 2, 3], engines=new_engines)
return new_model
def get_fake_model():
mapping_config = 'mapping_config.json'
exec_net = MockedExecNet()
net = MockedNet(
inputs={DEFAULT_INPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')},
outputs={DEFAULT_OUTPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')})
core = None
batching_info = BatchingInfo(None)
shape_info = ShapeInfo(None, net.inputs)
new_engines = {}
available_versions = [1, 2, 3]
requests_queue = queue.Queue()
free_ireq_index_queue = queue.Queue(maxsize=1)
free_ireq_index_queue.put(0)
for version in available_versions:
engine = IrEngine(model_name='test',
model_version=version,
mapping_config=mapping_config,
exec_net=exec_net,
net=net,
core=core,
batching_info=batching_info,
shape_info=shape_info,
target_device="CPU",
free_ireq_index_queue=free_ireq_index_queue,
plugin_config=None,
num_ireq=1,
requests_queue=requests_queue)
new_engines.update({version: engine})
model_name = "test"
versions_statuses = {}
batch_size_param, shape_param = None, None
for version in available_versions:
versions_statuses[version] = ModelVersionStatus(model_name, version)
new_model = LocalModel(model_name=model_name,
model_directory='fake_path/model/',
available_versions=available_versions,
engines=new_engines,
batch_size_param=batch_size_param,
shape_param=shape_param,
version_policy_filter=lambda versions: versions[:],
versions_statuses=versions_statuses,
update_locks={},
plugin_config=None,
target_device="CPU",
num_ireq=1)
return new_model
def get_fake_ir_engine():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
input_key = 'input'
output_key = 'output'
inputs = {input_key: Layer('FP32', [1, 1, 1], 'NCHW')}
outputs = {output_key: Layer('FP32', [1, 1, 1], 'NCHW')}
engine = IrEngine(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
exec_net=exec_net,
inputs=inputs,
outputs=outputs)
return engine
def get_fake_ir_engine():
mapping_config = 'mapping_config.json'
exec_net = None
net = MockedNet(
inputs={DEFAULT_INPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')},
outputs={DEFAULT_OUTPUT_KEY: MockedIOInfo('FP32', [1, 1, 1], 'NCHW')})
plugin = None
batching_info = BatchingInfo(None)
shape_info = ShapeInfo(None, net.inputs)
engine = IrEngine(model_name='test',
model_version=1,
mapping_config=mapping_config,
exec_net=exec_net,
net=net,
plugin=plugin,
batching_info=batching_info,
shape_info=shape_info)
return engine
def get_engines_for_model(versions):
inference_engines = {}
failures = []
for version in versions:
try:
logger.info("Creating inference engine object "
"for version: {}".format(version['version']))
inference_engines[version['version']] = IrEngine.build(
model_bin=version['bin_model_path'],
model_xml=version['xml_model_path'])
except Exception as e:
logger.error("Error occurred while loading model "
"version: {}".format(version))
logger.error("Content error: {}".format(str(e).rstrip()))
failures.append(version)
for failure in failures:
versions.remove(failure)
return inference_engines
def get_fake_ir_engine():
model_xml = 'model1.xml'
model_bin = 'model1.bin'
mapping_config = 'mapping_config.json'
exec_net = None
net = None
batch_size = None
plugin = None
input_key = DEFAULT_INPUT_KEY
output_key = DEFAULT_OUTPUT_KEY
inputs = {input_key: Layer('FP32', [1, 1, 1], 'NCHW')}
outputs = {output_key: Layer('FP32', [1, 1, 1], 'NCHW')}
engine = IrEngine(model_bin=model_bin,
model_xml=model_xml,
mapping_config=mapping_config,
exec_net=exec_net,
inputs=inputs,
outputs=outputs,
net=net,
plugin=plugin,
batch_size=batch_size)
return engine
def get_engine_for_version(cls, model_name, version_attributes):
engine_spec = cls._get_engine_spec(model_name, version_attributes)
engine = IrEngine.build(**engine_spec)
return engine
