def __init__(self,
ng_model_function,
device='CPU'): # type: (List[Function], str) -> None
super(NgraphBackendRep, self).__init__()
self.device = self._get_ngraph_device_name(device)
self.ng_model_function = ng_model_function
if 'IE' in device:
self.runtime = ng.runtime(backend_name=self.device,
mode=BackendMode.STATIC)
else:
self.runtime = ng.runtime(backend_name=self.device,
mode=BackendMode.DYNAMIC)
self.computation = self.runtime.computation(ng_model_function)
def predict(img_no, plot_result):
"""
Calculate the Dice and plot the predicted masks for image # img_no
"""
img = imgs_validation[[img_no], ]
msk = msks_validation[[img_no], ]
#TODO load onnx model in ngraph
if onnx:
onnx_protobuf = onnx.load('/data/Healthcare_app/output/unet_model_for_decathlon_100_iter.onnx')
ng_models = import_onnx_model(onnx_protobuf)
ng_model = ng_models[0]
runtime = ng.runtime(backend_name='CPU')
unet = runtime.computation(ng_model['output'], *ng_model['inputs'])
start_time = time.time()
pred_mask= unet(img)[0]
print ("Time for prediction ngraph: ", '%.0f'%((time.time()-start_time)*1000),"ms")
else:
start_time = time.time()
pred_mask = model.predict(img, verbose=0, steps=None)
#print ("Time for prediction TF: ", '\033[1m %.0f \033[0m'%((time.time()-start_time)*1000),"ms")
end_time = (time.time()-start_time)*1000
print(end_time)
plotDiceScore(img_no,img,msk,pred_mask,plot_result, round(end_time))
return end_time
def run(args):
onnx_filename = os.path.join(args.test_dir, 'model.onnx')
input_names, output_names = onnx_input_output_names(onnx_filename)
test_data_dir = os.path.join(args.test_dir, 'test_data_set_0')
inputs, outputs = load_test_data(test_data_dir, input_names, output_names)
model = onnx.load(onnx_filename)
ng_func = import_onnx_model(model)
runtime = ng.runtime(backend_name=args.backend)
computation = runtime.computation(ng_func)
inputs = [v for n, v in inputs]
outputs = [v for n, v in outputs]
actual_outputs = computation(*inputs)
for i, (name, expected,
actual) in enumerate(zip(output_names, outputs, actual_outputs)):
np.testing.assert_allclose(expected, actual, rtol=1e-3,
atol=1e-4), name
print('%s: OK' % name)
print('ALL OK')
def compute():
computation(*inputs)
return run_onnx_util.run_benchmark(compute, args.iterations)
def run(args):
onnx_filename = os.path.join(args.test_dir, 'model.onnx')
input_names, output_names = onnx_input_output_names(onnx_filename)
test_data_dir = os.path.join(args.test_dir, 'test_data_set_0')
inputs, outputs = load_test_data(test_data_dir, input_names, output_names)
model = onnx.load(onnx_filename)
ng_func = import_onnx_model(model)
runtime = ng.runtime(backend_name=args.backend)
computation = runtime.computation(ng_func)
inputs = [v for n, v in inputs]
outputs = [v for n, v in outputs]
actual_outputs = computation(*inputs)
for i, (name, expected,
actual) in enumerate(zip(output_names, outputs, actual_outputs)):
np.testing.assert_allclose(expected, actual, rtol=1e-3,
atol=1e-4), name
print('%s: OK' % name)
print('ALL OK')
if args.iterations > 1:
num_iterations = args.iterations - 1
start = time.time()
for t in range(num_iterations):
computation(*inputs)
elapsed = time.time() - start
print('Elapsed: %.3f msec' % (elapsed * 1000 / num_iterations))
def supports_ngraph_device(cls, ngraph_device_name): # type: (str) -> bool
"""Check whether particular nGraph device is supported by current nGraph library.
:param ngraph_device_name: Name of nGraph device.
:return: True if current nGraph library supports ngraph_device_name.
"""
try:
ng.runtime(backend_name=ngraph_device_name)
except RuntimeError as e:
# Catch error raised when backend isn't available:
# 'Backend {ngraph_device_name} not found in registered backends'
if str(ngraph_device_name) in str(e) and 'not found' in str(e):
return False
else:
raise e
return True
def __init__(self,
ng_model_function,
device='CPU'): # type: (List[Function], str) -> None
super(NgraphBackendRep, self).__init__()
self.device = self._get_ngraph_device_name(device)
self.ng_model_function = ng_model_function
self.runtime = ng.runtime(backend_name=self.device)
self.computation = self.runtime.computation(ng_model_function)
def __init__(self,
ng_model,
device='CPU'): # type: (List[Dict], str) -> None
super(NgraphBackendRep, self).__init__()
self.device = self._get_ngraph_device_name(device)
self.model = ng_model
self.runtime = ng.runtime(backend_name=self.device)
self.computations = [
self.runtime.computation(model['output'], *model['inputs'])
for model in ng_model
]
def test_bad_data_shape():
A = ng.parameter(shape=[2, 2], name='A', dtype=np.float32)
B = ng.parameter(shape=[2, 2], name='B')
model = (A + B)
runtime = ng.runtime(backend_name='INTERPRETER')
computation = runtime.computation(model, A, B)
value_a = np.array([[1, 2]], dtype=np.float32)
value_b = np.array([[5, 6], [7, 8]], dtype=np.float32)
with pytest.raises(UserInputError):
computation(value_a, value_b)
def evaluate(backend_name, ng_model, dataset, batch_size, print_freq):
runtime = ng.runtime(backend_name=backend_name)
computation = runtime.computation(ng_model)
batch_sys_time = AverageMeter()
batch_proc_time = AverageMeter()
batch_perf_time = AverageMeter()
# warm-up
for idx, (img, _) in enumerate(dataset):
computation(img)
if idx == WARM_UP_SIZE:
break
# Unix time (seconds since epoch). It is system-wide by definition
clock_sys_start = time.time()
# the sum of the system and user CPU time of the current process.
# It does not include time elapsed during sleep. It is process-wide
clock_proc_start = time.process_time()
# does include time elapsed during sleep and is system-wide,
# clock with highest available resolution
clock_perf_start = time.perf_counter()
for i, (batch, _) in enumerate(dataset):
computation(batch)
# measure elapsed time
batch_sys_time.update(time.time() - clock_sys_start)
batch_proc_time.update(time.process_time() - clock_proc_start)
batch_perf_time.update(time.perf_counter() - clock_perf_start)
if i % print_freq == 0:
print(
'Test: [{0}/{1}]\t'
'Time (sys) {batch_sys_time.val:.3f}s ({batch_sys_time.avg:.3f}s)\t'
'Time (proc) {batch_proc_time.val:.3f}s ({batch_proc_time.avg:.3f}s)\t'
'Time (perf) {batch_perf_time.val:.3f}s ({batch_perf_time.avg:.3f}s)\t'
''.format(i * batch_size,
len(dataset) * batch_size,
batch_sys_time=batch_sys_time,
batch_proc_time=batch_proc_time,
batch_perf_time=batch_perf_time))
clock_sys_start = time.time()
clock_proc_start = time.process_time()
clock_perf_start = time.perf_counter()
return {
'sys_time': batch_sys_time,
'proc_time': batch_proc_time,
'perf_time': batch_perf_time
}
def __init__(self):
model = import_onnx_file("model/model.onnx")
runtime = ng.runtime(backend_name="CPU")
self.inference = runtime.computation(model)
self.emotion_table = {
"0": "neutral",
"1": "happiness",
"2": "surprise",
"3": "sadness",
"4": "anger",
"5": "disgust",
"6": "fear",
"7": "contempt",
}
def test_reduction_ops(ng_api_helper, numpy_function, reduction_axes):
manager_name = pytest.config.getoption('backend', default='CPU')
runtime = ng.runtime(manager_name=manager_name)
shape = [2, 4, 3, 2]
parameter_a = ng.parameter(shape, name='A', dtype=np.float32)
model = ng_api_helper(parameter_a, reduction_axes)
computation = runtime.computation(model, parameter_a)
value_a = np.random.randn(*shape).astype(np.float32)
result = computation(value_a)
expected = numpy_function(value_a, axis=reduction_axes)
assert np.allclose(result, expected)
def test_serialization():
dtype = np.float32
manager_name = pytest.config.getoption('backend', default='CPU')
shape = [2, 2]
parameter_a = ng.parameter(shape, dtype=dtype, name='A')
parameter_b = ng.parameter(shape, dtype=dtype, name='B')
parameter_c = ng.parameter(shape, dtype=dtype, name='C')
model = (parameter_a + parameter_b) * parameter_c
runtime = ng.runtime(manager_name=manager_name)
computation = runtime.computation(model, parameter_a, parameter_b, parameter_c)
serialized = computation.serialize(2)
serial_json = json.loads(serialized)
assert serial_json[0]["name"] != ''
assert 10 == len(serial_json[0]["ops"])
def __init__(self):
# Initialize variables
training_id = os.environ.get("TRAINING_ID")
model_file_name = os.environ.get("MODEL_FILE_NAME")
mountPath = os.environ.get("MOUNT_PATH")
# Replace with path of trained model
model_path = mountPath + '/' + training_id + '/' + model_file_name
# Load model from the FLEXVolume
self.models = import_onnx_file(model_path)
self.runtime = ng.runtime(backend_name='CPU')
self.model = self.models[0]
self.net = self.runtime.computation(self.model['output'],
*self.model['inputs'])
print("Model loaded")
def test_avg_pooling_3d():
manager_name = pytest.config.getoption('backend', default='CPU')
rt = ng.runtime(manager_name=manager_name)
data = np.arange(11, 27, dtype=np.float32)
data = data.reshape((1, 1, 4, 4))
data = np.broadcast_to(data, (1, 1, 4, 4, 4))
param = ng.parameter(data.shape)
avgpool = ng.avg_pool(param, [2, 2, 2], [2, 2, 2])
comp = rt.computation(avgpool, param)
result = comp(data)
result_ref = [[[[[13.5, 15.5], [21.5, 23.5]], [[13.5, 15.5], [21.5,
23.5]]]]]
np.testing.assert_allclose(result, result_ref, rtol=0.001)
def test_binary_op_with_scalar(ng_api_helper, numpy_function):
manager_name = pytest.config.getoption('backend', default='CPU')
runtime = ng.runtime(manager_name=manager_name)
value_a = np.array([[1, 2], [3, 4]], dtype=np.float32)
value_b = np.array([[5, 6], [7, 8]], dtype=np.float32)
shape = [2, 2]
parameter_a = ng.parameter(shape, name='A', dtype=np.float32)
model = ng_api_helper(parameter_a, value_b)
computation = runtime.computation(model, parameter_a)
result = computation(value_a)
expected = numpy_function(value_a, value_b)
assert np.allclose(result, expected)
def test_simple_computation_on_ndarrays(dtype):
manager_name = pytest.config.getoption('backend', default='CPU')
shape = [2, 2]
parameter_a = ng.parameter(shape, dtype=dtype, name='A')
parameter_b = ng.parameter(shape, dtype=dtype, name='B')
parameter_c = ng.parameter(shape, dtype=dtype, name='C')
model = (parameter_a + parameter_b) * parameter_c
runtime = ng.runtime(manager_name=manager_name)
computation = runtime.computation(model, parameter_a, parameter_b, parameter_c)
value_a = np.array([[1, 2], [3, 4]], dtype=dtype)
value_b = np.array([[5, 6], [7, 8]], dtype=dtype)
value_c = np.array([[9, 10], [11, 12]], dtype=dtype)
result = computation(value_a, value_b, value_c)
assert np.allclose(result, np.array([[54, 80], [110, 144]], dtype=dtype))
def test_simple_graph():
node1 = make_node('Add', ['A', 'B'], ['X'], name='add_node1')
node2 = make_node('Add', ['X', 'C'], ['Y'], name='add_node2')
graph = make_graph([node1, node2], 'test_graph', [
make_tensor_value_info('A', onnx.TensorProto.FLOAT, [1]),
make_tensor_value_info('B', onnx.TensorProto.FLOAT, [1]),
make_tensor_value_info('C', onnx.TensorProto.FLOAT, [1])
], [make_tensor_value_info('Y', onnx.TensorProto.FLOAT, [1])])
model = make_model(graph, producer_name='ngraph ONNXImporter')
ng_model = import_onnx_model(model)[0]
runtime = ng.runtime(
manager_name=pytest.config.getoption('backend', default='CPU'))
computation = runtime.computation(ng_model['output'], *ng_model['inputs'])
assert np.array_equal(computation(4, 5, 6),
np.array([15.0], dtype=np.float32))
def test_serialization():
dtype = np.float32
backend_name = test.BACKEND_NAME
shape = [2, 2]
parameter_a = ng.parameter(shape, dtype=dtype, name="A")
parameter_b = ng.parameter(shape, dtype=dtype, name="B")
parameter_c = ng.parameter(shape, dtype=dtype, name="C")
model = (parameter_a + parameter_b) * parameter_c
runtime = ng.runtime(backend_name=backend_name)
computation = runtime.computation(model, parameter_a, parameter_b, parameter_c)
try:
serialized = computation.serialize(2)
serial_json = json.loads(serialized)
assert serial_json[0]["name"] != ""
assert 10 == len(serial_json[0]["ops"])
except Exception:
pass
def test_serialization():
dtype = np.float32
backend_name = test.BACKEND_NAME
shape = [2, 2]
parameter_a = ng.parameter(shape, dtype=dtype, name='A')
parameter_b = ng.parameter(shape, dtype=dtype, name='B')
parameter_c = ng.parameter(shape, dtype=dtype, name='C')
model = (parameter_a + parameter_b) * parameter_c
runtime = ng.runtime(backend_name=backend_name)
computation = runtime.computation(model, parameter_a, parameter_b, parameter_c)
try:
serialized = computation.serialize(2)
serial_json = json.loads(serialized)
assert serial_json[0]['name'] != ''
assert 10 == len(serial_json[0]['ops'])
except Exception:
pass
input_data = np.array([1, 2, 3])
new_shape = [3, 3]
expected = [[1, 2, 3],
[1, 2, 3],
[1, 2, 3]]
result = run_op_node([input_data], ng.broadcast_to, new_shape)
assert np.allclose(result, expected)
axis = 0
expected = [[1, 1, 1],
[2, 2, 2],
[3, 3, 3]]
result = run_op_node([input_data], ng.broadcast_to, new_shape, axis)
assert np.allclose(result, expected)
input_data = np.arange(4)
new_shape = [3, 4, 2, 4]
expected = np.broadcast_to(input_data, new_shape)
result = run_op_node([input_data], ng.broadcast_to, new_shape)
assert np.allclose(result, expected)
def __init__(self):
print("Loading model")
# Import the ONNX file
models = import_onnx_file('resnet.onnx')
# Create an nGraph runtime environment
runtime = ng.runtime(backend_name='CPU')
# Select the first model and compile it to a callable function
model = models[0]
self.resnet = runtime.computation(model['output'], *model['inputs'])
print("Model loaded")
#Do a test run to warm up and check all is ok
print("Running test on img of Zebra as warmup")
img = image.load_img('zebra.jpg', target_size=(224, 224))
img = image.img_to_array(img)
x = np.expand_dims(img.copy(), axis=0)
x = preprocess_input(x, mode='torch')
x = x.transpose(0, 3, 1, 2)
preds = self.resnet(x)
print(decode_predictions(preds, top=5))
def run_ngraph_inference(model_onnx, inputs, device):
onnx_protobuf = onnx.load(model_onnx)
ng_function = import_onnx_model(onnx_protobuf)
runtime = ng.runtime(backend_name=device)
model = runtime.computation(ng_function)
# TODO: doesnt work with fully connected, fix
if "fully" not in model_onnx:
inputs = np.expand_dims(
inputs, 1
) # add a dimension so slicing in the loop returns a properly shaped input for ngraph. pytorch too
n = len(inputs)
outputs = []
start = timer()
for i in range(n):
data = inputs[i]
outputs.append(model(data))
inference_time = (timer() - start) * 1000 / n
print(f'inference time (msec): {inference_time:.5f}')
return outputs, inference_time
def test_serialization():
dtype = np.float32
backend_name = pytest.config.getoption('backend', default='CPU')
shape = [2, 2]
parameter_a = ng.parameter(shape, dtype=dtype, name='A')
parameter_b = ng.parameter(shape, dtype=dtype, name='B')
parameter_c = ng.parameter(shape, dtype=dtype, name='C')
model = (parameter_a + parameter_b) * parameter_c
runtime = ng.runtime(backend_name=backend_name)
computation = runtime.computation(model, parameter_a, parameter_b, parameter_c)
serialized = computation.serialize(2)
serial_json = json.loads(serialized)
assert serial_json[0]['name'] != ''
assert 10 == len(serial_json[0]['ops'])
input_data = np.array([1, 2, 3])
new_shape = [3, 3]
expected = [[1, 2, 3],
[1, 2, 3],
[1, 2, 3]]
result = run_op_node([input_data], ng.broadcast, new_shape)
assert np.allclose(result, expected)
axis = 0
expected = [[1, 1, 1],
[2, 2, 2],
[3, 3, 3]]
result = run_op_node([input_data], ng.broadcast, new_shape, axis)
assert np.allclose(result, expected)
input_data = np.arange(4)
new_shape = [3, 4, 2, 4]
expected = np.broadcast_to(input_data, new_shape)
result = run_op_node([input_data], ng.broadcast, new_shape)
assert np.allclose(result, expected)
def test_serialization():
dtype = np.float32
manager_name = pytest.config.getoption('backend', default='CPU')
shape = [2, 2]
parameter_a = ng.parameter(shape, dtype=dtype, name='A')
parameter_b = ng.parameter(shape, dtype=dtype, name='B')
parameter_c = ng.parameter(shape, dtype=dtype, name='C')
model = (parameter_a + parameter_b) * parameter_c
runtime = ng.runtime(manager_name=manager_name)
computation = runtime.computation(model, parameter_a, parameter_b,
parameter_c)
serialized = computation.serialize(2)
serial_json = json.loads(serialized)
assert serial_json[0]['name'] != ''
assert 10 == len(serial_json[0]['ops'])
input_data = np.array([1, 2, 3])
new_shape = [3, 3]
expected = [[1, 2, 3], [1, 2, 3], [1, 2, 3]]
result = run_op_node([input_data], ng.broadcast, new_shape)
assert np.allclose(result, expected)
axis = 0
expected = [[1, 1, 1], [2, 2, 2], [3, 3, 3]]
result = run_op_node([input_data], ng.broadcast, new_shape, axis)
assert np.allclose(result, expected)
input_data = np.arange(4)
new_shape = [3, 4, 2, 4]
expected = np.broadcast_to(input_data, new_shape)
result = run_op_node([input_data], ng.broadcast, new_shape)
assert np.allclose(result, expected)
def test_backend_config():
dummy_config = {"dummy_option": "dummy_value"}
# Expect no throw
ng.runtime(backend_name=test.BACKEND_NAME).set_config(dummy_config)
def _get_runtime():
manager_name = pytest.config.getoption('backend', default='CPU')
return ng.runtime(manager_name=manager_name)
def get_runtime():
return ng.runtime(backend_name=pytest.config.getoption('backend', default='CPU'))
"""Usage example for the ngraph Pythonic API.""" import numpy as np import ngraph as ng A = ng.parameter(shape=[2, 2], name='A', dtype=np.float32) B = ng.parameter(shape=[2, 2], name='B') C = ng.parameter(shape=[2, 2], name='C') # >>> print(A) # <Parameter: 'A' ([2, 2], float)> model = (A + B) * C # >>> print(model) # <Multiply: 'Multiply_14' ([2, 2])> runtime = ng.runtime(backend_name='CPU') # >>> print(runtime) # <Runtime: Backend='CPU'> computation = runtime.computation(model, A, B, C) # >>> print(computation) # <Computation: Multiply_14(A, B, C)> value_a = np.array([[1, 2], [3, 4]], dtype=np.float32) value_b = np.array([[5, 6], [7, 8]], dtype=np.float32) value_c = np.array([[9, 10], [11, 12]], dtype=np.float32) result = computation(value_a, value_b, value_c) # >>> print(result) # [[ 54. 80.] # [110. 144.]]
def get_runtime():
"""Return runtime object."""
manager_name = pytest.config.getoption('backend', default='CPU')
return ng.runtime(manager_name=manager_name)
def get_runtime():
"""Return runtime object."""
return ng.runtime(backend_name=test.BACKEND_NAME)
def get_runtime():
"""Return runtime object."""
backend_name = pytest.config.getoption('backend', default='CPU')
return ng.runtime(backend_name=backend_name)
import sys
import timeit
import numpy as np
import onnx
import ngraph as ng
from ngraph_onnx.onnx_importer.importer import import_onnx_model
model = onnx.load(sys.argv[1])
ng_func = import_onnx_model(model)
#print(ng_model)
picture = np.ones([1, 3, 224, 224], dtype=np.float32)
runtime = ng.runtime(backend_name='CPU')
#runtime = ng.runtime(backend_name='GPU')
resnet = runtime.computation(ng_func)
#print(resnet)
def run():
resnet(picture)
n = 100
print(timeit.timeit('run()', globals=globals(), number=n) / n * 1000, 'msec')
def get_runtime():
return ng.runtime(backend_name=BACKEND_NAME)
def get_transformer():
return ng.runtime(
manager_name=pytest.config.getoption('backend', default='CPU'))
