def __init__(self, ng_model_function, device='CPU'): # type: (List[Function], str) -> None super(NgraphBackendRep, self).__init__() self.device = self._get_ngraph_device_name(device) self.ng_model_function = ng_model_function if 'IE' in device: self.runtime = ng.runtime(backend_name=self.device, mode=BackendMode.STATIC) else: self.runtime = ng.runtime(backend_name=self.device, mode=BackendMode.DYNAMIC) self.computation = self.runtime.computation(ng_model_function)
def predict(img_no, plot_result): """ Calculate the Dice and plot the predicted masks for image # img_no """ img = imgs_validation[[img_no], ] msk = msks_validation[[img_no], ] #TODO load onnx model in ngraph if onnx: onnx_protobuf = onnx.load('/data/Healthcare_app/output/unet_model_for_decathlon_100_iter.onnx') ng_models = import_onnx_model(onnx_protobuf) ng_model = ng_models[0] runtime = ng.runtime(backend_name='CPU') unet = runtime.computation(ng_model['output'], *ng_model['inputs']) start_time = time.time() pred_mask= unet(img)[0] print ("Time for prediction ngraph: ", '%.0f'%((time.time()-start_time)*1000),"ms") else: start_time = time.time() pred_mask = model.predict(img, verbose=0, steps=None) #print ("Time for prediction TF: ", '\033[1m %.0f \033[0m'%((time.time()-start_time)*1000),"ms") end_time = (time.time()-start_time)*1000 print(end_time) plotDiceScore(img_no,img,msk,pred_mask,plot_result, round(end_time)) return end_time
def run(args): onnx_filename = os.path.join(args.test_dir, 'model.onnx') input_names, output_names = onnx_input_output_names(onnx_filename) test_data_dir = os.path.join(args.test_dir, 'test_data_set_0') inputs, outputs = load_test_data(test_data_dir, input_names, output_names) model = onnx.load(onnx_filename) ng_func = import_onnx_model(model) runtime = ng.runtime(backend_name=args.backend) computation = runtime.computation(ng_func) inputs = [v for n, v in inputs] outputs = [v for n, v in outputs] actual_outputs = computation(*inputs) for i, (name, expected, actual) in enumerate(zip(output_names, outputs, actual_outputs)): np.testing.assert_allclose(expected, actual, rtol=1e-3, atol=1e-4), name print('%s: OK' % name) print('ALL OK') def compute(): computation(*inputs) return run_onnx_util.run_benchmark(compute, args.iterations)
def run(args): onnx_filename = os.path.join(args.test_dir, 'model.onnx') input_names, output_names = onnx_input_output_names(onnx_filename) test_data_dir = os.path.join(args.test_dir, 'test_data_set_0') inputs, outputs = load_test_data(test_data_dir, input_names, output_names) model = onnx.load(onnx_filename) ng_func = import_onnx_model(model) runtime = ng.runtime(backend_name=args.backend) computation = runtime.computation(ng_func) inputs = [v for n, v in inputs] outputs = [v for n, v in outputs] actual_outputs = computation(*inputs) for i, (name, expected, actual) in enumerate(zip(output_names, outputs, actual_outputs)): np.testing.assert_allclose(expected, actual, rtol=1e-3, atol=1e-4), name print('%s: OK' % name) print('ALL OK') if args.iterations > 1: num_iterations = args.iterations - 1 start = time.time() for t in range(num_iterations): computation(*inputs) elapsed = time.time() - start print('Elapsed: %.3f msec' % (elapsed * 1000 / num_iterations))
def supports_ngraph_device(cls, ngraph_device_name): # type: (str) -> bool """Check whether particular nGraph device is supported by current nGraph library. :param ngraph_device_name: Name of nGraph device. :return: True if current nGraph library supports ngraph_device_name. """ try: ng.runtime(backend_name=ngraph_device_name) except RuntimeError as e: # Catch error raised when backend isn't available: # 'Backend {ngraph_device_name} not found in registered backends' if str(ngraph_device_name) in str(e) and 'not found' in str(e): return False else: raise e return True
def __init__(self, ng_model_function, device='CPU'): # type: (List[Function], str) -> None super(NgraphBackendRep, self).__init__() self.device = self._get_ngraph_device_name(device) self.ng_model_function = ng_model_function self.runtime = ng.runtime(backend_name=self.device) self.computation = self.runtime.computation(ng_model_function)
def __init__(self, ng_model, device='CPU'): # type: (List[Dict], str) -> None super(NgraphBackendRep, self).__init__() self.device = self._get_ngraph_device_name(device) self.model = ng_model self.runtime = ng.runtime(backend_name=self.device) self.computations = [ self.runtime.computation(model['output'], *model['inputs']) for model in ng_model ]
def test_bad_data_shape(): A = ng.parameter(shape=[2, 2], name='A', dtype=np.float32) B = ng.parameter(shape=[2, 2], name='B') model = (A + B) runtime = ng.runtime(backend_name='INTERPRETER') computation = runtime.computation(model, A, B) value_a = np.array([[1, 2]], dtype=np.float32) value_b = np.array([[5, 6], [7, 8]], dtype=np.float32) with pytest.raises(UserInputError): computation(value_a, value_b)
def evaluate(backend_name, ng_model, dataset, batch_size, print_freq): runtime = ng.runtime(backend_name=backend_name) computation = runtime.computation(ng_model) batch_sys_time = AverageMeter() batch_proc_time = AverageMeter() batch_perf_time = AverageMeter() # warm-up for idx, (img, _) in enumerate(dataset): computation(img) if idx == WARM_UP_SIZE: break # Unix time (seconds since epoch). It is system-wide by definition clock_sys_start = time.time() # the sum of the system and user CPU time of the current process. # It does not include time elapsed during sleep. It is process-wide clock_proc_start = time.process_time() # does include time elapsed during sleep and is system-wide, # clock with highest available resolution clock_perf_start = time.perf_counter() for i, (batch, _) in enumerate(dataset): computation(batch) # measure elapsed time batch_sys_time.update(time.time() - clock_sys_start) batch_proc_time.update(time.process_time() - clock_proc_start) batch_perf_time.update(time.perf_counter() - clock_perf_start) if i % print_freq == 0: print( 'Test: [{0}/{1}]\t' 'Time (sys) {batch_sys_time.val:.3f}s ({batch_sys_time.avg:.3f}s)\t' 'Time (proc) {batch_proc_time.val:.3f}s ({batch_proc_time.avg:.3f}s)\t' 'Time (perf) {batch_perf_time.val:.3f}s ({batch_perf_time.avg:.3f}s)\t' ''.format(i * batch_size, len(dataset) * batch_size, batch_sys_time=batch_sys_time, batch_proc_time=batch_proc_time, batch_perf_time=batch_perf_time)) clock_sys_start = time.time() clock_proc_start = time.process_time() clock_perf_start = time.perf_counter() return { 'sys_time': batch_sys_time, 'proc_time': batch_proc_time, 'perf_time': batch_perf_time }
def __init__(self): model = import_onnx_file("model/model.onnx") runtime = ng.runtime(backend_name="CPU") self.inference = runtime.computation(model) self.emotion_table = { "0": "neutral", "1": "happiness", "2": "surprise", "3": "sadness", "4": "anger", "5": "disgust", "6": "fear", "7": "contempt", }
def test_reduction_ops(ng_api_helper, numpy_function, reduction_axes): manager_name = pytest.config.getoption('backend', default='CPU') runtime = ng.runtime(manager_name=manager_name) shape = [2, 4, 3, 2] parameter_a = ng.parameter(shape, name='A', dtype=np.float32) model = ng_api_helper(parameter_a, reduction_axes) computation = runtime.computation(model, parameter_a) value_a = np.random.randn(*shape).astype(np.float32) result = computation(value_a) expected = numpy_function(value_a, axis=reduction_axes) assert np.allclose(result, expected)
def test_serialization(): dtype = np.float32 manager_name = pytest.config.getoption('backend', default='CPU') shape = [2, 2] parameter_a = ng.parameter(shape, dtype=dtype, name='A') parameter_b = ng.parameter(shape, dtype=dtype, name='B') parameter_c = ng.parameter(shape, dtype=dtype, name='C') model = (parameter_a + parameter_b) * parameter_c runtime = ng.runtime(manager_name=manager_name) computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) serialized = computation.serialize(2) serial_json = json.loads(serialized) assert serial_json[0]["name"] != '' assert 10 == len(serial_json[0]["ops"])
def __init__(self): # Initialize variables training_id = os.environ.get("TRAINING_ID") model_file_name = os.environ.get("MODEL_FILE_NAME") mountPath = os.environ.get("MOUNT_PATH") # Replace with path of trained model model_path = mountPath + '/' + training_id + '/' + model_file_name # Load model from the FLEXVolume self.models = import_onnx_file(model_path) self.runtime = ng.runtime(backend_name='CPU') self.model = self.models[0] self.net = self.runtime.computation(self.model['output'], *self.model['inputs']) print("Model loaded")
def test_avg_pooling_3d(): manager_name = pytest.config.getoption('backend', default='CPU') rt = ng.runtime(manager_name=manager_name) data = np.arange(11, 27, dtype=np.float32) data = data.reshape((1, 1, 4, 4)) data = np.broadcast_to(data, (1, 1, 4, 4, 4)) param = ng.parameter(data.shape) avgpool = ng.avg_pool(param, [2, 2, 2], [2, 2, 2]) comp = rt.computation(avgpool, param) result = comp(data) result_ref = [[[[[13.5, 15.5], [21.5, 23.5]], [[13.5, 15.5], [21.5, 23.5]]]]] np.testing.assert_allclose(result, result_ref, rtol=0.001)
def test_binary_op_with_scalar(ng_api_helper, numpy_function): manager_name = pytest.config.getoption('backend', default='CPU') runtime = ng.runtime(manager_name=manager_name) value_a = np.array([[1, 2], [3, 4]], dtype=np.float32) value_b = np.array([[5, 6], [7, 8]], dtype=np.float32) shape = [2, 2] parameter_a = ng.parameter(shape, name='A', dtype=np.float32) model = ng_api_helper(parameter_a, value_b) computation = runtime.computation(model, parameter_a) result = computation(value_a) expected = numpy_function(value_a, value_b) assert np.allclose(result, expected)
def test_simple_computation_on_ndarrays(dtype): manager_name = pytest.config.getoption('backend', default='CPU') shape = [2, 2] parameter_a = ng.parameter(shape, dtype=dtype, name='A') parameter_b = ng.parameter(shape, dtype=dtype, name='B') parameter_c = ng.parameter(shape, dtype=dtype, name='C') model = (parameter_a + parameter_b) * parameter_c runtime = ng.runtime(manager_name=manager_name) computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) value_a = np.array([[1, 2], [3, 4]], dtype=dtype) value_b = np.array([[5, 6], [7, 8]], dtype=dtype) value_c = np.array([[9, 10], [11, 12]], dtype=dtype) result = computation(value_a, value_b, value_c) assert np.allclose(result, np.array([[54, 80], [110, 144]], dtype=dtype))
def test_simple_graph(): node1 = make_node('Add', ['A', 'B'], ['X'], name='add_node1') node2 = make_node('Add', ['X', 'C'], ['Y'], name='add_node2') graph = make_graph([node1, node2], 'test_graph', [ make_tensor_value_info('A', onnx.TensorProto.FLOAT, [1]), make_tensor_value_info('B', onnx.TensorProto.FLOAT, [1]), make_tensor_value_info('C', onnx.TensorProto.FLOAT, [1]) ], [make_tensor_value_info('Y', onnx.TensorProto.FLOAT, [1])]) model = make_model(graph, producer_name='ngraph ONNXImporter') ng_model = import_onnx_model(model)[0] runtime = ng.runtime( manager_name=pytest.config.getoption('backend', default='CPU')) computation = runtime.computation(ng_model['output'], *ng_model['inputs']) assert np.array_equal(computation(4, 5, 6), np.array([15.0], dtype=np.float32))
def test_serialization(): dtype = np.float32 backend_name = test.BACKEND_NAME shape = [2, 2] parameter_a = ng.parameter(shape, dtype=dtype, name="A") parameter_b = ng.parameter(shape, dtype=dtype, name="B") parameter_c = ng.parameter(shape, dtype=dtype, name="C") model = (parameter_a + parameter_b) * parameter_c runtime = ng.runtime(backend_name=backend_name) computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) try: serialized = computation.serialize(2) serial_json = json.loads(serialized) assert serial_json[0]["name"] != "" assert 10 == len(serial_json[0]["ops"]) except Exception: pass
def test_serialization(): dtype = np.float32 backend_name = test.BACKEND_NAME shape = [2, 2] parameter_a = ng.parameter(shape, dtype=dtype, name='A') parameter_b = ng.parameter(shape, dtype=dtype, name='B') parameter_c = ng.parameter(shape, dtype=dtype, name='C') model = (parameter_a + parameter_b) * parameter_c runtime = ng.runtime(backend_name=backend_name) computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) try: serialized = computation.serialize(2) serial_json = json.loads(serialized) assert serial_json[0]['name'] != '' assert 10 == len(serial_json[0]['ops']) except Exception: pass input_data = np.array([1, 2, 3]) new_shape = [3, 3] expected = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] result = run_op_node([input_data], ng.broadcast_to, new_shape) assert np.allclose(result, expected) axis = 0 expected = [[1, 1, 1], [2, 2, 2], [3, 3, 3]] result = run_op_node([input_data], ng.broadcast_to, new_shape, axis) assert np.allclose(result, expected) input_data = np.arange(4) new_shape = [3, 4, 2, 4] expected = np.broadcast_to(input_data, new_shape) result = run_op_node([input_data], ng.broadcast_to, new_shape) assert np.allclose(result, expected)
def __init__(self): print("Loading model") # Import the ONNX file models = import_onnx_file('resnet.onnx') # Create an nGraph runtime environment runtime = ng.runtime(backend_name='CPU') # Select the first model and compile it to a callable function model = models[0] self.resnet = runtime.computation(model['output'], *model['inputs']) print("Model loaded") #Do a test run to warm up and check all is ok print("Running test on img of Zebra as warmup") img = image.load_img('zebra.jpg', target_size=(224, 224)) img = image.img_to_array(img) x = np.expand_dims(img.copy(), axis=0) x = preprocess_input(x, mode='torch') x = x.transpose(0, 3, 1, 2) preds = self.resnet(x) print(decode_predictions(preds, top=5))
def run_ngraph_inference(model_onnx, inputs, device): onnx_protobuf = onnx.load(model_onnx) ng_function = import_onnx_model(onnx_protobuf) runtime = ng.runtime(backend_name=device) model = runtime.computation(ng_function) # TODO: doesnt work with fully connected, fix if "fully" not in model_onnx: inputs = np.expand_dims( inputs, 1 ) # add a dimension so slicing in the loop returns a properly shaped input for ngraph. pytorch too n = len(inputs) outputs = [] start = timer() for i in range(n): data = inputs[i] outputs.append(model(data)) inference_time = (timer() - start) * 1000 / n print(f'inference time (msec): {inference_time:.5f}') return outputs, inference_time
def test_serialization(): dtype = np.float32 backend_name = pytest.config.getoption('backend', default='CPU') shape = [2, 2] parameter_a = ng.parameter(shape, dtype=dtype, name='A') parameter_b = ng.parameter(shape, dtype=dtype, name='B') parameter_c = ng.parameter(shape, dtype=dtype, name='C') model = (parameter_a + parameter_b) * parameter_c runtime = ng.runtime(backend_name=backend_name) computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) serialized = computation.serialize(2) serial_json = json.loads(serialized) assert serial_json[0]['name'] != '' assert 10 == len(serial_json[0]['ops']) input_data = np.array([1, 2, 3]) new_shape = [3, 3] expected = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] result = run_op_node([input_data], ng.broadcast, new_shape) assert np.allclose(result, expected) axis = 0 expected = [[1, 1, 1], [2, 2, 2], [3, 3, 3]] result = run_op_node([input_data], ng.broadcast, new_shape, axis) assert np.allclose(result, expected) input_data = np.arange(4) new_shape = [3, 4, 2, 4] expected = np.broadcast_to(input_data, new_shape) result = run_op_node([input_data], ng.broadcast, new_shape) assert np.allclose(result, expected)
def test_serialization(): dtype = np.float32 manager_name = pytest.config.getoption('backend', default='CPU') shape = [2, 2] parameter_a = ng.parameter(shape, dtype=dtype, name='A') parameter_b = ng.parameter(shape, dtype=dtype, name='B') parameter_c = ng.parameter(shape, dtype=dtype, name='C') model = (parameter_a + parameter_b) * parameter_c runtime = ng.runtime(manager_name=manager_name) computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) serialized = computation.serialize(2) serial_json = json.loads(serialized) assert serial_json[0]['name'] != '' assert 10 == len(serial_json[0]['ops']) input_data = np.array([1, 2, 3]) new_shape = [3, 3] expected = [[1, 2, 3], [1, 2, 3], [1, 2, 3]] result = run_op_node([input_data], ng.broadcast, new_shape) assert np.allclose(result, expected) axis = 0 expected = [[1, 1, 1], [2, 2, 2], [3, 3, 3]] result = run_op_node([input_data], ng.broadcast, new_shape, axis) assert np.allclose(result, expected) input_data = np.arange(4) new_shape = [3, 4, 2, 4] expected = np.broadcast_to(input_data, new_shape) result = run_op_node([input_data], ng.broadcast, new_shape) assert np.allclose(result, expected)
def test_backend_config(): dummy_config = {"dummy_option": "dummy_value"} # Expect no throw ng.runtime(backend_name=test.BACKEND_NAME).set_config(dummy_config)
def _get_runtime(): manager_name = pytest.config.getoption('backend', default='CPU') return ng.runtime(manager_name=manager_name)
def get_runtime(): return ng.runtime(backend_name=pytest.config.getoption('backend', default='CPU'))
"""Usage example for the ngraph Pythonic API.""" import numpy as np import ngraph as ng A = ng.parameter(shape=[2, 2], name='A', dtype=np.float32) B = ng.parameter(shape=[2, 2], name='B') C = ng.parameter(shape=[2, 2], name='C') # >>> print(A) # <Parameter: 'A' ([2, 2], float)> model = (A + B) * C # >>> print(model) # <Multiply: 'Multiply_14' ([2, 2])> runtime = ng.runtime(backend_name='CPU') # >>> print(runtime) # <Runtime: Backend='CPU'> computation = runtime.computation(model, A, B, C) # >>> print(computation) # <Computation: Multiply_14(A, B, C)> value_a = np.array([[1, 2], [3, 4]], dtype=np.float32) value_b = np.array([[5, 6], [7, 8]], dtype=np.float32) value_c = np.array([[9, 10], [11, 12]], dtype=np.float32) result = computation(value_a, value_b, value_c) # >>> print(result) # [[ 54. 80.] # [110. 144.]]
def get_runtime(): """Return runtime object.""" manager_name = pytest.config.getoption('backend', default='CPU') return ng.runtime(manager_name=manager_name)
def get_runtime(): """Return runtime object.""" return ng.runtime(backend_name=test.BACKEND_NAME)
def get_runtime(): """Return runtime object.""" backend_name = pytest.config.getoption('backend', default='CPU') return ng.runtime(backend_name=backend_name)
import sys import timeit import numpy as np import onnx import ngraph as ng from ngraph_onnx.onnx_importer.importer import import_onnx_model model = onnx.load(sys.argv[1]) ng_func = import_onnx_model(model) #print(ng_model) picture = np.ones([1, 3, 224, 224], dtype=np.float32) runtime = ng.runtime(backend_name='CPU') #runtime = ng.runtime(backend_name='GPU') resnet = runtime.computation(ng_func) #print(resnet) def run(): resnet(picture) n = 100 print(timeit.timeit('run()', globals=globals(), number=n) / n * 1000, 'msec')
def get_runtime(): return ng.runtime(backend_name=BACKEND_NAME)
def get_transformer(): return ng.runtime( manager_name=pytest.config.getoption('backend', default='CPU'))