def test_allreduce_hint_gpu(config): pytest.xfail("Multi-GPU testing not enabled yet") if 'gpu' not in ngt.transformer_choices(): pytest.skip("GPUTransformer not available") c = config os.environ["HETR_SERVER_GPU_NUM"] = str(len(c['device_id'])) ax_A_length = 32 ax_B_length = 16 np_result = [np.full((ax_A_length, ax_B_length), c['expected_result'], np.float32)] parallel_axis = ng.make_axis(name='axis_parallel', length=16) with ng.metadata(device_id=c['device_id'], parallel=parallel_axis): axis_A = ng.make_axis(length=ax_A_length, name='axis_A') axis_B = ng.make_axis(length=ax_B_length, name='axis_B') var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)).named('var_A') var_B = ng.variable(initial_value=UniformInit(c['input'], c['input']), axes=[axis_B]).named('var_B') var_B.metadata['reduce_func'] = c['func'] var_minus = (var_A - var_B).named('var_minus') with closing(ngt.make_transformer_factory('hetr', device='gpu')()) as hetr: out_comp = hetr.computation([var_minus]).named('out_comp') result = out_comp() np.testing.assert_array_equal(result, np_result)
def transformer_factory(request): def set_and_get_factory(transformer_name): factory = ngt.make_transformer_factory(transformer_name) ngt.set_transformer_factory(factory) return factory transformer_name = request.param if pytest.config.getoption("--enable_flex"): if transformer_name == flex_gpu_transformer_name: if flex_gpu_transformer_name in ngt.transformer_choices(): yield set_and_get_factory(transformer_name) else: raise ValueError("GPU not found, should not set --enable_flex" "flag for py.test.") else: pytest.skip( 'Skip all other transformers since --enable_flex is set.') else: if transformer_name == flex_gpu_transformer_name: pytest.skip('Skip flex test since --enable_flex is not set.') else: yield set_and_get_factory(transformer_name) # Reset transformer factory to default ngt.set_transformer_factory(ngt.make_transformer_factory("numpy"))
def test_allreduce_hint(hetr_device, config): if hetr_device == 'gpu': if 'gpu' not in ngt.transformer_choices(): pytest.skip("GPUTransformer not available") input = config['input'] device_id = config['device_id'] axis_A = ng.make_axis(length=4, name='axis_A') parallel_axis = ng.make_axis(name='axis_parallel', length=16) with ng.metadata(device=hetr_device, device_id=device_id, parallel=parallel_axis): var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)) var_B = ng.variable(axes=[axis_A], initial_value=UniformInit(input, input)) var_B.metadata['reduce_func'] = 'sum' var_B_mean = var_B / len(device_id) var_minus = (var_A - var_B_mean) with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as hetr: out_comp = hetr.computation(var_minus) result = out_comp() np_result = np.full((axis_A.length), config['expected_result'], np.float32) np.testing.assert_array_equal(result, np_result)
def test_gpu_graph(config): pytest.xfail("Multi-GPU testing not enabled yet") if 'gpu' not in ngt.transformer_choices(): pytest.skip('GPUTransformer not available!') t = config with ng.metadata(device='gpu'): x = ng.placeholder(axes=t['axes']) with ng.metadata(device='gpu', device_id=t['device_id'], parallel=t['parallel_axis']): x_plus_one = x + 1 with ng.metadata(device='gpu'): x_plus_two = x_plus_one + 1 os.environ["HETR_SERVER_GPU_NUM"] = str(len(t['device_id'])) np_x = np.random.randint(100, size=t['axes'].full_lengths) with closing(ngt.make_transformer_factory('hetr')()) as transformer: computation = transformer.computation(x_plus_two, x) res = computation(np_x) np.testing.assert_array_equal(res, np_x + 2)
def pytest_addoption(parser): parser.addoption("--batch_size", type=int, default=8, help="Batch size for tests using input_tensor fixture.") parser.addoption("--transformer", default="cpu", choices=ngt.transformer_choices(), help="Select from available transformers") parser.addoption("--serialization_integration_test", action="store_true", help="Force all unit tests to serialize and deserialize the graph before \ transformer compilation.") parser.addoption('--hetr_device', action='append', default=[], help='Set hetr device (cpu, gpu, etc.)')
def test_multiple_gather_ops(hetr_device): if hetr_device == 'gpu': if 'gpu' not in ngt.transformer_choices(): pytest.skip("GPUTransformer not available") pytest.xfail( "Failure due to gather recv tensor being returned in wrong shape, " " possible mismatch between op layout and op.tensor layout") H = ng.make_axis(length=2, name='height') W = ng.make_axis(length=4, name='width') x = ng.placeholder(axes=[H, W]) with ng.metadata(device_id=('0', '1'), parallel=W): x_plus_one = x + 1 x_mul_two = x_plus_one * 2 input = np.random.randint(100, size=x.axes.lengths) with closing(ngt.make_transformer_factory('hetr', device=hetr_device)()) as hetr: plus = hetr.computation([x_mul_two, x_plus_one], x) result_mul_two, result_plus_one = plus(input) np.testing.assert_array_equal(result_plus_one, input + 1) np.testing.assert_array_equal(result_mul_two, (input + 1) * 2)
def backend_names(self): return ngt.transformer_choices()
# ---------------------------------------------------------------------------- import pytest import ngraph.transformers as ngt from ngraph.flex.names import flex_gpu_transformer_name from ngraph.testing.error_check import transformer_name def pytest_addoption(parser): parser.addoption("--enable_flex", action="store_true", help="Enable and *only* enable {} transformer.".format( flex_gpu_transformer_name)) @pytest.fixture(scope="module", params=ngt.transformer_choices()) def transformer_factory(request): def set_and_get_factory(transformer_name): factory = ngt.make_transformer_factory(transformer_name) ngt.set_transformer_factory(factory) return factory transformer_name = request.param if pytest.config.getoption("--enable_flex"): if transformer_name == flex_gpu_transformer_name: if flex_gpu_transformer_name in ngt.transformer_choices(): yield set_and_get_factory(transformer_name) else: raise ValueError("GPU not found, should not set --enable_flex" "flag for py.test.")
# ---------------------------------------------------------------------------- # Copyright 2016 Nervana Systems Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ---------------------------------------------------------------------------- import pytest import ngraph.transformers as ngt @pytest.fixture(scope="class", params=ngt.transformer_choices()) def transformer_factory(request): factory = ngt.make_transformer_factory(request.param) ngt.set_transformer_factory(factory) yield factory # Reset transformer factory to default ngt.set_transformer_factory( ngt.make_transformer_factory("numpy"))
def test_allreduce_gpu_op(config): class myProcess(Process): def __init__(self, y, device_id, queue): Process.__init__(self) self.y = y self.device_id = device_id self.exit = Event() self.queue = queue def run(self): with closing( ngt.make_transformer_factory( 'gpu', device_id=self.device_id)()) as t: comp = t.computation(self.y) self.queue.put(comp()) while not self.exit.is_set(): time.sleep(0.1) pytest.xfail("Multi-GPU testing not enabled yet") if 'gpu' not in ngt.transformer_choices(): pytest.skip('GPUTransformer not available!') c = config x = list() y = list() input_list = list() process_list = list() result_list = list() np_result_list = list() queue = Queue() with ng.metadata(device='gpu', device_id=c['device_id'], transformer='None', host_transformer='None'): for i in range(len(c['device_id'])): x_input = c['x_input'] * (i + 1) x.append(ng.constant(x_input)) input_list.append(x_input) for i in range(len(c['device_id'])): ar_op = GPUCudaAllReduceOp(x[i], c['func']) if (i != 0): ar_op._shared_queues = y[0].shared_queues y.append(ar_op) if c['func'] == 'mean': np_result = np.mean(input_list, axis=0) elif c['func'] == 'sum': np_result = np.sum(input_list, axis=0) for i, d in enumerate(c['device_id']): process_list.append(myProcess(y[i], d, queue)) process_list[i].start() for p in reversed(process_list): np_result_list.append(np_result) result_list.append(queue.get()) p.exit.set() p.join() np.testing.assert_array_equal(result_list, np_result_list)