def test_allreduce_hint_gpu(config):
pytest.xfail("Multi-GPU testing not enabled yet")
if 'gpu' not in ngt.transformer_choices():
pytest.skip("GPUTransformer not available")
c = config
os.environ["HETR_SERVER_GPU_NUM"] = str(len(c['device_id']))
ax_A_length = 32
ax_B_length = 16
np_result = [np.full((ax_A_length, ax_B_length), c['expected_result'], np.float32)]
parallel_axis = ng.make_axis(name='axis_parallel', length=16)
with ng.metadata(device_id=c['device_id'], parallel=parallel_axis):
axis_A = ng.make_axis(length=ax_A_length, name='axis_A')
axis_B = ng.make_axis(length=ax_B_length, name='axis_B')
var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1)).named('var_A')
var_B = ng.variable(initial_value=UniformInit(c['input'], c['input']),
axes=[axis_B]).named('var_B')
var_B.metadata['reduce_func'] = c['func']
var_minus = (var_A - var_B).named('var_minus')
with closing(ngt.make_transformer_factory('hetr', device='gpu')()) as hetr:
out_comp = hetr.computation([var_minus]).named('out_comp')
result = out_comp()
np.testing.assert_array_equal(result, np_result)
def transformer_factory(request):
def set_and_get_factory(transformer_name):
factory = ngt.make_transformer_factory(transformer_name)
ngt.set_transformer_factory(factory)
return factory
transformer_name = request.param
if pytest.config.getoption("--enable_flex"):
if transformer_name == flex_gpu_transformer_name:
if flex_gpu_transformer_name in ngt.transformer_choices():
yield set_and_get_factory(transformer_name)
else:
raise ValueError("GPU not found, should not set --enable_flex"
"flag for py.test.")
else:
pytest.skip(
'Skip all other transformers since --enable_flex is set.')
else:
if transformer_name == flex_gpu_transformer_name:
pytest.skip('Skip flex test since --enable_flex is not set.')
else:
yield set_and_get_factory(transformer_name)
# Reset transformer factory to default
ngt.set_transformer_factory(ngt.make_transformer_factory("numpy"))
def test_allreduce_hint(hetr_device, config):
if hetr_device == 'gpu':
if 'gpu' not in ngt.transformer_choices():
pytest.skip("GPUTransformer not available")
input = config['input']
device_id = config['device_id']
axis_A = ng.make_axis(length=4, name='axis_A')
parallel_axis = ng.make_axis(name='axis_parallel', length=16)
with ng.metadata(device=hetr_device,
device_id=device_id,
parallel=parallel_axis):
var_A = ng.variable(axes=[axis_A], initial_value=UniformInit(1, 1))
var_B = ng.variable(axes=[axis_A],
initial_value=UniformInit(input, input))
var_B.metadata['reduce_func'] = 'sum'
var_B_mean = var_B / len(device_id)
var_minus = (var_A - var_B_mean)
with closing(ngt.make_transformer_factory('hetr',
device=hetr_device)()) as hetr:
out_comp = hetr.computation(var_minus)
result = out_comp()
np_result = np.full((axis_A.length), config['expected_result'],
np.float32)
np.testing.assert_array_equal(result, np_result)
def test_gpu_graph(config):
pytest.xfail("Multi-GPU testing not enabled yet")
if 'gpu' not in ngt.transformer_choices():
pytest.skip('GPUTransformer not available!')
t = config
with ng.metadata(device='gpu'):
x = ng.placeholder(axes=t['axes'])
with ng.metadata(device='gpu',
device_id=t['device_id'],
parallel=t['parallel_axis']):
x_plus_one = x + 1
with ng.metadata(device='gpu'):
x_plus_two = x_plus_one + 1
os.environ["HETR_SERVER_GPU_NUM"] = str(len(t['device_id']))
np_x = np.random.randint(100, size=t['axes'].full_lengths)
with closing(ngt.make_transformer_factory('hetr')()) as transformer:
computation = transformer.computation(x_plus_two, x)
res = computation(np_x)
np.testing.assert_array_equal(res, np_x + 2)
def pytest_addoption(parser):
parser.addoption("--batch_size", type=int, default=8,
help="Batch size for tests using input_tensor fixture.")
parser.addoption("--transformer", default="cpu", choices=ngt.transformer_choices(),
help="Select from available transformers")
parser.addoption("--serialization_integration_test", action="store_true",
help="Force all unit tests to serialize and deserialize the graph before \
transformer compilation.")
parser.addoption('--hetr_device', action='append', default=[],
help='Set hetr device (cpu, gpu, etc.)')
def test_multiple_gather_ops(hetr_device):
if hetr_device == 'gpu':
if 'gpu' not in ngt.transformer_choices():
pytest.skip("GPUTransformer not available")
pytest.xfail(
"Failure due to gather recv tensor being returned in wrong shape, "
" possible mismatch between op layout and op.tensor layout")
H = ng.make_axis(length=2, name='height')
W = ng.make_axis(length=4, name='width')
x = ng.placeholder(axes=[H, W])
with ng.metadata(device_id=('0', '1'), parallel=W):
x_plus_one = x + 1
x_mul_two = x_plus_one * 2
input = np.random.randint(100, size=x.axes.lengths)
with closing(ngt.make_transformer_factory('hetr',
device=hetr_device)()) as hetr:
plus = hetr.computation([x_mul_two, x_plus_one], x)
result_mul_two, result_plus_one = plus(input)
np.testing.assert_array_equal(result_plus_one, input + 1)
np.testing.assert_array_equal(result_mul_two, (input + 1) * 2)
def backend_names(self):
return ngt.transformer_choices()
# ----------------------------------------------------------------------------
import pytest
import ngraph.transformers as ngt
from ngraph.flex.names import flex_gpu_transformer_name
from ngraph.testing.error_check import transformer_name
def pytest_addoption(parser):
parser.addoption("--enable_flex",
action="store_true",
help="Enable and *only* enable {} transformer.".format(
flex_gpu_transformer_name))
@pytest.fixture(scope="module", params=ngt.transformer_choices())
def transformer_factory(request):
def set_and_get_factory(transformer_name):
factory = ngt.make_transformer_factory(transformer_name)
ngt.set_transformer_factory(factory)
return factory
transformer_name = request.param
if pytest.config.getoption("--enable_flex"):
if transformer_name == flex_gpu_transformer_name:
if flex_gpu_transformer_name in ngt.transformer_choices():
yield set_and_get_factory(transformer_name)
else:
raise ValueError("GPU not found, should not set --enable_flex"
"flag for py.test.")
# ----------------------------------------------------------------------------
# Copyright 2016 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------
import pytest
import ngraph.transformers as ngt
@pytest.fixture(scope="class", params=ngt.transformer_choices())
def transformer_factory(request):
factory = ngt.make_transformer_factory(request.param)
ngt.set_transformer_factory(factory)
yield factory
# Reset transformer factory to default
ngt.set_transformer_factory(
ngt.make_transformer_factory("numpy"))
def test_allreduce_gpu_op(config):
class myProcess(Process):
def __init__(self, y, device_id, queue):
Process.__init__(self)
self.y = y
self.device_id = device_id
self.exit = Event()
self.queue = queue
def run(self):
with closing(
ngt.make_transformer_factory(
'gpu', device_id=self.device_id)()) as t:
comp = t.computation(self.y)
self.queue.put(comp())
while not self.exit.is_set():
time.sleep(0.1)
pytest.xfail("Multi-GPU testing not enabled yet")
if 'gpu' not in ngt.transformer_choices():
pytest.skip('GPUTransformer not available!')
c = config
x = list()
y = list()
input_list = list()
process_list = list()
result_list = list()
np_result_list = list()
queue = Queue()
with ng.metadata(device='gpu',
device_id=c['device_id'],
transformer='None',
host_transformer='None'):
for i in range(len(c['device_id'])):
x_input = c['x_input'] * (i + 1)
x.append(ng.constant(x_input))
input_list.append(x_input)
for i in range(len(c['device_id'])):
ar_op = GPUCudaAllReduceOp(x[i], c['func'])
if (i != 0):
ar_op._shared_queues = y[0].shared_queues
y.append(ar_op)
if c['func'] == 'mean':
np_result = np.mean(input_list, axis=0)
elif c['func'] == 'sum':
np_result = np.sum(input_list, axis=0)
for i, d in enumerate(c['device_id']):
process_list.append(myProcess(y[i], d, queue))
process_list[i].start()
for p in reversed(process_list):
np_result_list.append(np_result)
result_list.append(queue.get())
p.exit.set()
p.join()
np.testing.assert_array_equal(result_list, np_result_list)
