def check_vardrop(drop_inputs, drop_states, drop_outputs):
cell = contrib.rnn.VariationalDropoutCell(mx.gluon.rnn.RNNCell(100, prefix='rnn_'),
drop_outputs=drop_outputs,
drop_states=drop_states,
drop_inputs=drop_inputs)
cell.collect_params().initialize(init='xavier')
input_data = mx.nd.random_uniform(shape=(10, 3, 50), ctx=mx.context.current_context())
with mx.autograd.record():
outputs1, _ = cell.unroll(3, input_data, merge_outputs=True)
mx.nd.waitall()
outputs2, _ = cell.unroll(3, input_data, merge_outputs=True)
assert not almost_equal(outputs1.asnumpy(), outputs2.asnumpy())
inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
outputs, _ = cell.unroll(3, inputs, merge_outputs=False)
outputs = mx.sym.Group(outputs)
args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
assert outs == [(10, 100), (10, 100), (10, 100)]
cell.reset()
cell.hybridize()
with mx.autograd.record():
outputs3, _ = cell.unroll(3, input_data, merge_outputs=True)
mx.nd.waitall()
outputs4, _ = cell.unroll(3, input_data, merge_outputs=True)
assert not almost_equal(outputs3.asnumpy(), outputs4.asnumpy())
assert not almost_equal(outputs1.asnumpy(), outputs3.asnumpy())
def check_vardrop(drop_inputs, drop_states, drop_outputs):
cell = contrib.rnn.VariationalDropoutCell(mx.gluon.rnn.RNNCell(100, prefix='rnn_'),
drop_outputs=drop_outputs,
drop_states=drop_states,
drop_inputs=drop_inputs)
cell.collect_params().initialize(init='xavier')
input_data = mx.nd.random_uniform(shape=(10, 3, 50), ctx=mx.context.current_context())
with mx.autograd.record():
outputs1, _ = cell.unroll(3, input_data, merge_outputs=True)
mask1 = cell.drop_outputs_mask.asnumpy()
mx.nd.waitall()
outputs2, _ = cell.unroll(3, input_data, merge_outputs=True)
mask2 = cell.drop_outputs_mask.asnumpy()
assert not almost_equal(mask1, mask2)
assert not almost_equal(outputs1.asnumpy(), outputs2.asnumpy())
inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
outputs, _ = cell.unroll(3, inputs, merge_outputs=False)
outputs = mx.sym.Group(outputs)
args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
assert outs == [(10, 100), (10, 100), (10, 100)]
cell.reset()
cell.hybridize()
with mx.autograd.record():
outputs3, _ = cell.unroll(3, input_data, merge_outputs=True)
mx.nd.waitall()
outputs4, _ = cell.unroll(3, input_data, merge_outputs=True)
assert not almost_equal(outputs3.asnumpy(), outputs4.asnumpy())
assert not almost_equal(outputs1.asnumpy(), outputs3.asnumpy())
def check_vardrop(drop_inputs, drop_states, drop_outputs):
cell = gluon.rnn.VariationalDropoutCell(mx.gluon.rnn.RNNCell(100),
drop_outputs=drop_outputs,
drop_states=drop_states,
drop_inputs=drop_inputs)
input_data = mx.np.random.uniform(size=(10, 3, 50), ctx=mx.context.current_context())
cell.infer_shape(0, input_data, False)
cell.initialize(init='xavier')
with mx.autograd.record():
outputs1, _ = cell.unroll(3, input_data, merge_outputs=True)
mx.npx.waitall()
outputs2, _ = cell.unroll(3, input_data, merge_outputs=True)
assert not almost_equal(outputs1.asnumpy(), outputs2.asnumpy())
inputs = [mx.np.ones(shape=(10,50)) for i in range(3)]
cell.infer_shape(0, inputs[0], False)
cell.initialize()
outputs, _ = cell.unroll(3, inputs, merge_outputs=False)
outs = [o.shape for o in outputs]
assert outs == [(10, 100), (10, 100), (10, 100)]
cell.reset()
cell.hybridize()
with mx.autograd.record():
outputs3, _ = cell.unroll(3, input_data, merge_outputs=True)
mx.npx.waitall()
outputs4, _ = cell.unroll(3, input_data, merge_outputs=True)
assert not almost_equal(outputs3.asnumpy(), outputs4.asnumpy())
assert not almost_equal(outputs1.asnumpy(), outputs3.asnumpy())
def check_fluent_regular(func, kwargs, shape=(5, 17, 1), equal_nan=False):
with mx.name.NameManager():
data = mx.nd.random_uniform(shape=shape, ctx=default_context())
regular = getattr(mx.ndarray, func)(data, **kwargs)
fluent = getattr(data, func)(**kwargs)
if isinstance(regular, list):
for r, f in zip(regular, fluent):
assert almost_equal(r.asnumpy(), f.asnumpy(), equal_nan=equal_nan)
else:
assert almost_equal(regular.asnumpy(), fluent.asnumpy(), equal_nan=equal_nan)
def check_fluent_regular(func, kwargs, shape=(5, 17, 1), equal_nan=False):
with mx.name.NameManager():
data = mx.nd.random_uniform(shape=shape, ctx=default_context())
regular = getattr(mx.ndarray, func)(data, **kwargs)
fluent = getattr(data, func)(**kwargs)
if isinstance(regular, list):
for r, f in zip(regular, fluent):
assert almost_equal(r.asnumpy(), f.asnumpy(), equal_nan=equal_nan)
else:
assert almost_equal(regular.asnumpy(), fluent.asnumpy(), equal_nan=equal_nan)
def test_horovod_allreduce_postscale(self):
"""Test that the allreduce correctly sums 1D, 2D, 3D tensors with postscaling."""
hvd.init()
size = hvd.size()
dtypes = self.filter_supported_types(
['int32', 'int64', 'float16', 'float32', 'float64'])
int_types = ['int32', 'int64']
dims = [1, 2, 3]
ctx = self._current_context()
count = 1
shapes = [(), (17), (17, 17), (17, 17, 17)]
for dtype, dim in itertools.product(dtypes, dims):
mx.random.seed(1234, ctx=ctx)
np.random.seed(1234)
tensor = mx.nd.random.uniform(-100,
100,
shape=shapes[dim],
ctx=ctx)
tensor = tensor.astype(dtype)
factor = np.random.uniform()
scaled = hvd.allreduce(tensor,
average=False,
name=str(count),
postscale_factor=factor)
factor = mx.nd.array([factor], dtype='float64', ctx=ctx)
if ctx != mx.cpu() and not int(
os.environ.get('HOROVOD_MIXED_INSTALL', 0)):
# For integer types, scaling done in FP64
factor = factor.astype('float64' if dtype in
int_types else dtype)
tensor = tensor.astype('float64' if dtype in
int_types else dtype)
else:
# For integer types, scaling done in FP64, FP32 math for FP16 on CPU
factor = factor.astype('float32' if dtype ==
'float16' else 'float64' if dtype in
int_types else dtype)
tensor = tensor.astype('float32' if dtype ==
'float16' else 'float64' if dtype in
int_types else dtype)
expected = tensor * size
expected *= factor
expected = expected.astype(dtype)
count += 1
# Threshold for floating point equality depends on number of
# ranks, since we're comparing against precise multiplication.
if size <= 3 or dtype in int_types:
threshold = 0
elif size < 10:
threshold = 1e-4
elif size < 15:
threshold = 5e-4
else:
break
assert almost_equal(expected.asnumpy(), scaled.asnumpy(), atol=threshold), \
f'hvd.allreduce produces incorrect results for pre/post scaling: {hvd.rank()} {count} {dtype} {dim}'
def test_horovod_allreduce_inplace(self):
"""Test that the allreduce correctly sums 1D, 2D, 3D tensors."""
hvd.init()
size = hvd.size()
dtypes = self.filter_supported_types(
['int32', 'int64', 'float32', 'float64'])
dims = [1, 2, 3]
ctx = self._current_context()
count = 0
shapes = [(), (17), (17, 17), (17, 17, 17)]
for dtype, dim in itertools.product(dtypes, dims):
mx.random.seed(1234, ctx=ctx)
tensor = mx.nd.random.uniform(-100,
100,
shape=shapes[dim],
ctx=ctx)
tensor = tensor.astype(dtype)
multiplied = tensor * size
hvd.allreduce_(tensor, average=False, name=str(count))
count += 1
# Threshold for floating point equality depends on number of
# ranks, since we're comparing against precise multiplication.
if size <= 3 or dtype in ['int32', 'int64']:
threshold = 0
elif size < 10:
threshold = 1e-4
elif size < 15:
threshold = 5e-4
else:
break
assert almost_equal(tensor.asnumpy(), multiplied.asnumpy(), atol=threshold), \
f'hvd.allreduce produces incorrect results for self: {hvd.rank()} {count} {dtype} {dim}'
def test_horovod_grouped_allreduce_average(self):
"""Test that the grouped allreduce correctly averages 1D, 2D, 3D tensors."""
hvd.init()
size = hvd.size()
dtypes = self.filter_supported_types(
['int32', 'int64', 'float32', 'float64'])
dims = [1, 2, 3]
ctx = self._current_context()
count = 1
shapes = [(), (17), (17, 17), (17, 17, 17)]
for dtype, dim in itertools.product(dtypes, dims):
mx.random.seed(1234, ctx=ctx)
tensors = [
mx.nd.random.uniform(-100, 100, shape=shapes[dim], ctx=ctx)
for _ in range(5)
]
tensors = [tensor.astype(dtype) for tensor in tensors]
tensors = [tensor * size for tensor in tensors]
tensors = [tensor / size for tensor in tensors]
averaged = hvd.grouped_allreduce(tensors,
average=True,
name=str(count))
count += 1
# Threshold for floating point equality depends on number of
# ranks, since we're comparing against precise multiplication.
if size <= 3 or dtype in ['int32', 'int64']:
threshold = 0
elif size < 10:
threshold = 1e-4
elif size < 15:
threshold = 5e-4
else:
break
assert all([almost_equal(t1.asnumpy(), t2.asnumpy(), atol=threshold)
for t1, t2 in zip(averaged, tensors)]), \
f'hvd.grouped_allreduce produces incorrect results for average: {hvd.rank()} {count} {dtype} {dim}'
def verify(batch_size):
print('verifying batch size: ', batch_size)
fold = Fold()
num_samples = 100
inputs = []
fold_preds = []
for i in range(num_samples):
# get next batch
l_sent = l_sentences[i]
r_sent = r_sentences[i]
sent = mx.nd.concat(l_sent, r_sent, dim=0)
l_len = len(l_sent)
l_tree = l_trees[i]
r_tree = r_trees[i]
inputs.append((sent, l_len, l_tree, r_tree))
z_fold = net.fold_encode(fold, sent, l_len, l_tree, r_tree)
fold_preds.append(z_fold)
if (i + 1) % batch_size == 0 or (i + 1) == num_samples:
fold_outs = fold([fold_preds])[0]
outs = mx.nd.concat(*[
net(sent, l_len, l_tree, r_tree)
for sent, l_len, l_tree, r_tree in inputs
],
dim=0)
if not almost_equal(fold_outs.asnumpy(), outs.asnumpy()):
print(fold_preds)
print('l_sents: ', l_sent, l_sentences[i - 1])
print('r_sents: ', r_sent, r_sentences[i - 1])
print('\n'.join(
(str(l_tree), str_tree(l_tree), str(r_tree),
str_tree(r_tree), str(l_trees[i - 1]),
str_tree(l_trees[i - 1]), str(r_trees[i - 1]),
str_tree(r_trees[i - 1]), str(fold))))
assert_almost_equal(fold_outs.asnumpy(), outs.asnumpy())
fold_preds = []
inputs = []
fold.reset()
