def _test_resize_with_diff_type(dtype):
# test normal case
data_in = nd.random.uniform(0, 255, (300, 200, 3)).astype(dtype)
out_nd = transforms.Resize(200)(data_in)
data_expected = mx.image.imresize(data_in, 200, 200, 1)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test 4D input
data_bath_in = nd.random.uniform(0, 255, (3, 300, 200, 3)).astype(dtype)
out_batch_nd = transforms.Resize(200)(data_bath_in)
for i in range(len(out_batch_nd)):
assert_almost_equal(mx.image.imresize(data_bath_in[i], 200, 200, 1).asnumpy(),
out_batch_nd[i].asnumpy())
# test interp = 2
out_nd = transforms.Resize(200, interpolation=2)(data_in)
data_expected = mx.image.imresize(data_in, 200, 200, 2)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test height not equals to width
out_nd = transforms.Resize((200, 100))(data_in)
data_expected = mx.image.imresize(data_in, 200, 100, 1)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test keep_ratio
out_nd = transforms.Resize(150, keep_ratio=True)(data_in)
data_expected = mx.image.imresize(data_in, 150, 225, 1)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test size below zero
invalid_transform = transforms.Resize(-150, keep_ratio=True)
assertRaises(MXNetError, invalid_transform, data_in)
# test size more than 2:
invalid_transform = transforms.Resize((100, 100, 100), keep_ratio=True)
assertRaises(MXNetError, invalid_transform, data_in)
def _test_resize_with_diff_type(dtype):
# test normal case
data_in = nd.random.uniform(0, 255, (300, 200, 3)).astype(dtype)
out_nd = transforms.Resize(200)(data_in)
data_expected = mx.image.imresize(data_in, 200, 200, 1)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test 4D input
data_bath_in = nd.random.uniform(0, 255, (3, 300, 200, 3)).astype(dtype)
out_batch_nd = transforms.Resize(200)(data_bath_in)
for i in range(len(out_batch_nd)):
assert_almost_equal(mx.image.imresize(data_bath_in[i], 200, 200, 1).asnumpy(),
out_batch_nd[i].asnumpy())
# test interp = 2
out_nd = transforms.Resize(200, interpolation=2)(data_in)
data_expected = mx.image.imresize(data_in, 200, 200, 2)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test height not equals to width
out_nd = transforms.Resize((200, 100))(data_in)
data_expected = mx.image.imresize(data_in, 200, 100, 1)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test keep_ratio
out_nd = transforms.Resize(150, keep_ratio=True)(data_in)
data_expected = mx.image.imresize(data_in, 150, 225, 1)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test size below zero
invalid_transform = transforms.Resize(-150, keep_ratio=True)
assertRaises(MXNetError, invalid_transform, data_in)
# test size more than 2:
invalid_transform = transforms.Resize((100, 100, 100), keep_ratio=True)
assertRaises(MXNetError, invalid_transform, data_in)
def test_normalize():
# 3D Input
data_in_3d = nd.random.uniform(0, 1, (3, 300, 300))
out_nd_3d = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))(data_in_3d)
data_expected_3d = data_in_3d.asnumpy()
data_expected_3d[:][:][0] = data_expected_3d[:][:][0] / 3.0
data_expected_3d[:][:][1] = (data_expected_3d[:][:][1] - 1.0) / 2.0
data_expected_3d[:][:][2] = data_expected_3d[:][:][2] - 2.0
assert_almost_equal(data_expected_3d, out_nd_3d.asnumpy())
# 4D Input
data_in_4d = nd.random.uniform(0, 1, (2, 3, 300, 300))
out_nd_4d = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))(data_in_4d)
data_expected_4d = data_in_4d.asnumpy()
data_expected_4d[0][:][:][0] = data_expected_4d[0][:][:][0] / 3.0
data_expected_4d[0][:][:][1] = (data_expected_4d[0][:][:][1] - 1.0) / 2.0
data_expected_4d[0][:][:][2] = data_expected_4d[0][:][:][2] - 2.0
data_expected_4d[1][:][:][0] = data_expected_4d[1][:][:][0] / 3.0
data_expected_4d[1][:][:][1] = (data_expected_4d[1][:][:][1] - 1.0) / 2.0
data_expected_4d[1][:][:][2] = data_expected_4d[1][:][:][2] - 2.0
assert_almost_equal(data_expected_4d, out_nd_4d.asnumpy())
# Invalid Input - Neither 3D or 4D input
invalid_data_in = nd.random.uniform(0, 1, (5, 5, 3, 300, 300))
normalize_transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))
assertRaises(MXNetError, normalize_transformer, invalid_data_in)
# Invalid Input - Channel neither 1 or 3
invalid_data_in = nd.random.uniform(0, 1, (5, 4, 300, 300))
normalize_transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))
assertRaises(MXNetError, normalize_transformer, invalid_data_in)
def test_to_tensor():
# 3D Input
data_in = np.random.uniform(0, 255, (300, 300, 3)).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert_almost_equal(
out_nd.asnumpy(),
np.transpose(data_in.astype(dtype=np.float32) / 255.0, (2, 0, 1)))
# 4D Input
data_in_4d = nd.random.uniform(0, 1, (2, 3, 300, 300))
out_nd_4d = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))(data_in_4d)
data_expected_4d = data_in_4d.asnumpy()
data_expected_4d[0][:][:][0] = data_expected_4d[0][:][:][0] / 3.0
data_expected_4d[0][:][:][1] = (data_expected_4d[0][:][:][1] - 1.0) / 2.0
data_expected_4d[0][:][:][2] = data_expected_4d[0][:][:][2] - 2.0
data_expected_4d[1][:][:][0] = data_expected_4d[1][:][:][0] / 3.0
data_expected_4d[1][:][:][1] = (data_expected_4d[1][:][:][1] - 1.0) / 2.0
data_expected_4d[1][:][:][2] = data_expected_4d[1][:][:][2] - 2.0
assert_almost_equal(data_expected_4d, out_nd_4d.asnumpy())
# Default normalize values i.e., mean=0, std=1
data_in_3d_def = nd.random.uniform(0, 1, (3, 300, 300))
out_nd_3d_def = transforms.Normalize()(data_in_3d_def)
data_expected_3d_def = data_in_3d_def.asnumpy()
assert_almost_equal(data_expected_3d_def, out_nd_3d_def.asnumpy())
# Invalid Input - Neither 3D or 4D input
invalid_data_in = nd.random.uniform(0, 1, (5, 5, 3, 300, 300))
normalize_transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))
assertRaises(MXNetError, normalize_transformer, invalid_data_in)
# Invalid Input - Channel neither 1 or 3
invalid_data_in = nd.random.uniform(0, 1, (5, 4, 300, 300))
normalize_transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))
assertRaises(MXNetError, normalize_transformer, invalid_data_in)
def check_invalid_rsp_pull_list(kv, key):
dns_val = [mx.nd.ones(shape) * 2] * len(key)
assertRaises(MXNetError,
kv.row_sparse_pull,
key,
out=dns_val,
row_ids=[mx.nd.array([1])] * len(key))
def test_to_tensor():
# 3D Input
data_in = np.random.uniform(0, 255, (300, 300, 3)).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert_almost_equal(
out_nd.asnumpy(),
np.transpose(data_in.astype(dtype=np.float32) / 255.0, (2, 0, 1)))
# 4D Input
data_in = np.random.uniform(0, 255,
(5, 300, 300, 3)).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert_almost_equal(
out_nd.asnumpy(),
np.transpose(data_in.astype(dtype=np.float32) / 255.0, (0, 3, 1, 2)))
# Invalid Input
invalid_data_in = nd.random.uniform(
0, 255, (5, 5, 300, 300, 3)).astype(dtype=np.uint8)
transformer = transforms.ToTensor()
assertRaises(MXNetError, transformer, invalid_data_in)
# Bounds (0->0, 255->1)
data_in = np.zeros((10, 20, 3)).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert same(
out_nd.asnumpy(),
np.transpose(np.zeros(data_in.shape, dtype=np.float32), (2, 0, 1)))
data_in = np.full((10, 20, 3), 255).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert same(
out_nd.asnumpy(),
np.transpose(np.ones(data_in.shape, dtype=np.float32), (2, 0, 1)))
def check_invalid_rsp_pull_single(kv, key):
dns_val = mx.nd.ones(shape) * 2
assertRaises(MXNetError,
kv.row_sparse_pull,
key,
out=dns_val,
row_ids=mx.nd.array([1]))
def test_NDArrayIter_csr():
# creating toy data
num_rows = rnd.randint(5, 15)
num_cols = rnd.randint(1, 20)
batch_size = rnd.randint(1, num_rows)
shape = (num_rows, num_cols)
csr, _ = rand_sparse_ndarray(shape, 'csr')
dns = csr.asnumpy()
# CSRNDArray with last_batch_handle not equal to 'discard' will throw NotImplementedError
assertRaises(NotImplementedError, mx.io.NDArrayIter, {'data': csr}, dns, batch_size,
last_batch_handle='pad')
# CSRNDArray with shuffle
csr_iter = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, dns, batch_size,
shuffle=True, last_batch_handle='discard'))
num_batch = 0
for batch in csr_iter:
num_batch += 1
assert(num_batch == num_rows // batch_size)
# make iterators
csr_iter = iter(mx.io.NDArrayIter(csr, csr, batch_size, last_batch_handle='discard'))
begin = 0
for batch in csr_iter:
expected = np.zeros((batch_size, num_cols))
end = begin + batch_size
expected[:num_rows - begin] = dns[begin:end]
if end > num_rows:
expected[num_rows - begin:] = dns[0:end - num_rows]
assert_almost_equal(batch.data[0].asnumpy(), expected)
begin += batch_size
def test_normalize():
# 3D Input
data_in_3d = nd.random.uniform(0, 1, (3, 300, 300))
out_nd_3d = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))(data_in_3d)
data_expected_3d = data_in_3d.asnumpy()
data_expected_3d[:][:][0] = data_expected_3d[:][:][0] / 3.0
data_expected_3d[:][:][1] = (data_expected_3d[:][:][1] - 1.0) / 2.0
data_expected_3d[:][:][2] = data_expected_3d[:][:][2] - 2.0
assert_almost_equal(data_expected_3d, out_nd_3d.asnumpy())
# 4D Input
data_in_4d = nd.random.uniform(0, 1, (2, 3, 300, 300))
out_nd_4d = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))(data_in_4d)
data_expected_4d = data_in_4d.asnumpy()
data_expected_4d[0][:][:][0] = data_expected_4d[0][:][:][0] / 3.0
data_expected_4d[0][:][:][1] = (data_expected_4d[0][:][:][1] - 1.0) / 2.0
data_expected_4d[0][:][:][2] = data_expected_4d[0][:][:][2] - 2.0
data_expected_4d[1][:][:][0] = data_expected_4d[1][:][:][0] / 3.0
data_expected_4d[1][:][:][1] = (data_expected_4d[1][:][:][1] - 1.0) / 2.0
data_expected_4d[1][:][:][2] = data_expected_4d[1][:][:][2] - 2.0
assert_almost_equal(data_expected_4d, out_nd_4d.asnumpy())
# Invalid Input - Neither 3D or 4D input
invalid_data_in = nd.random.uniform(0, 1, (5, 5, 3, 300, 300))
normalize_transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))
assertRaises(MXNetError, normalize_transformer, invalid_data_in)
# Invalid Input - Channel neither 1 or 3
invalid_data_in = nd.random.uniform(0, 1, (5, 4, 300, 300))
normalize_transformer = transforms.Normalize(mean=(0, 1, 2), std=(3, 2, 1))
assertRaises(MXNetError, normalize_transformer, invalid_data_in)
def test_rotate():
transformer = transforms.Rotate(10.)
assertRaises(TypeError, transformer, mx.np.ones((3, 30, 60),
dtype='uint8'))
single_image = mx.np.ones((3, 30, 60), dtype='float32')
single_output = transformer(single_image)
assert same(single_output.shape, (3, 30, 60))
batch_image = mx.np.ones((3, 3, 30, 60), dtype='float32')
batch_output = transformer(batch_image)
assert same(batch_output.shape, (3, 3, 30, 60))
input_image = np.array([[[0., 0., 0.], [0., 0., 1.], [0., 0., 0.]]])
rotation_angles_expected_outs = [
(90., np.array([[[0., 1., 0.], [0., 0., 0.], [0., 0., 0.]]])),
(180., np.array([[[0., 0., 0.], [1., 0., 0.], [0., 0., 0.]]])),
(270., np.array([[[0., 0., 0.], [0., 0., 0.], [0., 1., 0.]]])),
(360., np.array([[[0., 0., 0.], [0., 0., 1.], [0., 0., 0.]]])),
]
for rot_angle, expected_result in rotation_angles_expected_outs:
transformer = transforms.Rotate(rot_angle)
ans = transformer(input_image)
print(type(ans), ans, type(expected_result), expected_result)
assert_almost_equal(ans.asnumpy(),
expected_result.asnumpy(),
atol=1e-6)
def test_custom_embed():
embed_root = 'embeddings'
embed_name = 'my_embed'
elem_delim = '\t'
pretrain_file = 'my_pretrain_file.txt'
_mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim, pretrain_file)
pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)
my_embed = text.embedding.CustomEmbedding(pretrain_file_path, elem_delim)
assert len(my_embed) == 3
assert my_embed.vec_len == 5
assert my_embed.token_to_idx['a'] == 1
assert my_embed.idx_to_token[1] == 'a'
first_vec = my_embed.idx_to_vec[0]
assert_almost_equal(first_vec.asnumpy(), np.array([0, 0, 0, 0, 0]))
unk_vec = my_embed.get_vecs_by_tokens('A')
assert_almost_equal(unk_vec.asnumpy(), np.array([0, 0, 0, 0, 0]))
a_vec = my_embed.get_vecs_by_tokens('A', lower_case_backup=True)
assert_almost_equal(a_vec.asnumpy(), np.array([0.1, 0.2, 0.3, 0.4, 0.5]))
unk_vecs = my_embed.get_vecs_by_tokens(['<unk$unk@unk>', '<unk$unk@unk>'])
assert_almost_equal(unk_vecs.asnumpy(), np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]))
# Test loaded unknown vectors.
pretrain_file2 = 'my_pretrain_file2.txt'
_mk_my_pretrain_file3(os.path.join(embed_root, embed_name), elem_delim, pretrain_file2)
pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file2)
my_embed2 = text.embedding.CustomEmbedding(pretrain_file_path, elem_delim,
init_unknown_vec=nd.ones, unknown_token='<unk>')
unk_vec2 = my_embed2.get_vecs_by_tokens('<unk>')
assert_almost_equal(unk_vec2.asnumpy(), np.array([1, 1, 1, 1, 1]))
unk_vec2 = my_embed2.get_vecs_by_tokens('<unk$unk@unk>')
assert_almost_equal(unk_vec2.asnumpy(), np.array([1, 1, 1, 1, 1]))
my_embed3 = text.embedding.CustomEmbedding(pretrain_file_path, elem_delim,
init_unknown_vec=nd.ones, unknown_token='<unk1>')
unk_vec3 = my_embed3.get_vecs_by_tokens('<unk1>')
assert_almost_equal(unk_vec3.asnumpy(), np.array([1.1, 1.2, 1.3, 1.4, 1.5]))
unk_vec3 = my_embed3.get_vecs_by_tokens('<unk$unk@unk>')
assert_almost_equal(unk_vec3.asnumpy(), np.array([1.1, 1.2, 1.3, 1.4, 1.5]))
# Test error handling.
invalid_pretrain_file = 'invalid_pretrain_file.txt'
_mk_my_invalid_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
invalid_pretrain_file)
pretrain_file_path = os.path.join(embed_root, embed_name, invalid_pretrain_file)
assertRaises(AssertionError, text.embedding.CustomEmbedding, pretrain_file_path, elem_delim)
invalid_pretrain_file2 = 'invalid_pretrain_file2.txt'
_mk_my_invalid_pretrain_file2(os.path.join(embed_root, embed_name), elem_delim,
invalid_pretrain_file2)
pretrain_file_path = os.path.join(embed_root, embed_name, invalid_pretrain_file2)
assertRaises(AssertionError, text.embedding.CustomEmbedding, pretrain_file_path, elem_delim)
def test_resize_gpu():
# Test with normal case 3D input float type
data_in_3d = mx.np.random.uniform(0, 255, (300, 300, 3))
out_nd_3d = transforms.Resize((100, 100))(data_in_3d)
data_in_4d_nchw = mx.np.moveaxis(mx.np.expand_dims(data_in_3d, axis=0), 3,
1)
data_expected_3d = (mx.np.moveaxis(
nd.contrib.BilinearResize2D(data_in_4d_nchw.as_nd_ndarray(),
height=100,
width=100,
align_corners=False), 1, 3))[0]
assert_almost_equal(out_nd_3d.asnumpy(), data_expected_3d.asnumpy())
# Test with normal case 4D input float type
data_in_4d = mx.np.random.uniform(0, 255, (2, 300, 300, 3))
out_nd_4d = transforms.Resize((100, 100))(data_in_4d)
data_in_4d_nchw = mx.np.moveaxis(data_in_4d, 3, 1)
data_expected_4d = mx.np.moveaxis(
nd.contrib.BilinearResize2D(data_in_4d_nchw.as_nd_ndarray(),
height=100,
width=100,
align_corners=False), 1, 3)
assert_almost_equal(out_nd_4d.asnumpy(), data_expected_4d.asnumpy())
# Test invalid interp
data_in_3d = mx.np.random.uniform(0, 255, (300, 300, 3))
invalid_transform = transforms.Resize(-150,
keep_ratio=False,
interpolation=2)
assertRaises(MXNetError, invalid_transform, data_in_3d)
# Credited to Hang Zhang
def py_bilinear_resize_nhwc(x, outputHeight, outputWidth):
batch, inputHeight, inputWidth, channel = x.shape
if outputHeight == inputHeight and outputWidth == inputWidth:
return x
y = np.empty([batch, outputHeight, outputWidth,
channel]).astype('uint8')
rheight = 1.0 * (inputHeight - 1) / (outputHeight -
1) if outputHeight > 1 else 0.0
rwidth = 1.0 * (inputWidth - 1) / (outputWidth -
1) if outputWidth > 1 else 0.0
for h2 in range(outputHeight):
h1r = 1.0 * h2 * rheight
h1 = int(np.floor(h1r))
h1lambda = h1r - h1
h1p = 1 if h1 < (inputHeight - 1) else 0
for w2 in range(outputWidth):
w1r = 1.0 * w2 * rwidth
w1 = int(np.floor(w1r))
w1lambda = w1r - w1
w1p = 1 if w1 < (inputHeight - 1) else 0
for b in range(batch):
for c in range(channel):
y[b][h2][w2][c] = (1-h1lambda)*((1-w1lambda)*x[b][h1][w1][c] + \
w1lambda*x[b][h1][w1+w1p][c]) + \
h1lambda*((1-w1lambda)*x[b][h1+h1p][w1][c] + \
w1lambda*x[b][h1+h1p][w1+w1p][c])
return y
def test_sparse_nd_exception():
""" test invalid sparse operator will throw a exception """
a = mx.nd.ones((2,2))
assertRaises(mx.base.MXNetError, mx.nd.sparse.retain, a, invalid_arg="garbage_value")
assertRaises(ValueError, mx.nd.sparse.csr_matrix, a, shape=(3,2))
assertRaises(ValueError, mx.nd.sparse.csr_matrix, (2,2), shape=(3,2))
assertRaises(ValueError, mx.nd.sparse.row_sparse_array, (2,2), shape=(3,2))
assertRaises(ValueError, mx.nd.sparse.zeros, "invalid_stype", (2,2))
def test_sparse_nd_exception():
""" test invalid sparse operator will throw a exception """
a = mx.nd.ones((2,2))
assertRaises(mx.base.MXNetError, mx.nd.sparse.retain, a, invalid_arg="garbage_value")
assertRaises(ValueError, mx.nd.sparse.csr_matrix, a, shape=(3,2))
assertRaises(ValueError, mx.nd.sparse.csr_matrix, (2,2), shape=(3,2))
assertRaises(ValueError, mx.nd.sparse.row_sparse_array, (2,2), shape=(3,2))
assertRaises(ValueError, mx.nd.sparse.zeros, "invalid_stype", (2,2))
def test_module_bind():
sym = mx.sym.Variable('data')
sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC')
mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)])
assertRaises(TypeError, mod.bind, data_shapes=[('data', mx.nd.array([10,10]))])
assert mod.binded == False
mod.bind(data_shapes=[('data', (10,10))])
assert mod.binded == True
def test_module_bind():
x = mx.sym.Variable("data")
net = mx.sym.FullyConnected(x, num_hidden=1)
mod = SVRGModule(symbol=net, data_names=['data'], label_names=None, update_freq=2)
assertRaises(TypeError, mod.bind, data_shapes=['data', mx.nd.zeros(shape=(2, 1))])
mod.bind(data_shapes=[('data', (2, 1))])
assert mod.binded == True
assert mod._mod_aux.binded == True
def test_module_bind():
sym = mx.sym.Variable('data')
sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC')
mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)])
assertRaises(TypeError, mod.bind, data_shapes=[('data', mx.nd.array([10,10]))])
assert mod.binded == False
mod.bind(data_shapes=[('data', (10,10))])
assert mod.binded == True
def test_module_bind():
x = mx.sym.Variable("data")
net = mx.sym.FullyConnected(x, num_hidden=1)
mod = SVRGModule(symbol=net, data_names=['data'], label_names=None, update_freq=2)
assertRaises(TypeError, mod.bind, data_shapes=['data', mx.nd.zeros(shape=(2, 1))])
mod.bind(data_shapes=[('data', (2, 1))])
assert mod.binded == True
assert mod._mod_aux.binded == True
def test_resize():
# Test with normal case 3D input float type
data_in_3d = nd.random.uniform(0, 255, (300, 300, 3))
out_nd_3d = transforms.Resize((100, 100))(data_in_3d)
data_in_4d_nchw = nd.moveaxis(nd.expand_dims(data_in_3d, axis=0), 3, 1)
data_expected_3d = (nd.moveaxis(nd.contrib.BilinearResize2D(data_in_4d_nchw, 100, 100), 1, 3))[0]
assert_almost_equal(out_nd_3d.asnumpy(), data_expected_3d.asnumpy())
# Test with normal case 4D input float type
data_in_4d = nd.random.uniform(0, 255, (2, 300, 300, 3))
out_nd_4d = transforms.Resize((100, 100))(data_in_4d)
data_in_4d_nchw = nd.moveaxis(data_in_4d, 3, 1)
data_expected_4d = nd.moveaxis(nd.contrib.BilinearResize2D(data_in_4d_nchw, 100, 100), 1, 3)
assert_almost_equal(out_nd_4d.asnumpy(), data_expected_4d.asnumpy())
# Test invalid interp
data_in_3d = nd.random.uniform(0, 255, (300, 300, 3))
invalid_transform = transforms.Resize(-150, keep_ratio=False, interpolation=2)
assertRaises(MXNetError, invalid_transform, data_in_3d)
# Credited to Hang Zhang
def py_bilinear_resize_nhwc(x, outputHeight, outputWidth):
batch, inputHeight, inputWidth, channel = x.shape
if outputHeight == inputHeight and outputWidth == inputWidth:
return x
y = np.empty([batch, outputHeight, outputWidth, channel]).astype('uint8')
rheight = 1.0 * (inputHeight - 1) / (outputHeight - 1) if outputHeight > 1 else 0.0
rwidth = 1.0 * (inputWidth - 1) / (outputWidth - 1) if outputWidth > 1 else 0.0
for h2 in range(outputHeight):
h1r = 1.0 * h2 * rheight
h1 = int(np.floor(h1r))
h1lambda = h1r - h1
h1p = 1 if h1 < (inputHeight - 1) else 0
for w2 in range(outputWidth):
w1r = 1.0 * w2 * rwidth
w1 = int(np.floor(w1r))
w1lambda = w1r - w1
w1p = 1 if w1 < (inputHeight - 1) else 0
for b in range(batch):
for c in range(channel):
y[b][h2][w2][c] = (1-h1lambda)*((1-w1lambda)*x[b][h1][w1][c] + \
w1lambda*x[b][h1][w1+w1p][c]) + \
h1lambda*((1-w1lambda)*x[b][h1+h1p][w1][c] + \
w1lambda*x[b][h1+h1p][w1+w1p][c])
return y
# Test with normal case 3D input int8 type
data_in_4d = nd.random.uniform(0, 255, (1, 300, 300, 3)).astype('uint8')
out_nd_3d = transforms.Resize((100, 100))(data_in_4d[0])
assert_almost_equal(out_nd_3d.asnumpy(), py_bilinear_resize_nhwc(data_in_4d.asnumpy(), 100, 100)[0], atol=1.0)
# Test with normal case 4D input int8 type
data_in_4d = nd.random.uniform(0, 255, (2, 300, 300, 3)).astype('uint8')
out_nd_4d = transforms.Resize((100, 100))(data_in_4d)
assert_almost_equal(out_nd_4d.asnumpy(), py_bilinear_resize_nhwc(data_in_4d.asnumpy(), 100, 100), atol=1.0)
def test_get_and_pretrain_file_names():
assert len(text.embedding.get_pretrained_file_names(
embedding_name='fasttext')) == 327
assert len(text.embedding.get_pretrained_file_names(embedding_name='glove')) == 10
reg = text.embedding.get_pretrained_file_names(embedding_name=None)
assert len(reg['glove']) == 10
assert len(reg['fasttext']) == 327
assertRaises(KeyError, text.embedding.get_pretrained_file_names, 'unknown$$')
def test_NDArrayIter_csr():
# creating toy data
num_rows = rnd.randint(5, 15)
num_cols = rnd.randint(1, 20)
batch_size = rnd.randint(1, num_rows)
shape = (num_rows, num_cols)
csr, _ = rand_sparse_ndarray(shape, 'csr')
dns = csr.asnumpy()
# CSRNDArray or scipy.sparse.csr_matrix with last_batch_handle not equal to 'discard' will throw NotImplementedError
assertRaises(NotImplementedError, mx.io.NDArrayIter,
{'data': csr}, dns, batch_size)
try:
import scipy.sparse as spsp
train_data = spsp.csr_matrix(dns)
assertRaises(NotImplementedError, mx.io.NDArrayIter,
{'data': train_data}, dns, batch_size)
except ImportError:
pass
# scipy.sparse.csr_matrix with shuffle
csr_iter = iter(mx.io.NDArrayIter({'data': train_data}, dns, batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_empty_list = iter(mx.io.NDArrayIter({'data': train_data}, [], batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_None = iter(mx.io.NDArrayIter({'data': train_data}, None, batch_size,
shuffle=True, last_batch_handle='discard'))
_test_NDArrayIter_csr(csr_iter, csr_iter_empty_list,
csr_iter_None, num_rows, batch_size)
# CSRNDArray with shuffle
csr_iter = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, dns, batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_empty_list = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, [], batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_None = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, None, batch_size,
shuffle=True, last_batch_handle='discard'))
_test_NDArrayIter_csr(csr_iter, csr_iter_empty_list,
csr_iter_None, num_rows, batch_size)
# make iterators
csr_iter = iter(mx.io.NDArrayIter(
csr, csr, batch_size, last_batch_handle='discard'))
begin = 0
for batch in csr_iter:
expected = np.zeros((batch_size, num_cols))
end = begin + batch_size
expected[:num_rows - begin] = dns[begin:end]
if end > num_rows:
expected[num_rows - begin:] = dns[0:end - num_rows]
assert_almost_equal(batch.data[0].asnumpy(), expected)
begin += batch_size
def test_NDArrayIter_csr():
# creating toy data
num_rows = rnd.randint(5, 15)
num_cols = rnd.randint(1, 20)
batch_size = rnd.randint(1, num_rows)
shape = (num_rows, num_cols)
csr, _ = rand_sparse_ndarray(shape, 'csr')
dns = csr.asnumpy()
# CSRNDArray or scipy.sparse.csr_matrix with last_batch_handle not equal to 'discard' will throw NotImplementedError
assertRaises(NotImplementedError, mx.io.NDArrayIter,
{'data': csr}, dns, batch_size)
try:
import scipy.sparse as spsp
train_data = spsp.csr_matrix(dns)
assertRaises(NotImplementedError, mx.io.NDArrayIter,
{'data': train_data}, dns, batch_size)
except ImportError:
pass
# scipy.sparse.csr_matrix with shuffle
csr_iter = iter(mx.io.NDArrayIter({'data': train_data}, dns, batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_empty_list = iter(mx.io.NDArrayIter({'data': train_data}, [], batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_None = iter(mx.io.NDArrayIter({'data': train_data}, None, batch_size,
shuffle=True, last_batch_handle='discard'))
_test_NDArrayIter_csr(csr_iter, csr_iter_empty_list,
csr_iter_None, num_rows, batch_size)
# CSRNDArray with shuffle
csr_iter = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, dns, batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_empty_list = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, [], batch_size,
shuffle=True, last_batch_handle='discard'))
csr_iter_None = iter(mx.io.NDArrayIter({'csr_data': csr, 'dns_data': dns}, None, batch_size,
shuffle=True, last_batch_handle='discard'))
_test_NDArrayIter_csr(csr_iter, csr_iter_empty_list,
csr_iter_None, num_rows, batch_size)
# make iterators
csr_iter = iter(mx.io.NDArrayIter(
csr, csr, batch_size, last_batch_handle='discard'))
begin = 0
for batch in csr_iter:
expected = np.zeros((batch_size, num_cols))
end = begin + batch_size
expected[:num_rows - begin] = dns[begin:end]
if end > num_rows:
expected[num_rows - begin:] = dns[0:end - num_rows]
assert_almost_equal(batch.data[0].asnumpy(), expected)
begin += batch_size
def _test_crop_resize_with_diff_type(dtype):
# test normal case
data_in = nd.arange(60).reshape((5, 4, 3)).astype(dtype)
out_nd = transforms.CropResize(0, 0, 3, 2)(data_in)
out_np = out_nd.asnumpy()
assert(out_np.sum() == 180)
assert((out_np[0:2,1,1].flatten() == [4, 16]).all())
# test 4D input
data_bath_in = nd.arange(180).reshape((2, 6, 5, 3)).astype(dtype)
out_batch_nd = transforms.CropResize(1, 2, 3, 4)(data_bath_in)
out_batch_np = out_batch_nd.asnumpy()
assert(out_batch_np.sum() == 7524)
assert((out_batch_np[0:2,0:4,1,1].flatten() == [37, 52, 67, 82, 127, 142, 157, 172]).all())
# test normal case with resize
data_in = nd.random.uniform(0, 255, (300, 200, 3)).astype(dtype)
out_nd = transforms.CropResize(0, 0, 100, 50, (25, 25), 2)(data_in)
data_expected = image.imresize(nd.slice(data_in, (0, 0, 0), (50, 100 , 3)), 25, 25, 2)
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test 4D input with resize
data_bath_in = nd.random.uniform(0, 255, (3, 300, 200, 3)).astype(dtype)
out_batch_nd = transforms.CropResize(0, 0, 100, 50, (25, 25), 2)(data_bath_in)
for i in range(len(out_batch_nd)):
assert_almost_equal(image.imresize(nd.slice(data_bath_in[i], (0, 0, 0), (50, 100, 3)), 25, 25, 2).asnumpy(),
out_batch_nd[i].asnumpy())
# test with resize height and width should be greater than 0
transformer = transforms.CropResize(0, 0, 100, 50, (-25, 25), 2)
assertRaises(MXNetError, transformer, data_in)
# test height and width should be greater than 0
transformer = transforms.CropResize(0, 0, -100, -50)
assertRaises(MXNetError, transformer, data_in)
# test cropped area is bigger than input data
transformer = transforms.CropResize(150, 200, 200, 500)
assertRaises(MXNetError, transformer, data_in)
assertRaises(MXNetError, transformer, data_bath_in)
def _test_crop_resize_with_diff_type(dtype):
# test normal case
data_in = nd.arange(60).reshape((5, 4, 3)).astype(dtype)
out_nd = transforms.CropResize(0, 0, 3, 2)(data_in)
out_np = out_nd.asnumpy()
assert(out_np.sum() == 180)
assert((out_np[0:2,1,1].flatten() == [4, 16]).all())
# test 4D input
data_bath_in = nd.arange(180).reshape((2, 6, 5, 3)).astype(dtype)
out_batch_nd = transforms.CropResize(1, 2, 3, 4)(data_bath_in)
out_batch_np = out_batch_nd.asnumpy()
assert(out_batch_np.sum() == 7524)
assert((out_batch_np[0:2,0:4,1,1].flatten() == [37, 52, 67, 82, 127, 142, 157, 172]).all())
# test normal case with resize
data_in = nd.random.uniform(0, 255, (300, 200, 3)).astype(dtype)
out_nd = transforms.CropResize(0, 0, 100, 50, (25, 25), 1)(data_in)
data_expected = transforms.Resize(size=25, interpolation=1)(nd.slice(data_in, (0, 0, 0), (50, 100, 3)))
assert_almost_equal(out_nd.asnumpy(), data_expected.asnumpy())
# test 4D input with resize
data_bath_in = nd.random.uniform(0, 255, (3, 300, 200, 3)).astype(dtype)
out_batch_nd = transforms.CropResize(0, 0, 100, 50, (25, 25), 1)(data_bath_in)
for i in range(len(out_batch_nd)):
actual = transforms.Resize(size=25, interpolation=1)(nd.slice(data_bath_in[i], (0, 0, 0), (50, 100, 3))).asnumpy()
expected = out_batch_nd[i].asnumpy()
assert_almost_equal(expected, actual)
# test with resize height and width should be greater than 0
transformer = transforms.CropResize(0, 0, 100, 50, (-25, 25), 1)
assertRaises(MXNetError, transformer, data_in)
# test height and width should be greater than 0
transformer = transforms.CropResize(0, 0, -100, -50)
assertRaises(MXNetError, transformer, data_in)
# test cropped area is bigger than input data
transformer = transforms.CropResize(150, 200, 200, 500)
assertRaises(MXNetError, transformer, data_in)
assertRaises(MXNetError, transformer, data_bath_in)
def test_module_set_params():
# data iter
data = mx.nd.array([[0.05, .10]]);
label = mx.nd.array([[.01, 0.99]]);
train_data = mx.io.NDArrayIter(data, label, batch_size=1)
# symbols
x = mx.symbol.Variable('data')
x = mx.symbol.FullyConnected(name='fc_0', data=x, num_hidden=2)
x = mx.symbol.Activation(name="act_0", data=x, act_type='sigmoid')
x = mx.symbol.FullyConnected(name='fc_1', data=x, num_hidden=2)
x = mx.symbol.Activation(name="act_1", data=x, act_type='sigmoid')
x = mx.symbol.LinearRegressionOutput(data=x, name='softmax', grad_scale=2)
# create module
mod = mx.mod.Module(x, context=[mx.cpu()]);
mod.bind(train_data.provide_data, label_shapes=train_data.provide_label,
for_training=True)
arg_params_correct = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]),
'fc_0_bias' : mx.nd.array([.35, .35]),
'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]),
'fc_1_bias' : mx.nd.array([.60, .60])}
arg_params_missing = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]),
'fc_0_bias' : mx.nd.array([.35, .35]),
'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]])}
arg_params_extra = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]),
'fc_0_bias' : mx.nd.array([.35, .35]),
'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]),
'fc_1_bias' : mx.nd.array([.60, .60]),
'fc_2_weight': mx.nd.array([.60, .60])}
arg_params_missing_extra = {'fc_2_weight': mx.nd.array([.60, .60])}
# test regular set_params
mod.set_params(force_init=True, arg_params=arg_params_correct, aux_params={})
# test allow missing
mod.set_params(force_init=True, arg_params=arg_params_missing, aux_params={}, allow_missing=True)
assertRaises(RuntimeError, mod.set_params,
force_init=True, arg_params=arg_params_missing,
aux_params={}, allow_missing=False)
# test allow extra
mod.set_params(force_init=True, arg_params=arg_params_extra, aux_params={}, allow_missing=True, allow_extra=True)
assertRaises(ValueError, mod.set_params,
force_init=True, arg_params=arg_params_extra,
aux_params={}, allow_missing=True, allow_extra=False)
# test allow missing + extra,
assertRaises(RuntimeError, mod.set_params,
force_init=True, arg_params=arg_params_missing_extra,
aux_params={}, allow_missing=False, allow_extra=False)
# test allow missing + extra, this will throw a runtime error
assertRaises(ValueError, mod.set_params,
force_init=True, arg_params=arg_params_missing_extra,
aux_params={}, allow_missing=True, allow_extra=False)
def test_module_set_params():
# data iter
data = mx.nd.array([[0.05, .10]]);
label = mx.nd.array([[.01, 0.99]]);
train_data = mx.io.NDArrayIter(data, label, batch_size=1)
# symbols
x = mx.symbol.Variable('data')
x = mx.symbol.FullyConnected(name='fc_0', data=x, num_hidden=2)
x = mx.symbol.Activation(name="act_0", data=x, act_type='sigmoid')
x = mx.symbol.FullyConnected(name='fc_1', data=x, num_hidden=2)
x = mx.symbol.Activation(name="act_1", data=x, act_type='sigmoid')
x = mx.symbol.LinearRegressionOutput(data=x, name='softmax', grad_scale=2)
# create module
mod = mx.mod.Module(x, context=[mx.cpu()]);
mod.bind(train_data.provide_data, label_shapes=train_data.provide_label,
for_training=True)
arg_params_correct = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]),
'fc_0_bias' : mx.nd.array([.35, .35]),
'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]),
'fc_1_bias' : mx.nd.array([.60, .60])}
arg_params_missing = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]),
'fc_0_bias' : mx.nd.array([.35, .35]),
'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]])}
arg_params_extra = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]),
'fc_0_bias' : mx.nd.array([.35, .35]),
'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]),
'fc_1_bias' : mx.nd.array([.60, .60]),
'fc_2_weight': mx.nd.array([.60, .60])}
arg_params_missing_extra = {'fc_2_weight': mx.nd.array([.60, .60])}
# test regular set_params
mod.set_params(force_init=True, arg_params=arg_params_correct, aux_params={})
# test allow missing
mod.set_params(force_init=True, arg_params=arg_params_missing, aux_params={}, allow_missing=True)
assertRaises(RuntimeError, mod.set_params,
force_init=True, arg_params=arg_params_missing,
aux_params={}, allow_missing=False)
# test allow extra
mod.set_params(force_init=True, arg_params=arg_params_extra, aux_params={}, allow_missing=True, allow_extra=True)
assertRaises(ValueError, mod.set_params,
force_init=True, arg_params=arg_params_extra,
aux_params={}, allow_missing=True, allow_extra=False)
# test allow missing + extra,
assertRaises(RuntimeError, mod.set_params,
force_init=True, arg_params=arg_params_missing_extra,
aux_params={}, allow_missing=False, allow_extra=False)
# test allow missing + extra, this will throw a runtime error
assertRaises(ValueError, mod.set_params,
force_init=True, arg_params=arg_params_missing_extra,
aux_params={}, allow_missing=True, allow_extra=False)
def check_invalid_key_types_single(kv, key):
dns_val = mx.nd.ones(shape) * 2
rsp_val = dns_val.tostype('row_sparse')
assertRaises(MXNetError, kv.init, key, dns_val)
assertRaises(MXNetError, kv.push, key, dns_val)
assertRaises(MXNetError, kv.pull, key, dns_val)
assertRaises(MXNetError, kv.row_sparse_pull, key, rsp_val,
row_ids=mx.nd.array([1]))
def check_invalid_key_types_list(kv, key):
dns_val = [mx.nd.ones(shape) * 2] * len(key)
rsp_val = [val.tostype('row_sparse') for val in dns_val]
assertRaises(MXNetError, kv.init, key, dns_val)
assertRaises(MXNetError, kv.push, key, dns_val)
assertRaises(MXNetError, kv.pull, key, dns_val)
assertRaises(MXNetError, kv.row_sparse_pull, key, rsp_val,
row_ids=[mx.nd.array([1])] * len(key))
def test_get_and_pretrain_file_names():
assert len(
text.embedding.get_pretrained_file_names(
embedding_name='fasttext')) == 327
assert len(
text.embedding.get_pretrained_file_names(embedding_name='glove')) == 10
reg = text.embedding.get_pretrained_file_names(embedding_name=None)
assert len(reg['glove']) == 10
assert len(reg['fasttext']) == 327
assertRaises(KeyError, text.embedding.get_pretrained_file_names,
'unknown$$')
def check_invalid_key_types_list(kv, key):
dns_val = [mx.nd.ones(shape) * 2] * len(key)
rsp_val = [val.tostype('row_sparse') for val in dns_val]
assertRaises(MXNetError, kv.init, key, dns_val)
assertRaises(MXNetError, kv.push, key, dns_val)
assertRaises(MXNetError, kv.pull, key, dns_val)
assertRaises(MXNetError, kv.row_sparse_pull, key, rsp_val,
row_ids=[mx.nd.array([1])] * len(key))
def check_invalid_key_types_single(kv, key):
dns_val = mx.nd.ones(shape) * 2
rsp_val = dns_val.tostype('row_sparse')
assertRaises(MXNetError, kv.init, key, dns_val)
assertRaises(MXNetError, kv.push, key, dns_val)
assertRaises(MXNetError, kv.pull, key, dns_val)
assertRaises(MXNetError, kv.row_sparse_pull, key, rsp_val,
row_ids=mx.nd.array([1]))
def test_to_tensor():
# 3D Input
data_in = np.random.uniform(0, 255, (300, 300, 3)).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert_almost_equal(out_nd.asnumpy(), np.transpose(
data_in.astype(dtype=np.float32) / 255.0, (2, 0, 1)))
# 4D Input
data_in = np.random.uniform(0, 255, (5, 300, 300, 3)).astype(dtype=np.uint8)
out_nd = transforms.ToTensor()(nd.array(data_in, dtype='uint8'))
assert_almost_equal(out_nd.asnumpy(), np.transpose(
data_in.astype(dtype=np.float32) / 255.0, (0, 3, 1, 2)))
# Invalid Input
invalid_data_in = nd.random.uniform(0, 255, (5, 5, 300, 300, 3)).astype(dtype=np.uint8)
transformer = transforms.ToTensor()
assertRaises(MXNetError, transformer, invalid_data_in)
def test_image_iter_exception():
def check_cifar10_exception():
get_cifar10()
dataiter = mx.io.ImageRecordIter(
path_imgrec="data/cifar/train.rec",
mean_img="data/cifar/cifar10_mean.bin",
rand_crop=False,
and_mirror=False,
shuffle=False,
data_shape=(5, 28, 28),
batch_size=100,
preprocess_threads=4,
prefetch_buffer=1)
labelcount = [0 for i in range(10)]
batchcount = 0
for batch in dataiter:
pass
assertRaises(MXNetError, check_cifar10_exception)
def test_indices_to_tokens():
counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])
vocab = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=1,
unknown_token='<unknown>', reserved_tokens=None)
i1 = vocab.to_tokens(1)
assert i1 == 'c'
i2 = vocab.to_tokens([1])
assert i2 == ['c']
i3 = vocab.to_tokens([0, 0])
assert i3 == ['<unknown>', '<unknown>']
i4 = vocab.to_tokens([3, 0, 3, 2])
assert i4 == ['a', '<unknown>', 'a', 'b']
assertRaises(ValueError, vocab.to_tokens, 100)
def test_image_iter_exception():
def check_cifar10_exception():
get_cifar10()
dataiter = mx.io.ImageRecordIter(
path_imgrec="data/cifar/train.rec",
mean_img="data/cifar/cifar10_mean.bin",
rand_crop=False,
and_mirror=False,
shuffle=False,
data_shape=(5, 28, 28),
batch_size=100,
preprocess_threads=4,
prefetch_buffer=1)
labelcount = [0 for i in range(10)]
batchcount = 0
for batch in dataiter:
pass
assertRaises(MXNetError, check_cifar10_exception)
def test_indices_to_tokens():
counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])
indexer = text.indexer.TokenIndexer(counter, most_freq_count=None, min_freq=1,
unknown_token='<unknown>', reserved_tokens=None)
i1 = indexer.to_tokens(1)
assert i1 == 'c'
i2 = indexer.to_tokens([1])
assert i2 == ['c']
i3 = indexer.to_tokens([0, 0])
assert i3 == ['<unknown>', '<unknown>']
i4 = indexer.to_tokens([3, 0, 3, 2])
assert i4 == ['a', '<unknown>', 'a', 'b']
assertRaises(ValueError, indexer.to_tokens, 100)
def test_get_embedding_names_and_pretrain_files():
assert len(
TokenEmbedding.get_embedding_and_pretrained_file_names(
embedding_name='fasttext')) == 294
assert len(
TokenEmbedding.get_embedding_and_pretrained_file_names(
embedding_name='glove')) == 10
reg = TokenEmbedding.get_embedding_and_pretrained_file_names(
embedding_name=None)
assert len(reg['glove']) == 10
assert len(reg['fasttext']) == 294
assertRaises(KeyError,
TokenEmbedding.get_embedding_and_pretrained_file_names,
'unknown$$')
def test_sparse_parameter():
p = gluon.Parameter('weight', shape=(10, 10), stype='row_sparse', grad_stype='row_sparse')
p.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)])
row_id = mx.nd.arange(0, 10, ctx=mx.cpu(1))
assert len(p.list_grad()) == 2
# getting row_sparse data without trainer throws an exception
assertRaises(RuntimeError, p.list_row_sparse_data, row_id)
trainer = mx.gluon.Trainer([p], 'sgd')
assert len(p.list_row_sparse_data(row_id)) == 2
weight = p.row_sparse_data(row_id)
assert weight.context == mx.cpu(1)
assert weight.shape == (10, 10)
assert weight.stype == 'row_sparse'
assert p.var().name == 'weight'
assert p.var().attr('__storage_type__') == str(_STORAGE_TYPE_STR_TO_ID['row_sparse'])
assert p.grad(mx.cpu(0)).stype == 'row_sparse'
p.reset_ctx(ctx=[mx.cpu(1), mx.cpu(2)])
assert p.list_ctx() == [mx.cpu(1), mx.cpu(2)]
def test_parameter_invalid_access():
# cannot call data on row_sparse parameters
p0 = gluon.Parameter('weight', shape=(10, 10), stype='row_sparse', grad_stype='row_sparse')
p0.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)])
assertRaises(RuntimeError, p0.data)
assertRaises(RuntimeError, p0.list_data)
row_id = mx.nd.arange(0, 10)
# cannot call row_sparse_data on dense parameters
p1 = gluon.Parameter('weight', shape=(10, 10))
p1.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)])
assertRaises(RuntimeError, p1.row_sparse_data, row_id.copyto(mx.cpu(0)))
assertRaises(RuntimeError, p1.list_row_sparse_data, row_id)
def test_parameter_invalid_access():
# cannot call data on row_sparse parameters
p0 = gluon.Parameter('weight', shape=(10, 10), stype='row_sparse', grad_stype='row_sparse')
p0.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)])
assertRaises(RuntimeError, p0.data)
assertRaises(RuntimeError, p0.list_data)
row_id = mx.nd.arange(0, 10)
# cannot call row_sparse_data on dense parameters
p1 = gluon.Parameter('weight', shape=(10, 10))
p1.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)])
assertRaises(RuntimeError, p1.row_sparse_data, row_id.copyto(mx.cpu(0)))
assertRaises(RuntimeError, p1.list_row_sparse_data, row_id)
def test_sparse_parameter():
p = gluon.Parameter('weight', shape=(10, 10), stype='row_sparse', grad_stype='row_sparse')
p.initialize(init='xavier', ctx=[mx.cpu(0), mx.cpu(1)])
row_id = mx.nd.arange(0, 10, ctx=mx.cpu(1))
assert len(p.list_grad()) == 2
# getting row_sparse data without trainer throws an exception
assertRaises(RuntimeError, p.list_row_sparse_data, row_id)
trainer = mx.gluon.Trainer([p], 'sgd')
assert len(p.list_row_sparse_data(row_id)) == 2
weight = p.row_sparse_data(row_id)
assert weight.context == mx.cpu(1)
assert weight.shape == (10, 10)
assert weight.stype == 'row_sparse'
assert p.var().name == 'weight'
assert p.var().attr('__storage_type__') == str(_STORAGE_TYPE_STR_TO_ID['row_sparse'])
assert p.grad(mx.cpu(0)).stype == 'row_sparse'
p.reset_ctx(ctx=[mx.cpu(1), mx.cpu(2)])
assert p.list_ctx() == [mx.cpu(1), mx.cpu(2)]
def test_indices_to_tokens():
counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])
vocab = text.vocab.Vocabulary(counter,
most_freq_count=None,
min_freq=1,
unknown_token='<unknown>',
reserved_tokens=None)
i1 = vocab.to_tokens(1)
assert i1 == 'c'
i2 = vocab.to_tokens([1])
assert i2 == ['c']
i3 = vocab.to_tokens([0, 0])
assert i3 == ['<unknown>', '<unknown>']
i4 = vocab.to_tokens([3, 0, 3, 2])
assert i4 == ['a', '<unknown>', 'a', 'b']
assertRaises(ValueError, vocab.to_tokens, 100)
def _test_NDArrayIter_csr(csr_iter, csr_iter_empty_list, csr_iter_None, num_rows, batch_size):
num_batch = 0
for _, batch_empty_list, batch_empty_None in zip(csr_iter, csr_iter_empty_list, csr_iter_None):
assert not batch_empty_list.label, 'label is not empty list'
assert not batch_empty_None.label, 'label is not empty list'
num_batch += 1
assert(num_batch == num_rows // batch_size)
assertRaises(StopIteration, csr_iter.next)
assertRaises(StopIteration, csr_iter_empty_list.next)
assertRaises(StopIteration, csr_iter_None.next)
def _test_NDArrayIter_csr(csr_iter, csr_iter_empty_list, csr_iter_None, num_rows, batch_size):
num_batch = 0
for _, batch_empty_list, batch_empty_None in zip(csr_iter, csr_iter_empty_list, csr_iter_None):
assert not batch_empty_list.label, 'label is not empty list'
assert not batch_empty_None.label, 'label is not empty list'
num_batch += 1
assert(num_batch == num_rows // batch_size)
assertRaises(StopIteration, csr_iter.next)
assertRaises(StopIteration, csr_iter_empty_list.next)
assertRaises(StopIteration, csr_iter_None.next)
def test_buffer_load():
nrepeat = 10
with TemporaryDirectory(prefix='test_buffer_load_') as tmpdir:
for repeat in range(nrepeat):
# test load_buffer as list
data = []
for i in range(10):
data.append(random_ndarray(np.random.randint(1, 5)))
fname = os.path.join(tmpdir, 'list_{0}.param'.format(repeat))
mx.nd.save(fname, data)
with open(fname, 'rb') as dfile:
buf_data = dfile.read()
data2 = mx.nd.load_frombuffer(buf_data)
assert len(data) == len(data2)
for x, y in zip(data, data2):
assert np.sum(x.asnumpy() != y.asnumpy()) == 0
# test garbage values
assertRaises(mx.base.MXNetError, mx.nd.load_frombuffer,
buf_data[:-10])
# test load_buffer as dict
dmap = {'ndarray xx %s' % i: x for i, x in enumerate(data)}
fname = os.path.join(tmpdir, 'dict_{0}.param'.format(repeat))
mx.nd.save(fname, dmap)
with open(fname, 'rb') as dfile:
buf_dmap = dfile.read()
dmap2 = mx.nd.load_frombuffer(buf_dmap)
assert len(dmap2) == len(dmap)
for k, x in dmap.items():
y = dmap2[k]
assert np.sum(x.asnumpy() != y.asnumpy()) == 0
# test garbage values
assertRaises(mx.base.MXNetError, mx.nd.load_frombuffer,
buf_dmap[:-10])
# we expect the single ndarray to be converted into a list containing the ndarray
single_ndarray = data[0]
fname = os.path.join(tmpdir, 'single_{0}.param'.format(repeat))
mx.nd.save(fname, single_ndarray)
with open(fname, 'rb') as dfile:
buf_single_ndarray = dfile.read()
single_ndarray_loaded = mx.nd.load_frombuffer(
buf_single_ndarray)
assert len(single_ndarray_loaded) == 1
single_ndarray_loaded = single_ndarray_loaded[0]
assert np.sum(single_ndarray.asnumpy() !=
single_ndarray_loaded.asnumpy()) == 0
# test garbage values
assertRaises(mx.base.MXNetError, mx.nd.load_frombuffer,
buf_single_ndarray[:-10])
def test_random_rotation():
# test exceptions for probability input outside of [0,1]
assertRaises(ValueError, transforms.RandomRotation, [-10, 10.], rotate_with_proba=1.1)
assertRaises(ValueError, transforms.RandomRotation, [-10, 10.], rotate_with_proba=-0.3)
# test `forward`
transformer = transforms.RandomRotation([-10, 10.])
assertRaises(TypeError, transformer, mx.nd.ones((3, 30, 60), dtype='uint8'))
single_image = mx.nd.ones((3, 30, 60), dtype='float32')
single_output = transformer(single_image)
assert same(single_output.shape, (3, 30, 60))
batch_image = mx.nd.ones((3, 3, 30, 60), dtype='float32')
batch_output = transformer(batch_image)
assert same(batch_output.shape, (3, 3, 30, 60))
# test identity (rotate_with_proba = 0)
transformer = transforms.RandomRotation([-100., 100.], rotate_with_proba=0.0)
data = mx.nd.random_normal(shape=(3, 30, 60))
assert_almost_equal(data, transformer(data))
def test_buffer_load():
nrepeat = 10
with TemporaryDirectory(prefix='test_buffer_load_') as tmpdir:
for repeat in range(nrepeat):
# test load_buffer as list
data = []
for i in range(10):
data.append(random_ndarray(np.random.randint(1, 5)))
fname = os.path.join(tmpdir, 'list_{0}.param'.format(repeat))
mx.nd.save(fname, data)
with open(fname, 'rb') as dfile:
buf_data = dfile.read()
data2 = mx.nd.load_frombuffer(buf_data)
assert len(data) == len(data2)
for x, y in zip(data, data2):
assert np.sum(x.asnumpy() != y.asnumpy()) == 0
# test garbage values
assertRaises(mx.base.MXNetError, mx.nd.load_frombuffer, buf_data[:-10])
# test load_buffer as dict
dmap = {'ndarray xx %s' % i : x for i, x in enumerate(data)}
fname = os.path.join(tmpdir, 'dict_{0}.param'.format(repeat))
mx.nd.save(fname, dmap)
with open(fname, 'rb') as dfile:
buf_dmap = dfile.read()
dmap2 = mx.nd.load_frombuffer(buf_dmap)
assert len(dmap2) == len(dmap)
for k, x in dmap.items():
y = dmap2[k]
assert np.sum(x.asnumpy() != y.asnumpy()) == 0
# test garbage values
assertRaises(mx.base.MXNetError, mx.nd.load_frombuffer, buf_dmap[:-10])
# we expect the single ndarray to be converted into a list containing the ndarray
single_ndarray = data[0]
fname = os.path.join(tmpdir, 'single_{0}.param'.format(repeat))
mx.nd.save(fname, single_ndarray)
with open(fname, 'rb') as dfile:
buf_single_ndarray = dfile.read()
single_ndarray_loaded = mx.nd.load_frombuffer(buf_single_ndarray)
assert len(single_ndarray_loaded) == 1
single_ndarray_loaded = single_ndarray_loaded[0]
assert np.sum(single_ndarray.asnumpy() != single_ndarray_loaded.asnumpy()) == 0
# test garbage values
assertRaises(mx.base.MXNetError, mx.nd.load_frombuffer, buf_single_ndarray[:-10])
def test_composite_embedding_with_one_embedding():
embed_root = 'embeddings'
embed_name = 'my_embed'
elem_delim = '\t'
pretrain_file = 'my_pretrain_file1.txt'
_mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim, pretrain_file)
pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)
my_embed = text.embedding.CustomEmbedding(pretrain_file_path, elem_delim,
init_unknown_vec=nd.ones)
counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])
v1 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=1, unknown_token='<unk>',
reserved_tokens=['<pad>'])
ce1 = text.embedding.CompositeEmbedding(v1, my_embed)
assert ce1.token_to_idx == {'<unk>': 0, '<pad>': 1, 'c': 2, 'b': 3, 'a': 4, 'some_word$': 5}
assert ce1.idx_to_token == ['<unk>', '<pad>', 'c', 'b', 'a', 'some_word$']
assert_almost_equal(ce1.idx_to_vec.asnumpy(),
np.array([[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[0.6, 0.7, 0.8, 0.9, 1],
[0.1, 0.2, 0.3, 0.4, 0.5],
[1, 1, 1, 1, 1]])
)
assert ce1.vec_len == 5
assert ce1.reserved_tokens == ['<pad>']
assert_almost_equal(ce1.get_vecs_by_tokens('c').asnumpy(),
np.array([1, 1, 1, 1, 1])
)
assert_almost_equal(ce1.get_vecs_by_tokens(['c']).asnumpy(),
np.array([[1, 1, 1, 1, 1]])
)
assert_almost_equal(ce1.get_vecs_by_tokens(['a', 'not_exist']).asnumpy(),
np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
[1, 1, 1, 1, 1]])
)
assert_almost_equal(ce1.get_vecs_by_tokens(['a', 'b']).asnumpy(),
np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
[0.6, 0.7, 0.8, 0.9, 1]])
)
assert_almost_equal(ce1.get_vecs_by_tokens(['A', 'b']).asnumpy(),
np.array([[1, 1, 1, 1, 1],
[0.6, 0.7, 0.8, 0.9, 1]])
)
assert_almost_equal(ce1.get_vecs_by_tokens(['A', 'b'], lower_case_backup=True).asnumpy(),
np.array([[0.1, 0.2, 0.3, 0.4, 0.5],
[0.6, 0.7, 0.8, 0.9, 1]])
)
ce1.update_token_vectors(['a', 'b'],
nd.array([[2, 2, 2, 2, 2],
[3, 3, 3, 3, 3]])
)
assert_almost_equal(ce1.idx_to_vec.asnumpy(),
np.array([[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[3, 3, 3, 3, 3],
[2, 2, 2, 2, 2],
[1, 1, 1, 1, 1]])
)
assertRaises(ValueError, ce1.update_token_vectors, 'unknown$$$', nd.array([0, 0, 0, 0, 0]))
assertRaises(AssertionError, ce1.update_token_vectors, '<unk>',
nd.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]))
assertRaises(AssertionError, ce1.update_token_vectors, '<unk>', nd.array([0]))
ce1.update_token_vectors(['<unk>'], nd.array([0, 0, 0, 0, 0]))
assert_almost_equal(ce1.idx_to_vec.asnumpy(),
np.array([[0, 0, 0, 0, 0],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[3, 3, 3, 3, 3],
[2, 2, 2, 2, 2],
[1, 1, 1, 1, 1]])
)
ce1.update_token_vectors(['<unk>'], nd.array([[10, 10, 10, 10, 10]]))
assert_almost_equal(ce1.idx_to_vec.asnumpy(),
np.array([[10, 10, 10, 10, 10],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[3, 3, 3, 3, 3],
[2, 2, 2, 2, 2],
[1, 1, 1, 1, 1]])
)
ce1.update_token_vectors('<unk>', nd.array([0, 0, 0, 0, 0]))
assert_almost_equal(ce1.idx_to_vec.asnumpy(),
np.array([[0, 0, 0, 0, 0],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[3, 3, 3, 3, 3],
[2, 2, 2, 2, 2],
[1, 1, 1, 1, 1]])
)
ce1.update_token_vectors('<unk>', nd.array([[10, 10, 10, 10, 10]]))
assert_almost_equal(ce1.idx_to_vec.asnumpy(),
np.array([[10, 10, 10, 10, 10],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[3, 3, 3, 3, 3],
[2, 2, 2, 2, 2],
[1, 1, 1, 1, 1]])
)
def check_invalid_rsp_pull_list(kv, key):
dns_val = [mx.nd.ones(shape) * 2] * len(key)
assertRaises(MXNetError, kv.row_sparse_pull, key, out=dns_val,
row_ids=[mx.nd.array([1])] * len(key))
def check_invalid_rsp_pull_single(kv, key):
dns_val = mx.nd.ones(shape) * 2
assertRaises(MXNetError, kv.row_sparse_pull,
key, out=dns_val, row_ids=mx.nd.array([1]))
def test_LibSVMIter():
def check_libSVMIter_synthetic():
cwd = os.getcwd()
data_path = os.path.join(cwd, 'data.t')
label_path = os.path.join(cwd, 'label.t')
with open(data_path, 'w') as fout:
fout.write('1.0 0:0.5 2:1.2\n')
fout.write('-2.0\n')
fout.write('-3.0 0:0.6 1:2.4 2:1.2\n')
fout.write('4 2:-1.2\n')
with open(label_path, 'w') as fout:
fout.write('1.0\n')
fout.write('-2.0 0:0.125\n')
fout.write('-3.0 2:1.2\n')
fout.write('4 1:1.0 2:-1.2\n')
data_dir = os.path.join(cwd, 'data')
data_train = mx.io.LibSVMIter(data_libsvm=data_path,
label_libsvm=label_path,
data_shape=(3, ),
label_shape=(3, ),
batch_size=3)
first = mx.nd.array([[0.5, 0., 1.2], [0., 0., 0.], [0.6, 2.4, 1.2]])
second = mx.nd.array([[0., 0., -1.2], [0.5, 0., 1.2], [0., 0., 0.]])
i = 0
for batch in iter(data_train):
expected = first.asnumpy() if i == 0 else second.asnumpy()
data = data_train.getdata()
data.check_format(True)
assert_almost_equal(data.asnumpy(), expected)
i += 1
def check_libSVMIter_news_data():
news_metadata = {
'name': 'news20.t',
'origin_name': 'news20.t.bz2',
'url':
"https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/news20.t.bz2",
'feature_dim': 62060 + 1,
'num_classes': 20,
'num_examples': 3993,
}
batch_size = 33
num_examples = news_metadata['num_examples']
data_dir = os.path.join(os.getcwd(), 'data')
get_bz2_data(data_dir, news_metadata['name'], news_metadata['url'],
news_metadata['origin_name'])
path = os.path.join(data_dir, news_metadata['name'])
data_train = mx.io.LibSVMIter(
data_libsvm=path,
data_shape=(news_metadata['feature_dim'], ),
batch_size=batch_size)
for epoch in range(2):
num_batches = 0
for batch in data_train:
# check the range of labels
data = batch.data[0]
label = batch.label[0]
data.check_format(True)
assert (np.sum(label.asnumpy() > 20) == 0)
assert (np.sum(label.asnumpy() <= 0) == 0)
num_batches += 1
expected_num_batches = num_examples / batch_size
assert (num_batches == int(expected_num_batches)), num_batches
data_train.reset()
def check_libSVMIter_exception():
cwd = os.getcwd()
data_path = os.path.join(cwd, 'data.t')
label_path = os.path.join(cwd, 'label.t')
with open(data_path, 'w') as fout:
fout.write('1.0 0:0.5 2:1.2\n')
fout.write('-2.0\n')
# Below line has a neg indice. Should throw an exception
fout.write('-3.0 -1:0.6 1:2.4 2:1.2\n')
fout.write('4 2:-1.2\n')
with open(label_path, 'w') as fout:
fout.write('1.0\n')
fout.write('-2.0 0:0.125\n')
fout.write('-3.0 2:1.2\n')
fout.write('4 1:1.0 2:-1.2\n')
data_dir = os.path.join(cwd, 'data')
data_train = mx.io.LibSVMIter(data_libsvm=data_path,
label_libsvm=label_path,
data_shape=(3, ),
label_shape=(3, ),
batch_size=3)
for batch in iter(data_train):
data_train.get_data().asnumpy()
check_libSVMIter_synthetic()
check_libSVMIter_news_data()
assertRaises(MXNetError, check_libSVMIter_exception)
def test_symbol_bool():
x = mx.symbol.Variable('x')
assertRaises(NotImplementedForSymbol, bool, x)
def test_composite_embedding_with_one_embedding():
embed_root = 'embeddings'
embed_name = 'my_embed'
elem_delim = '\t'
pretrain_file = 'my_pretrain_file1.txt'
_mk_my_pretrain_file(os.path.join(embed_root, embed_name), elem_delim,
pretrain_file)
pretrain_file_path = os.path.join(embed_root, embed_name, pretrain_file)
my_embed = text.embedding.CustomEmbedding(pretrain_file_path,
elem_delim,
init_unknown_vec=nd.ones)
counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])
v1 = text.vocab.Vocabulary(counter,
most_freq_count=None,
min_freq=1,
unknown_token='<unk>',
reserved_tokens=['<pad>'])
ce1 = text.embedding.CompositeEmbedding(v1, my_embed)
assert ce1.token_to_idx == {
'<unk>': 0,
'<pad>': 1,
'c': 2,
'b': 3,
'a': 4,
'some_word$': 5
}
assert ce1.idx_to_token == ['<unk>', '<pad>', 'c', 'b', 'a', 'some_word$']
assert_almost_equal(
ce1.idx_to_vec.asnumpy(),
np.array([[1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1],
[0.6, 0.7, 0.8, 0.9, 1], [0.1, 0.2, 0.3, 0.4, 0.5],
[1, 1, 1, 1, 1]]))
assert ce1.vec_len == 5
assert ce1.reserved_tokens == ['<pad>']
assert_almost_equal(
ce1.get_vecs_by_tokens('c').asnumpy(), np.array([1, 1, 1, 1, 1]))
assert_almost_equal(
ce1.get_vecs_by_tokens(['c']).asnumpy(), np.array([[1, 1, 1, 1, 1]]))
assert_almost_equal(
ce1.get_vecs_by_tokens(['a', 'not_exist']).asnumpy(),
np.array([[0.1, 0.2, 0.3, 0.4, 0.5], [1, 1, 1, 1, 1]]))
assert_almost_equal(
ce1.get_vecs_by_tokens(['a', 'b']).asnumpy(),
np.array([[0.1, 0.2, 0.3, 0.4, 0.5], [0.6, 0.7, 0.8, 0.9, 1]]))
assert_almost_equal(
ce1.get_vecs_by_tokens(['A', 'b']).asnumpy(),
np.array([[1, 1, 1, 1, 1], [0.6, 0.7, 0.8, 0.9, 1]]))
assert_almost_equal(
ce1.get_vecs_by_tokens(['A', 'b'], lower_case_backup=True).asnumpy(),
np.array([[0.1, 0.2, 0.3, 0.4, 0.5], [0.6, 0.7, 0.8, 0.9, 1]]))
ce1.update_token_vectors(['a', 'b'],
nd.array([[2, 2, 2, 2, 2], [3, 3, 3, 3, 3]]))
assert_almost_equal(
ce1.idx_to_vec.asnumpy(),
np.array([[1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1],
[3, 3, 3, 3, 3], [2, 2, 2, 2, 2], [1, 1, 1, 1, 1]]))
assertRaises(ValueError, ce1.update_token_vectors, 'unknown$$$',
nd.array([0, 0, 0, 0, 0]))
assertRaises(AssertionError, ce1.update_token_vectors, '<unk>',
nd.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]))
assertRaises(AssertionError, ce1.update_token_vectors, '<unk>',
nd.array([0]))
ce1.update_token_vectors(['<unk>'], nd.array([0, 0, 0, 0, 0]))
assert_almost_equal(
ce1.idx_to_vec.asnumpy(),
np.array([[0, 0, 0, 0, 0], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1],
[3, 3, 3, 3, 3], [2, 2, 2, 2, 2], [1, 1, 1, 1, 1]]))
ce1.update_token_vectors(['<unk>'], nd.array([[10, 10, 10, 10, 10]]))
assert_almost_equal(
ce1.idx_to_vec.asnumpy(),
np.array([[10, 10, 10, 10, 10], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1],
[3, 3, 3, 3, 3], [2, 2, 2, 2, 2], [1, 1, 1, 1, 1]]))
ce1.update_token_vectors('<unk>', nd.array([0, 0, 0, 0, 0]))
assert_almost_equal(
ce1.idx_to_vec.asnumpy(),
np.array([[0, 0, 0, 0, 0], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1],
[3, 3, 3, 3, 3], [2, 2, 2, 2, 2], [1, 1, 1, 1, 1]]))
ce1.update_token_vectors('<unk>', nd.array([[10, 10, 10, 10, 10]]))
assert_almost_equal(
ce1.idx_to_vec.asnumpy(),
np.array([[10, 10, 10, 10, 10], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1],
[3, 3, 3, 3, 3], [2, 2, 2, 2, 2], [1, 1, 1, 1, 1]]))
def test_LibSVMIter():
def check_libSVMIter_synthetic():
cwd = os.getcwd()
data_path = os.path.join(cwd, 'data.t')
label_path = os.path.join(cwd, 'label.t')
with open(data_path, 'w') as fout:
fout.write('1.0 0:0.5 2:1.2\n')
fout.write('-2.0\n')
fout.write('-3.0 0:0.6 1:2.4 2:1.2\n')
fout.write('4 2:-1.2\n')
with open(label_path, 'w') as fout:
fout.write('1.0\n')
fout.write('-2.0 0:0.125\n')
fout.write('-3.0 2:1.2\n')
fout.write('4 1:1.0 2:-1.2\n')
data_dir = os.path.join(cwd, 'data')
data_train = mx.io.LibSVMIter(data_libsvm=data_path, label_libsvm=label_path,
data_shape=(3, ), label_shape=(3, ), batch_size=3)
first = mx.nd.array([[0.5, 0., 1.2], [0., 0., 0.], [0.6, 2.4, 1.2]])
second = mx.nd.array([[0., 0., -1.2], [0.5, 0., 1.2], [0., 0., 0.]])
i = 0
for batch in iter(data_train):
expected = first.asnumpy() if i == 0 else second.asnumpy()
data = data_train.getdata()
data.check_format(True)
assert_almost_equal(data.asnumpy(), expected)
i += 1
def check_libSVMIter_news_data():
news_metadata = {
'name': 'news20.t',
'origin_name': 'news20.t.bz2',
'url': "https://apache-mxnet.s3-accelerate.dualstack.amazonaws.com/gluon/dataset/news20.t.bz2",
'feature_dim': 62060 + 1,
'num_classes': 20,
'num_examples': 3993,
}
batch_size = 33
num_examples = news_metadata['num_examples']
data_dir = os.path.join(os.getcwd(), 'data')
get_bz2_data(data_dir, news_metadata['name'], news_metadata['url'],
news_metadata['origin_name'])
path = os.path.join(data_dir, news_metadata['name'])
data_train = mx.io.LibSVMIter(data_libsvm=path, data_shape=(news_metadata['feature_dim'],),
batch_size=batch_size)
for epoch in range(2):
num_batches = 0
for batch in data_train:
# check the range of labels
data = batch.data[0]
label = batch.label[0]
data.check_format(True)
assert(np.sum(label.asnumpy() > 20) == 0)
assert(np.sum(label.asnumpy() <= 0) == 0)
num_batches += 1
expected_num_batches = num_examples / batch_size
assert(num_batches == int(expected_num_batches)), num_batches
data_train.reset()
def check_libSVMIter_exception():
cwd = os.getcwd()
data_path = os.path.join(cwd, 'data.t')
label_path = os.path.join(cwd, 'label.t')
with open(data_path, 'w') as fout:
fout.write('1.0 0:0.5 2:1.2\n')
fout.write('-2.0\n')
# Below line has a neg indice. Should throw an exception
fout.write('-3.0 -1:0.6 1:2.4 2:1.2\n')
fout.write('4 2:-1.2\n')
with open(label_path, 'w') as fout:
fout.write('1.0\n')
fout.write('-2.0 0:0.125\n')
fout.write('-3.0 2:1.2\n')
fout.write('4 1:1.0 2:-1.2\n')
data_dir = os.path.join(cwd, 'data')
data_train = mx.io.LibSVMIter(data_libsvm=data_path, label_libsvm=label_path,
data_shape=(3, ), label_shape=(3, ), batch_size=3)
for batch in iter(data_train):
data_train.get_data().asnumpy()
check_libSVMIter_synthetic()
check_libSVMIter_news_data()
assertRaises(MXNetError, check_libSVMIter_exception)
def test_sparse_nd_check_format():
""" test check_format for sparse ndarray """
shape = rand_shape_2d()
stypes = ["csr", "row_sparse"]
for stype in stypes:
arr, _ = rand_sparse_ndarray(shape, stype)
arr.check_format()
arr = mx.nd.sparse.zeros(stype, shape)
arr.check_format()
# CSR format index pointer array should be less than the number of rows
shape = (3, 4)
data_list = [7, 8, 9]
indices_list = [0, 2, 1]
indptr_list = [0, 5, 2, 3]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format indices should be in ascending order per row
indices_list = [2, 1, 1]
indptr_list = [0, 2, 2, 3]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format indptr should end with value equal with size of indices
indices_list = [1, 2, 1]
indptr_list = [0, 2, 2, 4]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format indices should not be negative
indices_list = [0, 2, 1]
indptr_list = [0, -2, 2, 3]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# Row Sparse format indices should be less than the number of rows
shape = (3, 2)
data_list = [[1, 2], [3, 4]]
indices_list = [1, 4]
a = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# Row Sparse format indices should be in ascending order
indices_list = [1, 0]
a = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# Row Sparse format indices should not be negative
indices_list = [1, -2]
a = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
def test_sparse_nd_check_format():
""" test check_format for sparse ndarray """
shape = rand_shape_2d()
stypes = ["csr", "row_sparse"]
for stype in stypes:
arr, _ = rand_sparse_ndarray(shape, stype)
arr.check_format()
arr = mx.nd.sparse.zeros(stype, shape)
arr.check_format()
# CSR format index pointer array should be less than the number of rows
shape = (3, 4)
data_list = [7, 8, 9]
indices_list = [0, 2, 1]
indptr_list = [0, 5, 2, 3]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list),
shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format indices should be in ascending order per row
indices_list = [2, 1, 1]
indptr_list = [0, 2, 2, 3]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list),
shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format indptr should end with value equal with size of indices
indices_list = [1, 2, 1]
indptr_list = [0, 2, 2, 4]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list),
shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format indices should not be negative
indices_list = [0, 2, 1]
indptr_list = [0, -2, 2, 3]
a = mx.nd.sparse.csr_matrix((data_list, indices_list, indptr_list),
shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# CSR format should be 2 Dimensional.
a = mx.nd.array([1, 2, 3])
assertRaises(ValueError, a.tostype, 'csr')
a = mx.nd.array([[[1, 2, 3]]])
assertRaises(ValueError, a.tostype, 'csr')
# Row Sparse format indices should be less than the number of rows
shape = (3, 2)
data_list = [[1, 2], [3, 4]]
indices_list = [1, 4]
a = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# Row Sparse format indices should be in ascending order
indices_list = [1, 0]
a = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
# Row Sparse format indices should not be negative
indices_list = [1, -2]
a = mx.nd.sparse.row_sparse_array((data_list, indices_list), shape=shape)
assertRaises(mx.base.MXNetError, a.check_format)
def test_vocabulary():
counter = Counter(['a', 'b', 'b', 'c', 'c', 'c', 'some_word$'])
v1 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=1, unknown_token='<unk>',
reserved_tokens=None)
assert len(v1) == 5
assert v1.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3, 'some_word$': 4}
assert v1.idx_to_token[1] == 'c'
assert v1.unknown_token == '<unk>'
assert v1.reserved_tokens is None
v2 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=2, unknown_token='<unk>',
reserved_tokens=None)
assert len(v2) == 3
assert v2.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2}
assert v2.idx_to_token[1] == 'c'
assert v2.unknown_token == '<unk>'
assert v2.reserved_tokens is None
v3 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=100, unknown_token='<unk>',
reserved_tokens=None)
assert len(v3) == 1
assert v3.token_to_idx == {'<unk>': 0}
assert v3.idx_to_token[0] == '<unk>'
assert v3.unknown_token == '<unk>'
assert v3.reserved_tokens is None
v4 = text.vocab.Vocabulary(counter, most_freq_count=2, min_freq=1, unknown_token='<unk>',
reserved_tokens=None)
assert len(v4) == 3
assert v4.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2}
assert v4.idx_to_token[1] == 'c'
assert v4.unknown_token == '<unk>'
assert v4.reserved_tokens is None
v5 = text.vocab.Vocabulary(counter, most_freq_count=3, min_freq=1, unknown_token='<unk>',
reserved_tokens=None)
assert len(v5) == 4
assert v5.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3}
assert v5.idx_to_token[1] == 'c'
assert v5.unknown_token == '<unk>'
assert v5.reserved_tokens is None
v6 = text.vocab.Vocabulary(counter, most_freq_count=100, min_freq=1, unknown_token='<unk>',
reserved_tokens=None)
assert len(v6) == 5
assert v6.token_to_idx == {'<unk>': 0, 'c': 1, 'b': 2, 'a': 3,
'some_word$': 4}
assert v6.idx_to_token[1] == 'c'
assert v6.unknown_token == '<unk>'
assert v6.reserved_tokens is None
v7 = text.vocab.Vocabulary(counter, most_freq_count=1, min_freq=2, unknown_token='<unk>',
reserved_tokens=None)
assert len(v7) == 2
assert v7.token_to_idx == {'<unk>': 0, 'c': 1}
assert v7.idx_to_token[1] == 'c'
assert v7.unknown_token == '<unk>'
assert v7.reserved_tokens is None
assertRaises(AssertionError, text.vocab.Vocabulary, counter, most_freq_count=None,
min_freq=0, unknown_token='<unknown>', reserved_tokens=['b'])
assertRaises(AssertionError, text.vocab.Vocabulary, counter, most_freq_count=None,
min_freq=1, unknown_token='<unknown>', reserved_tokens=['b', 'b'])
assertRaises(AssertionError, text.vocab.Vocabulary, counter, most_freq_count=None,
min_freq=1, unknown_token='<unknown>', reserved_tokens=['b', '<unknown>'])
v8 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=1, unknown_token='<unknown>',
reserved_tokens=['b'])
assert len(v8) == 5
assert v8.token_to_idx == {'<unknown>': 0, 'b': 1, 'c': 2, 'a': 3, 'some_word$': 4}
assert v8.idx_to_token[1] == 'b'
assert v8.unknown_token == '<unknown>'
assert v8.reserved_tokens == ['b']
v9 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=2, unknown_token='<unk>',
reserved_tokens=['b', 'a'])
assert len(v9) == 4
assert v9.token_to_idx == {'<unk>': 0, 'b': 1, 'a': 2, 'c': 3}
assert v9.idx_to_token[1] == 'b'
assert v9.unknown_token == '<unk>'
assert v9.reserved_tokens == ['b', 'a']
v10 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=100, unknown_token='<unk>',
reserved_tokens=['b', 'c'])
assert len(v10) == 3
assert v10.token_to_idx == {'<unk>': 0, 'b': 1, 'c': 2}
assert v10.idx_to_token[1] == 'b'
assert v10.unknown_token == '<unk>'
assert v10.reserved_tokens == ['b', 'c']
v11 = text.vocab.Vocabulary(counter, most_freq_count=1, min_freq=2, unknown_token='<unk>',
reserved_tokens=['<pad>', 'b'])
assert len(v11) == 4
assert v11.token_to_idx == {'<unk>': 0, '<pad>': 1, 'b': 2, 'c': 3}
assert v11.idx_to_token[1] == '<pad>'
assert v11.unknown_token == '<unk>'
assert v11.reserved_tokens == ['<pad>', 'b']
v12 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=2, unknown_token='b',
reserved_tokens=['<pad>'])
assert len(v12) == 3
assert v12.token_to_idx == {'b': 0, '<pad>': 1, 'c': 2}
assert v12.idx_to_token[1] == '<pad>'
assert v12.unknown_token == 'b'
assert v12.reserved_tokens == ['<pad>']
v13 = text.vocab.Vocabulary(counter, most_freq_count=None, min_freq=2, unknown_token='a',
reserved_tokens=['<pad>'])
assert len(v13) == 4
assert v13.token_to_idx == {'a': 0, '<pad>': 1, 'c': 2, 'b': 3}
assert v13.idx_to_token[1] == '<pad>'
assert v13.unknown_token == 'a'
assert v13.reserved_tokens == ['<pad>']
counter_tuple = Counter([('a', 'a'), ('b', 'b'), ('b', 'b'), ('c', 'c'), ('c', 'c'), ('c', 'c'),
('some_word$', 'some_word$')])
v14 = text.vocab.Vocabulary(counter_tuple, most_freq_count=None, min_freq=1,
unknown_token=('<unk>', '<unk>'), reserved_tokens=None)
assert len(v14) == 5
assert v14.token_to_idx == {('<unk>', '<unk>'): 0, ('c', 'c'): 1, ('b', 'b'): 2, ('a', 'a'): 3,
('some_word$', 'some_word$'): 4}
assert v14.idx_to_token[1] == ('c', 'c')
assert v14.unknown_token == ('<unk>', '<unk>')
assert v14.reserved_tokens is None