def test_full(): shape = (3, 4, 5) value = 7 dtype = "float32" for target, ctx in ctx_list(): data = sym.Variable("data", dtype=dtype) # full_like s = sym.full_like(data=data, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target, {"data": shape}) m = graph_runtime.create(graph, lib, ctx) m.run(data=np.random.uniform(size=shape).astype(dtype)) out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) np.testing.assert_allclose(out.asnumpy(), np.full(shape, fill_value=value, dtype=dtype), atol=1e-5, rtol=1e-5) # ones_like s = sym.ones_like(data=data, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target, {"data": shape}) m = graph_runtime.create(graph, lib, ctx) m.run(data=np.random.uniform(size=shape).astype(dtype)) out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) np.testing.assert_allclose(out.asnumpy(), np.full(shape, fill_value=1, dtype=dtype), atol=1e-5, rtol=1e-5) # zeros_like s = sym.zeros_like(data=data, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target, {"data": shape}) m = graph_runtime.create(graph, lib, ctx) m.run(data=np.random.uniform(size=shape).astype(dtype)) out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) np.testing.assert_allclose(out.asnumpy(), np.full(shape, fill_value=0, dtype=dtype), atol=1e-5, rtol=1e-5) # full s = sym.full(shape=shape, dtype=dtype, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target) m = graph_runtime.create(graph, lib, ctx) m.run() out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) np.testing.assert_allclose(out.asnumpy(), np.full(shape, fill_value=value, dtype=dtype), atol=1e-5, rtol=1e-5) # ones s = sym.ones(shape=shape, dtype=dtype, name="s") graph, lib, _ = nnvm.compiler.build(s, target) m = graph_runtime.create(graph, lib, ctx) m.run() out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) np.testing.assert_allclose(out.asnumpy(), np.full(shape, fill_value=1, dtype=dtype), atol=1e-5, rtol=1e-5) # zeros s = sym.zeros(shape=shape, dtype=dtype, name="s") graph, lib, _ = nnvm.compiler.build(s, target) m = graph_runtime.create(graph, lib, ctx) m.run() out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) np.testing.assert_allclose(out.asnumpy(), np.full(shape, fill_value=0, dtype=dtype), atol=1e-5, rtol=1e-5)
def test_full(): shape = (3, 4, 5) value = 7 dtype = "float32" for target, ctx in ctx_list(): data = sym.Variable("data", dtype=dtype) # full_like s = sym.full_like(data=data, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target, {"data": shape}) m = graph_runtime.create(graph, lib, ctx) m.run(data=np.random.uniform(size=shape).astype(dtype)) out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) tvm.testing.assert_allclose( out.asnumpy(), np.full(shape, fill_value=value, dtype=dtype), atol=1e-5, rtol=1e-5) # ones_like s = sym.ones_like(data=data, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target, {"data": shape}) m = graph_runtime.create(graph, lib, ctx) m.run(data=np.random.uniform(size=shape).astype(dtype)) out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) tvm.testing.assert_allclose( out.asnumpy(), np.full(shape, fill_value=1, dtype=dtype), atol=1e-5, rtol=1e-5) # zeros_like s = sym.zeros_like(data=data, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target, {"data": shape}) m = graph_runtime.create(graph, lib, ctx) m.run(data=np.random.uniform(size=shape).astype(dtype)) out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) tvm.testing.assert_allclose( out.asnumpy(), np.full(shape, fill_value=0, dtype=dtype), atol=1e-5, rtol=1e-5) # full s = sym.full(shape=shape, dtype=dtype, fill_value=value, name="s") graph, lib, _ = nnvm.compiler.build(s, target) m = graph_runtime.create(graph, lib, ctx) m.run() out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) tvm.testing.assert_allclose( out.asnumpy(), np.full(shape, fill_value=value, dtype=dtype), atol=1e-5, rtol=1e-5) # ones s = sym.ones(shape=shape, dtype=dtype, name="s") graph, lib, _ = nnvm.compiler.build(s, target) m = graph_runtime.create(graph, lib, ctx) m.run() out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) tvm.testing.assert_allclose( out.asnumpy(), np.full(shape, fill_value=1, dtype=dtype), atol=1e-5, rtol=1e-5) # zeros s = sym.zeros(shape=shape, dtype=dtype, name="s") graph, lib, _ = nnvm.compiler.build(s, target) m = graph_runtime.create(graph, lib, ctx) m.run() out = m.get_output(0, tvm.nd.empty(shape, dtype=dtype)) tvm.testing.assert_allclose( out.asnumpy(), np.full(shape, fill_value=0, dtype=dtype), atol=1e-5, rtol=1e-5)
def test_cnn_gradients(): # input data h = 128 w = 128 data_shape = (1000, 3, h, w) data = sym.Variable('data', shape=data_shape, dtype=0) # conv2d num_channels = 64 kernel_size = 32 conv_w_shape = (num_channels, 3, kernel_size, kernel_size) conv_b_shape = (num_channels, ) conv_w = sym.Variable('conv_w', shape=conv_w_shape) conv_b = sym.Variable('conv_b', shape=conv_b_shape) conv1 = sym.conv2d(data=data, weight=conv_w, bias=conv_b, channels=num_channels, kernel_size=(kernel_size, kernel_size), name='conv1') # relu1 relu1 = sym.relu(data=conv1, name='relu1') # max pooling max_pooling1 = sym.max_pool2d(data=relu1, pool_size=(2, 2), name='max_pooling1') # flatten flatten1 = sym.flatten(data=max_pooling1) # shape after flatten flatten_out_shape = (h - kernel_size) * (w - kernel_size) * num_channels # dense1 dense1_hidden_units = 100 dense1 = sym.dense(data=flatten1, name='dense1', units=dense1_hidden_units) # relu2 relu2 = sym.relu(data=dense1, name='relu2') # dense2 dense2_hidden_units = 10 dense2 = sym.dense(data=relu2, name='dense2', units=dense2_hidden_units) # softmax mlp = sym.softmax(data=dense2, name='softmax') # fake non-sparse label label = sym.full_like(mlp, fill_value=1) # cross entropy loss ce_loss = sym.sum(sym.elemwise_mul(sym.log_softmax(dense2), label), axis=1, keepdims=True, name="ce_loss") # input variables: # print grad_g.symbol.list_input_names() # >> ['data', 'conv_w', 'conv_b', # 'dense1_weight', 'dense1_bias', # 'dense2_weight', 'dense2_bias'] # output gradient variables: # print grad_g.symbol.list_output_names() # >> ['conv1_grad_data', 'conv1_grad_weight', 'conv1_grad_bias', # 'dense1_grad_weight', 'dense1_grad_bias', # 'dense2_grad_weight', 'dense2_grad_bias'] grad_g = graph_util.get_gradient_graph(ce_loss, ce_loss.list_input_variables()) # infer shape in_shapes, out_shapes = graph_util.infer_shape(grad_g) # forward graph shape assert in_shapes == [ list(data_shape), list(conv_w_shape), list(conv_b_shape), [dense1_hidden_units, flatten_out_shape], [dense1_hidden_units], [dense2_hidden_units, dense1_hidden_units], [dense2_hidden_units] ] # input grads shape should be equal with input shape assert in_shapes == out_shapes # output grads w.r.t input variables grads = graph_util.gradients(ce_loss, ce_loss.list_input_variables()) # gradients number should be equal with grad_input number assert len(grads) == len(ce_loss.list_input_variables()) # infer type in_dtypes, out_dtypes = graph_util.infer_dtype(grad_g) assert out_dtypes == [ 'float32', 'float32', 'float32', 'float32', 'float32', 'float32', 'float32' ]
def test_cnn_gradients(): # input data h = 128 w = 128 data_shape = (1000, 3, h, w) data = sym.Variable('data', shape=data_shape, dtype=0) # conv2d num_channels = 64 kernel_size = 32 conv_w_shape = (num_channels, 3, kernel_size, kernel_size) conv_b_shape = (num_channels,) conv_w = sym.Variable('conv_w', shape=conv_w_shape) conv_b = sym.Variable('conv_b', shape=conv_b_shape) conv1 = sym.conv2d(data=data, weight=conv_w, bias=conv_b, channels=num_channels, kernel_size=(kernel_size, kernel_size), name='conv1') # relu1 relu1 = sym.relu(data=conv1, name='relu1') # max pooling max_pooling1 = sym.max_pool2d(data=relu1, pool_size=(2, 2), name='max_pooling1') # flatten flatten1 = sym.flatten(data=max_pooling1) # shape after flatten flatten_out_shape = (h - kernel_size) * (w - kernel_size) * num_channels # dense1 dense1_hidden_units = 100 dense1 = sym.dense(data=flatten1, name='dense1', units=dense1_hidden_units) # relu2 relu2 = sym.relu(data=dense1, name='relu2') # dense2 dense2_hidden_units = 10 dense2 = sym.dense(data=relu2, name='dense2', units=dense2_hidden_units) # softmax mlp = sym.softmax(data=dense2, name='softmax') # fake non-sparse label label = sym.full_like(mlp, fill_value=1) # cross entropy loss ce_loss = sym.sum( sym.elemwise_mul(sym.log_softmax(dense2), label), axis=1, keepdims=True, name="ce_loss") # input variables: # print grad_g.symbol.list_input_names() # >> ['data', 'conv_w', 'conv_b', # 'dense1_weight', 'dense1_bias', # 'dense2_weight', 'dense2_bias'] # output gradient variables: # print grad_g.symbol.list_output_names() # >> ['conv1_grad_data', 'conv1_grad_weight', 'conv1_grad_bias', # 'dense1_grad_weight', 'dense1_grad_bias', # 'dense2_grad_weight', 'dense2_grad_bias'] grad_g = graph_util.get_gradient_graph(ce_loss, ce_loss.list_input_variables()) # infer shape in_shapes, out_shapes = graph_util.infer_shape(grad_g) # forward graph shape assert in_shapes == [list(data_shape), list(conv_w_shape), list(conv_b_shape), [dense1_hidden_units, flatten_out_shape], [dense1_hidden_units], [dense2_hidden_units, dense1_hidden_units], [dense2_hidden_units]] # input grads shape should be equal with input shape assert in_shapes == out_shapes # output grads w.r.t input variables grads = graph_util.gradients(ce_loss, ce_loss.list_input_variables()) # gradients number should be equal with grad_input number assert len(grads) == len(ce_loss.list_input_variables()) # infer type in_dtypes, out_dtypes = graph_util.infer_dtype(grad_g) assert out_dtypes == ['float32', 'float32', 'float32', 'float32', 'float32', 'float32', 'float32']