def test_conv_ewise_injective(): x = sym.Variable("x") y = sym.conv2d(x, channels=32, kernel_size=(3, 3), groups=32, name="y", padding=(1,1)) y = sym.flatten(y + 1) + 1 dtype = "float32" dshape = (1, 32, 18, 18) kshape = (32, 1, 3, 3) oshape = (1, 32* 18 * 18) shape_dict = {"x": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) m = graph_runtime.create(graph, lib, ctx) # print(graph.ir(join_entry_attrs=["shape"])) assert graph.index.num_nodes == 5 # set input data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype)) bias = tvm.nd.array(np.random.uniform(size=kshape[0]).astype(dtype)) m.run(x=data, y_weight=kernel, y_bias=bias) # get output out = m.get_output(0, tvm.nd.empty(oshape, dtype)) c_np = topi.testing.depthwise_conv2d_python_nchw( data.asnumpy(), kernel.asnumpy(), (1,1), 'SAME') c_np = c_np + bias.asnumpy().reshape(kshape[0], 1, 1) + 1 c_np = c_np.reshape(c_np.shape[0], np.prod(c_np.shape[1:])) + 1 np.testing.assert_allclose(out.asnumpy(), c_np, rtol=1e-5)
def test_fuse_conv2d_elu(): def elu(data): return -0.5 * sym.relu(1 - sym.exp(data)) + sym.relu(data) def get_sym(out_channel): data = sym.Variable(name="data") data = sym.conv2d(data=data, kernel_size=(3,3), channels=out_channel, padding=(1, 1), layout="NCHW", kernel_layout="OIHW", use_bias=True) data = sym.batch_norm(data) data = elu(data) return data in_channel = 8 out_channel = 16 size = 64 dshape = (1, in_channel, size, size) oshape = (1, out_channel, size, size) data = np.random.uniform(-1, 1, dshape).astype(np.float32) for target, ctx in ctx_list(): sym1 = get_sym(out_channel) sym2 = get_sym(out_channel) _, params1 = utils.create_workload(sym1, 1, dshape[1:], seed=0) _, params2 = utils.create_workload(sym2, 1, dshape[1:], seed=0) output1, g1 = build_and_run(sym1, params1, data, oshape, target, ctx, opt_level=2) output2, g2 = build_and_run(sym2, params2, data, oshape, target, ctx, opt_level=0) np.testing.assert_allclose(output1, output2, rtol=1e-5, atol=1e-5) # data, conv weight, bias, batch norm gamma, batch norm beta, conv op assert g1.index.num_nodes == 6
def test_conv_ewise_injective(): x = sym.Variable("x") y = sym.conv2d(x, channels=32, kernel_size=(3, 3), groups=32, name="y", padding=(1,1)) y = sym.flatten(y + 1) + 1 dtype = "float32" dshape = (1, 32, 18, 18) kshape = (32, 1, 3, 3) oshape = (1, 32* 18 * 18) shape_dict = {"x": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) m = graph_runtime.create(graph, lib, ctx) # print(graph.ir(join_entry_attrs=["shape"])) assert graph.index.num_nodes == 5 # set input data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype)) bias = tvm.nd.array(np.random.uniform(size=kshape[0]).astype(dtype)) m.run(x=data, y_weight=kernel, y_bias=bias) # get output out = m.get_output(0, tvm.nd.empty(oshape, dtype)) c_np = topi.testing.depthwise_conv2d_python_nchw( data.asnumpy(), kernel.asnumpy(), (1,1), 'SAME') c_np = c_np + bias.asnumpy().reshape(kshape[0], 1, 1) + 1 c_np = c_np.reshape(c_np.shape[0], np.prod(c_np.shape[1:])) + 1 np.testing.assert_allclose(out.asnumpy(), c_np, rtol=1e-5)
def test_concatenate_conv2d(): ch = 3 size = 8 data = sym.Variable(name="data") concat = sym.concatenate(data, data, axis=1) conv = sym.conv2d(data=concat, kernel_size=(1, 1), channels=ch * 2, use_bias=False, name="conv") net = sym.elemwise_add(concat, conv) dtype = "float32" dshape = (1, ch, size, size) kshape = (ch * 2, ch * 2, 1, 1) oshape = (1, ch * 2, size, size) shape_dict = {"data": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(net, target, shape_dict) # data, conv weight, conv op, concat assert graph.index.num_nodes == 4 data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype)) m = graph_runtime.create(graph, lib, ctx) m.run(data=data, conv_weight=kernel) # get output out = m.get_output(0, tvm.nd.empty(oshape, dtype)) concat = np.concatenate((data.asnumpy(), data.asnumpy()), axis=1) conv = topi.testing.conv2d_nchw_python(concat, kernel.asnumpy(), (1, 1), 'SAME') ref = concat + conv np.testing.assert_allclose(out.asnumpy(), ref, rtol=1e-5)
def test_injective_conv2d(): channels = 16 data = sym.Variable(name="data") pool = sym.global_avg_pool2d(data=data) weight = sym.reshape(pool, shape=[1, channels, 1, 1]) residual = sym.conv2d(data=data, kernel_size=(3,3), channels=channels, padding=(1, 1), layout="NCHW", kernel_layout="OIHW", use_bias=False, name="conv") net = weight * data + residual size = 56 dtype="float32" dshape = (1, channels, size, size) kshape = (channels, channels, 3, 3) oshape = dshape shape_dict = {"data": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(net, target, shape_dict) # data, global_avg_pool, conv weight, conv op, fused elemwise add assert graph.index.num_nodes == 5 data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype)) m = graph_runtime.create(graph, lib, ctx) m.run(data=data, conv_weight=kernel) # get output out = m.get_output(0, tvm.nd.empty(oshape, dtype)) residual = topi.testing.conv2d_nchw_python( data.asnumpy(), kernel.asnumpy(), (1,1), 'SAME') weight = np.mean(data.asnumpy(), axis=(2, 3)) c_np = weight[:, :, np.newaxis, np.newaxis] * data.asnumpy() + residual tvm.testing.assert_allclose(out.asnumpy(), c_np, rtol=1e-5)
def test_fuse_conv2d_elu(): def elu(data): return -0.5 * sym.relu(1 - sym.exp(data)) + sym.relu(data) def get_sym(out_channel): data = sym.Variable(name="data") data = sym.conv2d(data=data, kernel_size=(3,3), channels=out_channel, padding=(1, 1), layout="NCHW", kernel_layout="OIHW", use_bias=True) data = sym.batch_norm(data) data = elu(data) return data in_channel = 8 out_channel = 16 size = 64 dshape = (1, in_channel, size, size) oshape = (1, out_channel, size, size) data = np.random.uniform(-1, 1, dshape).astype(np.float32) for target, ctx in ctx_list(): sym1 = get_sym(out_channel) sym2 = get_sym(out_channel) _, params1 = utils.create_workload(sym1, 1, dshape[1:], seed=0) _, params2 = utils.create_workload(sym2, 1, dshape[1:], seed=0) output1, g1 = build_and_run(sym1, params1, data, oshape, target, ctx, opt_level=2) output2, g2 = build_and_run(sym2, params2, data, oshape, target, ctx, opt_level=0) tvm.testing.assert_allclose(output1, output2, rtol=1e-5, atol=1e-5) # data, conv weight, bias, batch norm gamma, batch norm beta, conv op assert g1.index.num_nodes == 6
def test_concatenate_conv2d(): ch = 3 size = 8 data = sym.Variable(name="data") concat = sym.concatenate(data, data, axis=1) conv = sym.conv2d(data=concat, kernel_size=(1,1), channels=ch*2, use_bias=False, name="conv") net = sym.elemwise_add(concat, conv) dtype="float32" dshape = (1, ch, size, size) kshape = (ch*2, ch*2, 1, 1) oshape = (1, ch*2, size, size) shape_dict = {"data": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(net, target, shape_dict) # data, conv weight, conv op, concat assert graph.index.num_nodes == 4 data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) kernel = tvm.nd.array(np.random.uniform(size=kshape).astype(dtype)) m = graph_runtime.create(graph, lib, ctx) m.run(data=data, conv_weight=kernel) # get output out = m.get_output(0, tvm.nd.empty(oshape, dtype)) concat = np.concatenate((data.asnumpy(), data.asnumpy()), axis=1) conv = topi.testing.conv2d_nchw_python( concat, kernel.asnumpy(), (1,1), 'SAME') ref = concat + conv tvm.testing.assert_allclose(out.asnumpy(), ref, rtol=1e-5)
def test_ewise_injective(): x = sym.Variable("x") y = x * 2 y = sym.flatten(y) + 1 dshape = (10, 2, 3) shape_dict = {"x": dshape} dtype = "float32" target = "llvm" for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) assert graph.index.num_nodes == 2 m = graph_runtime.create(graph, lib, ctx) x_np = np.random.uniform(size=dshape).astype(dtype) m.run(x=x_np) out = m.get_output(0, tvm.nd.empty((10, 6))) np.testing.assert_allclose( out.asnumpy(), x_np.reshape(out.shape) * 2 + 1, atol=1e-5, rtol=1e-5)
def test_injective_reduce_injective(): x = sym.Variable("x") x = sym.flatten(x) + 1 y = sym.sum(x, axis=1) dtype = "float32" dshape = (32, 1, 18, 18) shape_dict = {"x": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) m = graph_runtime.create(graph, lib, ctx) assert graph.index.num_nodes == 2 data = np.random.uniform(size=dshape).astype(dtype) m.run(x=data) c_np = np.sum(data.reshape(32, 18 * 18) + 1, axis=1) # get output out = m.get_output(0, tvm.nd.empty(c_np.shape, dtype)) np.testing.assert_allclose(out.asnumpy(), c_np, rtol=1e-5)
def test_ewise_injective(): x = sym.Variable("x") y = x * 2 y = sym.flatten(y) + 1 dshape = (10, 2, 3) shape_dict = {"x": dshape} dtype = "float32" target = "llvm" for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) assert graph.index.num_nodes == 2 m = graph_runtime.create(graph, lib, ctx) x_np = np.random.uniform(size=dshape).astype(dtype) m.run(x=x_np) out = m.get_output(0, tvm.nd.empty((10, 6))) np.testing.assert_allclose( out.asnumpy(), x_np.reshape(out.shape) * 2 + 1, atol=1e-5, rtol=1e-5)
def test_injective_reduce_injective(): x = sym.Variable("x") x = sym.flatten(x) + 1 y = sym.sum(x, axis=1) dtype = "float32" dshape = (32, 1, 18, 18) shape_dict = {"x": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) m = graph_runtime.create(graph, lib, ctx) assert graph.index.num_nodes == 2 data = np.random.uniform(size=dshape).astype(dtype) m.run(x=data) c_np = np.sum(data.reshape(32, 18 * 18) + 1, axis=1) # get output out = m.get_output(0, tvm.nd.empty(c_np.shape, dtype)) np.testing.assert_allclose(out.asnumpy(), c_np, rtol=1e-5)