def get_symbol(num_classes, version, **kwargs): """Get symbol of SqueezeNet Parameters ---------- num_classes: int The number of classification results version : str, optional "1.0" or "1.1" of SqueezeNet """ assert version == '1.1', ("Unsupported SqueezeNet version {version}:" "1.1 expected".format(version=version)) net = sym.Variable("data") net = sym.conv2d(net, channels=64, kernel_size=(3, 3), strides=(2, 2)) net = sym.relu(net) net = sym.max_pool2d(net, pool_size=(3, 3), strides=(2, 2)) net = _make_fire(net, 16, 64, 64) net = _make_fire(net, 16, 64, 64) net = sym.max_pool2d(net, pool_size=(3, 3), strides=(2, 2)) net = _make_fire(net, 32, 128, 128) net = _make_fire(net, 32, 128, 128) net = sym.max_pool2d(net, pool_size=(3, 3), strides=(2, 2)) net = _make_fire(net, 48, 192, 192) net = _make_fire(net, 48, 192, 192) net = _make_fire(net, 64, 256, 256) net = _make_fire(net, 64, 256, 256) net = sym.dropout(net, rate=0.5) net = sym.conv2d(net, channels=num_classes, kernel_size=(1, 1)) net = sym.relu(net) net = sym.global_avg_pool2d(net) return sym.softmax(net, axis=1)
def get_symbol(num_classes, version, **kwargs): """Get symbol of SqueezeNet Parameters ---------- num_classes: int The number of classification results version : str, optional "1.0" or "1.1" of SqueezeNet """ assert version == '1.1', ("Unsupported SqueezeNet version {version}:" "1.1 expected".format(version=version)) net = sym.Variable("data") net = sym.conv2d(net, channels=64, kernel_size=(3, 3), strides=(2, 2)) net = sym.relu(net) net = sym.max_pool2d(net, pool_size=(3, 3), strides=(2, 2)) net = _make_fire(net, 16, 64, 64) net = _make_fire(net, 16, 64, 64) net = sym.max_pool2d(net, pool_size=(3, 3), strides=(2, 2)) net = _make_fire(net, 32, 128, 128) net = _make_fire(net, 32, 128, 128) net = sym.max_pool2d(net, pool_size=(3, 3), strides=(2, 2)) net = _make_fire(net, 48, 192, 192) net = _make_fire(net, 48, 192, 192) net = _make_fire(net, 64, 256, 256) net = _make_fire(net, 64, 256, 256) net = sym.dropout(net, rate=0.5) net = sym.conv2d(net, channels=num_classes, kernel_size=(1, 1)) net = sym.relu(net) net = sym.global_avg_pool2d(net) return sym.softmax(net, axis=1)
def overfeat(num_classes=1000): data = sym.Variable("data") body = conv2d_block(data, "conv1", 96, kernel_size=(11, 11), strides=(4, 4), padding=(5, 5)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool1") body = conv2d_block(body, "conv2", 256, kernel_size=(5, 5), strides=(1, 1), padding=(2, 2)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool2") body = conv2d_block(body, "conv3", 512, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv4", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv5", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool3") flatten = sym.flatten(data=body, name="flatten") fc = sym.dense(data=flatten, units=3072, use_bias=False, name="fc1") fc = sym.dense(data=fc, units=4096, use_bias=False, name="fc2") fc = sym.dense(data=fc, units=num_classes, use_bias=False, name="fc3") return fc
def get_sym(layout, kernel_layout, channels): data = sym.Variable(name="data") data = sym.conv2d(data=data, kernel_size=(3,3), channels=channels, padding=(1, 1), layout=layout, kernel_layout=kernel_layout, use_bias=True) data = sym.max_pool2d(data=data, pool_size=(2, 2), strides=(2, 2), layout=layout) data = sym.upsampling(data=data, scale=2, layout=layout) softmax_axis = 1 if layout == "NHWC": softmax_axis = 3 data = sym.softmax(data=data, axis=softmax_axis) return data
def _alter_max_pool2d_layout(attrs, inputs, tinfo): import nnvm.symbol as sym copy_inputs = [s for s in inputs] new_attrs = {k: attrs[k] for k in attrs.keys()} # NHWC -> NCHW if attrs["layout"] != "NHWC": return None new_attrs["layout"] = "NCHW" if "target" in new_attrs: del new_attrs["target"] return sym.max_pool2d(*copy_inputs, **new_attrs)
def get_sym(layout, kernel_layout, channels): data = sym.Variable(name="data") data = sym.conv2d(data=data, kernel_size=(3,3), channels=channels, padding=(1, 1), layout=layout, kernel_layout=kernel_layout, use_bias=True) data = sym.max_pool2d(data=data, pool_size=(2, 2), strides=(2, 2), layout=layout) data = sym.upsampling(data=data, scale=2, layout=layout) softmax_axis = 1 if layout == "NHWC": softmax_axis = 3 data = sym.softmax(data=data, axis=softmax_axis) return data
def yolo(num_classes=1470): data = sym.Variable("data") body = conv2d_block(data, "conv1", 64, kernel_size=(7, 7), strides=(2, 2), padding=(3, 3)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool1") body = conv2d_block(body, "conv2", 192, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool2") body = conv2d_block(body, "conv3", 128, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv4", 256, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv5", 256, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv6", 512, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool3") body = conv2d_block(body, "conv7", 256, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv8", 512, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv9", 256, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv10", 512, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv11", 256, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv12", 512, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv13", 256, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv14", 512, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv15", 512, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv16", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = sym.max_pool2d(data=body, pool_size=(2, 2), strides=(2, 2), name="pool4") body = conv2d_block(body, "conv17", 512, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv18", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv19", 512, kernel_size=(1, 1), strides=(1, 1), padding=(0, 0)) body = conv2d_block(body, "conv20", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv21", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv22", 1024, kernel_size=(3, 3), strides=(2, 2), padding=(1, 1)) body = conv2d_block(body, "conv23", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) body = conv2d_block(body, "conv24", 1024, kernel_size=(3, 3), strides=(1, 1), padding=(1, 1)) flatten = sym.flatten(data=body, name="flatten") fc = sym.dense(data=flatten, units=4096, use_bias=False, name="fc1") act = sym.relu(data=fc, name="relu1") fc = sym.dense(data=act, units=num_classes, use_bias=False, name="fc2") return fc
def test_max_pool2d(): x = sym.Variable("x") y = sym.max_pool2d(x, pool_size=(2,2), strides=(2,2), padding=(0,0), name="y", ceil_mode=True) dtype = "float32" dshape = (1, 3, 28, 28) oshape = (1, 3, 14, 14) shape_dict = {"x": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) m = graph_runtime.create(graph, lib, ctx) data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) m.run(x=data) out = m.get_output(0, tvm.nd.empty(oshape, dtype)) b_np = np.max(data.asnumpy().reshape(1,3,14,2,14,2), axis=(3,5)) np.testing.assert_allclose(out.asnumpy(), b_np, rtol=1e-5)
def test_max_pool2d(): x = sym.Variable("x") y = sym.max_pool2d(x, pool_size=(2,2), strides=(2,2), padding=(0,0), name="y", ceil_mode=True) dtype = "float32" dshape = (1, 3, 28, 28) oshape = (1, 3, 14, 14) shape_dict = {"x": dshape} for target, ctx in ctx_list(): graph, lib, _ = nnvm.compiler.build(y, target, shape_dict) m = graph_runtime.create(graph, lib, ctx) data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype)) m.run(x=data) out = m.get_output(0, tvm.nd.empty(oshape, dtype)) b_np = np.max(data.asnumpy().reshape(1,3,14,2,14,2), axis=(3,5)) tvm.testing.assert_allclose(out.asnumpy(), b_np, rtol=1e-5)
def test_max_pool2d(): x = sym.Variable("data", shape=(1, 32, 512, 512)) y = sym.max_pool2d(x, name="pool", pool_size=(3,3), padding=(1,1), layout="NCHW") g, ldict = correct_layout(y) assert(ldict["data"][0] == "NCHW") assert(ldict["pool"][0] == "NCHW") # if index of H and W remain the same, # pool2d does not convert the layout. g, ldict = correct_layout(g, "NCHW16c") assert(ldict["data"][0] == "NCHW16c") assert(ldict["pool"][0] == "NCHW16c") # for other layout it requires a layout transform. g, ldict = correct_layout(g, "NHWC") assert(ldict["data"][0] == "NHWC") assert(ldict["data_NCHW"][0] == "NCHW") assert(ldict["pool"][0] == "NCHW")
def Pooling(data, kernel, stride, pad, pool_type, name): if pad[0] != 0 or pad[1] != 0: data = sym.pad(data=data, pad_width=((0, 0), (pad[0], pad[0]), (pad[1], pad[1]), (0, 0))) if pool_type == 'max': return sym.max_pool2d(data=data, pool_size=kernel, strides=stride, name=name, layout='NHWC') if pool_type == 'avg': return sym.avg_pool2d(data=data, pool_size=kernel, strides=stride, name=name, layout='NHWC') raise ValueError("Invalid pooling type: " + pool_type)
def test_max_pool2d(): x = sym.Variable("data", shape=(1, 32, 512, 512)) y = sym.max_pool2d(x, name="pool", pool_size=(3, 3), padding=(1, 1), layout="NCHW") g, ldict = correct_layout(y) assert (ldict["data"][0] == "NCHW") assert (ldict["pool"][0] == "NCHW") # if index of H and W remain the same, # pool2d does not convert the layout. g, ldict = correct_layout(g, "NCHW16c") assert (ldict["data"][0] == "NCHW16c") assert (ldict["pool"][0] == "NCHW16c") # for other layout it requires a layout transform. g, ldict = correct_layout(g, "NHWC") assert (ldict["data"][0] == "NHWC") assert (ldict["data_NCHW"][0] == "NCHW") assert (ldict["pool"][0] == "NCHW")
def get_feature(internel_layer, layers, filters, batch_norm=False): """ Get VGG feature body as stacks of convoltions. layers : [1, 1, 2, 2, 2] filters : [64, 128, 256, 512, 512] """ for i, num in enumerate(layers): """ i = 0, num = 1 i = 1, num = 1 i = 2, num = 2 i = 3, num = 2 i = 4, num = 2 """ for j in range(num): internel_layer = sym.pad(data=internel_layer, pad_width=((0, 0), (1, 1), (1, 1), (0, 0))) internel_layer = sym.conv2d(data=internel_layer, kernel_size=(3, 3), channels=filters[i], layout='NHWC', kernel_layout='HWOI', name="conv%s_%s" % (i + 1, j + 1)) if batch_norm: internel_layer = sym.batch_norm(data=internel_layer, axis=3, name="bn%s_%s" % (i + 1, j + 1)) internel_layer = sym.relu(data=internel_layer, name="relu%s_%s" % (i + 1, j + 1)) internel_layer = sym.max_pool2d(data=internel_layer, pool_size=(2, 2), strides=(2, 2), layout="NHWC", name="pool%s" % (i + 1)) return internel_layer
def test_max_pool2d(): x = sym.Variable('x') y = sym.max_pool2d(x, pool_size=(3, 3), name="y") y = sym.global_max_pool2d(y) assert y.list_input_names() == ["x"]
def check(in_shape, out_shape, **kwargs): x = sym.Variable("x", shape=in_shape) y = sym.max_pool2d(x, name="y", **kwargs) sdict = infer_shape(y) assert(tuple(sdict["y"][0]) == tuple(out_shape))
def test_max_pool2d(): x = sym.Variable('x') y = sym.max_pool2d(x, pool_size=(3, 3), name="y") y = sym.global_max_pool2d(y) assert y.list_input_names() == ["x"]
def max_pool2d_callback(attrs, inputs, tinfos): print("MAX_POOL2D!!!") new_attrs = {k: attrs[k] for k in attrs.keys()} new_attrs['layout'] = 'NCHWc' return sym.max_pool2d(inputs[0], **new_attrs)
def test_cnn_gradients(): # input data h = 128 w = 128 data_shape = (1000, 3, h, w) data = sym.Variable('data', shape=data_shape, dtype=0) # conv2d num_channels = 64 kernel_size = 32 conv_w_shape = (num_channels, 3, kernel_size, kernel_size) conv_b_shape = (num_channels, ) conv_w = sym.Variable('conv_w', shape=conv_w_shape) conv_b = sym.Variable('conv_b', shape=conv_b_shape) conv1 = sym.conv2d(data=data, weight=conv_w, bias=conv_b, channels=num_channels, kernel_size=(kernel_size, kernel_size), name='conv1') # relu1 relu1 = sym.relu(data=conv1, name='relu1') # max pooling max_pooling1 = sym.max_pool2d(data=relu1, pool_size=(2, 2), name='max_pooling1') # flatten flatten1 = sym.flatten(data=max_pooling1) # shape after flatten flatten_out_shape = (h - kernel_size) * (w - kernel_size) * num_channels # dense1 dense1_hidden_units = 100 dense1 = sym.dense(data=flatten1, name='dense1', units=dense1_hidden_units) # relu2 relu2 = sym.relu(data=dense1, name='relu2') # dense2 dense2_hidden_units = 10 dense2 = sym.dense(data=relu2, name='dense2', units=dense2_hidden_units) # softmax mlp = sym.softmax(data=dense2, name='softmax') # fake non-sparse label label = sym.full_like(mlp, fill_value=1) # cross entropy loss ce_loss = sym.sum(sym.elemwise_mul(sym.log_softmax(dense2), label), axis=1, keepdims=True, name="ce_loss") # input variables: # print grad_g.symbol.list_input_names() # >> ['data', 'conv_w', 'conv_b', # 'dense1_weight', 'dense1_bias', # 'dense2_weight', 'dense2_bias'] # output gradient variables: # print grad_g.symbol.list_output_names() # >> ['conv1_grad_data', 'conv1_grad_weight', 'conv1_grad_bias', # 'dense1_grad_weight', 'dense1_grad_bias', # 'dense2_grad_weight', 'dense2_grad_bias'] grad_g = graph_util.get_gradient_graph(ce_loss, ce_loss.list_input_variables()) # infer shape in_shapes, out_shapes = graph_util.infer_shape(grad_g) # forward graph shape assert in_shapes == [ list(data_shape), list(conv_w_shape), list(conv_b_shape), [dense1_hidden_units, flatten_out_shape], [dense1_hidden_units], [dense2_hidden_units, dense1_hidden_units], [dense2_hidden_units] ] # input grads shape should be equal with input shape assert in_shapes == out_shapes # output grads w.r.t input variables grads = graph_util.gradients(ce_loss, ce_loss.list_input_variables()) # gradients number should be equal with grad_input number assert len(grads) == len(ce_loss.list_input_variables()) # infer type in_dtypes, out_dtypes = graph_util.infer_dtype(grad_g) assert out_dtypes == [ 'float32', 'float32', 'float32', 'float32', 'float32', 'float32', 'float32' ]
def check(in_shape, out_shape, **kwargs): x = sym.Variable("x", shape=in_shape) y = sym.max_pool2d(x, name="y", **kwargs) sdict = infer_shape(y) assert(tuple(sdict["y"][0]) == tuple(out_shape))
def test_cnn_gradients(): # input data h = 128 w = 128 data_shape = (1000, 3, h, w) data = sym.Variable('data', shape=data_shape, dtype=0) # conv2d num_channels = 64 kernel_size = 32 conv_w_shape = (num_channels, 3, kernel_size, kernel_size) conv_b_shape = (num_channels,) conv_w = sym.Variable('conv_w', shape=conv_w_shape) conv_b = sym.Variable('conv_b', shape=conv_b_shape) conv1 = sym.conv2d(data=data, weight=conv_w, bias=conv_b, channels=num_channels, kernel_size=(kernel_size, kernel_size), name='conv1') # relu1 relu1 = sym.relu(data=conv1, name='relu1') # max pooling max_pooling1 = sym.max_pool2d(data=relu1, pool_size=(2, 2), name='max_pooling1') # flatten flatten1 = sym.flatten(data=max_pooling1) # shape after flatten flatten_out_shape = (h - kernel_size) * (w - kernel_size) * num_channels # dense1 dense1_hidden_units = 100 dense1 = sym.dense(data=flatten1, name='dense1', units=dense1_hidden_units) # relu2 relu2 = sym.relu(data=dense1, name='relu2') # dense2 dense2_hidden_units = 10 dense2 = sym.dense(data=relu2, name='dense2', units=dense2_hidden_units) # softmax mlp = sym.softmax(data=dense2, name='softmax') # fake non-sparse label label = sym.full_like(mlp, fill_value=1) # cross entropy loss ce_loss = sym.sum( sym.elemwise_mul(sym.log_softmax(dense2), label), axis=1, keepdims=True, name="ce_loss") # input variables: # print grad_g.symbol.list_input_names() # >> ['data', 'conv_w', 'conv_b', # 'dense1_weight', 'dense1_bias', # 'dense2_weight', 'dense2_bias'] # output gradient variables: # print grad_g.symbol.list_output_names() # >> ['conv1_grad_data', 'conv1_grad_weight', 'conv1_grad_bias', # 'dense1_grad_weight', 'dense1_grad_bias', # 'dense2_grad_weight', 'dense2_grad_bias'] grad_g = graph_util.get_gradient_graph(ce_loss, ce_loss.list_input_variables()) # infer shape in_shapes, out_shapes = graph_util.infer_shape(grad_g) # forward graph shape assert in_shapes == [list(data_shape), list(conv_w_shape), list(conv_b_shape), [dense1_hidden_units, flatten_out_shape], [dense1_hidden_units], [dense2_hidden_units, dense1_hidden_units], [dense2_hidden_units]] # input grads shape should be equal with input shape assert in_shapes == out_shapes # output grads w.r.t input variables grads = graph_util.gradients(ce_loss, ce_loss.list_input_variables()) # gradients number should be equal with grad_input number assert len(grads) == len(ce_loss.list_input_variables()) # infer type in_dtypes, out_dtypes = graph_util.infer_dtype(grad_g) assert out_dtypes == ['float32', 'float32', 'float32', 'float32', 'float32', 'float32', 'float32']