def get_super_resolution():
factor = 3
size = 224
data = sym.Variable(name='9')
conv1 = sym.conv2d(data,
channels=64,
kernel_size=(5, 5),
padding=(2, 2),
use_bias=False)
relu1 = sym.relu(conv1 + sym.Variable(name='2'))
conv2 = sym.conv2d(relu1,
channels=64,
kernel_size=(3, 3),
padding=(1, 1),
use_bias=False)
relu2 = sym.relu(conv2 + sym.Variable(name='4'))
conv3 = sym.conv2d(relu2,
channels=32,
kernel_size=(3, 3),
padding=(1, 1),
use_bias=False)
relu3 = sym.relu(conv3 + sym.Variable(name='6'))
conv4 = sym.conv2d(relu3,
channels=factor**2,
kernel_size=(3, 3),
padding=(1, 1),
use_bias=False)
conv4 = conv4 + sym.Variable(name='8')
# TODO(zhreshold): allow shape inference for batch size > 1
r1 = sym.reshape(conv4, shape=(1, 1, factor, factor, size, size))
t1 = sym.transpose(r1, axes=(0, 1, 4, 2, 5, 3))
r2 = sym.reshape(t1, shape=(1, 1, size * factor, size * factor))
return r2
def with_nnvm(nwarmup: int,
nloops: int,
args,
lam,
params={},
verbose: bool = False,
opt_level: int = 2,
debug: bool = False) -> Result:
""" Take numpy arrays as args, convert them to TVM tensors and call `lam`.
Result of lambda is converted back to numpy array and returned.
"""
runtime = graph_runtime if not debug else debug_runtime
tgt = 'llvm'
ctx = tvm.cpu(0)
inps = []
ishapes = {}
itypes = {}
idata = {}
for i, arg in enumerate(args):
nm = 'pl' + str(i)
inps.append(sym.Variable(name=nm))
ishapes.update({nm: arg.shape})
idata.update({nm: arg})
itypes.update({nm: "float32"})
out = lam(*inps)
with nnvm.compiler.build_config(opt_level=opt_level):
graph, lib, _ = nnvm.compiler.build(out, tgt, ishapes)
forward_graph,_,_,out_shapes,out_types = \
infer_shapes_dtypes(nnvm.graph.create(out), shape=ishapes, dtype=itypes, fallback_dtype='float32')
out_nd = tvm.nd.array(np.zeros(out_shapes[0], dtype=out_types[0]), ctx)
m = runtime.create(graph, lib, ctx)
m.set_input(**idata)
m.set_input(**params)
perfs: List[float] = []
for i in range(nwarmup + nloops):
tb = perf_counter()
m.run()
te = perf_counter()
if i >= nwarmup:
perfs.append(te - tb)
if verbose:
print("NNVM", te - tb)
out_nd = m.get_output(
0, tvm.nd.empty(shape=out_shapes[0], dtype=out_types[0], ctx=ctx))
return Result.fromPasses(out_nd.asnumpy(), perfs)
def nnvm_conv_test(nblocks=200,ks=1,w=54,h=6,c=256,verbose:bool=False,opt_level:int=2):
""" Test convolution performance for different shape """
shape=(1,h,w,c)
kshape=(ks,ks,c,c)
x=_sym.Variable(init=np.zeros(shape=shape),name='x')
k=_sym.Variable(init=np.zeros(shape=kshape),name='k')
t=x
def _print_shape(t):
print(run_nnvm(0,1,
{x:np.zeros(shape=shape),
k:np.zeros(shape=kshape)},
t).last_data.shape)
t=_sym.conv2d(t,k,
dilation=(1,1),
layout="NHWC",
strides=(1,1),
padding=[0,0],
kernel_size=(ks,ks),
channels=c,
kernel_layout="HWIO",
name="conv1",
use_bias=False)
# print(t.__dir__())
# print(t.list_attr())
t=_sym.strided_slice(t,begin=[0,0,0,0],end=[1,1,1,c])
t=_sym.expand_like(t,x,axis=[1,2])
r=run_nnvm(1,15,
{x:np.zeros(shape=shape)
,k:np.zeros(shape=kshape)}, t,
verbose=verbose,
opt_level=opt_level)
def block2_nnvm_run(nblocks:int=1,nwarmup:int=0,nloops:int=1,init_method='zeros',verbose=True,debug=False,**kwargs):
# sym_149080784 = tf.placeholder(shape=(1,108,21,32),dtype=tf.float32)
print("Warning: unused args:", kwargs) if kwargs != {} else None
inp=_sym.Variable(name='inp', init=np.zeros(shape=(1,54,6,192)))
block2,blockvars=block2_block_nnvm(nblocks,inp)
block2_params={sym:MODEL_PARAMS[k] for (k,sym) in blockvars.items()}
block2_params.update({inp:common_init('zeros',(1,54,6,192),np.float32)})
r=run_nnvm(
nwarmup,nloops,
block2_params,
block2,
verbose=verbose,
debug=debug)
return r
def get_super_resolution_deprecated():
factor = 3
size = 224
data = sym.Variable(name='9')
conv1 = sym.conv2d(data, channels=64, kernel_size=(5, 5), padding=(2, 2))
relu1 = sym.relu(conv1)
conv2 = sym.conv2d(relu1, channels=64, kernel_size=(3, 3), padding=(1, 1))
relu2 = sym.relu(conv2)
conv3 = sym.conv2d(relu2, channels=32, kernel_size=(3, 3), padding=(1, 1))
relu3 = sym.relu(conv3)
conv4 = sym.conv2d(relu3,
channels=factor**2,
kernel_size=(3, 3),
padding=(1, 1))
r1 = sym.reshape(conv4, shape=(0, 1, factor, factor, size, size))
t1 = sym.transpose(r1, axes=(0, 1, 4, 2, 5, 3))
r2 = sym.reshape(t1, shape=(0, 1, size * factor, size * factor))
return r2
def get_symbol(num_classes=1000, **kwargs):
data = sym.Variable(name="data")
# stage 1
conv = Conv(data, 32, kernel=(3, 3), stride=(2, 2), name="conv")
conv_1 = Conv(conv, 32, kernel=(3, 3), name="conv_1")
conv_2 = Conv(conv_1, 64, kernel=(3, 3), pad=(1, 1), name="conv_2")
pool = Pooling(data=conv_2,
kernel=(3, 3),
stride=(2, 2),
pool_type="max",
pad=(0, 0),
name="pool")
# stage 2
conv_3 = Conv(pool, 80, kernel=(1, 1), name="conv_3")
conv_4 = Conv(conv_3, 192, kernel=(3, 3), name="conv_4")
pool1 = Pooling(data=conv_4,
kernel=(3, 3),
stride=(2, 2),
pool_type="max",
pad=(0, 0),
name="pool1")
# stage 3
in3a = Inception7A(pool1, 64, 64, 96, 96, 48, 64, "avg", 32, "mixed")
in3b = Inception7A(in3a, 64, 64, 96, 96, 48, 64, "avg", 64, "mixed_1")
in3c = Inception7A(in3b, 64, 64, 96, 96, 48, 64, "avg", 64, "mixed_2")
in3d = Inception7B(in3c, 384, 64, 96, 96, "max", "mixed_3")
# stage 4
in4a = Inception7C(in3d, 192, 128, 128, 192, 128, 128, 128, 128, 192,
"avg", 192, "mixed_4")
in4b = Inception7C(in4a, 192, 160, 160, 192, 160, 160, 160, 160, 192,
"avg", 192, "mixed_5")
in4c = Inception7C(in4b, 192, 160, 160, 192, 160, 160, 160, 160, 192,
"avg", 192, "mixed_6")
in4d = Inception7C(in4c, 192, 192, 192, 192, 192, 192, 192, 192, 192,
"avg", 192, "mixed_7")
in4e = Inception7D(in4d, 192, 320, 192, 192, 192, 192, "max", "mixed_8")
# stage 5
in5a = Inception7E(in4e, 320, 384, 384, 384, 448, 384, 384, 384, "avg",
192, "mixed_9")
in5b = Inception7E(in5a, 320, 384, 384, 384, 448, 384, 384, 384, "max",
192, "mixed_10")
return in5b
def get_symbol(num_classes, num_layers=11, batch_norm=False):
"""
Parameters
----------
num_classes : int, default 1000
Number of classification classes.
num_layers : int
Number of layers for the variant of densenet. Options are 11, 13, 16, 19.
batch_norm : bool, default False
Use batch normalization.
"""
vgg_spec = {11: ([1, 1, 2, 2, 2], [64, 128, 256, 512, 512]),
13: ([2, 2, 2, 2, 2], [64, 128, 256, 512, 512]),
16: ([2, 2, 3, 3, 3], [64, 128, 256, 512, 512]),
19: ([2, 2, 4, 4, 4], [64, 128, 256, 512, 512]),
1:([1,0,0,0,0],[64,0,0,0,0])}
if num_layers not in vgg_spec:
raise ValueError("Invalide num_layers {}. Choices are 11,13,16,19.".format(num_layers))
layers, filters = vgg_spec[num_layers]
data = sym.Variable(name="data")
feature = get_feature(data, layers, filters, batch_norm)
return feature
def block1_block_nnvm_consts():
return {
"sym_75044384":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/conv2d_4/kernel",
shape=(1, 1, 32, 64)),
"sym_73427696":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/conv2d_4/bias",
shape=(64, )),
"sym_223382672":
_sym.Variable(
name="Rcnn_ctcV3/expand_conv1/activation/conv2d_6/kernel",
shape=(1, 1, 64, 64)),
"sym_356827536":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/activation/conv2d_6/bias",
shape=(64, )),
"sym_451228704":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/conv2d_5/kernel",
shape=(3, 3, 32, 64)),
"sym_88828560":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/conv2d_5/bias",
shape=(64, )),
"sym_379167584":
_sym.Variable(
name="Rcnn_ctcV3/expand_conv1/static_batch_normalization_3/gamma",
shape=(64, )),
"sym_492256464":
_sym.Variable(
name="Rcnn_ctcV3/expand_conv1/static_batch_normalization_3/beta",
shape=(64, )),
"sym_104779696":
_sym.Variable(
name=
"Rcnn_ctcV3/expand_conv1/static_batch_normalization_3/moving_mean",
shape=(64, )),
"sym_378983504":
_sym.Variable(
name=
"Rcnn_ctcV3/expand_conv1/static_batch_normalization_3/moving_variance",
shape=(64, )),
"sym_73418512":
_sym.Variable(
name=
"Rcnn_ctcV3/expand_conv1/static_batch_normalization_3/batchnorm/add/y",
shape=(1, )),
"sym_134609696":
_sym.Variable(
name="Rcnn_ctcV3/expand_conv1/activation/conv2d_7/kernel",
shape=(1, 1, 64, 64)),
"sym_131967104":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/activation/conv2d_7/bias",
shape=(64, )),
"sym_473740336":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/activation/max_2/mul/x",
shape=(1, )),
"sym_112766576":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/conv2d_8/kernel",
shape=(3, 3, 64, 64)),
"sym_105635760":
_sym.Variable(name="Rcnn_ctcV3/expand_conv1/conv2d_8/bias",
shape=(64, ))
}
def get_var(self, name, must_contain=True):
if must_contain:
assert name in self.vars
if name not in self.vars:
self.vars[name] = _sym.Variable(name=name)
return self.vars[name]
def new_const(self, value):
name = "_param_%d" % (self.const_ctr)
self.const_ctr += 1
self.params[name] = value
self.vars[name] = _sym.Variable(name=name)
return self.vars[name]
def resnet(units,
num_stages,
filter_list,
num_classes,
image_shape,
bottle_neck=True):
"""Return ResNet symbol of
Parameters
----------
units : list
Number of units in each stage
num_stages : int
Number of stage
filter_list : list
Channel size of each stage
num_classes : int
Ouput size of symbol
dataset : str
Dataset type, only cifar10 and imagenet supports
"""
num_unit = len(units)
assert num_unit == num_stages
data = sym.Variable(name='data')
#data = sym.batch_norm(data=data, epsilon=2e-5, scale=False, name='bn_data',axis=3)
(_, height, _) = image_shape
if height <= 32: # such as cifar10
body = sym.conv2d(data=data,
channels=filter_list[0],
kernel_size=(3, 3),
strides=(1, 1),
padding=(1, 1),
use_bias=False,
name="conv0")
else: # often expected to be 224 such as imagenet
body = sym.conv2d(data=data,
channels=filter_list[0],
kernel_size=(7, 7),
strides=(2, 2),
padding=(3, 3),
use_bias=False,
name="conv0")
body = sym.batch_norm(data=body, epsilon=2e-5, name='bn0', axis=3)
body = sym.relu(data=body, name='relu0')
body = sym.max_pool2d(data=body,
pool_size=(3, 3),
strides=(2, 2),
padding=(1, 1),
layout="NHWC")
#body = residual_unit2(
# body, filter_list[1], (1,1),
# False, name='stage%d_unit%d' % (1, 1), bottle_neck=bottle_neck,filter_last=filter_list[0])
for i in range(num_stages):
body = residual_unit(body,
filter_list[i + 1],
(1 if i == 0 else 2, 1 if i == 0 else 2),
False,
name='stage%d_unit%d' % (i + 1, 1),
bottle_neck=bottle_neck,
last_filter=filter_list[i])
for j in range(units[i] - 1):
body = residual_unit(body,
filter_list[i + 1], (1, 1),
True,
name='stage%d_unit%d' % (i + 1, j + 2),
bottle_neck=bottle_neck,
last_filter=filter_list[i + 1])
bn1 = sym.batch_norm(data=body, epsilon=2e-5, name='bn1', axis=3)
relu1 = sym.relu(data=bn1, name='relu1')
return relu1
def _get_model(dshape):
data = sym.Variable('data', shape=dshape)
fc1 = sym.dense(data, units=dshape[-1] * 2, use_bias=True)
left, right = sym.split(fc1, indices_or_sections=2, axis=1)
return sym.Group(((left + 1), (right - 1)))
def addvar(name,shape):
nonlocal varnames
s = _sym.Variable(name=name,shape=shape)
varnames.update({name:s})
return s
def Variable(name, **kwargs):
if name in _var_values:
kwargs["init"] = _var_values[name]
var = sym.Variable(name, **kwargs)
_global_vars[name] = var
return var
def drn(arch,
block,
layers,
num_classes=1000,
channels=(16, 32, 64, 128, 256, 512, 512, 512)):
data = sym.Variable(name='data')
if arch == 'C':
out = data
out = sym.conv2d(data=out,
channels=channels[0],
kernel_size=(7, 7),
strides=(1, 1),
padding=(3, 3),
use_bias=False)
out = int8_wrapper(sym.batch_norm, data=out)
out = sym.relu(data=out)
out = drn_unit(out,
basic_block,
channels[0],
channels[0],
layers[0],
stride=1)
out = drn_unit(out,
basic_block,
channels[0],
channels[1],
layers[1],
stride=2)
num_channel = channels[1]
else:
raise NotImplementedError()
out = drn_unit(out, block, num_channel, channels[2], layers[2], stride=2)
out = drn_unit(out, block, channels[2], channels[3], layers[3], stride=2)
out = drn_unit(out,
block,
channels[3],
channels[4],
layers[4],
dilation=2,
new_level=False)
num_channel = channels[4]
if layers[5] > 0:
out = drn_unit(out,
block,
num_channel,
channels[5],
layers[5],
dilation=4,
new_level=False)
num_channel = channels[5]
if arch == 'C':
if layers[6] > 0:
out = drn_unit(out,
block,
num_channel,
channels[6],
layers[6],
dilation=2,
new_level=False,
residual=False)
num_channel = channels[6]
if layers[7] > 0:
out = drn_unit(out,
block,
num_channel,
channels[7],
layers[7],
dilation=1,
new_level=False,
residual=False)
num_channel = channels[7]
return out
