def test_resnet152():
# Verify the memory allocation behavior on ResNet-152, the state-of-the-art
# model used for image classification.
import resnet
resnet_152 = resnet.get_symbol(num_classes=1000,
num_layers=152,
image_shape='3,224,224')
# We do the binding twice, one with the memory optimizations and one without.
# It is expected that the memory consumption of the former should be roughly
# half of that of the latter.
memory_opt_exec = resnet_152.simple_bind(mx.cpu(),
'write',
data=(32, 3, 224, 224))
os.environ["MXNET_MEMORY_OPT"] = '0'
no_opt_exec = resnet_152.simple_bind(mx.cpu(),
'write',
data=(32, 3, 224, 224))
os.environ["MXNET_MEMORY_OPT"] = '1'
memory_opt_alloc = grep_exec_memory_consumption(memory_opt_exec)
no_opt_alloc = grep_exec_memory_consumption(no_opt_exec)
assert memory_opt_alloc / no_opt_alloc < 0.6, \
"The ratio between the memory consumption with the memory optimizations " \
"enabled and disabled (%d vs. %d MB) is expected to be smaller than 0.6" \
% (memory_opt_alloc, no_opt_alloc)
def get_symbol(num_classes=20, nms_thresh=0.5, force_nms=False, **kwargs):
body = resnet.get_symbol(num_classes, 50, '3,224,224')
conv1 = body.get_internals()['_plus12_output']
conv2 = body.get_internals()['_plus15_output']
# anchors
anchors = [
1.3221, 1.73145, 3.19275, 4.00944, 5.05587, 8.09892, 9.47112, 4.84053,
11.2364, 10.0071
]
num_anchor = len(anchors) // 2
# extra layers
conv7_1 = conv_act_layer(conv2,
'conv7_1',
1024,
kernel=(3, 3),
pad=(1, 1),
act_type='leaky')
conv7_2 = conv_act_layer(conv7_1,
'conv7_2',
1024,
kernel=(3, 3),
pad=(1, 1),
act_type='leaky')
# re-organize
conv5_6 = mx.sym.stack_neighbor(data=conv1,
kernel=(2, 2),
name='stack_downsample')
concat = mx.sym.Concat(*[conv5_6, conv7_2], dim=1)
# concat = conv7_2
conv8_1 = conv_act_layer(concat,
'conv8_1',
1024,
kernel=(3, 3),
pad=(1, 1),
act_type='leaky')
pred = mx.symbol.Convolution(data=conv8_1,
name='conv_pred',
kernel=(1, 1),
num_filter=num_anchor * (num_classes + 4 + 1))
out = mx.contrib.symbol.YoloOutput(data=pred,
num_class=num_classes,
num_anchor=num_anchor,
object_grad_scale=5.0,
background_grad_scale=1.0,
coord_grad_scale=1.0,
class_grad_scale=1.0,
anchors=anchors,
nms_topk=400,
warmup_samples=12800,
name='yolo_output')
return out
import os
os.environ["MXNET_EXEC_PREFER_BULK_EXEC"] = sys.argv[1]
import numpy as np
import mxnet as mx
import resnet
import time
batch_size = 1
dshape = (batch_size, 3, 224, 224)
layers = [3, 24, 36, 3]
factor = 1
cfg = [int(z * factor) for z in layers]
nmodule = sum(cfg)
net = resnet.get_symbol(cfg)
print("Test resnet with %d modules.. check if bulk message appear" % nmodule)
def test_speed(net):
dev = mx.gpu(0)
dshape = (batch_size, 3, 224, 224)
lshape = (batch_size)
num_epoch = 100
tmp_data = np.random.uniform(-1, 1, dshape).astype("float32")
train_iter = mx.io.NDArrayIter(data=tmp_data,
batch_size=batch_size,
shuffle=False,
last_batch_handle='pad')
# train_imagenet.py --set-resnet-aug ...
# Finally, to get the legacy MXNet v1.2 training settings on a per-use basis, invoke as in:
# train_imagenet.py --set-data-aug-level 3
parser.set_defaults(
# network
num_layers = 50,
# data
resize = 256,
num_classes = 1000,
num_examples = 1281167,
image_shape = '3,224,224',
min_random_scale = 1, # if input image has min size k, suggest to use
# 256.0/x, e.g. 0.533 for 480
# train
num_epochs = 90,
lr_step_epochs = '30,60,80',
dtype = 'float32'
)
args = parser.parse_args()
if not args.use_dali:
data.set_data_aug_level(parser, 0)
# load network
import resnet as net
sym = net.get_symbol(**vars(args))
# train
fit.fit(args, sym, dali.get_rec_iter)
def multilayer_bi_lstm_unroll_new(seq_len,
num_hidden,
num_label,
batch_size,
num_lstm_layer,
dropout=0):
shape = {"data": (32, 3, 32, 32)}
data = mx.sym.Variable('data')
label = mx.sym.Variable('label')
from importlib import import_module
resnet = import_module('resnet')
conv = resnet.get_symbol(2, 34, '3,32,' + str(seq_len * 8))
column_features = mx.sym.SliceChannel(data=conv,
num_outputs=seq_len,
axis=3,
squeeze_axis=1)
forward_param_cells = []
backward_param_cells = []
last_states = []
for i in range(num_lstm_layer):
forward_param_cells.append(
LSTMParam(i2h_weight=mx.sym.Variable("f_l%d_i2h_weight" % i),
i2h_bias=mx.sym.Variable("f_l%d_i2h_bias" % i),
h2h_weight=mx.sym.Variable("f_l%d_h2h_weight" % i),
h2h_bias=mx.sym.Variable("f_l%d_h2h_bias" % i)))
backward_param_cells.append(
LSTMParam(i2h_weight=mx.sym.Variable("b_l%d_i2h_weight" % i),
i2h_bias=mx.sym.Variable("b_l%d_i2h_bias" % i),
h2h_weight=mx.sym.Variable("b_l%d_h2h_weight" % i),
h2h_bias=mx.sym.Variable("b_l%d_h2h_bias" % i)))
last_states.append(
LSTMState(c=mx.sym.Variable("f_l%d_init_c" % 0),
h=mx.sym.Variable("f_l%d_init_h" % 0)))
last_states.append(
LSTMState(c=mx.sym.Variable("b_l%d_init_c" % 0),
h=mx.sym.Variable("b_l%d_init_h" % 0)))
last_states.append(
LSTMState(c=mx.sym.Variable("f_l%d_init_c" % 1),
h=mx.sym.Variable("f_l%d_init_h" % 1)))
last_states.append(
LSTMState(c=mx.sym.Variable("b_l%d_init_c" % 1),
h=mx.sym.Variable("b_l%d_init_h" % 1)))
assert (len(last_states) == num_lstm_layer * 2)
hidden_all = []
forward_hidden = []
for seqidx in range(seq_len):
hidden = mx.sym.Flatten(data=column_features[seqidx])
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
next_state = lstm(num_hidden,
indata=hidden,
prev_state=last_states[0],
param=forward_param_cells[0],
seqidx=seqidx,
layeridx=0)
hidden = next_state.h
last_states[0] = next_state
forward_hidden.append(hidden)
backward_hidden = []
for seqidx in range(seq_len):
k = seq_len - seqidx - 1
hidden = mx.sym.Flatten(data=column_features[k])
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
next_state = lstm(num_hidden,
indata=hidden,
prev_state=last_states[1],
param=backward_param_cells[0],
seqidx=k,
layeridx=0)
hidden = next_state.h
last_states[1] = next_state
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
backward_hidden.insert(0, hidden)
hidden_all = []
for i in range(seq_len):
hidden_all.append(
mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1))
forward_hidden = []
for seqidx in range(seq_len):
hidden = mx.sym.Flatten(data=hidden_all[seqidx])
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
next_state = lstm(num_hidden,
indata=hidden,
prev_state=last_states[2],
param=forward_param_cells[1],
seqidx=seqidx,
layeridx=1)
hidden = next_state.h
last_states[2] = next_state
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
forward_hidden.append(hidden)
backward_hidden = []
for seqidx in range(seq_len):
k = seq_len - seqidx - 1
hidden = mx.sym.Flatten(data=hidden_all[k])
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
next_state = lstm(num_hidden,
indata=hidden,
prev_state=last_states[3],
param=backward_param_cells[1],
seqidx=k,
layeridx=1)
hidden = next_state.h
last_states[3] = next_state
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
backward_hidden.insert(0, hidden)
hidden_all = []
for i in range(seq_len):
hidden_all.append(
mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1))
hidden_concat = mx.sym.Concat(*hidden_all, dim=0)
pred = mx.sym.FullyConnected(name='fc1', data=hidden_concat, num_hidden=37)
softmax_pred = pred
pred = mx.sym.Reshape(data=pred, shape=(-4, seq_len, -1, 0))
loss = mx.contrib.sym.ctc_loss(data=pred, label=label)
ctc_loss = mx.sym.MakeLoss(loss)
softmax_class = mx.symbol.SoftmaxActivation(data=softmax_pred)
softmax_loss = mx.sym.BlockGrad(softmax_class)
sm = mx.sym.Group([softmax_loss, ctc_loss])
return sm
def bi_lstm_unroll(seq_len, num_hidden, num_label, dropout=0):
last_states = []
last_states.append(
LSTMState(c=mx.sym.Variable("l0_init_c"),
h=mx.sym.Variable("l0_init_h")))
last_states.append(
LSTMState(c=mx.sym.Variable("l1_init_c"),
h=mx.sym.Variable("l1_init_h")))
forward_param = LSTMParam(i2h_weight=mx.sym.Variable("l0_i2h_weight"),
i2h_bias=mx.sym.Variable("l0_i2h_bias"),
h2h_weight=mx.sym.Variable("l0_h2h_weight"),
h2h_bias=mx.sym.Variable("l0_h2h_bias"))
backward_param = LSTMParam(i2h_weight=mx.sym.Variable("l1_i2h_weight"),
i2h_bias=mx.sym.Variable("l1_i2h_bias"),
h2h_weight=mx.sym.Variable("l1_h2h_weight"),
h2h_bias=mx.sym.Variable("l1_h2h_bias"))
assert (len(last_states) == 2)
# embeding layer
data = mx.sym.Variable('data')
label = mx.sym.Variable('label')
from importlib import import_module
resnet = import_module('resnet')
conv = resnet.get_symbol(2, 18, '3,32,' + str(seq_len * 8))
print('seq_len : ', seq_len)
column_features = mx.sym.SliceChannel(data=conv,
num_outputs=seq_len,
axis=3,
squeeze_axis=1)
hidden_all = []
forward_hidden = []
for seqidx in range(seq_len):
hidden = mx.sym.Flatten(data=column_features[seqidx])
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
next_state = lstm(num_hidden,
indata=hidden,
prev_state=last_states[0],
param=forward_param,
seqidx=seqidx,
layeridx=0)
hidden = next_state.h
last_states[0] = next_state
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
forward_hidden.append(hidden)
backward_hidden = []
for seqidx in range(seq_len):
k = seq_len - seqidx - 1
hidden = mx.sym.Flatten(data=column_features[k])
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
next_state = lstm(num_hidden,
indata=hidden,
prev_state=last_states[1],
param=backward_param,
seqidx=k,
layeridx=1)
hidden = next_state.h
last_states[1] = next_state
if dropout > 0.:
hidden = mx.sym.Dropout(data=hidden, p=dropout)
backward_hidden.insert(0, hidden)
hidden_all = []
for i in range(seq_len):
hidden_all.append(
mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1))
hidden_concat = mx.sym.Concat(*hidden_all, dim=0)
pred = mx.sym.FullyConnected(name='fc1', data=hidden_concat, num_hidden=98)
label = mx.sym.Reshape(data=label, shape=(-1, ))
label = mx.sym.Cast(data=label, dtype='int32')
#does Warp-CTC support bucketing?
sm = mx.sym.WarpCTC(name='ctc-loss',
data=pred,
label=label,
label_length=num_label,
input_length=seq_len)
return sm
set_num=args.set_num,
per_set_num=args.per_set_num,
duplicate_num=args.duplicate_num,
max_num_of_duplicate_set=args.max_num_of_duplicate_set,
hdfs_path=args.hdfs_path,
rec_name=args.val_rec,
max_times_epoch=args.max_times_epoch,
data_type='val')
label = mx.sym.var("label")
arg_params, aux_params, all_layers = load_model()
data = all_layers['data']
feature_output = all_layers['fc5_xaiver_256_output']
# score = tiny_network(data)
score = resnet.get_symbol(data=data,
num_classes=1,
num_layers=18,
image_shape=data_shapes)
feature_set = mx.sym.Custom(score=score,
feature=feature_output,
data=data,
label=label,
set_num=args.set_num,
per_set_num=args.per_set_num,
ctx=ctx,
op_type='batch_pool')
metric_cost = train_info.get_cost(5 * 16)
l2_param = float(args.l2_param)
l2_penalty_total = mx.sym.Custom(score=score,
set_num=args.set_num,
per_set_num=args.per_set_num,