def test_resnet152(): # Verify the memory allocation behavior on ResNet-152, the state-of-the-art # model used for image classification. import resnet resnet_152 = resnet.get_symbol(num_classes=1000, num_layers=152, image_shape='3,224,224') # We do the binding twice, one with the memory optimizations and one without. # It is expected that the memory consumption of the former should be roughly # half of that of the latter. memory_opt_exec = resnet_152.simple_bind(mx.cpu(), 'write', data=(32, 3, 224, 224)) os.environ["MXNET_MEMORY_OPT"] = '0' no_opt_exec = resnet_152.simple_bind(mx.cpu(), 'write', data=(32, 3, 224, 224)) os.environ["MXNET_MEMORY_OPT"] = '1' memory_opt_alloc = grep_exec_memory_consumption(memory_opt_exec) no_opt_alloc = grep_exec_memory_consumption(no_opt_exec) assert memory_opt_alloc / no_opt_alloc < 0.6, \ "The ratio between the memory consumption with the memory optimizations " \ "enabled and disabled (%d vs. %d MB) is expected to be smaller than 0.6" \ % (memory_opt_alloc, no_opt_alloc)
def get_symbol(num_classes=20, nms_thresh=0.5, force_nms=False, **kwargs): body = resnet.get_symbol(num_classes, 50, '3,224,224') conv1 = body.get_internals()['_plus12_output'] conv2 = body.get_internals()['_plus15_output'] # anchors anchors = [ 1.3221, 1.73145, 3.19275, 4.00944, 5.05587, 8.09892, 9.47112, 4.84053, 11.2364, 10.0071 ] num_anchor = len(anchors) // 2 # extra layers conv7_1 = conv_act_layer(conv2, 'conv7_1', 1024, kernel=(3, 3), pad=(1, 1), act_type='leaky') conv7_2 = conv_act_layer(conv7_1, 'conv7_2', 1024, kernel=(3, 3), pad=(1, 1), act_type='leaky') # re-organize conv5_6 = mx.sym.stack_neighbor(data=conv1, kernel=(2, 2), name='stack_downsample') concat = mx.sym.Concat(*[conv5_6, conv7_2], dim=1) # concat = conv7_2 conv8_1 = conv_act_layer(concat, 'conv8_1', 1024, kernel=(3, 3), pad=(1, 1), act_type='leaky') pred = mx.symbol.Convolution(data=conv8_1, name='conv_pred', kernel=(1, 1), num_filter=num_anchor * (num_classes + 4 + 1)) out = mx.contrib.symbol.YoloOutput(data=pred, num_class=num_classes, num_anchor=num_anchor, object_grad_scale=5.0, background_grad_scale=1.0, coord_grad_scale=1.0, class_grad_scale=1.0, anchors=anchors, nms_topk=400, warmup_samples=12800, name='yolo_output') return out
import os os.environ["MXNET_EXEC_PREFER_BULK_EXEC"] = sys.argv[1] import numpy as np import mxnet as mx import resnet import time batch_size = 1 dshape = (batch_size, 3, 224, 224) layers = [3, 24, 36, 3] factor = 1 cfg = [int(z * factor) for z in layers] nmodule = sum(cfg) net = resnet.get_symbol(cfg) print("Test resnet with %d modules.. check if bulk message appear" % nmodule) def test_speed(net): dev = mx.gpu(0) dshape = (batch_size, 3, 224, 224) lshape = (batch_size) num_epoch = 100 tmp_data = np.random.uniform(-1, 1, dshape).astype("float32") train_iter = mx.io.NDArrayIter(data=tmp_data, batch_size=batch_size, shuffle=False, last_batch_handle='pad')
# train_imagenet.py --set-resnet-aug ... # Finally, to get the legacy MXNet v1.2 training settings on a per-use basis, invoke as in: # train_imagenet.py --set-data-aug-level 3 parser.set_defaults( # network num_layers = 50, # data resize = 256, num_classes = 1000, num_examples = 1281167, image_shape = '3,224,224', min_random_scale = 1, # if input image has min size k, suggest to use # 256.0/x, e.g. 0.533 for 480 # train num_epochs = 90, lr_step_epochs = '30,60,80', dtype = 'float32' ) args = parser.parse_args() if not args.use_dali: data.set_data_aug_level(parser, 0) # load network import resnet as net sym = net.get_symbol(**vars(args)) # train fit.fit(args, sym, dali.get_rec_iter)
def multilayer_bi_lstm_unroll_new(seq_len, num_hidden, num_label, batch_size, num_lstm_layer, dropout=0): shape = {"data": (32, 3, 32, 32)} data = mx.sym.Variable('data') label = mx.sym.Variable('label') from importlib import import_module resnet = import_module('resnet') conv = resnet.get_symbol(2, 34, '3,32,' + str(seq_len * 8)) column_features = mx.sym.SliceChannel(data=conv, num_outputs=seq_len, axis=3, squeeze_axis=1) forward_param_cells = [] backward_param_cells = [] last_states = [] for i in range(num_lstm_layer): forward_param_cells.append( LSTMParam(i2h_weight=mx.sym.Variable("f_l%d_i2h_weight" % i), i2h_bias=mx.sym.Variable("f_l%d_i2h_bias" % i), h2h_weight=mx.sym.Variable("f_l%d_h2h_weight" % i), h2h_bias=mx.sym.Variable("f_l%d_h2h_bias" % i))) backward_param_cells.append( LSTMParam(i2h_weight=mx.sym.Variable("b_l%d_i2h_weight" % i), i2h_bias=mx.sym.Variable("b_l%d_i2h_bias" % i), h2h_weight=mx.sym.Variable("b_l%d_h2h_weight" % i), h2h_bias=mx.sym.Variable("b_l%d_h2h_bias" % i))) last_states.append( LSTMState(c=mx.sym.Variable("f_l%d_init_c" % 0), h=mx.sym.Variable("f_l%d_init_h" % 0))) last_states.append( LSTMState(c=mx.sym.Variable("b_l%d_init_c" % 0), h=mx.sym.Variable("b_l%d_init_h" % 0))) last_states.append( LSTMState(c=mx.sym.Variable("f_l%d_init_c" % 1), h=mx.sym.Variable("f_l%d_init_h" % 1))) last_states.append( LSTMState(c=mx.sym.Variable("b_l%d_init_c" % 1), h=mx.sym.Variable("b_l%d_init_h" % 1))) assert (len(last_states) == num_lstm_layer * 2) hidden_all = [] forward_hidden = [] for seqidx in range(seq_len): hidden = mx.sym.Flatten(data=column_features[seqidx]) if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) next_state = lstm(num_hidden, indata=hidden, prev_state=last_states[0], param=forward_param_cells[0], seqidx=seqidx, layeridx=0) hidden = next_state.h last_states[0] = next_state forward_hidden.append(hidden) backward_hidden = [] for seqidx in range(seq_len): k = seq_len - seqidx - 1 hidden = mx.sym.Flatten(data=column_features[k]) if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) next_state = lstm(num_hidden, indata=hidden, prev_state=last_states[1], param=backward_param_cells[0], seqidx=k, layeridx=0) hidden = next_state.h last_states[1] = next_state if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) backward_hidden.insert(0, hidden) hidden_all = [] for i in range(seq_len): hidden_all.append( mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1)) forward_hidden = [] for seqidx in range(seq_len): hidden = mx.sym.Flatten(data=hidden_all[seqidx]) if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) next_state = lstm(num_hidden, indata=hidden, prev_state=last_states[2], param=forward_param_cells[1], seqidx=seqidx, layeridx=1) hidden = next_state.h last_states[2] = next_state if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) forward_hidden.append(hidden) backward_hidden = [] for seqidx in range(seq_len): k = seq_len - seqidx - 1 hidden = mx.sym.Flatten(data=hidden_all[k]) if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) next_state = lstm(num_hidden, indata=hidden, prev_state=last_states[3], param=backward_param_cells[1], seqidx=k, layeridx=1) hidden = next_state.h last_states[3] = next_state if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) backward_hidden.insert(0, hidden) hidden_all = [] for i in range(seq_len): hidden_all.append( mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1)) hidden_concat = mx.sym.Concat(*hidden_all, dim=0) pred = mx.sym.FullyConnected(name='fc1', data=hidden_concat, num_hidden=37) softmax_pred = pred pred = mx.sym.Reshape(data=pred, shape=(-4, seq_len, -1, 0)) loss = mx.contrib.sym.ctc_loss(data=pred, label=label) ctc_loss = mx.sym.MakeLoss(loss) softmax_class = mx.symbol.SoftmaxActivation(data=softmax_pred) softmax_loss = mx.sym.BlockGrad(softmax_class) sm = mx.sym.Group([softmax_loss, ctc_loss]) return sm
def bi_lstm_unroll(seq_len, num_hidden, num_label, dropout=0): last_states = [] last_states.append( LSTMState(c=mx.sym.Variable("l0_init_c"), h=mx.sym.Variable("l0_init_h"))) last_states.append( LSTMState(c=mx.sym.Variable("l1_init_c"), h=mx.sym.Variable("l1_init_h"))) forward_param = LSTMParam(i2h_weight=mx.sym.Variable("l0_i2h_weight"), i2h_bias=mx.sym.Variable("l0_i2h_bias"), h2h_weight=mx.sym.Variable("l0_h2h_weight"), h2h_bias=mx.sym.Variable("l0_h2h_bias")) backward_param = LSTMParam(i2h_weight=mx.sym.Variable("l1_i2h_weight"), i2h_bias=mx.sym.Variable("l1_i2h_bias"), h2h_weight=mx.sym.Variable("l1_h2h_weight"), h2h_bias=mx.sym.Variable("l1_h2h_bias")) assert (len(last_states) == 2) # embeding layer data = mx.sym.Variable('data') label = mx.sym.Variable('label') from importlib import import_module resnet = import_module('resnet') conv = resnet.get_symbol(2, 18, '3,32,' + str(seq_len * 8)) print('seq_len : ', seq_len) column_features = mx.sym.SliceChannel(data=conv, num_outputs=seq_len, axis=3, squeeze_axis=1) hidden_all = [] forward_hidden = [] for seqidx in range(seq_len): hidden = mx.sym.Flatten(data=column_features[seqidx]) if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) next_state = lstm(num_hidden, indata=hidden, prev_state=last_states[0], param=forward_param, seqidx=seqidx, layeridx=0) hidden = next_state.h last_states[0] = next_state if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) forward_hidden.append(hidden) backward_hidden = [] for seqidx in range(seq_len): k = seq_len - seqidx - 1 hidden = mx.sym.Flatten(data=column_features[k]) if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) next_state = lstm(num_hidden, indata=hidden, prev_state=last_states[1], param=backward_param, seqidx=k, layeridx=1) hidden = next_state.h last_states[1] = next_state if dropout > 0.: hidden = mx.sym.Dropout(data=hidden, p=dropout) backward_hidden.insert(0, hidden) hidden_all = [] for i in range(seq_len): hidden_all.append( mx.sym.Concat(*[forward_hidden[i], backward_hidden[i]], dim=1)) hidden_concat = mx.sym.Concat(*hidden_all, dim=0) pred = mx.sym.FullyConnected(name='fc1', data=hidden_concat, num_hidden=98) label = mx.sym.Reshape(data=label, shape=(-1, )) label = mx.sym.Cast(data=label, dtype='int32') #does Warp-CTC support bucketing? sm = mx.sym.WarpCTC(name='ctc-loss', data=pred, label=label, label_length=num_label, input_length=seq_len) return sm
set_num=args.set_num, per_set_num=args.per_set_num, duplicate_num=args.duplicate_num, max_num_of_duplicate_set=args.max_num_of_duplicate_set, hdfs_path=args.hdfs_path, rec_name=args.val_rec, max_times_epoch=args.max_times_epoch, data_type='val') label = mx.sym.var("label") arg_params, aux_params, all_layers = load_model() data = all_layers['data'] feature_output = all_layers['fc5_xaiver_256_output'] # score = tiny_network(data) score = resnet.get_symbol(data=data, num_classes=1, num_layers=18, image_shape=data_shapes) feature_set = mx.sym.Custom(score=score, feature=feature_output, data=data, label=label, set_num=args.set_num, per_set_num=args.per_set_num, ctx=ctx, op_type='batch_pool') metric_cost = train_info.get_cost(5 * 16) l2_param = float(args.l2_param) l2_penalty_total = mx.sym.Custom(score=score, set_num=args.set_num, per_set_num=args.per_set_num,