def main(): fcnxs = symbol_fcnxs.get_fcn32s_symbol(numclass=21, workspace_default=1536) fcnxs_model_prefix = "model_pascal/FCN32s_VGG16" if args.model == "fcn16s": fcnxs = symbol_fcnxs.get_fcn16s_symbol(numclass=21, workspace_default=1536) fcnxs_model_prefix = "model_pascal/FCN16s_VGG16" elif args.model == "fcn8s": fcnxs = symbol_fcnxs.get_fcn8s_symbol(numclass=21, workspace_default=1536) fcnxs_model_prefix = "model_pascal/FCN8s_VGG16" arg_names = fcnxs.list_arguments() _, fcnxs_args, fcnxs_auxs = mx.model.load_checkpoint(args.prefix, args.epoch) # print arg_names if not args.retrain: if args.init_type == "vgg16": fcnxs_args, fcnxs_auxs = init_fcnxs.init_from_vgg16(ctx, fcnxs, fcnxs_args, fcnxs_auxs) elif args.init_type == "fcnxs": fcnxs_args, fcnxs_auxs = init_fcnxs.init_from_fcnxs(ctx, fcnxs, fcnxs_args, fcnxs_auxs) train_dataiter = FileIter( root_dir = "./VOC2012", flist_name = "train.lst", # cut_off_size = 400, rgb_mean = (123.68, 116.779, 103.939), ) val_dataiter = FileIter( root_dir = "./VOC2012", flist_name = "val.lst", rgb_mean = (123.68, 116.779, 103.939), ) model = Solver( ctx = ctx, symbol = fcnxs, begin_epoch = 0, num_epoch = 50, arg_params = fcnxs_args, aux_params = fcnxs_auxs, learning_rate = 1e-10, momentum = 0.99, wd = 0.0005) model.fit( train_data = train_dataiter, eval_data = val_dataiter, batch_end_callback = mx.callback.Speedometer(1, 10), epoch_end_callback = mx.callback.do_checkpoint(fcnxs_model_prefix))
def main(): pretrain_prefix = "model_vgg16/VGG16" pretrain_epoch = 813001 vgg16_rpn = symbol_vgg16.get_vgg16_rpn() _, rpn_args, rpn_auxs = mx.model.load_checkpoint(pretrain_prefix, pretrain_epoch) rpn_args, rpn_auxs = init_from_vgg16(ctx, vgg16_rpn, rpn_args, rpn_auxs) ''' rpn_prefix = "model_vgg16/VGG16" epoch = 813001 vgg16_rpn, rpn_args, rpn_auxs = mx.model.load_checkpoint(rpn_prefix, epoch) ''' train_dataiter = FileIter( rgb_mean=(123.68, 116.779, 103.939), file_lst="./file/train.txt", class_names="./file/class.txt", file_mface="./file/model3d.txt", bgfg=bgfg, num_data_used=-1 ) eval_dataiter = FileIter( rgb_mean=(123.68, 116.779, 103.939), file_lst="./file/val.txt", class_names="./file/class.txt", file_mface="./file/model3d.txt", bgfg=bgfg, num_data_used=1000 ) rpn_model = Solver( ctx=ctx, symbol=vgg16_rpn, arg_params=rpn_args, aux_params=rpn_auxs, begin_epoch=8, num_epoch=50, learning_rate=0.000001, momentum=0.9, wd=0.00001, bgfg=bgfg ) rpn_model.fit( train_data=train_dataiter, eval_data=eval_dataiter, regression_metric=smoothl1_metric, softmax_metric=softmax_metric, epoch_end_callback=mx.callback.do_checkpoint("model_vgg16/VGG16") )
def main(): epoch = 74 prefix = "VGG_FC_ILSVRC_16_layers" pwd = os.getcwd() os.chdir("../") _, args, auxs = mx.model.load_checkpoint(prefix, epoch) os.chdir(pwd) deeplabv2_symbol_train = symbol_deeplab.get_deeplab_symbol_train() args = init_deeplab.init_from_vgg16(mx.gpu(), deeplabv2_symbol_train, args, auxs) logger = logging.getLogger() logger.setLevel(logging.INFO) ctx = mx.gpu() train_dataiter = FileIter( root_dir=parser_args.root, flist_name="train.txt", shrink=8, rgb_mean=(123.68, 116.779, 103.939), ) val_dataiter = FileIter( root_dir=parser_args.root, flist_name="test.txt", shrink=8, rgb_mean=(123.68, 116.779, 103.939), ) if parser_args.retrain: begin_epoch = 50 else: begin_epoch = 0 model = Solver(ctx=ctx, symbol=deeplabv2_symbol_train, begin_epoch=begin_epoch, num_epoch=50, arg_params=args, aux_params=auxs, learning_rate=0.001, momentum=0.9, wd=0.0005) deeplab_model_prefix = "DeepLab-V2" model.fit( train_data=train_dataiter, eval_data=val_dataiter, batch_end_callback=mx.callback.Speedometer(1, 10), epoch_end_callback=mx.callback.do_checkpoint(deeplab_model_prefix))
def _train_impl(args, model_specs, logger): if len(args.output) > 0: _make_dirs(args.output) # dataiter dataset_specs_tgt = get_dataset_specs_tgt(args, model_specs) scale, mean_, _ = _get_scalemeanstd() if scale > 0: mean_ /= scale margs = argparse.Namespace(**model_specs) dargs = argparse.Namespace(**dataset_specs_tgt) # number of list_lines split_filename = 'issegm/data_list/{}/{}.lst'.format( margs.dataset, args.split) num_source = 0 with open(split_filename) as f: for item in f.readlines(): num_source = num_source + 1 # batches_per_epoch = num_source // args.batch_images # optimizer assert args.to_epoch is not None if args.stop_epoch is not None: assert args.stop_epoch > args.from_epoch and args.stop_epoch <= args.to_epoch else: args.stop_epoch = args.to_epoch from_iter = args.from_epoch * batches_per_epoch to_iter = args.to_epoch * batches_per_epoch lr_params = model_specs['lr_params'] base_lr = lr_params['base'] if lr_params['type'] == 'fixed': scheduler = FixedScheduler() elif lr_params['type'] == 'step': left_step = [] for step in lr_params['args']['step']: if from_iter > step: base_lr *= lr_params['args']['factor'] continue left_step.append(step - from_iter) model_specs['lr_params']['step'] = left_step scheduler = mx.lr_scheduler.MultiFactorScheduler(**lr_params['args']) elif lr_params['type'] == 'linear': scheduler = LinearScheduler(updates=to_iter + 1, frequency=50, stop_lr=min(base_lr / 100., 1e-6), offset=from_iter) elif lr_params['type'] == 'poly': scheduler = PolyScheduler(updates=to_iter + 1, frequency=50, stop_lr=min(base_lr / 100., 1e-8), power=0.9, offset=from_iter) initializer = mx.init.Xavier(rnd_type='gaussian', factor_type='in', magnitude=2) optimizer_params = { 'learning_rate': base_lr, 'momentum': 0.9, 'wd': args.weight_decay, 'lr_scheduler': scheduler, 'rescale_grad': 1.0 / len(args.gpus.split(',')), } data_src_port = args.init_src_port data_src_num = int(num_source * data_src_port) mod = _get_module(args, margs, dargs) addr_weights = args.weights # first weights should be xxxx_ep-0000.params! addr_output = args.output # initializer net_args = None net_auxs = None ### if addr_weights is not None: net_args, net_auxs = mxutil.load_params_from_file(addr_weights) ####################################### training model to_model = osp.join(addr_output, str(args.idx_round), '{}_ep'.format(args.model)) dataiter = FileIter( dataset=margs.dataset, split=args.split, data_root=args.data_root, num_sel_source=data_src_num, num_source=num_source, seed_int=args.seed_int, dataset_tgt=args.dataset_tgt, split_tgt=args.split_tgt, data_root_tgt=args.data_root_tgt, sampler='random', batch_images=args.batch_images, meta=dataset_specs_tgt, rgb_mean=mean_, feat_stride=margs.feat_stride, label_stride=margs.feat_stride, origin_size=args.origin_size, origin_size_tgt=args.origin_size_tgt, crop_size=args.crop_size, scale_rate_range=[float(_) for _ in args.scale_rate_range.split(',')], transformer=None, transformer_image=ts.Compose(_get_transformer_image()), prefetch_threads=args.prefetch_threads, prefetcher_type=args.prefetcher, ) dataiter.reset() mod.fit( dataiter, eval_metric=_get_metric(), batch_end_callback=mx.callback.log_train_metric(10, auto_reset=False), epoch_end_callback=mx.callback.do_checkpoint(to_model), kvstore=args.kvstore, optimizer='sgd', optimizer_params=optimizer_params, initializer=initializer, arg_params=net_args, aux_params=net_auxs, allow_missing=args.from_epoch == 0, begin_epoch=args.from_epoch, num_epoch=args.stop_epoch, )
parser.add_argument('--load-epoch', type=int, help="load the model on an epoch using the model-prefix") parser.add_argument('--save-model-prefix', type=str, help='the prefix of the model to save') # todo statistic about mean data args = parser.parse_args() import symbol net = symbol.get_symbol(output_dim = 30) from data import FileIter train = FileIter( eval_ratio = 0.2, is_val = False, data_name = "data", batch_size = args.batch_size, label_name = "lr_label" ) val = FileIter( eval_ratio = 0.2, is_val = True, data_name = "data", batch_size = 1, label_name = "lr_label" ) from solver import Solver model = Solver(
def _train_impl(args, model_specs, logger): if len(args.output) > 0: _make_dirs(args.output) # dataiter dataset_specs = get_dataset_specs(args, model_specs) scale, mean_, _ = _get_scalemeanstd() if scale > 0: mean_ /= scale margs = argparse.Namespace(**model_specs) dargs = argparse.Namespace(**dataset_specs) dataiter = FileIter( dataset=margs.dataset, split=args.split, data_root=args.data_root, sampler='random', batch_images=args.batch_images, meta=dataset_specs, rgb_mean=mean_, feat_stride=margs.feat_stride, label_stride=margs.feat_stride, origin_size=args.origin_size, crop_size=args.crop_size, scale_rate_range=[float(_) for _ in args.scale_rate_range.split(',')], transformer=None, transformer_image=ts.Compose(_get_transformer_image()), prefetch_threads=args.prefetch_threads, prefetcher_type=args.prefetcher, ) dataiter.reset() # optimizer assert args.to_epoch is not None if args.stop_epoch is not None: assert args.stop_epoch > args.from_epoch and args.stop_epoch <= args.to_epoch else: args.stop_epoch = args.to_epoch from_iter = args.from_epoch * dataiter.batches_per_epoch to_iter = args.to_epoch * dataiter.batches_per_epoch lr_params = model_specs['lr_params'] base_lr = lr_params['base'] if lr_params['type'] == 'fixed': scheduler = FixedScheduler() elif lr_params['type'] == 'step': left_step = [] for step in lr_params['args']['step']: if from_iter > step: base_lr *= lr_params['args']['factor'] continue left_step.append(step - from_iter) model_specs['lr_params']['step'] = left_step scheduler = mx.lr_scheduler.MultiFactorScheduler(**lr_params['args']) elif lr_params['type'] == 'linear': scheduler = LinearScheduler(updates=to_iter + 1, frequency=50, stop_lr=min(base_lr / 100., 1e-6), offset=from_iter) optimizer_params = { 'learning_rate': base_lr, 'momentum': 0.9, 'wd': args.weight_decay, 'lr_scheduler': scheduler, 'rescale_grad': 1.0 / len(args.gpus.split(',')), } # initializer net_args = None net_auxs = None if args.weights is not None: net_args, net_auxs = mxutil.load_params_from_file(args.weights) initializer = mx.init.Xavier(rnd_type='gaussian', factor_type='in', magnitude=2) # to_model = osp.join(args.output, '{}_ep'.format(args.model)) mod = _get_module(args, margs, dargs) mod.fit( dataiter, eval_metric=_get_metric(), batch_end_callback=mx.callback.Speedometer(dataiter.batch_size, 1), epoch_end_callback=mx.callback.do_checkpoint(to_model), kvstore=args.kvstore, optimizer='sgd', optimizer_params=optimizer_params, initializer=initializer, arg_params=net_args, aux_params=net_auxs, allow_missing=args.from_epoch == 0, begin_epoch=args.from_epoch, num_epoch=args.stop_epoch, )
def _val_impl(args, model_specs, logger): assert args.prefetch_threads == 1 assert args.weights is not None net_args, net_auxs = util.load_params_from_file(args.weights) mod = _get_module(model_specs) has_gt = args.split in ( 'train', 'val', ) scale_, mean_, std_ = _get_scalemeanstd() if args.test_scales is None: crop_sizes = [model_specs['crop_size']] else: crop_sizes = sorted([int(_) for _ in args.test_scales.split(',')])[::-1] batch_images = args.batch_images if has_gt: gt_labels = np.array( parse_split_file(model_specs['split_filename'], args.data_root)[1]) save_dir = os.path.join(args.output, os.path.splitext(args.log_file)[0]) if not os.path.isdir(save_dir): os.makedirs(save_dir) preds = [] for crop_size in crop_sizes: save_path = os.path.join(save_dir, 'preds_sz{}'.format(crop_size)) if os.path.isfile(save_path): logger.info('File %s exists, skipped crop size %d', save_path, crop_size) with open(save_path) as f: preds.append(cPickle.load(f)) continue ts_list = [ ts.Scale(crop_size), ts.ThreeCrops(crop_size) if args.test_3crops else ts.CenterCrop(crop_size), ] if scale_ > 0: ts_list.append(ts.ListInput(ts.ColorScale(np.single(scale_)))) ts_list += [ts.ListInput(ts.ColorNormalize(mean_, std_))] transformer = ts.Compose(ts_list) dataiter = FileIter( split_filename=model_specs['split_filename'], data_root=args.data_root, has_gt=has_gt, batch_images=batch_images, transformer=transformer, prefetch_threads=args.prefetch_threads, prefetcher_type=args.prefetcher, ) dataiter.reset() mod.bind(dataiter.provide_data, dataiter.provide_label, for_training=False, force_rebind=True) if not mod.params_initialized: mod.init_params(arg_params=net_args, aux_params=net_auxs) this_call_preds = [] start = time.time() counter = [0, 0] for nbatch, batch in enumerate(dataiter): mod.forward(batch, is_train=False) outputs = mod.get_outputs()[0].asnumpy() outputs = outputs.reshape( (batch_images, -1, model_specs['classes'])).mean(1) this_call_preds.append(outputs) if args.test_flipping: batch.data[0] = mx.nd.flip(batch.data[0], axis=3) mod.forward(batch, is_train=False) outputs = mod.get_outputs()[0].asnumpy() outputs = outputs.reshape( (batch_images, -1, model_specs['classes'])).mean(1) this_call_preds[-1] = (this_call_preds[-1] + outputs) / 2 score_str = '' if has_gt: counter[0] += batch_images counter[1] += (this_call_preds[-1].argmax(1) == gt_labels[nbatch * batch_images:(nbatch + 1) * batch_images]).sum() score_str = ', Top1 {:.4f}%'.format(100.0 * counter[1] / counter[0]) logger.info('Crop size {}, done {}/{} at speed: {:.2f}/s{}'.\ format(crop_size, nbatch+1, dataiter.batches_per_epoch, 1.*(nbatch+1)*batch_images / (time.time()-start), score_str)) logger.info('Done crop size {} in {:.4f}s.'.format( crop_size, time.time() - start)) this_call_preds = np.vstack(this_call_preds) with open(save_path, 'wb') as f: cPickle.dump(this_call_preds, f) preds.append(this_call_preds) for num_sizes in set(( 1, len(crop_sizes), )): for this_pred_inds in itertools.combinations(xrange(len(crop_sizes)), num_sizes): this_pred = np.mean([preds[_] for _ in this_pred_inds], axis=0) this_pred_label = this_pred.argsort(1)[:, -1 - np.arange(5)] logger.info('Done testing crop_size %s', [crop_sizes[_] for _ in this_pred_inds]) if has_gt: top1 = 100. * (this_pred_label[:, 0] == gt_labels).sum() / gt_labels.size top5 = 100. * sum( map(lambda x, y: y in x.tolist(), this_pred_label, gt_labels)) / gt_labels.size logger.info('Top1 %.4f%%, Top5 %.4f%%', top1, top5) else: # TODO: Save predictions for submission raise NotImplementedError('Save predictions for submission')
def _train_impl(args, model_specs, logger): if len(args.output) > 0: _make_dirs(args.output) # dataiter dataset_specs = get_dataset_specs(args, model_specs) scale, mean_, _ = _get_scalemeanstd() if scale > 0: mean_ /= scale margs = argparse.Namespace(**model_specs) dargs = argparse.Namespace(**dataset_specs) dataiter = FileIter(dataset=margs.dataset, split=args.split, data_root=args.data_root, sampler='random', batch_images=args.batch_images, meta=dataset_specs, rgb_mean=mean_, feat_stride=margs.feat_stride, label_stride=margs.feat_stride, origin_size=args.origin_size, crop_size=args.crop_size, scale_rate_range=[float(_) for _ in args.scale_rate_range.split(',')], transformer=None, transformer_image=ts.Compose(_get_transformer_image()), prefetch_threads=args.prefetch_threads, prefetcher_type=args.prefetcher,) dataiter.reset() # optimizer assert args.to_epoch is not None if args.stop_epoch is not None: assert args.stop_epoch > args.from_epoch and args.stop_epoch <= args.to_epoch else: args.stop_epoch = args.to_epoch from_iter = args.from_epoch * dataiter.batches_per_epoch to_iter = args.to_epoch * dataiter.batches_per_epoch lr_params = model_specs['lr_params'] base_lr = lr_params['base'] if lr_params['type'] == 'fixed': scheduler = FixedScheduler() elif lr_params['type'] == 'step': left_step = [] for step in lr_params['args']['step']: if from_iter > step: base_lr *= lr_params['args']['factor'] continue left_step.append(step - from_iter) model_specs['lr_params']['step'] = left_step scheduler = mx.lr_scheduler.MultiFactorScheduler(**lr_params['args']) elif lr_params['type'] == 'linear': scheduler = LinearScheduler(updates=to_iter+1, frequency=50, stop_lr=min(base_lr/100., 1e-6), offset=from_iter) optimizer_params = { 'learning_rate': base_lr, 'momentum': 0.9, 'wd': args.weight_decay, 'lr_scheduler': scheduler, 'rescale_grad': 1.0/len(args.gpus.split(',')), } # initializer net_args = None net_auxs = None if args.weights is not None: net_args, net_auxs = mxutil.load_params_from_file(args.weights) initializer = mx.init.Xavier(rnd_type='gaussian', factor_type='in', magnitude=2) # to_model = osp.join(args.output, '{}_ep'.format(args.model)) mod = _get_module(args, margs, dargs) mod.fit( dataiter, eval_metric=_get_metric(), batch_end_callback=mx.callback.Speedometer(dataiter.batch_size, 1), epoch_end_callback=mx.callback.do_checkpoint(to_model), kvstore=args.kvstore, optimizer='sgd', optimizer_params=optimizer_params, initializer=initializer, arg_params=net_args, aux_params=net_auxs, allow_missing=args.from_epoch == 0, begin_epoch=args.from_epoch, num_epoch=args.stop_epoch, )
def main(): gpu_list = [] _gpus = args.gpu.split(',') for _gpu in _gpus: _gpu = _gpu.strip() if len(_gpu)==0: continue gpu_list.append(int(_gpu)) assert len(gpu_list)>0 ctx = mx.gpu(gpu_list[0]) carvn_root = '' num_classes = 2 cutoff = None if args.cutoff==0 else args.cutoff epochs = [74,31,27,19] model_prefixes = ['VGG_FC_ILSVRC_16_layers', args.model_dir+"/FCN32s_VGG16", args.model_dir+"/FCN16s_VGG16", args.model_dir+"/FCN8s_VGG16"] if args.model == "fcn16s": fcnxs = symbol_fcnxs.get_fcn16s_symbol(numclass=num_classes, workspace_default=1536) fcnxs_model_prefix = model_prefixes[2] load_prefix = model_prefixes[1] lr = 1e-5 run_epochs = epochs[2] load_epoch = epochs[1] elif args.model == "fcn8s": fcnxs = symbol_fcnxs.get_fcn8s_symbol(numclass=num_classes, workspace_default=1536) fcnxs_model_prefix = model_prefixes[3] load_prefix = model_prefixes[2] lr = 1e-6 run_epochs = epochs[3] load_epoch = epochs[2] else: fcnxs = symbol_fcnxs.get_fcn32s_symbol(numclass=num_classes, workspace_default=1536) fcnxs_model_prefix = model_prefixes[1] load_prefix = model_prefixes[0] lr = 1e-4 run_epochs = epochs[1] load_epoch = epochs[0] arg_names = fcnxs.list_arguments() print('loading', load_prefix, load_epoch) print('lr', lr) print('model_prefix', fcnxs_model_prefix) print('running epochs', run_epochs) _, fcnxs_args, fcnxs_auxs = mx.model.load_checkpoint(load_prefix, load_epoch) if not args.retrain: if args.model == "fcn16s" or args.model == "fcn8s": fcnxs_args, fcnxs_auxs = init_fcnxs.init_from_fcnxs(ctx, fcnxs, fcnxs_args, fcnxs_auxs) else: fcnxs_args, fcnxs_auxs = init_fcnxs.init_from_vgg16(ctx, fcnxs, fcnxs_args, fcnxs_auxs) train_dataiter = FileIter( root_dir = carvn_root, flist_name = "../data/train.lst", cut_off_size = cutoff, rgb_mean = (123.68, 116.779, 103.939), ) val_dataiter = FileIter( root_dir = carvn_root, flist_name = "../data/val.lst", cut_off_size = cutoff, rgb_mean = (123.68, 116.779, 103.939), ) model = Solver( ctx = ctx, symbol = fcnxs, begin_epoch = 0, num_epoch = run_epochs, arg_params = fcnxs_args, aux_params = fcnxs_auxs, learning_rate = lr, momentum = 0.99, wd = 0.0005) _metric = DiceMetric() model.fit( train_data = train_dataiter, eval_data = val_dataiter, eval_metric = _metric, batch_end_callback = mx.callback.Speedometer(1, 10), epoch_end_callback = mx.callback.do_checkpoint(fcnxs_model_prefix))
import numpy as np import logging logging.basicConfig(level=logging.DEBUG) import mxnet as mx from create_network import get_symbol from data import FileIter if __name__ == "__main__": batch_size = 32 lr = 0.001 beta1 = 0.5 dataiter = FileIter(r'E:\clean_data', batch_size) network = get_symbol() ctx = mx.gpu(1) # =============module network============= mod_network = mx.mod.Module(symbol=network, data_names=('data',), label_names=('l2_label',), context=ctx) mod_network.bind(data_shapes=dataiter.provide_data, label_shapes=dataiter.provide_label) mod_network.init_params(initializer=mx.init.Normal(0.02)) mod_network.init_optimizer( optimizer='adam', optimizer_params={ 'learning_rate': lr, 'wd': 0., 'beta1': beta1, })
def train_net(args, ctx, pretrained, epoch, prefix, lr=0.001): root_dir = args.dataset train_dir = os.path.join(root_dir, "keypoint_train_images_20170902") anno_file = os.path.join(root_dir, "keypoint_train_annotations_20170909.json") if args.network == 'vgg': mean_value = 127 div_num = 255. else: mean_value = 0 div_num = 1. train_data = FileIter(train_dir, anno_file, batch_size=args.batch_size, no_shuffle=args.no_shuffle, mean_value=mean_value, div_num=div_num) if args.network == 'vgg': sym = symbol_vgg.get_vgg_train() else: sym = symbol_resnet.get_resnet_train() arg_params, aux_params = load_param(pretrained, epoch, convert=True) # infer shape data_shape_dict = dict(train_data.provide_data + train_data.provide_label) arg_shape, out_shape, aux_shape = sym.infer_shape(**data_shape_dict) arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape)) out_shape_dict = dict(zip(sym.list_outputs(), out_shape)) aux_shape_dict = dict(zip(sym.list_auxiliary_states(), aux_shape)) print('output shape %s' % pprint.pformat(out_shape_dict)) # check parameter shapes for k in sym.list_arguments(): if k in data_shape_dict: continue if k not in arg_params: print('reset not in model params ' + k) arg_params[k] = mx.random.normal(0, 0.01, shape=arg_shape_dict[k]) assert k in arg_params, k + ' not initialized' if arg_params[k].shape != arg_shape_dict[k]: arg_params[k] = mx.random.normal(0, 0.01, shape=arg_shape_dict[k]) print 'need init', k # assert arg_params[k].shape == arg_shape_dict[k], \ # 'shape inconsistent for ' + k + ' inferred ' + str(arg_shape_dict[k]) + ' provided ' + str( # arg_params[k].shape) for k in sym.list_auxiliary_states(): if k not in aux_params: print('reset not in model params ' + k) aux_params[k] = mx.nd.zeros(shape=aux_shape_dict[k]) assert k in aux_params, k + ' not initialized' if aux_params[k].shape != aux_shape_dict[k]: aux_params[k] = mx.nd.zeros(shape=aux_shape_dict[k]) print 'need init', k # assert aux_params[k].shape == aux_shape_dict[k], \ # 'shape inconsistent for ' + k + ' inferred ' + str(aux_shape_dict[k]) + ' provided ' + str( # aux_params[k].shape) data_names = [k[0] for k in train_data.provide_data] label_names = [k[0] for k in train_data.provide_label] if args.network == 'resnet': fixed_param_prefix = ['conv0', 'stage1', 'gamma', 'beta'] fixed_param_names = list() if fixed_param_prefix is not None: for name in sym.list_arguments(): for prefix_ in fixed_param_prefix: if prefix_ in name: fixed_param_names.append(name) print('fix params ' + str(fixed_param_names)) else: fixed_param_prefix = [ 'conv1_1', 'conv1_2', 'conv2_1', 'conv2_2', 'conv3_1', 'conv3_2', 'conv3_3', 'conv3_4', 'conv4_1', 'conv4_2', 'conv4_3', 'conv4_4' ] fixed_param_prefix = [] fixed_param_names = list() if fixed_param_prefix is not None: for name in sym.list_arguments(): for prefix_ in fixed_param_prefix: if prefix_ in name: fixed_param_names.append(name) print('fix params ' + str(fixed_param_names)) mod = CustomModule(symbol=sym, data_names=data_names, label_names=label_names, context=ctx, logger=logger, fixed_param_names=fixed_param_names) batch_end_callback = mx.callback.Speedometer(train_data.batch_size, frequent=10, auto_reset=False) epoch_end_callback = mx.callback.do_checkpoint(prefix) eval_metrics = mx.metric.CompositeEvalMetric() for branch in range(1, 3): for stage in range(1, 7): eval_metrics.add( AICRMSE(train_data.batch_size / len(ctx), stage=stage, branch=branch)) # optimizer optimizer_params = { 'learning_rate': lr, 'lr_scheduler': mx.lr_scheduler.FactorScheduler(100000, factor=0.3), 'rescale_grad': 1.0 / train_data.batch_size / len(ctx) } mod.fit(train_data, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, eval_metric=eval_metrics, optimizer='adam', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, num_epoch=100)
def _val_impl(args, model_specs, logger): assert args.prefetch_threads == 1 assert args.weights is not None net_args, net_auxs = mxutil.load_params_from_file(args.weights) mod = _get_module(args, model_specs) has_gt = args.split in ('train', 'val',) scale_, mean_, std_ = _get_scalemeanstd() if args.test_scales is None: crop_sizes = [model_specs['crop_size']] else: crop_sizes = sorted([int(_) for _ in args.test_scales.split(',')])[::-1] batch_images = args.batch_images if has_gt: gt_labels = np.array(parse_split_file(model_specs['dataset'], args.split, args.data_root)[1]) save_dir = os.path.join(args.output, os.path.splitext(args.log_file)[0]) if not os.path.isdir(save_dir): os.makedirs(save_dir) preds = [] for crop_size in crop_sizes: save_path = os.path.join(save_dir, 'preds_sz{}'.format(crop_size)) if os.path.isfile(save_path): logger.info('File %s exists, skipped crop size %d', save_path, crop_size) with open(save_path) as f: preds.append(cPickle.load(f)) continue ts_list = [ts.Scale(crop_size), ts.ThreeCrops(crop_size) if args.test_3crops else ts.CenterCrop(crop_size),] if scale_ > 0: ts_list.append(ts.ListInput(ts.ColorScale(np.single(scale_)))) ts_list += [ts.ListInput(ts.ColorNormalize(mean_, std_))] transformer = ts.Compose(ts_list) dataiter = FileIter(dataset=model_specs['dataset'], split=args.split, data_root=args.data_root, sampler='fixed', has_gt=has_gt, batch_images=batch_images, transformer=transformer, prefetch_threads=args.prefetch_threads, prefetcher_type=args.prefetcher,) dataiter.reset() mod.bind(dataiter.provide_data, dataiter.provide_label, for_training=False, force_rebind=True) if not mod.params_initialized: mod.init_params(arg_params=net_args, aux_params=net_auxs) this_call_preds = [] start = time.time() counter = [0, 0] for nbatch, batch in enumerate(dataiter): mod.forward(batch, is_train=False) outputs = mod.get_outputs()[0].asnumpy() outputs = outputs.reshape((batch_images, -1, model_specs['classes'])).mean(1) this_call_preds.append(outputs) if args.test_flipping: batch.data[0] = mx.nd.flip(batch.data[0], axis=3) mod.forward(batch, is_train=False) outputs = mod.get_outputs()[0].asnumpy() outputs = outputs.reshape((batch_images, -1, model_specs['classes'])).mean(1) this_call_preds[-1] = (this_call_preds[-1] + outputs) / 2 score_str = '' if has_gt: counter[0] += batch_images counter[1] += (this_call_preds[-1].argmax(1) == gt_labels[nbatch*batch_images : (nbatch+1)*batch_images]).sum() score_str = ', Top1 {:.4f}%'.format(100.0*counter[1] / counter[0]) logger.info('Crop size {}, done {}/{} at speed: {:.2f}/s{}'.\ format(crop_size, nbatch+1, dataiter.batches_per_epoch, 1.*(nbatch+1)*batch_images / (time.time()-start), score_str)) logger.info('Done crop size {} in {:.4f}s.'.format(crop_size, time.time() - start)) this_call_preds = np.vstack(this_call_preds) with open(save_path, 'wb') as f: cPickle.dump(this_call_preds, f) preds.append(this_call_preds) for num_sizes in set((1, len(crop_sizes),)): for this_pred_inds in itertools.combinations(xrange(len(crop_sizes)), num_sizes): this_pred = np.mean([preds[_] for _ in this_pred_inds], axis=0) this_pred_label = this_pred.argsort(1)[:, -1 - np.arange(5)] logger.info('Done testing crop_size %s', [crop_sizes[_] for _ in this_pred_inds]) if has_gt: top1 = 100. * (this_pred_label[:, 0] == gt_labels).sum() / gt_labels.size top5 = 100. * sum(map(lambda x, y: y in x.tolist(), this_pred_label, gt_labels)) / gt_labels.size logger.info('Top1 %.4f%%, Top5 %.4f%%', top1, top5) else: # TODO: Save predictions for submission raise NotImplementedError('Save predictions for submission')
def _train_impl(args, model_specs, logger): # dataiter scale_, mean_, std_ = _get_scalemeanstd() assert scale_ == 1./255 pca = (np.array([0.2175, 0.0188, 0.0045]), np.array([[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.814], [-0.5836, -0.6948, 0.4203]])) crop_size = model_specs['crop_size'] transformer = ts.Compose([ts.RandomSizedCrop(crop_size), ts.ColorScale(np.single(1./255)), ts.ColorJitter(crop_size, 0.4, 0.4, 0.4), ts.Lighting(0.1, pca[0], pca[1]), ts.Bound(lower=0., upper=1.), ts.HorizontalFlip(), ts.ColorNormalize(mean_, std_),]) if model_specs['dataset'] == 'ilsvrc-cls': dataiter = FileIter(dataset=model_specs['dataset'], split=args.split, data_root=args.data_root, sampler='random', batch_images=model_specs['batch_images'], transformer=transformer, prefetch_threads=args.prefetch_threads, prefetcher_type=args.prefetcher,) else: raise NotImplementedError('Unknown dataset: {}'.format(model_specs['dataset'])) dataiter.reset() # optimizer assert args.to_epoch is not None if args.stop_epoch is not None: assert args.stop_epoch > args.from_epoch and args.stop_epoch <= args.to_epoch else: args.stop_epoch = args.to_epoch from_iter = args.from_epoch * dataiter.batches_per_epoch to_iter = args.to_epoch * dataiter.batches_per_epoch lr_params = model_specs['lr_params'] base_lr = lr_params['base'] if lr_params['type'] == 'fixed': scheduler = FixedScheduler() elif lr_params['type'] == 'step': left_step = [] for step in lr_params['args']['step']: if from_iter > step: base_lr *= lr_params['args']['factor'] continue left_step.append(step - from_iter) model_specs['lr_params']['step'] = left_step scheduler = mx.lr_scheduler.MultiFactorScheduler(**lr_params['args']) elif lr_params['type'] == 'linear': scheduler = LinearScheduler(updates=to_iter+1, frequency=50, stop_lr=1e-6, offset=from_iter) optimizer_params = { 'learning_rate': base_lr, 'momentum': 0.9, 'wd': 0.0001, 'lr_scheduler': scheduler, } # initializer net_args = None net_auxs = None if args.weights is not None: net_args, net_auxs = mxutil.load_params_from_file(args.weights) initializer = mx.init.Mixed( ['linear.*', '.*',], [TorchXavier_Linear(rnd_type='uniform', factor_type='in', magnitude=1), mx.init.Xavier(rnd_type='gaussian', factor_type='in', magnitude=2),] ) # fit to_model = os.path.join(args.output, '{}_ep'.format(args.model)) mod = _get_module(args, model_specs) mod.fit( dataiter, eval_metric=_get_metric(), batch_end_callback=mx.callback.Speedometer(dataiter.batch_size, 1), epoch_end_callback=mx.callback.do_checkpoint(to_model), kvstore=args.kvstore, optimizer='TorchNesterov', optimizer_params=optimizer_params, initializer=initializer, arg_params=net_args, aux_params=net_auxs, allow_missing=args.from_epoch == 0, begin_epoch=args.from_epoch, num_epoch=args.stop_epoch, )
from collections import namedtuple BatchEndParam = namedtuple('BatchEndParams', ['epoch', 'nbatch', 'eval_metric']) if __name__ == "__main__": ctx = mx.gpu(1) network = get_symbol() model_prefix = 'facealignment' save_model_prefix = "facealignment" # _, network_args, network_auxs = mx.model.load_checkpoint(model_prefix, 1) network_args = {} network_auxs = {} network_args, network_auxs = init_params(ctx, network, network_args, network_auxs, (1, 3, 128, 128)) dataiter = FileIter(r'E:\mxnet_tutorial_data') # prepare grad_params arg_shapes, out_shapes, aux_shapes = network.infer_shape( data=dataiter.provide_data[0][1]) arg_names = network.list_arguments() grad_params = {} for name, shape in zip(arg_names, arg_shapes): if not (name.endswith('data') or name.endswith('label')): grad_params[name] = mx.nd.zeros(shape, ctx) # prepare aux_params aux_names = network.list_auxiliary_states() aux_params = { k: mx.nd.zeros(s, ctx) for k, s in zip(aux_names, aux_shapes)