def train_net(image_set, year, root_path, devkit_path, pretrained, epoch,
prefix, ctx, begin_epoch, end_epoch, frequent, kv_store, work_load_list=None, resume=False):
# set up logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# load symbol
sym = get_vgg_rcnn()
# setup multi-gpu
config.TRAIN.BATCH_IMAGES *= len(ctx)
config.TRAIN.BATCH_SIZE *= len(ctx)
# load training data
voc, roidb, means, stds = load_rpn_roidb(image_set, year, root_path, devkit_path, flip=True)
train_data = ROIIter(roidb, batch_size=config.TRAIN.BATCH_IMAGES, shuffle=True, mode='train',
ctx=ctx, work_load_list=work_load_list)
# infer max shape
max_data_shape = [('data', (1, 3, 1000, 1000))]
# load pretrained
args, auxs = load_param(pretrained, epoch, convert=True)
# initialize params
if not resume:
input_shapes = {k: v for k, v in train_data.provide_data + train_data.provide_label}
arg_shape, _, _ = sym.infer_shape(**input_shapes)
arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape))
args['cls_score_weight'] = mx.random.normal(mean=0, stdvar=0.01, shape=arg_shape_dict['cls_score_weight'])
args['cls_score_bias'] = mx.nd.zeros(shape=arg_shape_dict['cls_score_bias'])
args['bbox_pred_weight'] = mx.random.normal(mean=0, stdvar=0.001, shape=arg_shape_dict['bbox_pred_weight'])
args['bbox_pred_bias'] = mx.nd.zeros(shape=arg_shape_dict['bbox_pred_bias'])
# train
solver = Solver(prefix, sym, ctx, begin_epoch, end_epoch, kv_store, args, auxs, momentum=0.9, wd=0.0005,
learning_rate=1e-3, lr_scheduler=mx.lr_scheduler.FactorScheduler(30000, 0.1),
mutable_data_shape=True, max_data_shape=max_data_shape)
solver.fit(train_data, frequent=frequent)
# edit params and save
for epoch in range(begin_epoch + 1, end_epoch + 1):
arg_params, aux_params = load_checkpoint(prefix, epoch)
arg_params['bbox_pred_weight'] = (arg_params['bbox_pred_weight'].T * mx.nd.array(stds)).T
arg_params['bbox_pred_bias'] = arg_params['bbox_pred_bias'] * mx.nd.array(stds) + \
mx.nd.array(means)
save_checkpoint(prefix, epoch, arg_params, aux_params)
def combine_model(prefix1, epoch1, prefix2, epoch2, prefix_out, epoch_out):
args1, auxs1 = load_checkpoint(prefix1, epoch1)
args2, auxs2 = load_checkpoint(prefix2, epoch2)
arg_names = args1.keys() + args2.keys()
aux_names = auxs1.keys() + auxs2.keys()
args = dict()
for arg in arg_names:
if arg in args1:
args[arg] = args1[arg]
if arg in args2:
args[arg] = args2[arg]
auxs = dict()
for aux in aux_names:
if aux in auxs1:
auxs[aux] = auxs1[aux]
if aux in auxs2:
auxs[aux] = auxs2[aux]
save_checkpoint(prefix_out, epoch_out, args, auxs)
def train_rcnn(image_set,
year,
root_path,
devkit_path,
pretrained,
epoch,
prefix,
ctx,
begin_epoch,
end_epoch,
frequent,
kv_store,
work_load_list=None,
resume=False,
proposal='rpn'):
# set up logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# load symbol
sym = get_vgg_rcnn()
# setup multi-gpu
config.TRAIN.BATCH_IMAGES *= len(ctx)
config.TRAIN.BATCH_SIZE *= len(ctx)
# load training data
voc, roidb, means, stds = eval('load_' + proposal + '_roidb')(image_set,
year,
root_path,
devkit_path,
flip=True)
train_data = ROIIter(roidb,
batch_size=config.TRAIN.BATCH_IMAGES,
shuffle=True,
mode='train',
ctx=ctx,
work_load_list=work_load_list)
# infer max shape
max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3, 1000, 1000))]
# load pretrained
args, auxs, _ = load_param(pretrained, epoch, convert=True)
# initialize params
if not resume:
input_shapes = {
k: v
for k, v in train_data.provide_data + train_data.provide_label
}
arg_shape, _, _ = sym.infer_shape(**input_shapes)
arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape))
args['cls_score_weight'] = mx.random.normal(
0, 0.01, shape=arg_shape_dict['cls_score_weight'])
args['cls_score_bias'] = mx.nd.zeros(
shape=arg_shape_dict['cls_score_bias'])
args['bbox_pred_weight'] = mx.random.normal(
0, 0.001, shape=arg_shape_dict['bbox_pred_weight'])
args['bbox_pred_bias'] = mx.nd.zeros(
shape=arg_shape_dict['bbox_pred_bias'])
# prepare training
if config.TRAIN.FINETUNE:
fixed_param_prefix = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5']
else:
fixed_param_prefix = ['conv1', 'conv2']
data_names = [k[0] for k in train_data.provide_data]
label_names = [k[0] for k in train_data.provide_label]
batch_end_callback = Speedometer(train_data.batch_size, frequent=frequent)
epoch_end_callback = mx.callback.do_checkpoint(prefix)
if config.TRAIN.HAS_RPN is True:
eval_metric = AccuracyMetric(use_ignore=True, ignore=-1)
cls_metric = LogLossMetric(use_ignore=True, ignore=-1)
else:
eval_metric = AccuracyMetric()
cls_metric = LogLossMetric()
bbox_metric = SmoothL1LossMetric()
eval_metrics = mx.metric.CompositeEvalMetric()
for child_metric in [eval_metric, cls_metric, bbox_metric]:
eval_metrics.add(child_metric)
optimizer_params = {
'momentum': 0.9,
'wd': 0.0005,
'learning_rate': 0.001,
'lr_scheduler': mx.lr_scheduler.FactorScheduler(30000, 0.1),
'rescale_grad': (1.0 / config.TRAIN.BATCH_SIZE)
}
# train
mod = MutableModule(sym,
data_names=data_names,
label_names=label_names,
logger=logger,
context=ctx,
work_load_list=work_load_list,
max_data_shapes=max_data_shape,
fixed_param_prefix=fixed_param_prefix)
mod.fit(train_data,
eval_metric=eval_metrics,
epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback,
kvstore=kv_store,
optimizer='sgd',
optimizer_params=optimizer_params,
arg_params=args,
aux_params=auxs,
begin_epoch=begin_epoch,
num_epoch=end_epoch)
# edit params and save
for epoch in range(begin_epoch + 1, end_epoch + 1):
arg_params, aux_params = load_checkpoint(prefix, epoch)
arg_params['bbox_pred_weight'] = (arg_params['bbox_pred_weight'].T *
mx.nd.array(stds)).T
arg_params['bbox_pred_bias'] = arg_params['bbox_pred_bias'] * mx.nd.array(stds) + \
mx.nd.array(means)
save_checkpoint(prefix, epoch, arg_params, aux_params)
def train_rcnn(
image_set,
year,
root_path,
devkit_path,
pretrained,
epoch,
prefix,
ctx,
begin_epoch,
end_epoch,
frequent,
kv_store,
work_load_list=None,
):
# load symbol
sym = get_vgg_rcnn()
# setup multi-gpu
config.TRAIN.BATCH_IMAGES *= len(ctx)
config.TRAIN.BATCH_SIZE *= len(ctx)
# load training data
voc, roidb, means, stds = load_rpn_roidb(image_set, year, root_path, devkit_path, flip=True)
train_data = ROIIter(
roidb, batch_size=config.TRAIN.BATCH_IMAGES, shuffle=True, mode="train", ctx=ctx, work_load_list=work_load_list
)
# infer max shape
max_data_shape = [("data", (1, 3, 1000, 1000))]
# load pretrained
args, auxs = load_param(pretrained, epoch, convert=True)
# initialize params
arg_shape, _, _ = sym.infer_shape(data=(1, 3, 224, 224), rois=(1, 5))
arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape))
args["cls_score_weight"] = mx.random.normal(mean=0, stdvar=0.01, shape=arg_shape_dict["cls_score_weight"])
args["cls_score_bias"] = mx.nd.zeros(shape=arg_shape_dict["cls_score_bias"])
args["bbox_pred_weight"] = mx.random.normal(mean=0, stdvar=0.001, shape=arg_shape_dict["bbox_pred_weight"])
args["bbox_pred_bias"] = mx.nd.zeros(shape=arg_shape_dict["bbox_pred_bias"])
# train
solver = Solver(
prefix,
sym,
ctx,
begin_epoch,
end_epoch,
kv_store,
args,
auxs,
momentum=0.9,
wd=0.0005,
learning_rate=1e-3,
lr_scheduler=mx.lr_scheduler.FactorScheduler(30000, 0.1),
mutable_data_shape=True,
max_data_shape=max_data_shape,
)
solver.fit(train_data, frequent=frequent)
# edit params and save
for epoch in range(begin_epoch + 1, end_epoch + 1):
arg_params, aux_params = load_checkpoint(prefix, epoch)
arg_params["bbox_pred_weight"] = (arg_params["bbox_pred_weight"].T * mx.nd.array(stds)).T
arg_params["bbox_pred_bias"] = arg_params["bbox_pred_bias"] * mx.nd.array(stds) + mx.nd.array(means)
save_checkpoint(prefix, epoch, arg_params, aux_params)
def train_rcnn(image_set, year, root_path, devkit_path, pretrained, epoch,
prefix, ctx, begin_epoch, end_epoch, frequent, kv_store,
work_load_list=None, resume=False, proposal='rpn'):
# set up logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# load symbol
sym = get_vgg_rcnn()
# setup multi-gpu
config.TRAIN.BATCH_IMAGES *= len(ctx)
config.TRAIN.BATCH_SIZE *= len(ctx)
# load training data
voc, roidb, means, stds = eval('load_' + proposal + '_roidb')(image_set, year, root_path, devkit_path, flip=True)
train_data = ROIIter(roidb, batch_size=config.TRAIN.BATCH_IMAGES, shuffle=True, mode='train',
ctx=ctx, work_load_list=work_load_list)
# infer max shape
max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3, 1000, 1000))]
# load pretrained
args, auxs = load_param(pretrained, epoch, convert=True)
# initialize params
if not resume:
input_shapes = {k: v for k, v in train_data.provide_data + train_data.provide_label}
arg_shape, _, _ = sym.infer_shape(**input_shapes)
arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape))
args['cls_score_weight'] = mx.random.normal(0, 0.01, shape=arg_shape_dict['cls_score_weight'])
args['cls_score_bias'] = mx.nd.zeros(shape=arg_shape_dict['cls_score_bias'])
args['bbox_pred_weight'] = mx.random.normal(0, 0.001, shape=arg_shape_dict['bbox_pred_weight'])
args['bbox_pred_bias'] = mx.nd.zeros(shape=arg_shape_dict['bbox_pred_bias'])
# prepare training
if config.TRAIN.FINETUNE:
fixed_param_prefix = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5']
else:
fixed_param_prefix = ['conv1', 'conv2']
data_names = [k[0] for k in train_data.provide_data]
label_names = [k[0] for k in train_data.provide_label]
#batch_end_callback = Speedometer(train_data.batch_size, frequent=frequent)
epoch_end_callback = mx.callback.do_checkpoint(prefix)
if config.TRAIN.HAS_RPN is True:
eval_metric = AccuracyMetric(use_ignore=True, ignore=-1)
cls_metric = LogLossMetric(use_ignore=True, ignore=-1)
else:
eval_metric = AccuracyMetric()
cls_metric = LogLossMetric()
bbox_metric = SmoothL1LossMetric()
eval_metrics = mx.metric.CompositeEvalMetric()
for child_metric in [eval_metric, cls_metric, bbox_metric]:
eval_metrics.add(child_metric)
optimizer_params = {'momentum': 0.9,
'wd': 0.0005,
'learning_rate': 0.001,
'lr_scheduler': mx.lr_scheduler.FactorScheduler(30000, 0.1),
'rescale_grad': (1.0 / config.TRAIN.BATCH_SIZE)}
# train
# mod = MutableModule(sym, data_names=data_names, label_names=label_names,
# logger=logger, context=ctx, work_load_list=work_load_list,
# max_data_shapes=max_data_shape, fixed_param_prefix=fixed_param_prefix)
# mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback,
# batch_end_callback=batch_end_callback, kvstore=kv_store,
# optimizer='sgd', optimizer_params=optimizer_params,
# arg_params=args, aux_params=auxs, begin_epoch=begin_epoch, num_epoch=end_epoch)
from mxnet.executor import Executor
def batch_end_callback(batch_size, frequent):
call_back = mx.callback.Speedometer(batch_size, frequent)
def AverageL2Norm(d):
return (mx.nd.norm(d) / np.sqrt(d.size)).asnumpy()[0]
def decorator(parameter):
call_back(parameter)
if parameter.locals['nbatch'] % frequent == 0:
executor_manager = parameter.locals['executor_manager']
d = Executor._get_dict(executor_manager.aux_names, executor_manager.aux_arrays)
for key, value in sorted(list(zip(d.keys(), d.values()))):
print(key, 'AverageL2Norm(aux):', AverageL2Norm(value[0]))
for (index, value) in enumerate(executor_manager.param_names):
print (value, 'AverageL2Norm(param,grad):', (AverageL2Norm(executor_manager.param_arrays[index][0]),
AverageL2Norm(executor_manager.grad_arrays[index][0])))
return decorator
mod = mx.model.FeedForward(sym, data_names=data_names, label_names=label_names,
logger=logger, ctx=ctx, work_load_list=work_load_list,
fixed_param_prefix=fixed_param_prefix,
optimizer=mx.optimizer.SGD(**optimizer_params),
arg_params=args, aux_params=auxs, begin_epoch=begin_epoch, num_epoch=end_epoch
)
mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback(train_data.batch_size, frequent=frequent), kvstore=kv_store)
# edit params and save
for epoch in range(begin_epoch + 1, end_epoch + 1):
arg_params, aux_params = load_checkpoint(prefix, epoch)
arg_params['bbox_pred_weight'] = (arg_params['bbox_pred_weight'].T * mx.nd.array(stds)).T
arg_params['bbox_pred_bias'] = arg_params['bbox_pred_bias'] * mx.nd.array(stds) + \
mx.nd.array(means)
save_checkpoint(prefix, epoch, arg_params, aux_params)
def train_rcnn(image_set, year, root_path, devkit_path, pretrained, epoch,
prefix, ctx, begin_epoch, end_epoch, frequent, kv_store,
work_load_list=None, resume=False, proposal='rpn'):
# set up logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
# load symbol
sym = get_vgg_rcnn()
# setup multi-gpu
config.TRAIN.BATCH_IMAGES *= len(ctx)
config.TRAIN.BATCH_SIZE *= len(ctx)
# load training data
voc, roidb, means, stds = eval('load_' + proposal + '_roidb')(image_set, year, root_path, devkit_path, flip=True)
train_data = ROIIter(roidb, batch_size=config.TRAIN.BATCH_IMAGES, shuffle=True, mode='train',
ctx=ctx, work_load_list=work_load_list)
# infer max shape
max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3, 1000, 1000))]
# load pretrained
args, auxs = load_param(pretrained, epoch, convert=True)
# initialize params
if not resume:
input_shapes = {k: v for k, v in train_data.provide_data + train_data.provide_label}
arg_shape, _, _ = sym.infer_shape(**input_shapes)
arg_shape_dict = dict(zip(sym.list_arguments(), arg_shape))
args['cls_score_weight'] = mx.random.normal(0, 0.01, shape=arg_shape_dict['cls_score_weight'])
args['cls_score_bias'] = mx.nd.zeros(shape=arg_shape_dict['cls_score_bias'])
args['bbox_pred_weight'] = mx.random.normal(0, 0.001, shape=arg_shape_dict['bbox_pred_weight'])
args['bbox_pred_bias'] = mx.nd.zeros(shape=arg_shape_dict['bbox_pred_bias'])
# prepare training
if config.TRAIN.FINETUNE:
fixed_param_prefix = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5']
else:
fixed_param_prefix = ['conv1', 'conv2']
data_names = [k[0] for k in train_data.provide_data]
label_names = [k[0] for k in train_data.provide_label]
batch_end_callback = Speedometer(train_data.batch_size, frequent=frequent)
epoch_end_callback = mx.callback.do_checkpoint(prefix)
if config.TRAIN.HAS_RPN is True:
eval_metric = AccuracyMetric(use_ignore=True, ignore=-1)
cls_metric = LogLossMetric(use_ignore=True, ignore=-1)
else:
eval_metric = AccuracyMetric()
cls_metric = LogLossMetric()
bbox_metric = SmoothL1LossMetric()
eval_metrics = mx.metric.CompositeEvalMetric()
for child_metric in [eval_metric, cls_metric, bbox_metric]:
eval_metrics.add(child_metric)
optimizer_params = {'momentum': 0.9,
'wd': 0.0005,
'learning_rate': 0.001,
'lr_scheduler': mx.lr_scheduler.FactorScheduler(30000, 0.1),
'rescale_grad': (1.0 / config.TRAIN.BATCH_SIZE)}
# train
mod = MutableModule(sym, data_names=data_names, label_names=label_names,
logger=logger, context=ctx, work_load_list=work_load_list,
max_data_shapes=max_data_shape, fixed_param_prefix=fixed_param_prefix)
mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback, kvstore=kv_store,
optimizer='sgd', optimizer_params=optimizer_params,
arg_params=args, aux_params=auxs, begin_epoch=begin_epoch, num_epoch=end_epoch)
# edit params and save
for epoch in range(begin_epoch + 1, end_epoch + 1):
arg_params, aux_params = load_checkpoint(prefix, epoch)
arg_params['bbox_pred_weight'] = (arg_params['bbox_pred_weight'].T * mx.nd.array(stds)).T
arg_params['bbox_pred_bias'] = arg_params['bbox_pred_bias'] * mx.nd.array(stds) + \
mx.nd.array(means)
save_checkpoint(prefix, epoch, arg_params, aux_params)