def get_net(cfg, ctx, prefix, epoch, has_rpn):
try:
if has_rpn:
sym_instance = eval(cfg.symbol + '.' + cfg.symbol)()
sym = sym_instance.get_symbol(cfg, is_train=False)
else:
sym_instance = eval(cfg.symbol + '.' + cfg.symbol)()
sym = sym_instance.get_symbol_rcnn(cfg, is_train=False)
# load model
arg_params, aux_params = load_param(prefix, epoch, process=True)
# infer shape
SHORT_SIDE = config.SCALES[0][0]
LONG_SIDE = config.SCALES[0][1]
DATA_NAMES = ['data', 'im_info']
LABEL_NAMES = None
DATA_SHAPES = [('data', (1, 3, LONG_SIDE, SHORT_SIDE)),
('im_info', (1, 3))]
LABEL_SHAPES = None
data_shape_dict = dict(DATA_SHAPES)
sym_instance.infer_shape(data_shape_dict)
sym_instance.check_parameter_shapes(arg_params,
aux_params,
data_shape_dict,
is_train=False)
# decide maximum shape
max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]),
max([v[1] for v in cfg.SCALES])))]]
if not has_rpn:
max_data_shape.append(
('rois', (cfg.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5)))
# create predictor
predictor = Predictor(sym,
DATA_NAMES,
LABEL_NAMES,
context=ctx,
max_data_shapes=max_data_shape,
provide_data=[DATA_SHAPES],
provide_label=[LABEL_SHAPES],
arg_params=arg_params,
aux_params=aux_params)
except Exception, e:
print(traceback.format_exc())
predictor = None
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
# ctx_id = [mx.cpu()]
print(ctx_id)
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 2
classes = ['nuclei']
# load demo data
# image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/stage1_test_images'
image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/LikeVOC/img'
image_names = glob.glob(os.path.join(image_root, '*.png'))
data = []
for im_name in image_names:
im = cv2.imread(im_name,
cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im,
target_size,
max_size,
stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array(
[[im_tensor.shape[2], im_tensor.shape[3], im_scale]],
dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names]
for i in range(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])]
for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(
'/home/daiab/machine_disk/projects/FCIS/'
'output/fcis/nuclei/nuclei_end2end/SDS_train/e2e',
2,
process=True)
data = data
max_data_shape = max_data_shape
provide_data = provide_data
predictor = Predictor(sym,
data_names,
label_names,
context=[mx.gpu(ctx_id[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for i in range(2):
data_batch = mx.io.DataBatch(data=[data[0]],
label=[],
pad=0,
index=0,
provide_data=[[
(k, v.shape)
for k, v in zip(data_names, data[0])
]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
# print('-----------', data_batch)
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales,
config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(
data=[data[idx]],
label=[],
pad=0,
index=idx,
provide_data=[[(k, v.shape)
for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [
data_batch.data[i][1].asnumpy()[0, 2]
for i in range(len(data_batch.data))
]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch,
data_names, scales, config)
im_shapes = [
data_batch.data[i][0].shape[2:4]
for i in range(len(data_batch.data))
]
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in range(num_classes)]
all_masks = [[] for _ in range(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print(im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = gpu_mask_voting(
masks, boxes, scores[0], num_classes, 100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [
result_masks[j][:, 0, :, :] for j in range(1, num_classes)
]
# print(dets)
# print(masks)
print('testing {} {:.4f}s'.format(im_name, toc()))
# visualize
for i in range(len(dets)):
keep = np.where(dets[i][:, -1] > 0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, classes, config)
print('done')
def main():
# get symbol
ctx_id = [int(i) for i in config.gpus.split(',')]
# ctx_id = [mx.cpu()]
print(ctx_id)
pprint.pprint(config)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
# set up class names
num_classes = 81
classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat',
'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse',
'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie',
'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove',
'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon',
'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut',
'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse',
'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book',
'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush']
# load demo data
image_names = ['COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg', 'COCO_test2015_000000073428.jpg',
'COCO_test2015_000000393281.jpg']
data = []
for im_name in image_names:
assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name))
im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION)
target_size = config.SCALES[0][0]
max_size = config.SCALES[0][1]
im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE)
im_tensor = transform(im, config.network.PIXEL_MEANS)
im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32)
data.append({'data': im_tensor, 'im_info': im_info})
# get predictor
data_names = ['data', 'im_info']
label_names = []
data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))]
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]]
provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))]
provide_label = [None for i in range(len(data))]
arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco', 0, process=True)
data = data
max_data_shape = max_data_shape
provide_data = provide_data
predictor = Predictor(sym, data_names, label_names,
context=[mx.gpu(ctx_id[0])],
max_data_shapes=max_data_shape,
provide_data=provide_data,
provide_label=provide_label,
arg_params=arg_params,
aux_params=aux_params)
# warm up
for i in range(2):
data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))]
# print('-----------', data_batch)
_, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config)
# test
for idx, im_name in enumerate(image_names):
data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx,
provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]],
provide_label=[None])
scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))]
tic()
scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config)
im_shapes = [data_batch.data[i][0].shape[2:4] for i in range(len(data_batch.data))]
if not config.TEST.USE_MASK_MERGE:
all_boxes = [[] for _ in range(num_classes)]
all_masks = [[] for _ in range(num_classes)]
nms = py_nms_wrapper(config.TEST.NMS)
for j in range(1, num_classes):
indexes = np.where(scores[0][:, j] > 0.7)[0]
cls_scores = scores[0][indexes, j, np.newaxis]
cls_masks = masks[0][indexes, 1, :, :]
try:
if config.CLASS_AGNOSTIC:
cls_boxes = boxes[0][indexes, :]
else:
raise Exception()
except:
cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4]
cls_dets = np.hstack((cls_boxes, cls_scores))
keep = nms(cls_dets)
all_boxes[j] = cls_dets[keep, :]
all_masks[j] = cls_masks[keep, :]
dets = [all_boxes[j] for j in range(1, num_classes)]
masks = [all_masks[j] for j in range(1, num_classes)]
else:
print('=============')
masks = masks[0][:, 1:, :, :]
im_height = np.round(im_shapes[0][0] / scales[0]).astype('int')
im_width = np.round(im_shapes[0][1] / scales[0]).astype('int')
print(im_height, im_width)
boxes = clip_boxes(boxes[0], (im_height, im_width))
result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes,
100, im_width, im_height,
config.TEST.NMS, config.TEST.MASK_MERGE_THRESH,
config.BINARY_THRESH, ctx_id[0])
dets = [result_dets[j] for j in range(1, num_classes)]
masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)]
# print(dets)
# print(masks)
print('testing {} {:.4f}s'.format(im_name, toc()))
# visualize
for i in range(len(dets)):
keep = np.where(dets[i][:,-1]>0.7)
dets[i] = dets[i][keep]
masks[i] = masks[i][keep]
im = cv2.imread(cur_path + '/../demo/' + im_name)
im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
show_masks(im, dets, masks, classes, config)
print('done')
def test_deepim():
config.TRAIN.MASK_SYN = False
if args.vis or args.vis_video or args.vis_video_zoom:
config.TEST.VISUALIZE = True
config.TEST.FAST_TEST = False
epoch = config.TEST.test_epoch
ctx = [mx.gpu(int(i)) for i in args.gpus.split(",")]
image_set = config.dataset.test_image_set
root_path = config.dataset.root_path
dataset = config.dataset.dataset.split("+")[0]
dataset_path = config.dataset.dataset_path
new_args_name = args.cfg
logger, final_output_path = create_logger(
config.output_path, new_args_name, image_set
)
prefix = os.path.join(
final_output_path,
"..",
"_".join([iset for iset in config.dataset.image_set.split("+")]),
config.TRAIN.model_prefix,
)
pprint.pprint(config)
logger.info("testing config:{}\n".format(pprint.pformat(config)))
# load symbol and testing data
sym_instance = eval(config.symbol + "." + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
if config.dataset.dataset.startswith("ModelNet"):
imdb_test = eval(dataset)(
config,
image_set + config.dataset.class_name[0].split("/")[-1],
root_path,
dataset_path,
class_name=config.dataset.class_name[0],
result_path=final_output_path,
)
print(imdb_test)
pairdbs = [
load_gt_pairdb(
config,
dataset,
image_set + class_name.split("/")[-1],
config.dataset.root_path,
config.dataset.dataset_path,
class_name=class_name,
result_path=final_output_path,
)
for class_name in config.dataset.class_name
]
pairdb = merge_pairdb(pairdbs)
else:
imdb_test = eval(dataset)(
config,
image_set + config.dataset.class_name[0],
root_path,
dataset_path,
class_name=config.dataset.class_name[0],
result_path=final_output_path,
)
print(imdb_test)
pairdbs = [
load_gt_pairdb(
config,
dataset,
image_set + class_name,
config.dataset.root_path,
config.dataset.dataset_path,
class_name=class_name,
result_path=final_output_path,
)
for class_name in config.dataset.class_name
]
pairdb = merge_pairdb(pairdbs)
# get test data iter
test_data = TestDataLoader(pairdb, config=config, batch_size=len(ctx))
# infer shape
data_shape_dict = dict(test_data.provide_data_single)
sym_instance.infer_shape(data_shape_dict)
# load model and check parameters
arg_params, aux_params = load_param(prefix, epoch, process=True)
sym_instance.check_parameter_shapes(
arg_params, aux_params, data_shape_dict, is_train=False
)
# decide maximum shape
data_names = [k[0] for k in test_data.provide_data_single]
label_names = None
max_data_shape = [
[
(
"data",
(
1,
3,
max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES]),
),
)
]
]
# create predictor
predictor = Predictor(
config,
sym,
data_names,
label_names,
context=ctx,
max_data_shapes=max_data_shape,
provide_data=test_data.provide_data,
provide_label=test_data.provide_label,
arg_params=arg_params,
aux_params=aux_params,
)
# start detection
pred_eval(
config,
predictor,
test_data,
imdb_test,
vis=args.vis,
ignore_cache=args.ignore_cache,
logger=logger,
pairdb=pairdb,
)
print(args.cfg, config.TEST.test_epoch)
import mxnet as mx
from lib.utils.load_model import load_param
import sys
sys.path.append('CFN')
import CFNet
import numpy as np
import caffe
#caffe.set_mode_gpu()
#caffe.set_device(2)
#args, auxs = load_param('output/colornet/CFN', 1)
#args, auxs = load_param('output/CFNpretrain', 0)
sym, args, auxs = mx.model.load_checkpoint('output/colornet/CFN', 1)
arg_flow, aux_flow = load_param('model/pretrained_model/flownet', 0)
sym = CFNet.CFNet().get_train_symbol()
#prototxt = '../colorization/flownet-release/models/flownet/model_simple/deploy.tpl.prototxt'
#model = '../colorization/flownet-release/models/flownet/model_simple/flownet_official.caffemodel'
#net = caffe.Net(prototxt, model, caffe.TEST)
'''
for item in net.params.items():
name, layer = item
num = 0
for p in net.params[name]:
np.save('flownet_weights/'+str(name)+'_'+str(num), p.data)
num += 1
'''
for k, v in args.items():
print k
#if 'conv' in k or 'Conv' in k:
# args['flow_'+k] = v
def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, lr,
lr_step):
new_args_name = args.cfg
if args.vis:
config.TRAIN.VISUALIZE = True
logger, final_output_path = create_logger(config.output_path,
new_args_name,
config.dataset.image_set,
args.temp)
prefix = os.path.join(final_output_path, prefix)
logger.info('called with args {}'.format(args))
print(config.train_iter.SE3_PM_LOSS)
if config.train_iter.SE3_PM_LOSS:
print("SE3_PM_LOSS == True")
else:
print("SE3_PM_LOSS == False")
if not config.network.STANDARD_FLOW_REP:
print_and_log("[h, w] representation for flow is dep", logger)
# load dataset and prepare imdb for training
image_sets = [iset for iset in config.dataset.image_set.split('+')]
datasets = [dset for dset in config.dataset.dataset.split('+')]
print("config.dataset.class_name: {}".format(config.dataset.class_name))
print("image_sets: {}".format(image_sets))
if datasets[0].startswith('ModelNet'):
pairdbs = [
load_gt_pairdb(config,
datasets[i],
image_sets[i] + class_name.split('/')[-1],
config.dataset.root_path,
config.dataset.dataset_path,
class_name=class_name,
result_path=final_output_path)
for class_name in config.dataset.class_name
for i in range(len(image_sets))
]
else:
pairdbs = [
load_gt_pairdb(config,
datasets[i],
image_sets[i] + class_name,
config.dataset.root_path,
config.dataset.dataset_path,
class_name=class_name,
result_path=final_output_path)
for class_name in config.dataset.class_name
for i in range(len(image_sets))
]
pairdb = merge_pairdb(pairdbs)
if not args.temp:
src_file = os.path.join(curr_path, 'symbols', config.symbol + '.py')
dst_file = os.path.join(
final_output_path,
'{}_{}.py'.format(config.symbol, time.strftime('%Y-%m-%d-%H-%M')))
os.popen('cp {} {}'.format(src_file, dst_file))
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=True)
# setup multi-gpu
batch_size = len(ctx)
input_batch_size = config.TRAIN.BATCH_PAIRS * batch_size
pprint.pprint(config)
logger.info('training config:{}\n'.format(pprint.pformat(config)))
# load training data
train_data = TrainDataLoader(sym,
pairdb,
config,
batch_size=input_batch_size,
shuffle=config.TRAIN.SHUFFLE,
ctx=ctx)
train_data.get_batch_parallel()
max_scale = [
max([v[0] for v in config.SCALES]),
max(v[1] for v in config.SCALES)
]
max_data_shape = [('image_observed', (config.TRAIN.BATCH_PAIRS, 3,
max_scale[0], max_scale[1])),
('image_rendered', (config.TRAIN.BATCH_PAIRS, 3,
max_scale[0], max_scale[1])),
('depth_gt_observed', (config.TRAIN.BATCH_PAIRS, 1,
max_scale[0], max_scale[1])),
('src_pose', (config.TRAIN.BATCH_PAIRS, 3, 4)),
('tgt_pose', (config.TRAIN.BATCH_PAIRS, 3, 4))]
if config.network.INPUT_DEPTH:
max_data_shape.append(('depth_observed', (config.TRAIN.BATCH_PAIRS, 1,
max_scale[0], max_scale[1])))
max_data_shape.append(('depth_rendered', (config.TRAIN.BATCH_PAIRS, 1,
max_scale[0], max_scale[1])))
if config.network.INPUT_MASK:
max_data_shape.append(('mask_observed', (config.TRAIN.BATCH_PAIRS, 1,
max_scale[0], max_scale[1])))
max_data_shape.append(('mask_rendered', (config.TRAIN.BATCH_PAIRS, 1,
max_scale[0], max_scale[1])))
rot_param = 3 if config.network.ROT_TYPE == "EULER" else 4
max_label_shape = [('rot', (config.TRAIN.BATCH_PAIRS, rot_param)),
('trans', (config.TRAIN.BATCH_PAIRS, 3))]
if config.network.PRED_FLOW:
max_label_shape.append(('flow', (config.TRAIN.BATCH_PAIRS, 2,
max_scale[0], max_scale[1])))
max_label_shape.append(('flow_weights', (config.TRAIN.BATCH_PAIRS, 2,
max_scale[0], max_scale[1])))
if config.train_iter.SE3_PM_LOSS:
max_label_shape.append(
('point_cloud_model', (config.TRAIN.BATCH_PAIRS, 3,
config.train_iter.NUM_3D_SAMPLE)))
max_label_shape.append(
('point_cloud_weights', (config.TRAIN.BATCH_PAIRS, 3,
config.train_iter.NUM_3D_SAMPLE)))
max_label_shape.append(
('point_cloud_observed', (config.TRAIN.BATCH_PAIRS, 3,
config.train_iter.NUM_3D_SAMPLE)))
if config.network.PRED_MASK:
max_label_shape.append(
('mask_gt_observed', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0],
max_scale[1])))
# max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape, max_label_shape)
print_and_log(
'providing maximum shape, {}, {}'.format(max_data_shape,
max_label_shape), logger)
# infer max shape
'''
max_label_shape = [('label', (config.TRAIN.BATCH_IMAGES, 1,
max([v[0] for v in max_scale]),
max([v[1] for v in max_scale])))]
max_data_shape, max_label_shape = train_data.infer_shape(
max_data_shape, max_label_shape)
print('providing maximum shape', max_data_shape, max_label_shape)
'''
# infer shape
data_shape_dict = dict(train_data.provide_data_single +
train_data.provide_label_single)
print_and_log('\ndata_shape_dict: {}\n'.format(data_shape_dict), logger)
sym_instance.infer_shape(data_shape_dict)
print('************(wg): infering shape **************')
internals = sym.get_internals()
_, out_shapes, _ = internals.infer_shape(**data_shape_dict)
print(sym.list_outputs())
shape_dict = dict(zip(internals.list_outputs(), out_shapes))
pprint.pprint(shape_dict)
# load and initialize params
if config.TRAIN.RESUME:
print('continue training from ', begin_epoch)
arg_params, aux_params = load_param(prefix, begin_epoch, convert=True)
elif pretrained == 'xavier':
print('xavier')
# arg_params = {}
# aux_params = {}
# sym_instance.init_weights(config, arg_params, aux_params)
else:
print(pretrained)
arg_params, aux_params = load_param(pretrained, epoch, convert=True)
print('arg_params: ', arg_params.keys())
print('aux_params: ', aux_params.keys())
if not config.network.skip_initialize:
sym_instance.init_weights(config, arg_params, aux_params)
# check parameter shapes
if pretrained != 'xavier':
sym_instance.check_parameter_shapes(arg_params, aux_params,
data_shape_dict)
# create solver
fixed_param_prefix = config.network.FIXED_PARAMS
data_names = [k[0] for k in train_data.provide_data_single]
label_names = [k[0] for k in train_data.provide_label_single]
mod = MutableModule(
sym,
data_names=data_names,
label_names=label_names,
logger=logger,
context=ctx,
max_data_shapes=[max_data_shape for _ in range(batch_size)],
max_label_shapes=[max_label_shape for _ in range(batch_size)],
fixed_param_prefix=fixed_param_prefix,
config=config)
# decide training params
# metrics
eval_metrics = mx.metric.CompositeEvalMetric()
metric_list = []
iter_idx = 0
if config.network.PRED_FLOW:
metric_list.append(metric.Flow_L2LossMetric(config, iter_idx))
metric_list.append(metric.Flow_CurLossMetric(config, iter_idx))
if config.train_iter.SE3_DIST_LOSS:
metric_list.append(metric.Rot_L2LossMetric(config, iter_idx))
metric_list.append(metric.Trans_L2LossMetric(config, iter_idx))
if config.train_iter.SE3_PM_LOSS:
metric_list.append(metric.PointMatchingLossMetric(config, iter_idx))
if config.network.PRED_MASK:
metric_list.append(metric.MaskLossMetric(config, iter_idx))
# Visualize Training Batches
if config.TRAIN.VISUALIZE:
metric_list.append(metric.SimpleVisualize(config))
# metric_list.append(metric.MaskVisualize(config, save_dir = final_output_path))
metric_list.append(
metric.MinibatchVisualize(config)) # flow visualization
for child_metric in metric_list:
eval_metrics.add(child_metric)
# callback
batch_end_callback = callback.Speedometer(train_data.batch_size,
frequent=args.frequent)
epoch_end_callback = mx.callback.module_checkpoint(
mod, prefix, period=1, save_optimizer_states=True)
# decide learning rate
base_lr = lr
lr_factor = 0.1
lr_epoch = [float(epoch) for epoch in lr_step.split(',')]
lr_epoch_diff = [
epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch
]
lr = base_lr * (lr_factor**(len(lr_epoch) - len(lr_epoch_diff)))
lr_iters = [
int(epoch * len(pairdb) / batch_size) for epoch in lr_epoch_diff
]
print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters)
lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor,
config.TRAIN.warmup,
config.TRAIN.warmup_lr,
config.TRAIN.warmup_step)
if not isinstance(train_data, PrefetchingIter):
train_data = PrefetchingIter(train_data)
# train
if config.TRAIN.optimizer == 'adam':
optimizer_params = {'learning_rate': lr}
if pretrained == 'xavier':
init = mx.init.Mixed(['rot_weight|trans_weight', '.*'], [
mx.init.Zero(),
mx.init.Xavier(
rnd_type='gaussian', factor_type="in", magnitude=2)
])
mod.fit(train_data,
eval_metric=eval_metrics,
epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback,
kvstore=config.default.kvstore,
optimizer='adam',
optimizer_params=optimizer_params,
begin_epoch=begin_epoch,
num_epoch=end_epoch,
prefix=prefix,
initializer=init,
force_init=True)
else:
mod.fit(train_data,
eval_metric=eval_metrics,
epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback,
kvstore=config.default.kvstore,
optimizer='adam',
arg_params=arg_params,
aux_params=aux_params,
begin_epoch=begin_epoch,
num_epoch=end_epoch,
prefix=prefix)
elif config.TRAIN.optimizer == 'sgd':
# optimizer
optimizer_params = {
'momentum': config.TRAIN.momentum,
'wd': config.TRAIN.wd,
'learning_rate': lr,
'lr_scheduler': lr_scheduler,
'rescale_grad': 1.0,
'clip_gradient': None
}
if pretrained == 'xavier':
init = mx.init.Mixed(['rot_weight|trans_weight', '.*'], [
mx.init.Zero(),
mx.init.Xavier(
rnd_type='gaussian', factor_type="in", magnitude=2)
])
mod.fit(train_data,
eval_metric=eval_metrics,
epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback,
kvstore=config.default.kvstore,
optimizer='sgd',
optimizer_params=optimizer_params,
begin_epoch=begin_epoch,
num_epoch=end_epoch,
prefix=prefix,
initializer=init,
force_init=True)
else:
mod.fit(train_data,
eval_metric=eval_metrics,
epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback,
kvstore=config.default.kvstore,
optimizer='sgd',
optimizer_params=optimizer_params,
arg_params=arg_params,
aux_params=aux_params,
begin_epoch=begin_epoch,
num_epoch=end_epoch,
prefix=prefix)
def test_fcis(config,
dataset,
image_set,
root_path,
dataset_path,
ctx,
prefix,
epoch,
vis,
ignore_cache,
shuffle,
has_rpn,
proposal,
thresh,
logger=None,
output_path=None):
if not logger:
assert False, 'require a logger'
# print config
pprint.pprint(config)
logger.info('testing config:{}\n'.format(pprint.pformat(config)))
# load symbol and testing data
if has_rpn:
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
imdb = eval(dataset)(image_set,
root_path,
dataset_path,
result_path=output_path,
binary_thresh=config.BINARY_THRESH,
mask_size=config.MASK_SIZE)
sdsdb = imdb.gt_sdsdb()
else:
raise NotImplementedError
# get test data iter
test_data = TestLoader(sdsdb,
config,
batch_size=len(ctx),
shuffle=shuffle,
has_rpn=has_rpn)
# load model
arg_params, aux_params = load_param(prefix, epoch, process=False)
# infer shape
data_shape_dict = dict(test_data.provide_data_single)
sym_instance.infer_shape(data_shape_dict)
sym_instance.check_parameter_shapes(arg_params,
aux_params,
data_shape_dict,
is_train=False)
# decide maximum shape
data_names = [k[0] for k in test_data.provide_data_single]
label_names = []
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
if not has_rpn:
raise NotImplementedError()
# create predictor
predictor = Predictor(sym,
data_names,
label_names,
context=ctx,
max_data_shapes=max_data_shape,
provide_data=test_data.provide_data,
provide_label=test_data.provide_label,
arg_params=arg_params,
aux_params=aux_params)
# start detection
pred_eval(predictor,
test_data,
imdb,
config,
vis=vis,
ignore_cache=ignore_cache,
thresh=thresh,
logger=logger)
def test_deepim():
config.TRAIN.MASK_SYN = False
if args.vis or args.vis_video or args.vis_video_zoom:
config.TEST.VISUALIZE = True
config.TEST.FAST_TEST = False
if args.iter_test:
config.TEST.test_iter = 5
if args.refine:
config.TEST.test_iter = 1
if args.skip_flow:
config.network.FLOW_I2R = False
config.network.FLOW_R2I = False
config.train_iter0.FLOW_I2R = False
config.train_iter0.FLOW_R2I = False
config.train_iter1.FLOW_I2R = False
config.train_iter1.FLOW_R2I = False
config.train_iter2.FLOW_I2R = False
config.train_iter2.FLOW_R2I = False
config.train_iter3.FLOW_I2R = False
config.train_iter3.FLOW_R2I = False
epoch = config.TEST.test_epoch
ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')]
if len(ctx) != config.NUM_GPUS:
print("********** WARNING: length of context doesn't match num_gpus set in config, {} vs. {} **********".\
format(len(ctx), config.NUM_GPUS))
image_set = config.dataset.test_image_set
root_path = config.dataset.root_path
dataset = config.dataset.dataset.split('+')[0]
dataset_path = config.dataset.dataset_path
if not os.path.basename(args.cfg).split('.')[0].endswith('temp'):
new_args_name = os.path.basename(
args.cfg).split('.')[0] + '_{}gpus.yaml'.format(config.NUM_GPUS)
else:
new_args_name = args.cfg
if config.TEST.VISUALIZE or args.temp:
logger, final_output_path = create_logger(config.output_path,
new_args_name, image_set,
True)
else:
logger, final_output_path = create_logger(config.output_path,
new_args_name, image_set)
prefix = os.path.join(
final_output_path, '..',
'_'.join([iset for iset in config.dataset.image_set.split('+')]),
config.TRAIN.model_prefix)
pprint.pprint(config)
logger.info('testing config:{}\n'.format(pprint.pformat(config)))
# load symbol and testing data
sym_instance = eval(config.symbol + '.' + config.symbol)()
sym = sym_instance.get_symbol(config, is_train=False)
if config.dataset.dataset.startswith('ModelNet'):
imdb_test = eval(dataset)(config,
image_set +
config.dataset.class_name[0].split('/')[-1],
root_path,
dataset_path,
class_name=config.dataset.class_name[0],
result_path=final_output_path)
print(imdb_test)
pairdbs = [
load_gt_pairdb(config,
dataset,
image_set + class_name.split('/')[-1],
config.dataset.root_path,
config.dataset.dataset_path,
class_name=class_name,
result_path=final_output_path)
for class_name in config.dataset.class_name
]
pairdb = merge_pairdb(pairdbs)
else:
imdb_test = eval(dataset)(config,
image_set + config.dataset.class_name[0],
root_path,
dataset_path,
class_name=config.dataset.class_name[0],
result_path=final_output_path)
print(imdb_test)
pairdbs = [
load_gt_pairdb(config,
dataset,
image_set + class_name,
config.dataset.root_path,
config.dataset.dataset_path,
class_name=class_name,
result_path=final_output_path)
for class_name in config.dataset.class_name
]
pairdb = merge_pairdb(pairdbs)
# get test data iter
test_data = TestDataLoader(pairdb, config=config, batch_size=len(ctx))
# infer shape
data_shape_dict = dict(test_data.provide_data_single)
sym_instance.infer_shape(data_shape_dict)
# load model and check parameters
arg_params, aux_params = load_param(prefix, epoch, process=True)
sym_instance.check_parameter_shapes(arg_params,
aux_params,
data_shape_dict,
is_train=False)
# decide maximum shape
data_names = [k[0] for k in test_data.provide_data_single]
label_names = None
max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]),
max([v[1] for v in config.SCALES])))]]
# create predictor
predictor = Predictor(config,
sym,
data_names,
label_names,
context=ctx,
max_data_shapes=max_data_shape,
provide_data=test_data.provide_data,
provide_label=test_data.provide_label,
arg_params=arg_params,
aux_params=aux_params)
# start detection
pred_eval(config,
predictor,
test_data,
imdb_test,
vis=args.vis,
ignore_cache=args.ignore_cache,
logger=logger,
pairdb=pairdb)
print(args.cfg, config.TEST.test_epoch)
def train_net(args, ctx, pretrained, epoch, prefix,
begin_epoch, end_epoch, lr, lr_step):
mx.random.seed(3)
np.random.seed(3)
logger, final_output_path = create_logger(config.output_path, args.cfg, config.dataset.image_set)
prefix = os.path.join(final_output_path, prefix)
# load symbol
shutil.copy2(os.path.join(curr_path, 'symbols', config.symbol + '.py'), final_output_path)
sym_instance = eval(config.symbol)()
sym = sym_instance.get_symbol(config, is_train=True)
feat_sym = sym.get_internals()['rpn_cls_score_output']
# setup multi-gpu
batch_size = len(ctx)
input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size
# print config
pprint.pprint(config)
logger.info('training config:{}\n'.format(pprint.pformat(config)))
# load dataset and prepare imdb for training
image_sets = [iset for iset in config.dataset.image_set.split('+')]
sdsdbs = [load_gt_sdsdb(config.dataset.dataset,
image_set,
config.dataset.root_path,
config.dataset.dataset_path,
mask_size=config.MASK_SIZE,
binary_thresh=config.BINARY_THRESH,
result_path=final_output_path,
flip=config.TRAIN.FLIP)
for image_set in image_sets]
sdsdb = merge_roidb(sdsdbs)
sdsdb = filter_roidb(sdsdb, config)
# load training data
train_data = AnchorLoader(feat_sym, sdsdb, config,
batch_size=input_batch_size,
shuffle=config.TRAIN.SHUFFLE,
ctx=ctx, feat_stride=config.network.RPN_FEAT_STRIDE,
anchor_scales=config.network.ANCHOR_SCALES,
anchor_ratios=config.network.ANCHOR_RATIOS,
aspect_grouping=config.TRAIN.ASPECT_GROUPING,
allowed_border=config.TRAIN.RPN_ALLOWED_BORDER)
# infer max shape
max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3,
max([v[0] for v in config.SCALES]),
max(v[1] for v in config.SCALES)))]
max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape)
max_data_shape.append(('gt_boxes', (config.TRAIN.BATCH_IMAGES, 100, 5)))
max_data_shape.append(('gt_masks', (config.TRAIN.BATCH_IMAGES, 100,
max([v[0] for v in config.SCALES]),
max(v[1] for v in config.SCALES))))
print('providing maximum shape', max_data_shape, max_label_shape)
# infer shape
data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single)
print('data shape:')
pprint.pprint(data_shape_dict)
sym_instance.infer_shape(data_shape_dict)
# load and initialize params
if config.TRAIN.RESUME:
print('continue training from ', begin_epoch)
arg_params, aux_params = load_param(prefix, begin_epoch, convert=True)
else:
arg_params, aux_params = load_param(pretrained, epoch, convert=True)
sym_instance.init_weight(config, arg_params, aux_params)
# check parameter shapes
sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict)
# create solver
fixed_param_prefix = config.network.FIXED_PARAMS
data_names = [k[0] for k in train_data.provide_data_single]
label_names = [k[0] for k in train_data.provide_label_single]
mod = MutableModule(sym, data_names=data_names,
label_names=label_names,
logger=logger, context=ctx,
max_data_shapes=[max_data_shape for _ in range(batch_size)],
max_label_shapes=[max_label_shape for _ in range(batch_size)],
fixed_param_prefix=fixed_param_prefix)
# decide training metric
# RPN, classification accuracy/loss, regression loss
rpn_acc = metric.RPNAccMetric()
rpn_cls_loss = metric.RPNLogLossMetric()
rpn_bbox_loss = metric.RPNL1LossMetric()
fcis_acc = metric.FCISAccMetric(config)
fcis_acc_fg = metric.FCISAccFGMetric(config)
fcis_cls_loss = metric.FCISLogLossMetric(config)
fcis_bbox_loss = metric.FCISL1LossMetric(config)
fcis_mask_loss = metric.FCISMaskLossMetric(config)
eval_metrics = mx.metric.CompositeEvalMetric()
for child_metric in [rpn_acc, rpn_cls_loss, rpn_bbox_loss,
fcis_acc, fcis_acc_fg, fcis_cls_loss, fcis_bbox_loss, fcis_mask_loss]:
eval_metrics.add(child_metric)
batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=args.frequent)
means = np.tile(np.array(config.TRAIN.BBOX_MEANS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES)
stds = np.tile(np.array(config.TRAIN.BBOX_STDS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES)
epoch_end_callback = callback.do_checkpoint(prefix, means, stds)
# print epoch, begin_epoch, end_epoch, lr_step
base_lr = lr
lr_factor = 0.1
lr_epoch = [float(epoch) for epoch in lr_step.split(',')]
lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch]
lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff)))
lr_iters = [int(epoch * len(sdsdb) / batch_size) for epoch in lr_epoch_diff]
print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters)
lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, config.TRAIN.warmup,
config.TRAIN.warmup_lr, config.TRAIN.warmup_step)
# optimizer
optimizer_params = {'momentum': config.TRAIN.momentum,
'wd': config.TRAIN.wd,
'learning_rate': lr,
'lr_scheduler': lr_scheduler,
'rescale_grad': 1.0,
'clip_gradient': None}
if not isinstance(train_data, PrefetchingIter):
train_data = PrefetchingIter(train_data)
# del sdsdb
# a = mx.viz.plot_network(sym)
# a.render('../example', view=True)
# print 'prepare sds finished'
mod.fit(train_data, eval_metric=eval_metrics,
epoch_end_callback=epoch_end_callback,
batch_end_callback=batch_end_callback,
kvstore=config.default.kvstore,
optimizer='sgd',
optimizer_params=optimizer_params,
arg_params=arg_params,
aux_params=aux_params,
begin_epoch=begin_epoch,
num_epoch=end_epoch)