def get_net(cfg, ctx, prefix, epoch, has_rpn): try: if has_rpn: sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol(cfg, is_train=False) else: sym_instance = eval(cfg.symbol + '.' + cfg.symbol)() sym = sym_instance.get_symbol_rcnn(cfg, is_train=False) # load model arg_params, aux_params = load_param(prefix, epoch, process=True) # infer shape SHORT_SIDE = config.SCALES[0][0] LONG_SIDE = config.SCALES[0][1] DATA_NAMES = ['data', 'im_info'] LABEL_NAMES = None DATA_SHAPES = [('data', (1, 3, LONG_SIDE, SHORT_SIDE)), ('im_info', (1, 3))] LABEL_SHAPES = None data_shape_dict = dict(DATA_SHAPES) sym_instance.infer_shape(data_shape_dict) sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape max_data_shape = [[('data', (1, 3, max([v[0] for v in cfg.SCALES]), max([v[1] for v in cfg.SCALES])))]] if not has_rpn: max_data_shape.append( ('rois', (cfg.TEST.PROPOSAL_POST_NMS_TOP_N + 30, 5))) # create predictor predictor = Predictor(sym, DATA_NAMES, LABEL_NAMES, context=ctx, max_data_shapes=max_data_shape, provide_data=[DATA_SHAPES], provide_label=[LABEL_SHAPES], arg_params=arg_params, aux_params=aux_params) except Exception, e: print(traceback.format_exc()) predictor = None
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] # ctx_id = [mx.cpu()] print(ctx_id) pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 2 classes = ['nuclei'] # load demo data # image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/stage1_test_images' image_root = '/home/daiab/machine_disk/work/kaggle_nuclei/data/LikeVOC/img' image_names = glob.glob(os.path.join(image_root, '*.png')) data = [] for im_name in image_names: im = cv2.imread(im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array( [[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))] provide_label = [None for i in range(len(data))] arg_params, aux_params = load_param( '/home/daiab/machine_disk/projects/FCIS/' 'output/fcis/nuclei/nuclei_end2end/SDS_train/e2e', 2, process=True) data = data max_data_shape = max_data_shape provide_data = provide_data predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in range(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[ (k, v.shape) for k, v in zip(data_names, data[0]) ]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] # print('-----------', data_batch) _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch( data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [ data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data)) ] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [ data_batch.data[i][0].shape[2:4] for i in range(len(data_batch.data)) ] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in range(num_classes)] all_masks = [[] for _ in range(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting( masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [ result_masks[j][:, 0, :, :] for j in range(1, num_classes) ] # print(dets) # print(masks) print('testing {} {:.4f}s'.format(im_name, toc())) # visualize for i in range(len(dets)): keep = np.where(dets[i][:, -1] > 0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config) print('done')
def main(): # get symbol ctx_id = [int(i) for i in config.gpus.split(',')] # ctx_id = [mx.cpu()] print(ctx_id) pprint.pprint(config) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) # set up class names num_classes = 81 classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'] # load demo data image_names = ['COCO_test2015_000000000275.jpg', 'COCO_test2015_000000001412.jpg', 'COCO_test2015_000000073428.jpg', 'COCO_test2015_000000393281.jpg'] data = [] for im_name in image_names: assert os.path.exists(cur_path + '/../demo/' + im_name), ('%s does not exist'.format('../demo/' + im_name)) im = cv2.imread(cur_path + '/../demo/' + im_name, cv2.IMREAD_COLOR | cv2.IMREAD_IGNORE_ORIENTATION) target_size = config.SCALES[0][0] max_size = config.SCALES[0][1] im, im_scale = resize(im, target_size, max_size, stride=config.network.IMAGE_STRIDE) im_tensor = transform(im, config.network.PIXEL_MEANS) im_info = np.array([[im_tensor.shape[2], im_tensor.shape[3], im_scale]], dtype=np.float32) data.append({'data': im_tensor, 'im_info': im_info}) # get predictor data_names = ['data', 'im_info'] label_names = [] data = [[mx.nd.array(data[i][name]) for name in data_names] for i in range(len(data))] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] provide_data = [[(k, v.shape) for k, v in zip(data_names, data[i])] for i in range(len(data))] provide_label = [None for i in range(len(data))] arg_params, aux_params = load_param(cur_path + '/../model/fcis_coco', 0, process=True) data = data max_data_shape = max_data_shape provide_data = provide_data predictor = Predictor(sym, data_names, label_names, context=[mx.gpu(ctx_id[0])], max_data_shapes=max_data_shape, provide_data=provide_data, provide_label=provide_label, arg_params=arg_params, aux_params=aux_params) # warm up for i in range(2): data_batch = mx.io.DataBatch(data=[data[0]], label=[], pad=0, index=0, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[0])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))] # print('-----------', data_batch) _, _, _, _ = im_detect(predictor, data_batch, data_names, scales, config) # test for idx, im_name in enumerate(image_names): data_batch = mx.io.DataBatch(data=[data[idx]], label=[], pad=0, index=idx, provide_data=[[(k, v.shape) for k, v in zip(data_names, data[idx])]], provide_label=[None]) scales = [data_batch.data[i][1].asnumpy()[0, 2] for i in range(len(data_batch.data))] tic() scores, boxes, masks, data_dict = im_detect(predictor, data_batch, data_names, scales, config) im_shapes = [data_batch.data[i][0].shape[2:4] for i in range(len(data_batch.data))] if not config.TEST.USE_MASK_MERGE: all_boxes = [[] for _ in range(num_classes)] all_masks = [[] for _ in range(num_classes)] nms = py_nms_wrapper(config.TEST.NMS) for j in range(1, num_classes): indexes = np.where(scores[0][:, j] > 0.7)[0] cls_scores = scores[0][indexes, j, np.newaxis] cls_masks = masks[0][indexes, 1, :, :] try: if config.CLASS_AGNOSTIC: cls_boxes = boxes[0][indexes, :] else: raise Exception() except: cls_boxes = boxes[0][indexes, j * 4:(j + 1) * 4] cls_dets = np.hstack((cls_boxes, cls_scores)) keep = nms(cls_dets) all_boxes[j] = cls_dets[keep, :] all_masks[j] = cls_masks[keep, :] dets = [all_boxes[j] for j in range(1, num_classes)] masks = [all_masks[j] for j in range(1, num_classes)] else: print('=============') masks = masks[0][:, 1:, :, :] im_height = np.round(im_shapes[0][0] / scales[0]).astype('int') im_width = np.round(im_shapes[0][1] / scales[0]).astype('int') print(im_height, im_width) boxes = clip_boxes(boxes[0], (im_height, im_width)) result_masks, result_dets = gpu_mask_voting(masks, boxes, scores[0], num_classes, 100, im_width, im_height, config.TEST.NMS, config.TEST.MASK_MERGE_THRESH, config.BINARY_THRESH, ctx_id[0]) dets = [result_dets[j] for j in range(1, num_classes)] masks = [result_masks[j][:, 0, :, :] for j in range(1, num_classes)] # print(dets) # print(masks) print('testing {} {:.4f}s'.format(im_name, toc())) # visualize for i in range(len(dets)): keep = np.where(dets[i][:,-1]>0.7) dets[i] = dets[i][keep] masks[i] = masks[i][keep] im = cv2.imread(cur_path + '/../demo/' + im_name) im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) show_masks(im, dets, masks, classes, config) print('done')
def test_deepim(): config.TRAIN.MASK_SYN = False if args.vis or args.vis_video or args.vis_video_zoom: config.TEST.VISUALIZE = True config.TEST.FAST_TEST = False epoch = config.TEST.test_epoch ctx = [mx.gpu(int(i)) for i in args.gpus.split(",")] image_set = config.dataset.test_image_set root_path = config.dataset.root_path dataset = config.dataset.dataset.split("+")[0] dataset_path = config.dataset.dataset_path new_args_name = args.cfg logger, final_output_path = create_logger( config.output_path, new_args_name, image_set ) prefix = os.path.join( final_output_path, "..", "_".join([iset for iset in config.dataset.image_set.split("+")]), config.TRAIN.model_prefix, ) pprint.pprint(config) logger.info("testing config:{}\n".format(pprint.pformat(config))) # load symbol and testing data sym_instance = eval(config.symbol + "." + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) if config.dataset.dataset.startswith("ModelNet"): imdb_test = eval(dataset)( config, image_set + config.dataset.class_name[0].split("/")[-1], root_path, dataset_path, class_name=config.dataset.class_name[0], result_path=final_output_path, ) print(imdb_test) pairdbs = [ load_gt_pairdb( config, dataset, image_set + class_name.split("/")[-1], config.dataset.root_path, config.dataset.dataset_path, class_name=class_name, result_path=final_output_path, ) for class_name in config.dataset.class_name ] pairdb = merge_pairdb(pairdbs) else: imdb_test = eval(dataset)( config, image_set + config.dataset.class_name[0], root_path, dataset_path, class_name=config.dataset.class_name[0], result_path=final_output_path, ) print(imdb_test) pairdbs = [ load_gt_pairdb( config, dataset, image_set + class_name, config.dataset.root_path, config.dataset.dataset_path, class_name=class_name, result_path=final_output_path, ) for class_name in config.dataset.class_name ] pairdb = merge_pairdb(pairdbs) # get test data iter test_data = TestDataLoader(pairdb, config=config, batch_size=len(ctx)) # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) # load model and check parameters arg_params, aux_params = load_param(prefix, epoch, process=True) sym_instance.check_parameter_shapes( arg_params, aux_params, data_shape_dict, is_train=False ) # decide maximum shape data_names = [k[0] for k in test_data.provide_data_single] label_names = None max_data_shape = [ [ ( "data", ( 1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES]), ), ) ] ] # create predictor predictor = Predictor( config, sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params, ) # start detection pred_eval( config, predictor, test_data, imdb_test, vis=args.vis, ignore_cache=args.ignore_cache, logger=logger, pairdb=pairdb, ) print(args.cfg, config.TEST.test_epoch)
import mxnet as mx from lib.utils.load_model import load_param import sys sys.path.append('CFN') import CFNet import numpy as np import caffe #caffe.set_mode_gpu() #caffe.set_device(2) #args, auxs = load_param('output/colornet/CFN', 1) #args, auxs = load_param('output/CFNpretrain', 0) sym, args, auxs = mx.model.load_checkpoint('output/colornet/CFN', 1) arg_flow, aux_flow = load_param('model/pretrained_model/flownet', 0) sym = CFNet.CFNet().get_train_symbol() #prototxt = '../colorization/flownet-release/models/flownet/model_simple/deploy.tpl.prototxt' #model = '../colorization/flownet-release/models/flownet/model_simple/flownet_official.caffemodel' #net = caffe.Net(prototxt, model, caffe.TEST) ''' for item in net.params.items(): name, layer = item num = 0 for p in net.params[name]: np.save('flownet_weights/'+str(name)+'_'+str(num), p.data) num += 1 ''' for k, v in args.items(): print k #if 'conv' in k or 'Conv' in k: # args['flow_'+k] = v
def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, lr, lr_step): new_args_name = args.cfg if args.vis: config.TRAIN.VISUALIZE = True logger, final_output_path = create_logger(config.output_path, new_args_name, config.dataset.image_set, args.temp) prefix = os.path.join(final_output_path, prefix) logger.info('called with args {}'.format(args)) print(config.train_iter.SE3_PM_LOSS) if config.train_iter.SE3_PM_LOSS: print("SE3_PM_LOSS == True") else: print("SE3_PM_LOSS == False") if not config.network.STANDARD_FLOW_REP: print_and_log("[h, w] representation for flow is dep", logger) # load dataset and prepare imdb for training image_sets = [iset for iset in config.dataset.image_set.split('+')] datasets = [dset for dset in config.dataset.dataset.split('+')] print("config.dataset.class_name: {}".format(config.dataset.class_name)) print("image_sets: {}".format(image_sets)) if datasets[0].startswith('ModelNet'): pairdbs = [ load_gt_pairdb(config, datasets[i], image_sets[i] + class_name.split('/')[-1], config.dataset.root_path, config.dataset.dataset_path, class_name=class_name, result_path=final_output_path) for class_name in config.dataset.class_name for i in range(len(image_sets)) ] else: pairdbs = [ load_gt_pairdb(config, datasets[i], image_sets[i] + class_name, config.dataset.root_path, config.dataset.dataset_path, class_name=class_name, result_path=final_output_path) for class_name in config.dataset.class_name for i in range(len(image_sets)) ] pairdb = merge_pairdb(pairdbs) if not args.temp: src_file = os.path.join(curr_path, 'symbols', config.symbol + '.py') dst_file = os.path.join( final_output_path, '{}_{}.py'.format(config.symbol, time.strftime('%Y-%m-%d-%H-%M'))) os.popen('cp {} {}'.format(src_file, dst_file)) sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=True) # setup multi-gpu batch_size = len(ctx) input_batch_size = config.TRAIN.BATCH_PAIRS * batch_size pprint.pprint(config) logger.info('training config:{}\n'.format(pprint.pformat(config))) # load training data train_data = TrainDataLoader(sym, pairdb, config, batch_size=input_batch_size, shuffle=config.TRAIN.SHUFFLE, ctx=ctx) train_data.get_batch_parallel() max_scale = [ max([v[0] for v in config.SCALES]), max(v[1] for v in config.SCALES) ] max_data_shape = [('image_observed', (config.TRAIN.BATCH_PAIRS, 3, max_scale[0], max_scale[1])), ('image_rendered', (config.TRAIN.BATCH_PAIRS, 3, max_scale[0], max_scale[1])), ('depth_gt_observed', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0], max_scale[1])), ('src_pose', (config.TRAIN.BATCH_PAIRS, 3, 4)), ('tgt_pose', (config.TRAIN.BATCH_PAIRS, 3, 4))] if config.network.INPUT_DEPTH: max_data_shape.append(('depth_observed', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0], max_scale[1]))) max_data_shape.append(('depth_rendered', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0], max_scale[1]))) if config.network.INPUT_MASK: max_data_shape.append(('mask_observed', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0], max_scale[1]))) max_data_shape.append(('mask_rendered', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0], max_scale[1]))) rot_param = 3 if config.network.ROT_TYPE == "EULER" else 4 max_label_shape = [('rot', (config.TRAIN.BATCH_PAIRS, rot_param)), ('trans', (config.TRAIN.BATCH_PAIRS, 3))] if config.network.PRED_FLOW: max_label_shape.append(('flow', (config.TRAIN.BATCH_PAIRS, 2, max_scale[0], max_scale[1]))) max_label_shape.append(('flow_weights', (config.TRAIN.BATCH_PAIRS, 2, max_scale[0], max_scale[1]))) if config.train_iter.SE3_PM_LOSS: max_label_shape.append( ('point_cloud_model', (config.TRAIN.BATCH_PAIRS, 3, config.train_iter.NUM_3D_SAMPLE))) max_label_shape.append( ('point_cloud_weights', (config.TRAIN.BATCH_PAIRS, 3, config.train_iter.NUM_3D_SAMPLE))) max_label_shape.append( ('point_cloud_observed', (config.TRAIN.BATCH_PAIRS, 3, config.train_iter.NUM_3D_SAMPLE))) if config.network.PRED_MASK: max_label_shape.append( ('mask_gt_observed', (config.TRAIN.BATCH_PAIRS, 1, max_scale[0], max_scale[1]))) # max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape, max_label_shape) print_and_log( 'providing maximum shape, {}, {}'.format(max_data_shape, max_label_shape), logger) # infer max shape ''' max_label_shape = [('label', (config.TRAIN.BATCH_IMAGES, 1, max([v[0] for v in max_scale]), max([v[1] for v in max_scale])))] max_data_shape, max_label_shape = train_data.infer_shape( max_data_shape, max_label_shape) print('providing maximum shape', max_data_shape, max_label_shape) ''' # infer shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) print_and_log('\ndata_shape_dict: {}\n'.format(data_shape_dict), logger) sym_instance.infer_shape(data_shape_dict) print('************(wg): infering shape **************') internals = sym.get_internals() _, out_shapes, _ = internals.infer_shape(**data_shape_dict) print(sym.list_outputs()) shape_dict = dict(zip(internals.list_outputs(), out_shapes)) pprint.pprint(shape_dict) # load and initialize params if config.TRAIN.RESUME: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) elif pretrained == 'xavier': print('xavier') # arg_params = {} # aux_params = {} # sym_instance.init_weights(config, arg_params, aux_params) else: print(pretrained) arg_params, aux_params = load_param(pretrained, epoch, convert=True) print('arg_params: ', arg_params.keys()) print('aux_params: ', aux_params.keys()) if not config.network.skip_initialize: sym_instance.init_weights(config, arg_params, aux_params) # check parameter shapes if pretrained != 'xavier': sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # create solver fixed_param_prefix = config.network.FIXED_PARAMS data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] mod = MutableModule( sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in range(batch_size)], max_label_shapes=[max_label_shape for _ in range(batch_size)], fixed_param_prefix=fixed_param_prefix, config=config) # decide training params # metrics eval_metrics = mx.metric.CompositeEvalMetric() metric_list = [] iter_idx = 0 if config.network.PRED_FLOW: metric_list.append(metric.Flow_L2LossMetric(config, iter_idx)) metric_list.append(metric.Flow_CurLossMetric(config, iter_idx)) if config.train_iter.SE3_DIST_LOSS: metric_list.append(metric.Rot_L2LossMetric(config, iter_idx)) metric_list.append(metric.Trans_L2LossMetric(config, iter_idx)) if config.train_iter.SE3_PM_LOSS: metric_list.append(metric.PointMatchingLossMetric(config, iter_idx)) if config.network.PRED_MASK: metric_list.append(metric.MaskLossMetric(config, iter_idx)) # Visualize Training Batches if config.TRAIN.VISUALIZE: metric_list.append(metric.SimpleVisualize(config)) # metric_list.append(metric.MaskVisualize(config, save_dir = final_output_path)) metric_list.append( metric.MinibatchVisualize(config)) # flow visualization for child_metric in metric_list: eval_metrics.add(child_metric) # callback batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=args.frequent) epoch_end_callback = mx.callback.module_checkpoint( mod, prefix, period=1, save_optimizer_states=True) # decide learning rate base_lr = lr lr_factor = 0.1 lr_epoch = [float(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [ epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch ] lr = base_lr * (lr_factor**(len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [ int(epoch * len(pairdb) / batch_size) for epoch in lr_epoch_diff ] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, config.TRAIN.warmup, config.TRAIN.warmup_lr, config.TRAIN.warmup_step) if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) # train if config.TRAIN.optimizer == 'adam': optimizer_params = {'learning_rate': lr} if pretrained == 'xavier': init = mx.init.Mixed(['rot_weight|trans_weight', '.*'], [ mx.init.Zero(), mx.init.Xavier( rnd_type='gaussian', factor_type="in", magnitude=2) ]) mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='adam', optimizer_params=optimizer_params, begin_epoch=begin_epoch, num_epoch=end_epoch, prefix=prefix, initializer=init, force_init=True) else: mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='adam', arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch, prefix=prefix) elif config.TRAIN.optimizer == 'sgd': # optimizer optimizer_params = { 'momentum': config.TRAIN.momentum, 'wd': config.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None } if pretrained == 'xavier': init = mx.init.Mixed(['rot_weight|trans_weight', '.*'], [ mx.init.Zero(), mx.init.Xavier( rnd_type='gaussian', factor_type="in", magnitude=2) ]) mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='sgd', optimizer_params=optimizer_params, begin_epoch=begin_epoch, num_epoch=end_epoch, prefix=prefix, initializer=init, force_init=True) else: mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch, prefix=prefix)
def test_fcis(config, dataset, image_set, root_path, dataset_path, ctx, prefix, epoch, vis, ignore_cache, shuffle, has_rpn, proposal, thresh, logger=None, output_path=None): if not logger: assert False, 'require a logger' # print config pprint.pprint(config) logger.info('testing config:{}\n'.format(pprint.pformat(config))) # load symbol and testing data if has_rpn: sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) imdb = eval(dataset)(image_set, root_path, dataset_path, result_path=output_path, binary_thresh=config.BINARY_THRESH, mask_size=config.MASK_SIZE) sdsdb = imdb.gt_sdsdb() else: raise NotImplementedError # get test data iter test_data = TestLoader(sdsdb, config, batch_size=len(ctx), shuffle=shuffle, has_rpn=has_rpn) # load model arg_params, aux_params = load_param(prefix, epoch, process=False) # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data_single] label_names = [] max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] if not has_rpn: raise NotImplementedError() # create predictor predictor = Predictor(sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) # start detection pred_eval(predictor, test_data, imdb, config, vis=vis, ignore_cache=ignore_cache, thresh=thresh, logger=logger)
def test_deepim(): config.TRAIN.MASK_SYN = False if args.vis or args.vis_video or args.vis_video_zoom: config.TEST.VISUALIZE = True config.TEST.FAST_TEST = False if args.iter_test: config.TEST.test_iter = 5 if args.refine: config.TEST.test_iter = 1 if args.skip_flow: config.network.FLOW_I2R = False config.network.FLOW_R2I = False config.train_iter0.FLOW_I2R = False config.train_iter0.FLOW_R2I = False config.train_iter1.FLOW_I2R = False config.train_iter1.FLOW_R2I = False config.train_iter2.FLOW_I2R = False config.train_iter2.FLOW_R2I = False config.train_iter3.FLOW_I2R = False config.train_iter3.FLOW_R2I = False epoch = config.TEST.test_epoch ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] if len(ctx) != config.NUM_GPUS: print("********** WARNING: length of context doesn't match num_gpus set in config, {} vs. {} **********".\ format(len(ctx), config.NUM_GPUS)) image_set = config.dataset.test_image_set root_path = config.dataset.root_path dataset = config.dataset.dataset.split('+')[0] dataset_path = config.dataset.dataset_path if not os.path.basename(args.cfg).split('.')[0].endswith('temp'): new_args_name = os.path.basename( args.cfg).split('.')[0] + '_{}gpus.yaml'.format(config.NUM_GPUS) else: new_args_name = args.cfg if config.TEST.VISUALIZE or args.temp: logger, final_output_path = create_logger(config.output_path, new_args_name, image_set, True) else: logger, final_output_path = create_logger(config.output_path, new_args_name, image_set) prefix = os.path.join( final_output_path, '..', '_'.join([iset for iset in config.dataset.image_set.split('+')]), config.TRAIN.model_prefix) pprint.pprint(config) logger.info('testing config:{}\n'.format(pprint.pformat(config))) # load symbol and testing data sym_instance = eval(config.symbol + '.' + config.symbol)() sym = sym_instance.get_symbol(config, is_train=False) if config.dataset.dataset.startswith('ModelNet'): imdb_test = eval(dataset)(config, image_set + config.dataset.class_name[0].split('/')[-1], root_path, dataset_path, class_name=config.dataset.class_name[0], result_path=final_output_path) print(imdb_test) pairdbs = [ load_gt_pairdb(config, dataset, image_set + class_name.split('/')[-1], config.dataset.root_path, config.dataset.dataset_path, class_name=class_name, result_path=final_output_path) for class_name in config.dataset.class_name ] pairdb = merge_pairdb(pairdbs) else: imdb_test = eval(dataset)(config, image_set + config.dataset.class_name[0], root_path, dataset_path, class_name=config.dataset.class_name[0], result_path=final_output_path) print(imdb_test) pairdbs = [ load_gt_pairdb(config, dataset, image_set + class_name, config.dataset.root_path, config.dataset.dataset_path, class_name=class_name, result_path=final_output_path) for class_name in config.dataset.class_name ] pairdb = merge_pairdb(pairdbs) # get test data iter test_data = TestDataLoader(pairdb, config=config, batch_size=len(ctx)) # infer shape data_shape_dict = dict(test_data.provide_data_single) sym_instance.infer_shape(data_shape_dict) # load model and check parameters arg_params, aux_params = load_param(prefix, epoch, process=True) sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict, is_train=False) # decide maximum shape data_names = [k[0] for k in test_data.provide_data_single] label_names = None max_data_shape = [[('data', (1, 3, max([v[0] for v in config.SCALES]), max([v[1] for v in config.SCALES])))]] # create predictor predictor = Predictor(config, sym, data_names, label_names, context=ctx, max_data_shapes=max_data_shape, provide_data=test_data.provide_data, provide_label=test_data.provide_label, arg_params=arg_params, aux_params=aux_params) # start detection pred_eval(config, predictor, test_data, imdb_test, vis=args.vis, ignore_cache=args.ignore_cache, logger=logger, pairdb=pairdb) print(args.cfg, config.TEST.test_epoch)
def train_net(args, ctx, pretrained, epoch, prefix, begin_epoch, end_epoch, lr, lr_step): mx.random.seed(3) np.random.seed(3) logger, final_output_path = create_logger(config.output_path, args.cfg, config.dataset.image_set) prefix = os.path.join(final_output_path, prefix) # load symbol shutil.copy2(os.path.join(curr_path, 'symbols', config.symbol + '.py'), final_output_path) sym_instance = eval(config.symbol)() sym = sym_instance.get_symbol(config, is_train=True) feat_sym = sym.get_internals()['rpn_cls_score_output'] # setup multi-gpu batch_size = len(ctx) input_batch_size = config.TRAIN.BATCH_IMAGES * batch_size # print config pprint.pprint(config) logger.info('training config:{}\n'.format(pprint.pformat(config))) # load dataset and prepare imdb for training image_sets = [iset for iset in config.dataset.image_set.split('+')] sdsdbs = [load_gt_sdsdb(config.dataset.dataset, image_set, config.dataset.root_path, config.dataset.dataset_path, mask_size=config.MASK_SIZE, binary_thresh=config.BINARY_THRESH, result_path=final_output_path, flip=config.TRAIN.FLIP) for image_set in image_sets] sdsdb = merge_roidb(sdsdbs) sdsdb = filter_roidb(sdsdb, config) # load training data train_data = AnchorLoader(feat_sym, sdsdb, config, batch_size=input_batch_size, shuffle=config.TRAIN.SHUFFLE, ctx=ctx, feat_stride=config.network.RPN_FEAT_STRIDE, anchor_scales=config.network.ANCHOR_SCALES, anchor_ratios=config.network.ANCHOR_RATIOS, aspect_grouping=config.TRAIN.ASPECT_GROUPING, allowed_border=config.TRAIN.RPN_ALLOWED_BORDER) # infer max shape max_data_shape = [('data', (config.TRAIN.BATCH_IMAGES, 3, max([v[0] for v in config.SCALES]), max(v[1] for v in config.SCALES)))] max_data_shape, max_label_shape = train_data.infer_shape(max_data_shape) max_data_shape.append(('gt_boxes', (config.TRAIN.BATCH_IMAGES, 100, 5))) max_data_shape.append(('gt_masks', (config.TRAIN.BATCH_IMAGES, 100, max([v[0] for v in config.SCALES]), max(v[1] for v in config.SCALES)))) print('providing maximum shape', max_data_shape, max_label_shape) # infer shape data_shape_dict = dict(train_data.provide_data_single + train_data.provide_label_single) print('data shape:') pprint.pprint(data_shape_dict) sym_instance.infer_shape(data_shape_dict) # load and initialize params if config.TRAIN.RESUME: print('continue training from ', begin_epoch) arg_params, aux_params = load_param(prefix, begin_epoch, convert=True) else: arg_params, aux_params = load_param(pretrained, epoch, convert=True) sym_instance.init_weight(config, arg_params, aux_params) # check parameter shapes sym_instance.check_parameter_shapes(arg_params, aux_params, data_shape_dict) # create solver fixed_param_prefix = config.network.FIXED_PARAMS data_names = [k[0] for k in train_data.provide_data_single] label_names = [k[0] for k in train_data.provide_label_single] mod = MutableModule(sym, data_names=data_names, label_names=label_names, logger=logger, context=ctx, max_data_shapes=[max_data_shape for _ in range(batch_size)], max_label_shapes=[max_label_shape for _ in range(batch_size)], fixed_param_prefix=fixed_param_prefix) # decide training metric # RPN, classification accuracy/loss, regression loss rpn_acc = metric.RPNAccMetric() rpn_cls_loss = metric.RPNLogLossMetric() rpn_bbox_loss = metric.RPNL1LossMetric() fcis_acc = metric.FCISAccMetric(config) fcis_acc_fg = metric.FCISAccFGMetric(config) fcis_cls_loss = metric.FCISLogLossMetric(config) fcis_bbox_loss = metric.FCISL1LossMetric(config) fcis_mask_loss = metric.FCISMaskLossMetric(config) eval_metrics = mx.metric.CompositeEvalMetric() for child_metric in [rpn_acc, rpn_cls_loss, rpn_bbox_loss, fcis_acc, fcis_acc_fg, fcis_cls_loss, fcis_bbox_loss, fcis_mask_loss]: eval_metrics.add(child_metric) batch_end_callback = callback.Speedometer(train_data.batch_size, frequent=args.frequent) means = np.tile(np.array(config.TRAIN.BBOX_MEANS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES) stds = np.tile(np.array(config.TRAIN.BBOX_STDS), 2 if config.CLASS_AGNOSTIC else config.dataset.NUM_CLASSES) epoch_end_callback = callback.do_checkpoint(prefix, means, stds) # print epoch, begin_epoch, end_epoch, lr_step base_lr = lr lr_factor = 0.1 lr_epoch = [float(epoch) for epoch in lr_step.split(',')] lr_epoch_diff = [epoch - begin_epoch for epoch in lr_epoch if epoch > begin_epoch] lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) lr_iters = [int(epoch * len(sdsdb) / batch_size) for epoch in lr_epoch_diff] print('lr', lr, 'lr_epoch_diff', lr_epoch_diff, 'lr_iters', lr_iters) lr_scheduler = WarmupMultiFactorScheduler(lr_iters, lr_factor, config.TRAIN.warmup, config.TRAIN.warmup_lr, config.TRAIN.warmup_step) # optimizer optimizer_params = {'momentum': config.TRAIN.momentum, 'wd': config.TRAIN.wd, 'learning_rate': lr, 'lr_scheduler': lr_scheduler, 'rescale_grad': 1.0, 'clip_gradient': None} if not isinstance(train_data, PrefetchingIter): train_data = PrefetchingIter(train_data) # del sdsdb # a = mx.viz.plot_network(sym) # a.render('../example', view=True) # print 'prepare sds finished' mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, batch_end_callback=batch_end_callback, kvstore=config.default.kvstore, optimizer='sgd', optimizer_params=optimizer_params, arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch)