def main(): args = sys.argv print('Called with args:') print(args) if 'train' in args: # train set blobs_all = GetBlobs('../image_1000/') print('{:d} images'.format(len(blobs_all))) # output directory where the models are saved output_dir = 'output/models' if not os.path.exists(output_dir): os.makedirs(output_dir) print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = 'output/tensorboard' if not os.path.exists(tb_dir): os.makedirs(tb_dir) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) # load network net = vgg16() train_net(net, blobs_all, output_dir, tb_dir) else: pre_trained = 'output/models/' if os.file.exists(pre_trained): print('For test.') else: print('No pre_trained model yet.')
def main(args): opt = vars(args) # initialize opt['dataset_splitBy'] = opt['dataset'] + '_' + opt['splitBy'] checkpoint_dir = osp.join(opt['checkpoint_path'], opt['dataset_splitBy']) if not osp.isdir(checkpoint_dir): os.makedirs(checkpoint_dir) # set random seed torch.manual_seed(opt['seed']) random.seed(opt['seed']) # set up loader data_json = osp.join('cache/prepro', opt['dataset_splitBy'], 'data.json') data_h5 = osp.join('cache/prepro', opt['dataset_splitBy'], 'data.h5') loader = GtMRCNLoader(data_json, data_h5) # set up model opt['vocab_size'] = loader.vocab_size opt['C4_feat_dim'] = 1024 net = resnetv1(opt, batch_size=opt['batch_size'], num_layers=101) # determine batch size in opt.py # output directory where the models are saved output_dir = osp.join(opt['dataset_splitBy'], 'output_{}'.format(opt['output_postfix'])) print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = osp.join(opt['dataset_splitBy'], 'tb_{}'.format(opt['output_postfix'])) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) # also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False cfg.TRAIN.USE_FLIPPED = orgflip if args.cfg_file is not None: cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) #train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, train_net( net, loader, output_dir, tb_dir, pretrained_model= 'pyutils/mask-faster-rcnn/output/res101/coco_2014_train_minus_refer_valtest+coco_2014_valminusminival/notime/res101_mask_rcnn_iter_1250000.pth', max_iters=args.max_iters)
# also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False _, valroidb = combined_roidb(args.imdbval_name) print('{:d} validation roidb entries'.format(len(valroidb))) cfg.TRAIN.USE_FLIPPED = orgflip # load network if args.net == 'vgg16': net = vgg16() elif args.net == 'res50': net = resnetv1(num_layers=50) elif args.net == 'res101': net = resnetv1(num_layers=101) elif args.net == 'res152': net = resnetv1(num_layers=152) elif args.net == 'mobile': net = mobilenetv1() else: raise NotImplementedError train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=args.weight, max_iters=args.max_iters)
if args.net == 'vgg16': net = vgg16() elif args.net == 'res34': net = imagenet(num_layers=34) elif args.net == 'res50': net = imagenet(num_layers=50) elif args.net == 'res101': net = imagenet(num_layers=101) elif args.net == 'res152': net = imagenet(num_layers=152) elif args.net == 'mobile': net = mobilenetv1() else: raise NotImplementedError elif (cfg.NET_TYPE == 'lidar'): net = lidarnet(num_layers=101) train_net(net, db, output_dir, tb_dir, pretrained_model=args.weights_file, max_iters=args.max_iters, sum_size=256, val_sum_size=5000, batch_size=cfg.TRAIN.BATCH_SIZE, val_batch_size=cfg.TRAIN.VAL_BATCH_SIZE, val_thresh=0.4, augment_en=cfg.TRAIN.AUGMENT_EN, val_augment_en=cfg.TRAIN.VAL_AUGMENT_EN)
print('Called with args:') print(args) if args.cfg_file is not None: cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) print('Using config:') pprint.pprint(cfg) np.random.seed(cfg.RNG_SEED) # Choose net if args.net == 'adgm': net = Adgm if args.net == 'vae': net = Vae # output directory where the models are saved output_dir = get_output_dir(args.dataset, args.tag) print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = get_output_tb_dir(args.dataset, args.tag) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) train_net(net, Mnist, output_dir, tb_dir, max_iters=args.max_iters)
roidb = get_training_roidb(imdb) return roidb roidbs = [get_roidb(s) for s in imdb_names.split('+')] roidb = roidbs[0] if len(roidbs) > 1: for r in roidbs[1:]: roidb.extend(r) tmp = get_imdb(imdb_names.split('+')[1]) imdb = datasets.imdb.imdb(imdb_names, tmp.classes) else: imdb = get_imdb(imdb_names) return imdb, roidb save_path = '/home/vador/Documents/project/AI/drl-rpn-tf-video/output-weights/voc_2007_train/' # Cannot use weights_path because it leads to dimensions mismatch weigths_path = '/home/vador/Documents/project/AI/drl-rpn-tf-video/pretrained-data/data3D/vgg16_drl_rpn_iter_1.ckpt' imdb_name = 'voc_2007_train' imdbval_name = 'voc_2007_trainval' imdb, roidb = combined_roidb(imdb_name) print('{:d} roidb entries'.format(len(roidb))) output_dir = get_output_dir(imdb, None, save_path) _, valroidb = combined_roidb(imdbval_name) train_net(net, imdb, roidb, valroidb, output_dir, pretrained_model=weigths_path, max_iters=1)
# also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False _, valroidb = combined_roidb(args.imdbval_name) print('{:d} validation roidb entries'.format(len(valroidb))) cfg.TRAIN.USE_FLIPPED = orgflip # load network if args.net == 'vgg16': net = vgg16() elif args.net == 'res50': net = resnetv1(num_layers=50) elif args.net == 'res101': net = resnetv1(num_layers=101) elif args.net == 'res152': net = resnetv1(num_layers=152) elif args.net == 'mobile': net = mobilenetv1() else: raise NotImplementedError train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=args.weight, max_iters=args.max_iters, gpu_fraction=0.4)
if args.cfg_file is not None: cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) cfg.CLASSES_FILE = args.com_classes args = parser.reload_or_save_args(cfg, args) # print('Using config:') # pprint.pprint(cfg) np.random.seed(cfg.RNG_SEED) # load network base_net = load_base_network() # train and valid data imdb, roidb, valroidb = prepare_datas(args.package_name) # output data directory output_dir, tb_dir = prepare_params() train_net(args, base_net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=args.weight, max_iters=args.max_iters * args.epochs)
from __future__ import absolute_import from __future__ import division from __future__ import print_function import numpy as np from model.train_val import train_net from nets.vgg16 import vgg16 class imdb(object): def __init__(self, num_classes): self.num_classes = num_classes output_dir = 'output/vgg16/vg/default' tb_dir = 'output/vgg16/vg/tb' N_obj = 201 vg_roidb = np.load('vg_roidb.npz') roidb_temp = vg_roidb['roidb'] roidb = roidb_temp[()] train_roidb = roidb['train_roidb'] test_roidb = roidb['test_roidb'] vg_imdb = imdb(N_obj) net = vgg16() roidb = train_roidb valroidb = test_roidb[0:1000] pretrained_model = 'output/vgg16/coco_2014_train+coco_2014_valminusminival/default/vgg16_faster_rcnn_iter_1190000.ckpt' train_net(net, vg_imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=pretrained_model, max_iters=800000)
print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = get_output_tb_dir(imdb, args.tag) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) # also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False _, valroidb = combined_roidb(args.imdbval_name) print('{:d} validation roidb entries'.format(len(valroidb))) cfg.TRAIN.USE_FLIPPED = orgflip # load network if args.net == 'vgg16': net = vgg16() elif args.net == 'res50': net = resnetv1(num_layers=50) elif args.net == 'res101': net = resnetv1(num_layers=101) elif args.net == 'res152': net = resnetv1(num_layers=152) elif args.net == 'mobile': net = mobilenetv1() else: raise NotImplementedError train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=args.weight, max_iters=args.max_iters)
for i in range(args.model_iters): if i == 0: iters = len(roidb) * args.epochs roidb_final = roidb else: iters = (len(roidb_im_pred) + len(roidb)) * args.epochs roidb_final = roidb + roidb_im_pred print("len(roidb): ", len(roidb)) print("len(roidb_im_pred): ", len(roidb_im_pred)) print("len(roidb_final): ", len(roidb_final)) total_step += iters steps_for_stage.append((i, total_step)) with open(output_dir + '/steps_for_stage.txt', 'w') as fp: fp.write("iter, step\n") for i, s in steps_for_stage: fp.write("%s, %s\n" % (i, s)) print("=== start training model {}, steps: {} ===".format(i, iters)) train_net( net, imdb, roidb_final, valroidb, output_dir, tb_dir, pretrained_model=args.weight, max_iters=total_step) roidb_im_pred = mine(roidb_im, model_iters=i, tag=args.tag, net=net, mine_threshold=args.t_mine, classes=imdb._classes)
cluster=cluster_spec)): # train set imdb, roidb = combined_roidb(args.imdb_name) print('{:d} roidb entries'.format(len(roidb))) # output directory where the models are saved output_dir = get_output_dir(imdb, args.tag) print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = get_output_tb_dir(imdb, args.tag) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) # also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False _, valroidb = combined_roidb(args.imdbval_name) print('{:d} validation roidb entries'.format(len(valroidb))) cfg.TRAIN.USE_FLIPPED = orgflip if args.net == 'vgg16': net = vgg16(batch_size=cfg.TRAIN.IMS_PER_BATCH) elif args.net == 'res101': net = Resnet101(batch_size=cfg.TRAIN.IMS_PER_BATCH) else: raise NotImplementedError train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=args.weight, max_iters=args.max_iters, master=server.target, is_chief=is_chief)
print('Called with args:') print(args) if args.cfg_file is not None: cfg_from_file(args.cfg_file) if args.set_cfgs is not None: cfg_from_list(args.set_cfgs) print('Using config:') pprint.pprint(cfg) np.random.seed(cfg.RNG_SEED) # test set orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False imdb, roidb = combined_roidb(args.imdbtest_name) print('{:d} test roidb entries'.format(len(roidb))) cfg.TRAIN.USE_FLIPPED = orgflip cfg.TRAIN.SNAPSHOT_PREFIX = "" cfg.TRAIN.SNAPSHOT_LOAD_PREFIX = "" net = resnetv1(imdb.nof_ent_classes, imdb.nof_rel_classes, num_layers=101) train_net(net, imdb, [], roidb, "", "", pretrained_model=args.model, max_iters=1, just_test=True)
def launch_train(self, conf): ''' ''' args = {} args['cfg_file'] = conf.frcnn_cfg args['weight'] = conf.starting_weights args['imdb_name'] = conf.train_set args['imdbval_name'] = conf.valid_set args['max_iters'] = conf.iters args['tag'] = conf.frcnn_tag args['net'] = conf.frcnn_net args['set_cfgs'] = None print('Called with args:') print(args) if args['cfg_file'] is not None: cfg_from_file(args['cfg_file']) if args['set_cfgs'] is not None: cfg_from_list(args['set_cfgs']) print('Using config:') pprint.pprint(cfg) np.random.seed(cfg.RNG_SEED) # train set imdb, roidb = combined_roidb(args['imdb_name'], conf) print('{:d} roidb entries'.format(len(roidb))) # output directory where the models are saved output_dir = conf.backup_folder #get_output_dir(imdb, args.tag) print('Output will be saved to `{:s}`'.format(output_dir)) # tensorboard directory where the summaries are saved during training tb_dir = conf.backup_folder # get_output_tb_dir(imdb, args.tag) print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir)) # also add the validation set, but with no flipping images orgflip = cfg.TRAIN.USE_FLIPPED cfg.TRAIN.USE_FLIPPED = False _, valroidb = combined_roidb(args['imdbval_name'], conf) print('{:d} validation roidb entries'.format(len(valroidb))) cfg.TRAIN.USE_FLIPPED = orgflip if args['net'] == 'vgg16': net = vgg16(batch_size=cfg.TRAIN.IMS_PER_BATCH) elif args['net'] == 'res50': net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=50) elif args['net'] == 'res101': net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=101) # load network elif args['net'] == 'res152': net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=152) elif args['net'] == 'mobile': net = mobilenetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH) else: raise NotImplementedError train_net(net, imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=args['weight'], max_iters=args['max_iters'])