def main():
args = sys.argv
print('Called with args:')
print(args)
if 'train' in args:
# train set
blobs_all = GetBlobs('../image_1000/')
print('{:d} images'.format(len(blobs_all)))
# output directory where the models are saved
output_dir = 'output/models'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
print('Output will be saved to `{:s}`'.format(output_dir))
# tensorboard directory where the summaries are saved during training
tb_dir = 'output/tensorboard'
if not os.path.exists(tb_dir):
os.makedirs(tb_dir)
print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))
# load network
net = vgg16()
train_net(net, blobs_all, output_dir, tb_dir)
else:
pre_trained = 'output/models/'
if os.file.exists(pre_trained):
print('For test.')
else:
print('No pre_trained model yet.')
def main(args):
opt = vars(args)
# initialize
opt['dataset_splitBy'] = opt['dataset'] + '_' + opt['splitBy']
checkpoint_dir = osp.join(opt['checkpoint_path'], opt['dataset_splitBy'])
if not osp.isdir(checkpoint_dir): os.makedirs(checkpoint_dir)
# set random seed
torch.manual_seed(opt['seed'])
random.seed(opt['seed'])
# set up loader
data_json = osp.join('cache/prepro', opt['dataset_splitBy'], 'data.json')
data_h5 = osp.join('cache/prepro', opt['dataset_splitBy'], 'data.h5')
loader = GtMRCNLoader(data_json, data_h5)
# set up model
opt['vocab_size'] = loader.vocab_size
opt['C4_feat_dim'] = 1024
net = resnetv1(opt, batch_size=opt['batch_size'],
num_layers=101) # determine batch size in opt.py
# output directory where the models are saved
output_dir = osp.join(opt['dataset_splitBy'],
'output_{}'.format(opt['output_postfix']))
print('Output will be saved to `{:s}`'.format(output_dir))
# tensorboard directory where the summaries are saved during training
tb_dir = osp.join(opt['dataset_splitBy'],
'tb_{}'.format(opt['output_postfix']))
print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))
# also add the validation set, but with no flipping images
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
cfg.TRAIN.USE_FLIPPED = orgflip
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
#train_net(net, imdb, roidb, valroidb, output_dir, tb_dir,
train_net(
net,
loader,
output_dir,
tb_dir,
pretrained_model=
'pyutils/mask-faster-rcnn/output/res101/coco_2014_train_minus_refer_valtest+coco_2014_valminusminival/notime/res101_mask_rcnn_iter_1250000.pth',
max_iters=args.max_iters)
# also add the validation set, but with no flipping images
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
_, valroidb = combined_roidb(args.imdbval_name)
print('{:d} validation roidb entries'.format(len(valroidb)))
cfg.TRAIN.USE_FLIPPED = orgflip
# load network
if args.net == 'vgg16':
net = vgg16()
elif args.net == 'res50':
net = resnetv1(num_layers=50)
elif args.net == 'res101':
net = resnetv1(num_layers=101)
elif args.net == 'res152':
net = resnetv1(num_layers=152)
elif args.net == 'mobile':
net = mobilenetv1()
else:
raise NotImplementedError
train_net(net,
imdb,
roidb,
valroidb,
output_dir,
tb_dir,
pretrained_model=args.weight,
max_iters=args.max_iters)
if args.net == 'vgg16':
net = vgg16()
elif args.net == 'res34':
net = imagenet(num_layers=34)
elif args.net == 'res50':
net = imagenet(num_layers=50)
elif args.net == 'res101':
net = imagenet(num_layers=101)
elif args.net == 'res152':
net = imagenet(num_layers=152)
elif args.net == 'mobile':
net = mobilenetv1()
else:
raise NotImplementedError
elif (cfg.NET_TYPE == 'lidar'):
net = lidarnet(num_layers=101)
train_net(net,
db,
output_dir,
tb_dir,
pretrained_model=args.weights_file,
max_iters=args.max_iters,
sum_size=256,
val_sum_size=5000,
batch_size=cfg.TRAIN.BATCH_SIZE,
val_batch_size=cfg.TRAIN.VAL_BATCH_SIZE,
val_thresh=0.4,
augment_en=cfg.TRAIN.AUGMENT_EN,
val_augment_en=cfg.TRAIN.VAL_AUGMENT_EN)
print('Called with args:')
print(args)
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
print('Using config:')
pprint.pprint(cfg)
np.random.seed(cfg.RNG_SEED)
# Choose net
if args.net == 'adgm':
net = Adgm
if args.net == 'vae':
net = Vae
# output directory where the models are saved
output_dir = get_output_dir(args.dataset, args.tag)
print('Output will be saved to `{:s}`'.format(output_dir))
# tensorboard directory where the summaries are saved during training
tb_dir = get_output_tb_dir(args.dataset, args.tag)
print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))
train_net(net, Mnist, output_dir, tb_dir, max_iters=args.max_iters)
roidb = get_training_roidb(imdb)
return roidb
roidbs = [get_roidb(s) for s in imdb_names.split('+')]
roidb = roidbs[0]
if len(roidbs) > 1:
for r in roidbs[1:]:
roidb.extend(r)
tmp = get_imdb(imdb_names.split('+')[1])
imdb = datasets.imdb.imdb(imdb_names, tmp.classes)
else:
imdb = get_imdb(imdb_names)
return imdb, roidb
save_path = '/home/vador/Documents/project/AI/drl-rpn-tf-video/output-weights/voc_2007_train/'
# Cannot use weights_path because it leads to dimensions mismatch
weigths_path = '/home/vador/Documents/project/AI/drl-rpn-tf-video/pretrained-data/data3D/vgg16_drl_rpn_iter_1.ckpt'
imdb_name = 'voc_2007_train'
imdbval_name = 'voc_2007_trainval'
imdb, roidb = combined_roidb(imdb_name)
print('{:d} roidb entries'.format(len(roidb)))
output_dir = get_output_dir(imdb, None, save_path)
_, valroidb = combined_roidb(imdbval_name)
train_net(net, imdb, roidb, valroidb, output_dir, pretrained_model=weigths_path, max_iters=1)
# also add the validation set, but with no flipping images
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
_, valroidb = combined_roidb(args.imdbval_name)
print('{:d} validation roidb entries'.format(len(valroidb)))
cfg.TRAIN.USE_FLIPPED = orgflip
# load network
if args.net == 'vgg16':
net = vgg16()
elif args.net == 'res50':
net = resnetv1(num_layers=50)
elif args.net == 'res101':
net = resnetv1(num_layers=101)
elif args.net == 'res152':
net = resnetv1(num_layers=152)
elif args.net == 'mobile':
net = mobilenetv1()
else:
raise NotImplementedError
train_net(net,
imdb,
roidb,
valroidb,
output_dir,
tb_dir,
pretrained_model=args.weight,
max_iters=args.max_iters,
gpu_fraction=0.4)
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
cfg.CLASSES_FILE = args.com_classes
args = parser.reload_or_save_args(cfg, args)
# print('Using config:')
# pprint.pprint(cfg)
np.random.seed(cfg.RNG_SEED)
# load network
base_net = load_base_network()
# train and valid data
imdb, roidb, valroidb = prepare_datas(args.package_name)
# output data directory
output_dir, tb_dir = prepare_params()
train_net(args,
base_net,
imdb,
roidb,
valroidb,
output_dir,
tb_dir,
pretrained_model=args.weight,
max_iters=args.max_iters * args.epochs)
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
from model.train_val import train_net
from nets.vgg16 import vgg16
class imdb(object):
def __init__(self, num_classes):
self.num_classes = num_classes
output_dir = 'output/vgg16/vg/default'
tb_dir = 'output/vgg16/vg/tb'
N_obj = 201
vg_roidb = np.load('vg_roidb.npz')
roidb_temp = vg_roidb['roidb']
roidb = roidb_temp[()]
train_roidb = roidb['train_roidb']
test_roidb = roidb['test_roidb']
vg_imdb = imdb(N_obj)
net = vgg16()
roidb = train_roidb
valroidb = test_roidb[0:1000]
pretrained_model = 'output/vgg16/coco_2014_train+coco_2014_valminusminival/default/vgg16_faster_rcnn_iter_1190000.ckpt'
train_net(net, vg_imdb, roidb, valroidb, output_dir, tb_dir, pretrained_model=pretrained_model, max_iters=800000)
print('Output will be saved to `{:s}`'.format(output_dir))
# tensorboard directory where the summaries are saved during training
tb_dir = get_output_tb_dir(imdb, args.tag)
print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))
# also add the validation set, but with no flipping images
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
_, valroidb = combined_roidb(args.imdbval_name)
print('{:d} validation roidb entries'.format(len(valroidb)))
cfg.TRAIN.USE_FLIPPED = orgflip
# load network
if args.net == 'vgg16':
net = vgg16()
elif args.net == 'res50':
net = resnetv1(num_layers=50)
elif args.net == 'res101':
net = resnetv1(num_layers=101)
elif args.net == 'res152':
net = resnetv1(num_layers=152)
elif args.net == 'mobile':
net = mobilenetv1()
else:
raise NotImplementedError
train_net(net, imdb, roidb, valroidb, output_dir, tb_dir,
pretrained_model=args.weight,
max_iters=args.max_iters)
for i in range(args.model_iters):
if i == 0:
iters = len(roidb) * args.epochs
roidb_final = roidb
else:
iters = (len(roidb_im_pred) + len(roidb)) * args.epochs
roidb_final = roidb + roidb_im_pred
print("len(roidb): ", len(roidb))
print("len(roidb_im_pred): ", len(roidb_im_pred))
print("len(roidb_final): ", len(roidb_final))
total_step += iters
steps_for_stage.append((i, total_step))
with open(output_dir + '/steps_for_stage.txt', 'w') as fp:
fp.write("iter, step\n")
for i, s in steps_for_stage:
fp.write("%s, %s\n" % (i, s))
print("=== start training model {}, steps: {} ===".format(i, iters))
train_net(
net,
imdb,
roidb_final,
valroidb,
output_dir,
tb_dir,
pretrained_model=args.weight,
max_iters=total_step)
roidb_im_pred = mine(roidb_im, model_iters=i, tag=args.tag, net=net, mine_threshold=args.t_mine, classes=imdb._classes)
cluster=cluster_spec)):
# train set
imdb, roidb = combined_roidb(args.imdb_name)
print('{:d} roidb entries'.format(len(roidb)))
# output directory where the models are saved
output_dir = get_output_dir(imdb, args.tag)
print('Output will be saved to `{:s}`'.format(output_dir))
# tensorboard directory where the summaries are saved during training
tb_dir = get_output_tb_dir(imdb, args.tag)
print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))
# also add the validation set, but with no flipping images
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
_, valroidb = combined_roidb(args.imdbval_name)
print('{:d} validation roidb entries'.format(len(valroidb)))
cfg.TRAIN.USE_FLIPPED = orgflip
if args.net == 'vgg16':
net = vgg16(batch_size=cfg.TRAIN.IMS_PER_BATCH)
elif args.net == 'res101':
net = Resnet101(batch_size=cfg.TRAIN.IMS_PER_BATCH)
else:
raise NotImplementedError
train_net(net, imdb, roidb, valroidb, output_dir, tb_dir,
pretrained_model=args.weight,
max_iters=args.max_iters, master=server.target, is_chief=is_chief)
print('Called with args:')
print(args)
if args.cfg_file is not None:
cfg_from_file(args.cfg_file)
if args.set_cfgs is not None:
cfg_from_list(args.set_cfgs)
print('Using config:')
pprint.pprint(cfg)
np.random.seed(cfg.RNG_SEED)
# test set
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
imdb, roidb = combined_roidb(args.imdbtest_name)
print('{:d} test roidb entries'.format(len(roidb)))
cfg.TRAIN.USE_FLIPPED = orgflip
cfg.TRAIN.SNAPSHOT_PREFIX = ""
cfg.TRAIN.SNAPSHOT_LOAD_PREFIX = ""
net = resnetv1(imdb.nof_ent_classes, imdb.nof_rel_classes, num_layers=101)
train_net(net, imdb, [], roidb, "", "",
pretrained_model=args.model,
max_iters=1,
just_test=True)
def launch_train(self, conf):
'''
'''
args = {}
args['cfg_file'] = conf.frcnn_cfg
args['weight'] = conf.starting_weights
args['imdb_name'] = conf.train_set
args['imdbval_name'] = conf.valid_set
args['max_iters'] = conf.iters
args['tag'] = conf.frcnn_tag
args['net'] = conf.frcnn_net
args['set_cfgs'] = None
print('Called with args:')
print(args)
if args['cfg_file'] is not None:
cfg_from_file(args['cfg_file'])
if args['set_cfgs'] is not None:
cfg_from_list(args['set_cfgs'])
print('Using config:')
pprint.pprint(cfg)
np.random.seed(cfg.RNG_SEED)
# train set
imdb, roidb = combined_roidb(args['imdb_name'], conf)
print('{:d} roidb entries'.format(len(roidb)))
# output directory where the models are saved
output_dir = conf.backup_folder #get_output_dir(imdb, args.tag)
print('Output will be saved to `{:s}`'.format(output_dir))
# tensorboard directory where the summaries are saved during training
tb_dir = conf.backup_folder # get_output_tb_dir(imdb, args.tag)
print('TensorFlow summaries will be saved to `{:s}`'.format(tb_dir))
# also add the validation set, but with no flipping images
orgflip = cfg.TRAIN.USE_FLIPPED
cfg.TRAIN.USE_FLIPPED = False
_, valroidb = combined_roidb(args['imdbval_name'], conf)
print('{:d} validation roidb entries'.format(len(valroidb)))
cfg.TRAIN.USE_FLIPPED = orgflip
if args['net'] == 'vgg16':
net = vgg16(batch_size=cfg.TRAIN.IMS_PER_BATCH)
elif args['net'] == 'res50':
net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=50)
elif args['net'] == 'res101':
net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=101)
# load network
elif args['net'] == 'res152':
net = resnetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH, num_layers=152)
elif args['net'] == 'mobile':
net = mobilenetv1(batch_size=cfg.TRAIN.IMS_PER_BATCH)
else:
raise NotImplementedError
train_net(net,
imdb,
roidb,
valroidb,
output_dir,
tb_dir,
pretrained_model=args['weight'],
max_iters=args['max_iters'])
