def main():
opt = TrainOptions().parse()
train_history = TrainHistoryFace()
checkpoint = Checkpoint()
visualizer = Visualizer(opt)
exp_dir = os.path.join(opt.exp_dir, opt.exp_id)
log_name = opt.vis_env + 'log.txt'
visualizer.log_name = os.path.join(exp_dir, log_name)
num_classes = opt.class_num
if not opt.slurm:
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_id
layer_num = opt.layer_num
order = opt.order
net = create_cu_net(neck_size= 4, growth_rate= 32, init_chan_num= 128,
class_num= num_classes, layer_num= layer_num, order= order,
loss_num= layer_num, use_spatial_transformer= opt.stn,
mlp_tot_layers= opt.mlp_tot_layers, mlp_hidden_units= opt.mlp_hidden_units,
get_mean_from_mlp= opt.get_mean_from_mlp)
# Load the pre-trained model
saved_wt_file = opt.saved_wt_file
if saved_wt_file == "":
print("=> Training from scratch")
else:
print("=> Loading weights from " + saved_wt_file)
checkpoint_t = torch.load(saved_wt_file)
state_dict = checkpoint_t['state_dict']
tt_names=[]
for names in net.state_dict():
tt_names.append(names)
for name, param in state_dict.items():
name = name[7:]
if name not in net.state_dict():
print("=> not load weights '{}'".format(name))
continue
if isinstance(param, Parameter):
param = param.data
if (net.state_dict()[name].shape[0] == param.shape[0]):
net.state_dict()[name].copy_(param)
else:
print("First dim different. Not loading weights {}".format(name))
if (opt.freeze):
print("\n\t\tFreezing basenet parameters\n")
for param in net.parameters():
param.requires_grad = False
"""
for i in range(layer_num):
net.choleskys[i].fc_1.bias.requires_grad = True
net.choleskys[i].fc_1.weight.requires_grad = True
net.choleskys[i].fc_2.bias.requires_grad = True
net.choleskys[i].fc_2.weight.requires_grad = True
net.choleskys[i].fc_3.bias.requires_grad = True
net.choleskys[i].fc_3.weight.requires_grad = True
"""
net.cholesky.fc_1.bias.requires_grad = True
net.cholesky.fc_1.weight.requires_grad = True
net.cholesky.fc_2.bias.requires_grad = True
net.cholesky.fc_2.weight.requires_grad = True
net.cholesky.fc_3.bias.requires_grad = True
net.cholesky.fc_3.weight.requires_grad = True
else:
print("\n\t\tNot freezing anything. Tuning every parameter\n")
for param in net.parameters():
param.requires_grad = True
net = torch.nn.DataParallel(net).cuda() # use multiple GPUs
# Optimizer
if opt.optimizer == "rmsprop":
optimizer = torch.optim.RMSprop(filter(lambda p: p.requires_grad, net.parameters()), lr=opt.lr, alpha=0.99,
eps=1e-8, momentum=0, weight_decay=0)
elif opt.optimizer == "adam":
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=opt.lr)
else:
print("Unknown Optimizer. Aborting!!!")
sys.exit(0)
print type(optimizer)
# Optionally resume from a checkpoint
if opt.resume_prefix != '':
# if 'pth' in opt.resume_prefix:
# trunc_index = opt.resume_prefix.index('pth')
# opt.resume_prefix = opt.resume_prefix[0:trunc_index - 1]
checkpoint.save_prefix = os.path.join(exp_dir, opt.resume_prefix)
checkpoint.load_prefix = os.path.join(exp_dir, opt.resume_prefix)[0:-1]
checkpoint.load_checkpoint(net, optimizer, train_history)
else:
checkpoint.save_prefix = exp_dir + '/'
print("Save prefix = {}".format(checkpoint.save_prefix))
# Load data
json_path = opt.json_path
train_json = opt.train_json
val_json = opt.val_json
print("Path added to each image path in JSON = {}".format(json_path))
print("Train JSON path = {}".format(train_json))
print("Val JSON path = {}".format(val_json))
if opt.bulat_aug:
# Use Bulat et al Augmentation Scheme
train_loader = torch.utils.data.DataLoader(
FACE(train_json, json_path, is_train= True, scale_factor= 0.2, rot_factor= 50, use_occlusion= True, keep_pts_inside= True),
batch_size=opt.bs, shuffle= True,
num_workers=opt.nThreads, pin_memory= True)
else:
train_loader = torch.utils.data.DataLoader(
FACE(train_json, json_path, is_train= True, keep_pts_inside= True),
batch_size=opt.bs, shuffle= True,
num_workers=opt.nThreads, pin_memory= True)
val_loader = torch.utils.data.DataLoader(
FACE(val_json, json_path, is_train=False),
batch_size=opt.bs, shuffle=False,
num_workers=opt.nThreads, pin_memory=True)
logger = Logger(os.path.join(opt.exp_dir, opt.exp_id, opt.resume_prefix+'face-training-log.txt'),
title='face-training-summary')
logger.set_names(['Epoch', 'LR', 'Train Loss', 'Val Loss', 'Train RMSE', 'Val RMSE', 'Train RMSE Box', 'Val RMSE Box', 'Train RMSE Meta', 'Val RMSE Meta'])
if not opt.is_train:
visualizer.log_path = os.path.join(opt.exp_dir, opt.exp_id, 'val_log.txt')
val_loss, val_rmse, predictions = validate(val_loader, net,
train_history.epoch[-1]['epoch'], visualizer, num_classes, flip_index)
checkpoint.save_preds(predictions)
return
global weights_HG
weights_HG = [float(x) for x in opt.hg_wt.split(",")]
if opt.is_covariance:
print("Covariance used from the heatmap")
else:
print("Covariance calculated from MLP")
if opt.stn:
print("Using spatial transformer on heatmaps")
print ("Postprocessing applied = {}".format(opt.pp))
if (opt.smax):
print("Scaled softmax used with tau = {}".format(opt.tau))
else:
print("No softmax used")
print("Individual Hourglass loss weights")
print(weights_HG)
print("wt_MSE (tradeoff between GLL and MSE in each hourglass)= " + str(opt.wt_mse))
print("wt_gauss_regln (tradeoff between GLL and Gaussian Regularisation in each hourglass)= " + str(opt.wt_gauss_regln))
if opt.bulat_aug:
print("Using Bulat et al, ICCV 2017 Augmentation Scheme")
print("Using Learning Policy {}".format(opt.lr_policy))
chosen_lr_policy = dict_of_functions[opt.lr_policy]
# Optionally resume from a checkpoint
start_epoch = 0
if opt.resume_prefix != '':
start_epoch = train_history.epoch[-1]['epoch'] + 1
# Training and validation
start_epoch = 0
if opt.resume_prefix != '':
start_epoch = train_history.epoch[-1]['epoch'] + 1
train_loss_orig_epoch = []
train_loss_gau_t1_epoch = []
train_loss_gau_t2_epoch = []
train_nme_orig_epoch = []
train_nme_gau_epoch = []
train_nme_new_epoch = []
val_loss_orig_epoch = []
val_loss_gau_t1_epoch = []
val_loss_gau_t2_epoch = []
val_nme_orig_epoch = []
val_nme_gau_epoch = []
val_nme_new_epoch = []
for epoch in range(start_epoch, opt.nEpochs):
chosen_lr_policy(opt, optimizer, epoch)
# Train for one epoch
train_loss, train_loss_mse,train_loss_gau_t1, train_loss_gau_t2,train_rmse_orig, train_rmse_gau, train_rmse_new_gd_box, train_rmse_new_meta_box = train(train_loader, net, optimizer, epoch, visualizer, opt)
#train_loss_gau_epoch.append(train_loss_gau)
train_loss_gau_t1_epoch.append(train_loss_gau_t1)
train_loss_gau_t2_epoch.append(train_loss_gau_t2)
train_nme_orig_epoch.append(train_rmse_orig)
train_nme_gau_epoch.append(train_rmse_gau)
train_loss_orig_epoch.append(train_loss_mse)
# Evaluate on validation set
val_loss, val_loss_mse, val_loss_gau_t1, val_loss_gau_t2 , val_rmse_orig, val_rmse_gau, val_rmse_new_gd_box, val_rmse_new_meta_box, predictions= validate(val_loader, net, epoch, visualizer, opt, num_classes, flip_index)
val_loss_orig_epoch.append(val_loss_mse)
val_loss_gau_t1_epoch.append(val_loss_gau_t1)
val_loss_gau_t2_epoch.append(val_loss_gau_t2)
val_nme_orig_epoch.append(val_rmse_orig)
val_nme_gau_epoch.append(val_rmse_gau)
# Update training history
e = OrderedDict( [('epoch', epoch)] )
lr = OrderedDict( [('lr', optimizer.param_groups[0]['lr'])] )
loss = OrderedDict( [('train_loss', train_loss),('val_loss', val_loss)] )
rmse = OrderedDict( [('val_rmse', val_rmse_gau)] )
train_history.update(e, lr, loss, rmse)
checkpoint.save_checkpoint(net, optimizer, train_history, predictions)
visualizer.plot_train_history_face(train_history)
logger.append([epoch, optimizer.param_groups[0]['lr'], train_loss, val_loss, train_rmse_gau, val_rmse_gau, train_rmse_new_gd_box, val_rmse_new_gd_box, train_rmse_new_meta_box, val_rmse_new_meta_box])
logger.close()
def main():
opt = TrainOptions().parse()
train_history = TrainHistory()
checkpoint = Checkpoint()
visualizer = Visualizer(opt)
exp_dir = os.path.join(opt.exp_dir, opt.exp_id)
log_name = opt.vis_env + 'log.txt'
visualizer.log_name = os.path.join(exp_dir, log_name)
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_id
# if opt.dataset == 'mpii':
num_classes = 16
# layer_num = 2
net = create_cu_net(neck_size=4,
growth_rate=32,
init_chan_num=128,
num_classes=num_classes,
layer_num=opt.layer_num,
max_link=1,
inter_loss_num=opt.layer_num)
# num1 = get_n_params(net)
# num2 = get_n_trainable_params(net)
# num3 = get_n_conv_params(net)
# print 'number of params: ', num1
# print 'number of trainalbe params: ', num2
# print 'number of conv params: ', num3
# torch.save(net.state_dict(), 'test-model-size.pth.tar')
# exit()
# device = torch.device("cuda:0")
# net = net.to(device)
net = torch.nn.DataParallel(net).cuda()
global quan_op
quan_op = QuanOp(net)
optimizer = torch.optim.RMSprop(net.parameters(),
lr=opt.lr,
alpha=0.99,
eps=1e-8,
momentum=0,
weight_decay=0)
"""optionally resume from a checkpoint"""
if opt.resume_prefix != '':
# if 'pth' in opt.resume_prefix:
# trunc_index = opt.resume_prefix.index('pth')
# opt.resume_prefix = opt.resume_prefix[0:trunc_index - 1]
# checkpoint.save_prefix = os.path.join(exp_dir, opt.resume_prefix)
checkpoint.save_prefix = exp_dir + '/'
checkpoint.load_prefix = os.path.join(exp_dir, opt.resume_prefix)[0:-1]
checkpoint.load_checkpoint(net, optimizer, train_history)
opt.lr = optimizer.param_groups[0]['lr']
resume_log = True
else:
checkpoint.save_prefix = exp_dir + '/'
resume_log = False
print 'save prefix: ', checkpoint.save_prefix
# model = {'state_dict': net.state_dict()}
# save_path = checkpoint.save_prefix + 'test-model-size.pth.tar'
# torch.save(model, save_path)
# exit()
"""load data"""
train_loader = torch.utils.data.DataLoader(MPII(
'dataset/mpii-hr-lsp-normalizer.json',
'/bigdata1/zt53/data',
is_train=True),
batch_size=opt.bs,
shuffle=True,
num_workers=opt.nThreads,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(MPII(
'dataset/mpii-hr-lsp-normalizer.json',
'/bigdata1/zt53/data',
is_train=False),
batch_size=opt.bs,
shuffle=False,
num_workers=opt.nThreads,
pin_memory=True)
"""optimizer"""
# optimizer = torch.optim.SGD( net.parameters(), lr=opt.lr,
# momentum=opt.momentum,
# weight_decay=opt.weight_decay )
# optimizer = torch.optim.RMSprop(net.parameters(), lr=opt.lr, alpha=0.99,
# eps=1e-8, momentum=0, weight_decay=0)
print type(optimizer)
# idx = range(0, 16)
# idx = [e for e in idx if e not in (6, 7, 8, 9, 12, 13)]
idx = [0, 1, 2, 3, 4, 5, 10, 11, 14, 15]
logger = Logger(os.path.join(opt.exp_dir, opt.exp_id,
'training-summary.txt'),
title='training-summary',
resume=resume_log)
logger.set_names(
['Epoch', 'LR', 'Train Loss', 'Val Loss', 'Train Acc', 'Val Acc'])
if not opt.is_train:
visualizer.log_path = os.path.join(opt.exp_dir, opt.exp_id,
'val_log.txt')
val_loss, val_pckh, predictions = validate(
val_loader, net, train_history.epoch[-1]['epoch'], visualizer, idx,
joint_flip_index, num_classes)
checkpoint.save_preds(predictions)
return
"""training and validation"""
start_epoch = 0
if opt.resume_prefix != '':
start_epoch = train_history.epoch[-1]['epoch'] + 1
for epoch in range(start_epoch, opt.nEpochs):
adjust_lr(opt, optimizer, epoch)
# # train for one epoch
train_loss, train_pckh = train(train_loader, net, optimizer, epoch,
visualizer, idx, opt)
# evaluate on validation set
val_loss, val_pckh, predictions = validate(val_loader, net, epoch,
visualizer, idx,
joint_flip_index,
num_classes)
# visualizer.display_imgpts(imgs, pred_pts, 4)
# exit()
# update training history
e = OrderedDict([('epoch', epoch)])
lr = OrderedDict([('lr', optimizer.param_groups[0]['lr'])])
loss = OrderedDict([('train_loss', train_loss),
('val_loss', val_loss)])
pckh = OrderedDict([('val_pckh', val_pckh)])
train_history.update(e, lr, loss, pckh)
checkpoint.save_checkpoint(net, optimizer, train_history, predictions)
# visualizer.plot_train_history(train_history)
logger.append([
epoch, optimizer.param_groups[0]['lr'], train_loss, val_loss,
train_pckh, val_pckh
])
logger.close()
def main():
opt = TrainOptions().parse()
train_history = TrainHistoryFace()
checkpoint = Checkpoint()
visualizer = Visualizer(opt)
exp_dir = os.path.join(opt.exp_dir, opt.exp_id)
log_name = opt.vis_env + '_val_log.txt'
visualizer.log_name = os.path.join(exp_dir, log_name)
num_classes = opt.class_num
if not opt.slurm:
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpu_id
layer_num = opt.layer_num
order = opt.order
net = create_cu_net(neck_size=4,
growth_rate=32,
init_chan_num=128,
class_num=num_classes,
layer_num=layer_num,
order=order,
loss_num=layer_num,
use_spatial_transformer=opt.stn,
mlp_tot_layers=opt.mlp_tot_layers,
mlp_hidden_units=opt.mlp_hidden_units,
get_mean_from_mlp=opt.get_mean_from_mlp)
# Load the pre-trained model
saved_wt_file = opt.saved_wt_file
print("Loading weights from " + saved_wt_file)
checkpoint_t = torch.load(saved_wt_file)
state_dict = checkpoint_t['state_dict']
for name, param in state_dict.items():
name = name[7:]
if name not in net.state_dict():
print("=> not load weights '{}'".format(name))
continue
if isinstance(param, Parameter):
param = param.data
net.state_dict()[name].copy_(param)
net = torch.nn.DataParallel(net).cuda() # use multiple GPUs
# Optimizer
if opt.optimizer == "rmsprop":
optimizer = torch.optim.RMSprop(filter(lambda p: p.requires_grad,
net.parameters()),
lr=opt.lr,
alpha=0.99,
eps=1e-8,
momentum=0,
weight_decay=0)
elif opt.optimizer == "adam":
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
net.parameters()),
lr=opt.lr)
else:
print("Unknown Optimizer. Aborting!!!")
sys.exit(0)
print(type(optimizer))
# Optionally resume from a checkpoint
if opt.resume_prefix != '':
# if 'pth' in opt.resume_prefix:
# trunc_index = opt.resume_prefix.index('pth')
# opt.resume_prefix = opt.resume_prefix[0:trunc_index - 1]
checkpoint.save_prefix = os.path.join(exp_dir, opt.resume_prefix)
checkpoint.load_prefix = os.path.join(exp_dir, opt.resume_prefix)[0:-1]
checkpoint.load_checkpoint(net, optimizer, train_history)
else:
checkpoint.save_prefix = exp_dir + '/'
print("Save prefix = {}".format(
checkpoint.save_prefix))
# Load data
json_path = opt.json_path
train_json = opt.train_json
val_json = opt.val_json
print("Path added to each image path in JSON = {}".format(json_path))
print("Train JSON path = {}".format(train_json))
print("Val JSON path = {}".format(val_json))
# This train loader is useless
train_loader = torch.utils.data.DataLoader(FACE(train_json,
json_path,
is_train=True),
batch_size=opt.bs,
shuffle=True,
num_workers=opt.nThreads,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(FACE(val_json,
json_path,
is_train=False),
batch_size=opt.bs,
shuffle=False,
num_workers=opt.nThreads,
pin_memory=True)
if not opt.is_train:
visualizer.log_path = os.path.join(opt.exp_dir, opt.exp_id,
'val_log.txt')
val_loss, val_rmse, predictions = validate(
val_loader, net, train_history.epoch[-1]['epoch'], visualizer,
num_classes, flip_index)
checkpoint.save_preds(predictions)
return
global f_path
global weights_HG
f_path = exp_dir
weights_HG = [float(x) for x in opt.hg_wt.split(",")]
print("Postprocessing applied = {}".format(opt.pp))
if (opt.smax):
print("Scaled softmax used with tau = {}".format(opt.tau))
else:
print("No softmax used")
if opt.is_covariance:
print("Covariance used from the heatmap")
else:
print("Covariance calculated from MLP")
print("Individual Hourglass loss weights")
print(weights_HG)
print("wt_MSE (tradeoff between GLL and MSE in each hourglass)= " +
str(opt.wt_mse))
print(
"wt_gauss_regln (tradeoff between GLL and Gaussian Regularisation in each hourglass)= "
+ str(opt.wt_gauss_regln))
# Optionally resume from a checkpoint
start_epoch = 0
if opt.resume_prefix != '':
start_epoch = train_history.epoch[-1]['epoch'] + 1
# Training and validation
start_epoch = 0
if opt.resume_prefix != '':
start_epoch = train_history.epoch[-1]['epoch'] + 1
train_loss_orig_epoch = []
train_loss_gau_t1_epoch = []
train_loss_gau_t2_epoch = []
train_nme_orig_epoch = []
train_nme_gau_epoch = []
train_nme_new_epoch = []
val_loss_orig_epoch = []
val_loss_gau_t1_epoch = []
val_loss_gau_t2_epoch = []
val_nme_orig_epoch = []
val_nme_gau_epoch = []
val_nme_new_epoch = []
for epoch in range(1):
# Evaluate on validation set
val_loss, val_loss_mse, val_loss_gau_t1, val_loss_gau_t2, val_rmse_orig, val_rmse_gau, val_rmse_new_box, predictions = validate(
val_loader, net, epoch, visualizer, opt, num_classes, flip_index)
val_loss_orig_epoch.append(val_loss_mse)
val_loss_gau_t1_epoch.append(val_loss_gau_t1)
val_loss_gau_t2_epoch.append(val_loss_gau_t2)
val_nme_orig_epoch.append(val_rmse_orig)
val_nme_gau_epoch.append(val_rmse_gau)
