def train_3d_pose_gan(args):
"""
Train a GAN for 3D poses, with arguments 'args'. This just makes some data loaders, using the 3D Pose dataset object
above. Really this just calls the train_loop function from utilz.train_utils.
:param args: Arguments from an ArgParser specifying how to run the trianing
"""
actions = misc.define_actions(args.action)
train_dataset = Human36M3DPoseDataset(actions=actions,
data_path=args.data_dir,
is_train=True)
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.train_batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_dataset = Human36M3DPoseDataset(actions=actions,
data_path=args.data_dir,
is_train=False)
val_loader = DataLoader(dataset=val_dataset,
batch_size=args.test_batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
model = FullyConnectedGan(clip_max=args.clip_max).cuda()
train_loop(model, train_loader, val_loader, _make_optimizer_fn, _load_fn,
_checkpoint_fn, _update_op, _validation_loss, args)
def main(hps):
tf.set_random_seed(hps.seed)
np.random.seed(hps.seed)
x_train, y_train, x_valid, y_valid, x_test, y_test, S = load_data(hps)
M = Model(hps, S)
"""
data_path = os.path.join(hps.data_dir, hps.dataset + '.data')
data_func = dataset.data_dict()[hps.dataset]
x_train, y_train, x_valid, y_valid, x_test, y_test = data_func(data_path)
if not hps.use_valid:
x_train = np.vstack([x_train, x_valid])
y_train = np.hstack([y_train, y_valid])
x_valid = y_valid = None
n_train, x_dim = x_train.shape
x_train, y_train, x_valid, y_valid, x_test, y_test, S = dataset.\
standardize_new(x_train, y_train, x_valid, y_valid, x_test, y_test, True)
"""
# Run the inference
sms = 60 if hps.save_model else 0
with wrapped_supervisor.create_sv(hps,
save_model_secs=sms,
save_summaries_secs=0,
global_step=M.global_step) as sv:
sess = sv.sess_
train_loop(sess=sess,
hps=hps,
x_train=x_train,
y_train=y_train,
x_test=x_test,
y_test=y_test,
infer_op=M.infer_op,
rmse_op=M.rmse,
loglh_op=M.log_likelihood,
ystd_op=M.ystd_avg,
x_ph=M.x,
y_ph=M.y,
x_extra_ph=None,
x_valid=x_valid,
y_valid=y_valid)
if len(hps.dump_pred_dir) > 0:
pred_out = sv.sess_.run([M.var_bn['y_mean'], M.ystd_avg], {
M.x: x_test,
M.y: y_test
})
pred_out[0] = pred_out[0] * S.std_y_train + S.mean_y_train
pred_out[1] *= S.std_y_train
f = lambda a, b: [
a * S.std_x_train + S.mean_x_train, b * S.std_y_train + S.
mean_y_train
]
todump = pred_out + f(x_test, y_test) + f(x_train, y_train)
with open(hps.dump_pred_dir, 'wb') as fout:
pickle.dump(todump, fout)
def train_stitched_fine_tune(args):
"""
Fine tune the stitched network, with arguments 'args'. This just makes some data loaders, using the 3D Pose dataset object
above. Really this just calls the train_loop function from utilz.train_utils.
We assume that the stacked hourglass network was pre-trained on MPII dataset, so we use the color normalization
from this dataset.
:param args: Arguments from an ArgParser specifying how to run the trianing
"""
# Unpack options
hg_model_file = args.load_hourglass
threed_baseline_model_file = args.load_2d3d
data_input_dir = args.data_dir
data_output_dir = args.output_dir
dataset_normalization = args.dataset_normalization
# Create the model, and load the pre-trained subnetworks (loading is more complex, because the stitched
# network needs to be initialized with the correct color means etc). So just re-use from the run.py
# TODO: refactor this, so it's somewhere else, and used in both train and run. HAve a stitched/utils.py probs
model, _ = load_model_and_dataset_mpii(hg_model_file,
threed_baseline_model_file,
data_input_dir, args)
model.train()
# Make data loaders, correcting the color norm and std (as the hourglass was pre-trained on MPII)
train_dataset = Human36mDataset(
dataset_path=data_input_dir,
is_train=True,
dataset_normalization=dataset_normalization,
load_image_data=True)
train_dataset.set_color_mean(model.hg_mean)
train_dataset.set_color_std(model.hg_std)
train_loader = DataLoader(dataset=train_dataset,
batch_size=args.train_batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_dataset = Human36mDataset(dataset_path=data_input_dir,
is_train=False,
dataset_normalization=dataset_normalization,
load_image_data=True)
val_dataset.set_color_mean(model.hg_mean)
val_dataset.set_color_std(model.hg_std)
val_loader = DataLoader(dataset=val_dataset,
batch_size=args.test_batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
# Run the training loop
train_loop(model, train_loader, val_loader, _make_optimizer_fn, _load_fn,
_checkpoint_fn, _update_op, _validation_loss, args)
optimizer_ft = optim.Adam(model.parameters(), lr=10e-4)
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
if __name__ == '__main__':
# Running the model
print('Starting Training with params: \nN_EPOCH = {},'
' \nBATCH_SIZE = {},'
' \nTRAIN_ON_GPU = {},'
' \nSAVE_FILE_NAME = {},'
' \nPATH_TO_SAVE = {}'.format(N_EPOCH, BATCH_SIZE, train_on_gpu,
SAVE_FILE_NAME, PATH))
model, history = train_loop(model,
criterion,
optimizer_ft,
train_loader,
valid_loader,
train_on_gpu,
save_file_name=SAVE_FILE_NAME,
max_epochs_stop=3,
n_epochs=N_EPOCH,
print_every=1,
path_to_save=PATH)
print("Starting check result on test data")
# Check test result
check_result(model, test_loader, criterion, train_on_gpu)
def train(config):
# print('config settings:')
# for key in config.keys():
# print(key,confi)
print('Training a {} model.'.format(config.model))
# set device
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('Device: {}'.format(device))
# When running on the CuDNN backend two further options must be set for reproducibility
if device == 'cuda':
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# load data
print('Loading data...')
print('Creating subtrees!' if config.create_subtrees else '')
train_data, dev_data, test_data = utils.get_train_test_dev(config.data_dir, create_subtrees=config.create_subtrees)
####
# load vocabulary and embedding
if config.use_pt_embed:
print('Loading pretrained embedding...')
v_pt, vectors = utils.get_pretrained_voc_vec(config.embed_path)
else:
print('Loading vocabulary from training data')
v = utils.create_voc_from_train(train_data)
# set sentiment dict
i2t = ["very negative", "negative", "neutral", "positive", "very positive"]
t2i = OrderedDict({p : i for p, i in zip(i2t, range(len(i2t)))})
if config.model == 'LSTM':
if not config.use_pt_embed:
raise NotImplementedError
lstm_kwargs = {
'v_pt':v_pt, 'embed_vectors':vectors, 'embed_dim':vectors.shape[1], 'hidden_dim':config.hidden_dim, 'output_dim':len(t2i)
}
losses, accuracies, best_accs = utils.train_loop(utils.generate_lstm,
lambda model: optim.Adam(model.parameters(), lr=config.learning_rate),
train_data, dev_data, test_data,
model_generator_kwargs = lstm_kwargs,
num_iterations=config.num_iterations, print_every=config.print_every, eval_every=config.eval_every, patience=config.patience,
batch_size=config.batch_size,
batch_fn=utils.get_minibatch,
prep_fn=utils.prepare_minibatch,
eval_fn=utils.evaluate,
permute=config.permute,
add_suffix=config.suffix,
result_dir=config.result_dir)
elif config.model == 'TreeLSTM':
if config.permute or not config.use_pt_embed:
raise NotImplementedError('Permute not implemented for TreeLSTM')
lstm_kwargs ={
'v_pt':v_pt, 'embed_vectors':vectors, 'embed_dim':vectors.shape[1], 'hidden_dim':config.hidden_dim, 'output_dim':len(t2i),
'childsum':config.childsum
}
losses, accuracies, best_accs = utils.train_loop(utils.generate_treelstm,
lambda model: optim.Adam(model.parameters(), lr=config.learning_rate),
train_data, dev_data, test_data,
model_generator_kwargs = lstm_kwargs,
num_iterations=config.num_iterations, print_every=config.print_every, eval_every=config.eval_every, patience=config.patience,
batch_size=config.batch_size,
batch_fn=utils.get_minibatch,
prep_fn=utils.prepare_treelstm_minibatch,
eval_fn=utils.evaluate,
add_suffix=config.suffix,
result_dir=config.result_dir)
elif config.model == 'BOW':
if config.use_pt_embed or config.permute:
raise NotImplementedError
bow_kwargs = {'vocab_size':len(v.w2i), 'embedding_dim':len(t2i), 'vocab':v}
losses, accuracies, best_accs = utils.train_loop(lambda **kwargs: models.BOW(**kwargs).to(device),
lambda model: optim.Adam(model.parameters(), lr=config.learning_rate),
train_data, dev_data, test_data,
model_generator_kwargs = bow_kwargs,
num_iterations=config.num_iterations, print_every=config.print_every,
eval_every=config.eval_every, patience=config.patience,
result_dir=config.result_dir)
elif config.model == 'CBOW':
if config.use_pt_embed or config.permute:
raise NotImplementedError
bow_kwargs = {'vocab_size':len(v.w2i), 'embedding_dim':config.embed_dim, 'vocab':v, 'num_classes':len(t2i)}
losses, accuracies, best_accs = utils.train_loop(lambda **kwargs: models.CBOW(**kwargs).to(device),
lambda model: optim.Adam(model.parameters(), lr=config.learning_rate),
train_data, dev_data, test_data,
model_generator_kwargs = bow_kwargs,
num_iterations=config.num_iterations, print_every=config.print_every,
eval_every=config.eval_every, patience=config.patience,
result_dir=config.result_dir)
elif config.model == 'DeepCBOW':
if config.use_pt_embed or config.permute:
raise NotImplementedError
bow_kwargs = {'vocab_size':len(v.w2i), 'embedding_dim':config.embed_dim, 'vocab':v, 'output_dim':len(t2i), 'num_hidden':config.hidden_dim}
losses, accuracies, best_accs = utils.train_loop(lambda **kwargs: models.DeepCBOW(**kwargs).to(device),
lambda model: optim.Adam(model.parameters(), lr=config.learning_rate),
train_data, dev_data, test_data,
model_generator_kwargs = bow_kwargs,
num_iterations=config.num_iterations, print_every=config.print_every,
eval_every=config.eval_every, patience=config.patience,
result_dir=config.result_dir)
elif config.model == 'PTDeepCBOW':
if config.permute or not config.use_pt_embed:
raise NotImplementedError
bow_kwargs = {
'v_pt':v_pt, 'embed_vectors':vectors, 'embed_dim':vectors.shape[1],
'num_hidden':config.hidden_dim, 'num_classes':len(t2i)
}
losses, accuracies, best_accs = utils.train_loop(utils.generate_pt_deep_cbow,
lambda model: optim.Adam(model.parameters(), lr=config.learning_rate),
train_data, dev_data, test_data,
model_generator_kwargs = bow_kwargs,
num_iterations=config.num_iterations, print_every=config.print_every,
eval_every=config.eval_every, patience=config.patience,
result_dir=config.result_dir)
# save data
np.savez_compressed(file=config.result_dir+'results.npz', losses=losses, accs=accuracies, best_accs=best_accs)
with open(config.result_dir + 'config.json', "w") as write_file:
json.dump(config.__dict__, write_file)
# plot results
utils.plot_results(losses, accuracies, filename=config.plot_dir+'plot.pdf', eval_every=config.eval_every,
print_every=config.print_every)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# Save configs
model_desc_output = [
": ".join([str(k), str(v)]) for k, v in vars(args).items()
]
with open(os.path.join(save_dir, 'configs.txt'), 'w') as file:
file.writelines("\n".join(model_desc_output))
if not TEST:
writer = SummaryWriter(f"runs/{TASK}/{MODEL_TYPE}-{time_stamp}")
# Train network
net, logs = train_loop(
net, optimizer, criterion, train_data, valid_data, save_dir=save_dir, task=TASK, n_epochs=N_EPOCHS, \
batch_size=BATCH_SIZE, verbose=True, scheduler=scheduler, save_every=args.save_every, \
eval_every=args.eval_every, writer=writer,
)
else:
# Test
logs = {k: [] for k in ['test_loss', 'test_metric']}
if TASK == 'mimic-diagnosis':
logs['test_top_5_recall'] = []
logs['test_top_30_recall'] = []
test_loader = DataLoader(range(len(test)),
shuffle=False,
batch_size=BATCH_SIZE)
if TASK == 'mimic-mortality':
test_preds = predict(net, test_loader, test_data,
verbose=True).cpu().numpy()
np.save(os.path.join(save_dir, 'test_preds.npy'), test_preds)
# Create save directory
time_stamp = time.strftime("%m-%d-%H-%M", time.localtime())
save_dir = os.path.join(EXPERIMENT_DIR, time_stamp)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# Save configs
model_desc_output = [
": ".join([str(k), str(v)]) for k, v in vars(args).items()
]
with open(os.path.join(save_dir, 'configs.txt'), 'w') as file:
file.writelines("\n".join(model_desc_output))
# Set up tensorboard
writer = SummaryWriter(f"runs/{MODEL_TYPE}-{time_stamp}")
# Train network
net, logs = train_loop(
net, MODEL_TYPE, optimizer, criterion, train_data, valid_data, save_dir=save_dir, n_epochs=N_EPOCHS, \
batch_size=BATCH_SIZE, verbose=True, scheduler=scheduler, save_every=args.save_every, \
eval_every=args.eval_every, clip=args.clip, writer=writer, accum_num=args.accum_num,
)
# Save Model
torch.save(net, os.path.join(save_dir, 'model.pt'))
# Save Logs
log_df = pd.DataFrame(logs)
log_df.to_csv(os.path.join(save_dir, 'logs.csv'))