def batch_test_train_1job_seq2seq(input_args):
#pdb.set_trace()
param_dic, param_desc, args = input_args #packed to input_args for parallel purpose
logLabel = '[train_1job_seq2seq_%s] ' % param_desc
logPrint(logLabel + 'Start')
root_dir = param_dic['root_dir']
data_label = param_dic['data_label']
model_label = param_dic['model_label']
data_dir = '%s/%s/train/data/' % (root_dir, data_label)
run_cmd('mkdir -p %s' % data_dir)
logPrint('%s created' % data_dir)
model_dir = '%s/%s/train/%s/%s/' % (root_dir, data_label, model_label,
param_desc)
run_cmd('mkdir -p %s' % model_dir)
logPrint('%s created' % model_dir)
dst_config = '%s/%s/train/%s/config.txt' % (root_dir, data_label,
model_label)
if os.path.exists(dst_config) == False:
run_cmd('cp %s %s' % (args.config_file, dst_config))
#sample_fa = '%s/sample.fa'%data_dir
#sample_dist = '%s/sample.dist'%data_dir
seq2seq_pair = '%s/seq2seq_cgk.txt' % data_dir
#to be passed to train_seq2seq function
param_dic['seq2seq_pair_path'] = seq2seq_pair
param_dic['model_dir_path'] = model_dir
if param_dic['n_clusters_validation'] > 0:
#pdb.set_trace()
param_dic[
'seq2seq_pair_path_validation'] = '%s/seq2seq_cgk_validation.txt' % data_dir
train_seq2seq(param_dic)
logPrint(logLabel + 'End')
return
model = Seq2Seq(encoder=encoder, decoder=decoder, device=device).to(device)
# Run the model parallelly
if torch.cuda.device_count() > 1:
logger.info("Using {} GPUs".format(torch.cuda.device_count()))
model = nn.DataParallel(model)
# Create loss criterion & optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
# Start training
logger.info("Training Started".center(60, '#'))
for epoch in range(epochs):
# Train the model
train_seq2seq(model, criterion, optimizer, clip, train_loader, device,
epoch, logger, log_interval, writer)
# Validate the model
val_seq2seq(model, criterion, val_loader, device, epoch, logger,
writer)
# Save model
torch.save(
model.state_dict(),
os.path.join(model_path,
"slr_seq2seq_epoch{:03d}.pth".format(epoch + 1)))
logger.info("Epoch {} Model Saved".format(epoch + 1).center(60, '#'))
logger.info("Training Finished".center(60, '#'))
yvocab_size=yvocab_size,
emb_dim=emb_dim,
num_layers=FLAGS.num_layers,
learning_rate=FLAGS.learning_rate,
use_lstm=FLAGS.use_lstm)
eval_batch_gen = data_utils.rand_batch_gen(test_q, test_a, batch_size)
train_batch_gen = data_utils.rand_batch_gen(train_q, train_a, batch_size)
# create session for training
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.memory_usage_percentage / 100)
session_conf = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess = tf.Session(config=session_conf)
# init all variables
# In[9]:
# sess = model.restore_last_session()
sess = train_seq2seq(model,
train_batch_gen,
eval_batch_gen,
sess,
int(FLAGS.num_epochs * train_q.shape[0] / batch_size),
FLAGS.checkpoint_every,
FLAGS.evaluate_every,
logdir=logdir,
vocab=vocab,
beam_length=FLAGS.beam_length)
#sess.close()