def main(_):
config_ = tf.ConfigProto()
config_.gpu_options.allow_growth = True
config_.allow_soft_placement = True
sess = tf.Session(config=config_)
current_model = None
domains = [
'basketball', 'blocks', 'calendar', 'housing', 'publications',
'recipes', 'restaurants', 'socialnetwork'
]
checkpoints = [
os.path.basename(f) for f in os.listdir(FLAGS.data_dir)
if f.startswith('parse') and f.endswith('.index')
]
accuracies = {}
for domain in domains:
accuracies[domain] = {}
data_dir = os.path.join(FLAGS.data_dir, 'all_domain_pruned', domain)
from_test_data = os.path.join(data_dir, "test_q.txt")
to_test_data = os.path.join(data_dir, "test_f.txt")
data_utils.splitToFrom(data_dir, domain + '_test.tsv.prune',
"test") # split to-from data
for ckpt in checkpoints:
logging.info("computing test accuracy for %s at checkpoint %s" %
(domain, ckpt))
test_acc, current_model = test_accuracy(from_test_data,
to_test_data, sess, ckpt,
current_model)
accuracies[domain][ckpt] = test_acc
logging.info(
"[test accuracy][domain %s][ckpt %s][accuracy = %1.3f]" %
(domain, ckpt, test_acc))
with open(
os.path.join(FLAGS.train_dir, 'test_accuracies_multidomain.json'),
'w') as f:
json.dump(accuracies, f, indent=1)
def main(_):
if FLAGS.self_test:
self_test()
elif FLAGS.decode:
decode()
elif FLAGS.test:
from_test_data = os.path.join(FLAGS.data_dir,"test_q.txt")
to_test_data = os.path.join(FLAGS.data_dir,"test_f.txt")
if not (os.path.exists(from_test_data) and os.path.exists(to_test_data)):
if FLAGS.test_file:
data_utils.splitToFrom(FLAGS.data_dir,FLAGS.test_file,"test") # split to-from data
else:
print("test data file missing!")
return
print("computing test accuracy...")
test_acc = test_accuracy(from_test_data,to_test_data)
print("test accuracy =",test_acc)
else:
train()
def main(_):
if FLAGS.self_test:
self_test()
elif FLAGS.decode:
decode()
elif FLAGS.test:
from_test_data = os.path.join(FLAGS.data_dir,"test_q.txt")
to_test_data = os.path.join(FLAGS.data_dir,"test_f.txt")
if True:#not (os.path.exists(from_test_data) and os.path.exists(to_test_data)):
logging.info('Creating splits')
if FLAGS.test_file:
data_utils.splitToFrom(FLAGS.data_dir,FLAGS.test_file,"test") # split to-from data
else:
logging.info("test data file missing!")
return
logging.info("computing test accuracy...")
test_acc = test_accuracy(from_test_data,to_test_data)
logging.info("test accuracy = %1.3f"%test_acc)
else:
train()
def train():
"""Train a query->logical form semantic parser model"""
from_train = None
to_train = None
from_dev = None
to_dev = None
# process training data
from_train_data = os.path.join(FLAGS.data_dir,"train_q.txt")
to_train_data = os.path.join(FLAGS.data_dir,"train_f.txt")
if FLAGS.combine:
from_train_data_vocab = os.path.join(FLAGS.data_dir,"train_q_"+FLAGS.combine+".txt")
to_train_data_vocab = os.path.join(FLAGS.data_dir,"train_f_"+FLAGS.combine+".txt")
else:
from_train_data_vocab = from_train_data
to_train_data_vocab = to_train_data
if FLAGS.train_file:
logging.info("using training files %s and %s " % (from_train_data,to_train_data))
from_dev_data = from_train_data
to_dev_data = to_train_data
data_utils.splitToFrom(FLAGS.data_dir,FLAGS.train_file,"train") # split to-from data
if FLAGS.dev_file:
data_utils.splitToFrom(FLAGS.data_dir,FLAGS.dev_file,"valid") # split to-from data
from_dev_data = os.path.join(FLAGS.data_dir,"valid_q.txt")
to_dev_data = os.path.join(FLAGS.data_dir,"valid_f.txt")
logging.info("using validation files %s and %s for validation" %(from_dev_data,to_dev_data))
elif FLAGS.test_file:
data_utils.splitToFrom(FLAGS.data_dir,FLAGS.test_file,"test") # split to-from data
from_dev_data = os.path.join(FLAGS.data_dir,"test_q.txt")
to_dev_data = os.path.join(FLAGS.data_dir,"test_f.txt")
logging.info("using test files %s and %s for validation" %(from_dev_data,to_dev_data))
else:
logging.info("using train files %s and %s for validation" %(from_dev_data,to_dev_data))
from_train, to_train, from_dev, to_dev, _, _ = data_utils.prepare_data(
FLAGS.data_dir,
from_train_data,
to_train_data,
from_dev_data,
to_dev_data,
FLAGS.from_vocab_size,
FLAGS.to_vocab_size,
from_train_data_vocab,
to_train_data_vocab)
else:
# Prepare data.
logging.info("Preparing data in %s" % FLAGS.data_dir)
logging.info("data preparation complete!")
config_ = tf.ConfigProto()
config_.gpu_options.allow_growth = True
with tf.Session(config=config_) as sess:
# Create model.
logging.info("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.size))
model = create_model(sess, False, True)
# Read data into buckets and compute their sizes.
logging.info ("Reading development and training data (limit: %d)."
% FLAGS.max_train_data_size)
dev_set = read_data(from_dev, to_dev)
train_set = read_data(from_train, to_train, FLAGS.max_train_data_size)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
logging.info("train_bucket_sizes: %s" % ','.join([str(x) for x in train_bucket_sizes]))
train_total_size = float(sum(train_bucket_sizes))
logging.info("train_total_size: %1.2f" % train_total_size)
# create data batches for training
all_train_batches = create_batches(train_set)
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = 0
epoch_count = 1
epoch_size = math.ceil(train_total_size/model.batch_size)
previous_losses = []
logging.info("------------------------------------------")
logging.info("Starting training: [max epochs=%d][epoch_size=%d][train_total_size=%1.2f]" % (FLAGS.max_epoch,epoch_size,train_total_size))
logging.info("------------------------------------------")
# generate random permutation of batch numbers for epoch
batch_permutations = [np.random.permutation(len(all_train_batches[bkt_idx])) for bkt_idx in xrange(len(_buckets))]
batch_pidx = [0 for bkt_idx in xrange(len(_buckets))]
while True:
bucket_id = 0 # we just have one bucket
# Get a batch and make a step.
# select batch for this iteration
bidx = batch_permutations[bucket_id][batch_pidx[bucket_id]]
encoder_inputs, decoder_inputs, target_weights, soft_prob_inputs = all_train_batches[bucket_id][bidx]
batch_pidx[bucket_id] = (batch_pidx[bucket_id] + 1) % len(all_train_batches[bucket_id])
# execute gradient descent step
start_time = time.time()
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, soft_prob_inputs,
target_weights, bucket_id, False)
step_time += (time.time() - start_time) / FLAGS.display_every
loss += step_loss / FLAGS.display_every
current_step += 1
# Once in a while, we save checkpoint, logging.info statistics, and run evals.
if current_step % FLAGS.display_every == 0:
# Print statistics for the previous epoch.
logging.info ("epoch %d/%d step %d learning rate %.6f time/batch %.2f loss "
"%2.5f" % (epoch_count,FLAGS.max_epoch,model.global_step.eval(), model.learning_rate.eval(),
step_time, loss))
previous_losses.append(loss)
# zero timer and loss.
step_time, loss = 0.0, 0.0
sys.stdout.flush()
# all epoch updates
if current_step%epoch_size==0:
epoch_count += 1
# generate random permutation of batch numbers for epoch
batch_permutations = [np.random.permutation(len(all_train_batches[bkt_idx])) for bkt_idx in xrange(len(_buckets))]
batch_pidx = [0 for bkt_idx in xrange(len(_buckets))]
# Decrease learning rate
if epoch_count>5:
sess.run(model.learning_rate_decay_op)
# Run evals on development set and logging.info their perplexity.
if False:#(epoch_count-1)%FLAGS.eval_per_epoch==0 or epoch_count>FLAGS.max_epoch:
logging.info(" epoch %d eval accuracy %.2f" % (epoch_count-1,test_accuracy(from_dev_data, to_dev_data, sess, model)))
if False:
val_acc = 0
for bucket_id in xrange(len(_buckets)):
if len(dev_set[bucket_id]) == 0:
logging.info(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights, soft_probabilities = model.get_batch(
dev_set, bucket_id)
_, eval_loss, output_logits = model.step(sess, encoder_inputs, decoder_inputs, soft_probabilities,
target_weights, bucket_id, True)
outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]
#print(len(outputs))
# If there is an EOS symbol in outputs, cut them at that point.
if data_utils.EOS_ID in outputs:
outputs = outputs[:outputs.index(data_utils.EOS_ID)]
outputs = post_process(outputs,to_vocab)
#print(outputs)
if compute_tree_accuracy(outputs,to_token_ids,to_vocab,rev_to_vocab_dict,comm_dict,FLAGS.display):
val_acc+= 1
eval_ppx = math.exp(float(eval_loss)) if eval_loss < 300 else float("inf")
logging.info(" epoch %d eval: bucket %d perplexity %.2f loss %2.5f" % (epoch_count-1,bucket_id, eval_ppx, eval_loss))
if FLAGS.save_every:
if epoch_count % FLAGS.save_every == 0:
checkpoint_path = os.path.join(FLAGS.train_dir, "parse.ckpt")
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
logging.info("Model saved at %d", epoch_count)
# save model at the end of last training epoch
if epoch_count > FLAGS.max_epoch:
logging.info("Max epoch reached!")
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, "parse.ckpt")
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
logging.info("Model saved.")
break
logging.info("Training complete!")
return
def train():
"""Train a query->logical form semantic parser model"""
from_train = None
to_train = None
from_dev = None
to_dev = None
# process training data
from_train_data = os.path.join(FLAGS.data_dir, "train_q.txt")
to_train_data = os.path.join(FLAGS.data_dir, "train_f.txt")
if FLAGS.train_file:
print("using training files", from_train_data, "and", to_train_data)
from_dev_data = from_train_data
to_dev_data = to_train_data
data_utils.splitToFrom(FLAGS.data_dir, FLAGS.train_file,
"train") # split to-from data
if FLAGS.dev_file:
data_utils.splitToFrom(FLAGS.data_dir, FLAGS.dev_file,
"valid") # split to-from data
from_dev_data = os.path.join(FLAGS.data_dir, "valid_q.txt")
to_dev_data = os.path.join(FLAGS.data_dir, "valid_f.txt")
print("using validation files", from_dev_data, "and", to_dev_data,
"for validation")
elif FLAGS.test_file:
data_utils.splitToFrom(FLAGS.data_dir, FLAGS.test_file,
"test") # split to-from data
from_dev_data = os.path.join(FLAGS.data_dir, "test_q.txt")
to_dev_data = os.path.join(FLAGS.data_dir, "test_f.txt")
print("using test files", from_dev_data, "and", to_dev_data,
"for validation")
else:
print("using train files", from_dev_data, "and", to_dev_data,
"for validation")
from_train, to_train, from_dev, to_dev, _, _ = data_utils.prepare_data(
FLAGS.data_dir, from_train_data, to_train_data, from_dev_data,
to_dev_data, FLAGS.from_vocab_size, FLAGS.to_vocab_size)
else:
# Prepare data.
print("Preparing data in %s" % FLAGS.data_dir)
print("data preparation complete!")
config_ = tf.ConfigProto()
config_.gpu_options.allow_growth = True
config_.allow_soft_placement = True
with tf.Session(config=config_) as sess:
# Create model.
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.size))
model = create_model(sess, False, True)
# Read data into buckets and compute their sizes.
print("Reading development and training data (limit: %d)." %
FLAGS.max_train_data_size)
dev_set = read_data(from_dev, to_dev)
train_set = read_data(from_train, to_train, FLAGS.max_train_data_size)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
print("train_bucket_sizes =", train_bucket_sizes)
train_total_size = float(sum(train_bucket_sizes))
print("train_total_size =", train_total_size)
# create data batches for training
all_train_batches = create_batches(train_set)
# This is the training loop.
step_time, loss = 0.0, 0.0
current_step = 0
epoch_count = 1
epoch_size = math.ceil(train_total_size / model.batch_size)
previous_losses = []
print("------------------------------------------")
print("Starting training: max epochs =", FLAGS.max_epoch,
"; epoch size =", epoch_size, "; train_total_size =",
train_total_size)
print("------------------------------------------")
# generate random permutation of batch numbers for epoch
batch_permutations = [
np.random.permutation(len(all_train_batches[bkt_idx]))
for bkt_idx in xrange(len(_buckets))
]
batch_pidx = [0 for bkt_idx in xrange(len(_buckets))]
while True:
bucket_id = 0 # we just have one bucket
# Get a batch and make a step.
# select batch for this iteration
bidx = batch_permutations[bucket_id][batch_pidx[bucket_id]]
encoder_inputs, decoder_inputs, target_weights = all_train_batches[
bucket_id][bidx]
batch_pidx[bucket_id] = (batch_pidx[bucket_id] + 1) % len(
all_train_batches[bucket_id])
# execute gradient descent step
start_time = time.time()
_, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, False)
step_time += (time.time() - start_time) / FLAGS.display_every
loss += step_loss / FLAGS.display_every
current_step += 1
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.display_every == 0:
# Print statistics for the previous epoch.
print(
"epoch %d/%d step %d learning rate %.6f time/batch %.2f loss "
"%2.5f" %
(epoch_count, FLAGS.max_epoch, model.global_step.eval(),
model.learning_rate.eval(), step_time, loss))
previous_losses.append(loss)
# zero timer and loss.
step_time, loss = 0.0, 0.0
sys.stdout.flush()
# all epoch updates
if current_step % epoch_size == 0:
epoch_count += 1
# generate random permutation of batch numbers for epoch
batch_permutations = [
np.random.permutation(len(all_train_batches[bkt_idx]))
for bkt_idx in xrange(len(_buckets))
]
batch_pidx = [0 for bkt_idx in xrange(len(_buckets))]
# Decrease learning rate
if epoch_count > 5:
sess.run(model.learning_rate_decay_op)
# Run evals on development set and print their perplexity.
if (
epoch_count - 1
) % FLAGS.eval_per_epoch == 0 or epoch_count > FLAGS.max_epoch:
for bucket_id in xrange(len(_buckets)):
if len(dev_set[bucket_id]) == 0:
print(" eval: empty bucket %d" % (bucket_id))
continue
encoder_inputs, decoder_inputs, target_weights = model.get_batch(
dev_set, bucket_id)
_, eval_loss, output_logits = model.step(
sess, encoder_inputs, decoder_inputs,
target_weights, bucket_id, True)
eval_ppx = math.exp(float(
eval_loss)) if eval_loss < 300 else float("inf")
print(
" epoch %d eval: bucket %d perplexity %.2f loss %2.5f"
%
(epoch_count - 1, bucket_id, eval_ppx, eval_loss))
# save model at the end of last training epoch
if epoch_count > FLAGS.max_epoch:
print("Max epoch reached!")
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir,
"parse.ckpt")
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
print("Model saved.")
break
print("Training complete!")
return