def run_experiment(experiment_config):
dataset = Dataset('data/autism.tsv')
num_epochs = 1000
eval_every = 10
for fold_id, (train_idxs, test_idxs) in dataset.cross_validation():
data_train_fold = dataset.get_data(train_idxs)
num_instances, labels_train_fold = dataset.get_labels(train_idxs)
data_test_fold = dataset.get_data(test_idxs)
_, labels_test_fold = dataset.get_labels(test_idxs)
with tf.Graph().as_default() as graph:
experiment = Experiment(experiment_config, num_instances,
NeuralNetworkClassifier, data_train_fold)
with tf.Session() as session:
global_step = 0
session.run(tf.global_variables_initializer())
log_saver = LogSaver('logs', 'fisher_fold{}'.format(fold_id),
session.graph)
train_selected_data = session.run(
experiment.selection_wrapper.selected_data)
test_selected_data = session.run(
experiment.selection_wrapper.select(data_test_fold))
tqdm_iter = tqdm(range(num_epochs), desc='Epochs')
for epoch in tqdm_iter:
feed_dict = {
experiment.clf.x: train_selected_data,
experiment.clf.y: labels_train_fold
}
loss, _ = session.run(
[experiment.clf.loss, experiment.clf.opt],
feed_dict=feed_dict)
if epoch % eval_every == 0:
summary = session.run(experiment.clf.summary_op,
feed_dict=feed_dict)
log_saver.log_train(summary, epoch)
feed_dict = {
experiment.clf.x: test_selected_data,
experiment.clf.y: labels_test_fold
}
summary = session.run(experiment.clf.summary_op,
feed_dict=feed_dict)
log_saver.log_test(summary, epoch)
tqdm_iter.set_postfix(loss='{:.2f}'.format(float(loss)),
epoch=epoch)
def train(
main_config,
model_config,
model_name,
experiment_name,
dataset_name,
):
main_cfg = MainConfig(main_config)
model = MODELS[model_name]
dataset = dataset_type.get_dataset(dataset_name)
train_data = dataset.train_set_pairs()
vectorizer = DatasetVectorizer(main_cfg.model_dir,
raw_sentence_pairs=train_data)
dataset_helper = Dataset(vectorizer, dataset, main_cfg.batch_size)
max_sentence_len = vectorizer.max_sentence_len
vocabulary_size = vectorizer.vocabulary_size
train_mini_sen1, train_mini_sen2, train_mini_labels = dataset_helper.pick_train_mini_batch(
)
train_mini_labels = train_mini_labels.reshape(-1, 1)
test_sentence1, test_sentence2 = dataset_helper.test_instances()
test_labels = dataset_helper.test_labels()
test_labels = test_labels.reshape(-1, 1)
num_batches = dataset_helper.num_batches
model = model(
max_sentence_len,
vocabulary_size,
main_config,
model_config,
)
model_saver = ModelSaver(
main_cfg.model_dir,
experiment_name,
main_cfg.checkpoints_to_keep,
)
config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=main_cfg.log_device_placement,
)
with tf.Session(config=config) as session:
global_step = 0
init = tf.global_variables_initializer()
session.run(init)
log_saver = LogSaver(
main_cfg.logs_path,
experiment_name,
dataset_name,
session.graph,
)
model_evaluator = ModelEvaluator(model, session)
metrics = {'acc': 0.0}
time_per_epoch = []
log('Training model for {} epochs'.format(main_cfg.num_epochs))
for epoch in tqdm(range(main_cfg.num_epochs), desc='Epochs'):
start_time = time.time()
train_sentence1, train_sentence2 = dataset_helper.train_instances(
shuffle=True)
train_labels = dataset_helper.train_labels()
train_batch_helper = BatchHelper(
train_sentence1,
train_sentence2,
train_labels,
main_cfg.batch_size,
)
# small eval set for measuring dev accuracy
dev_sentence1, dev_sentence2, dev_labels = dataset_helper.dev_instances(
)
dev_labels = dev_labels.reshape(-1, 1)
tqdm_iter = tqdm(range(num_batches),
total=num_batches,
desc="Batches",
leave=False,
postfix=metrics)
for batch in tqdm_iter:
global_step += 1
sentence1_batch, sentence2_batch, labels_batch = train_batch_helper.next(
batch)
feed_dict_train = {
model.x1: sentence1_batch,
model.x2: sentence2_batch,
model.is_training: True,
model.labels: labels_batch,
}
loss, _ = session.run([model.loss, model.opt],
feed_dict=feed_dict_train)
if batch % main_cfg.eval_every == 0:
feed_dict_train = {
model.x1: train_mini_sen1,
model.x2: train_mini_sen2,
model.is_training: False,
model.labels: train_mini_labels,
}
train_accuracy, train_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_train,
)
log_saver.log_train(train_summary, global_step)
feed_dict_dev = {
model.x1: dev_sentence1,
model.x2: dev_sentence2,
model.is_training: False,
model.labels: dev_labels
}
dev_accuracy, dev_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_dev,
)
log_saver.log_dev(dev_summary, global_step)
tqdm_iter.set_postfix(
dev_acc='{:.2f}'.format(float(dev_accuracy)),
train_acc='{:.2f}'.format(float(train_accuracy)),
loss='{:.2f}'.format(float(loss)),
epoch=epoch)
if global_step % main_cfg.save_every == 0:
model_saver.save(session, global_step=global_step)
model_evaluator.evaluate_dev(dev_sentence1, dev_sentence2,
dev_labels)
end_time = time.time()
total_time = timer(start_time, end_time)
time_per_epoch.append(total_time)
model_saver.save(session, global_step=global_step)
model_evaluator.evaluate_test(test_sentence1, test_sentence2,
test_labels)
model_evaluator.save_evaluation(
'{}/{}'.format(main_cfg.model_dir, experiment_name),
time_per_epoch[-1], dataset)
def train(main_config, args):
main_cfg = MainConfig(main_config, args)
main_cfg.max_sequence_len = args.max_seq_length
main_cfg.syn_weight = args.syn_weight
main_cfg.vocab_size = args.vocab_size
batch_size = tf.placeholder(tf.int64)
vocab = tf.contrib.lookup.index_table_from_file(vocabulary_file=os.path.join(main_cfg.data_dir,
main_cfg.vocab_file),
num_oov_buckets=0,
default_value=1)
with tf.device('/cpu:0'):
train_iter, dev_iter, test_iter, dev_run_iter = get_quora_datasets(main_cfg, batch_size, vocab)
train_handle, test_handle, dev_handle, dev_run_handle = train_iter.string_handle(), \
test_iter.string_handle(), \
dev_iter.string_handle(), dev_run_iter.string_handle()
model_name = 'bilstm_{}'.format(main_config['PARAMS']['embedding_size'])
# Switcher handle placeholder, iterator
handle = tf.placeholder(tf.string, shape=[], name='handle')
quora_iter = tf.data.Iterator.from_string_handle(handle,
train_iter.output_types,
train_iter.output_shapes)
quora_example = quora_iter.get_next()
# obtaining finished
step = tf.train.get_or_create_global_step()
main_config['DATA']['emb_path'] = os.path.join(args.data_dir, main_config['DATA']['embeddings'])
model = BiLSTMSiamese(quora_example, args, main_config)
best_loss = float("inf")
best_ckpt_saver = BestCheckpointSaver(
save_dir=args.model_dir,
num_to_keep=main_cfg.checkpoints_to_keep,
maximize=True
)
best_accuracy = best_ckpt_saver.get_best_accuracy()
config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
with tf.train.MonitoredTrainingSession(checkpoint_dir=main_cfg.model_dir,
save_checkpoint_steps=main_cfg.save_every,
config=config,
save_summaries_steps=0, save_summaries_secs=None,
log_step_count_steps=0,
hooks=[
SaveAtEnd(args.model_dir, model_name,
main_cfg.checkpoints_to_keep)]) as session:
model.set_session(session)
log_saver = LogSaver(args.model_dir, 'summaries', session.graph)
model_evaluator = EvalHelper(model, session, logger)
start_time = time.time()
patience_left = main_cfg.patience
train_handle, dev_handle, dev_run_handle = session.run([train_handle, dev_handle, dev_run_handle])
session.run(train_iter.initializer, feed_dict={batch_size: main_cfg.batch_size})
session.run(dev_run_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
logger.info('Starting training...')
while patience_left:
try:
# run train batch
loss, _, current_quora_ids, _ = model.run_train_batch(train_handle)
global_step = session.run(step)
# Use paraphrases to help model on difficult examples
if global_step % main_cfg.eval_every == 0:
model.evaluation_stats(train_handle, dev_run_handle, global_step, logger, log_saver)
except tf.errors.OutOfRangeError:
session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
all_dev_acc, all_dev_loss = model_evaluator.evaluate_dev(dev_handle, global_step, logger)
session.run(model.inc_gstep) # decay LR
if all_dev_acc > best_accuracy:
best_accuracy = all_dev_acc
best_ckpt_saver.handle(all_dev_acc, session, step)
patience_left = main_cfg.patience
else:
patience_left -= 1
if best_loss > all_dev_loss:
best_loss = all_dev_loss
session.run(dev_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
session.run(train_iter.initializer, feed_dict={batch_size: main_cfg.batch_size})
logger.info('No improvement observed over {} epochs. Initiating early stopping'.format(main_cfg.patience))
end_time = time.time()
total_time = timer(start_time, end_time)
logger.info('Training took {}, best accuracy is {}'.format(total_time, best_accuracy))
# model_evaluator.save_dev_evaluation(args.model_dir, total_time)
with open(os.path.join(args.model_dir, 'run_config.ini'), 'w') as configfile: # save
main_config.write(configfile)
saver = tf.train.Saver()
with tf.Session(config=config) as session:
session.run(tf.tables_initializer())
saver.restore(session, get_best_checkpoint(args.model_dir, select_maximum_value=True))
model.set_session(session)
model_evaluator = EvalHelper(model, session, logger)
test_handle = session.run(test_handle)
session.run(test_iter.initializer, feed_dict={batch_size: main_cfg.eval_batch_size})
test_acc, test_loss = model_evaluator.evaluate_test(test_handle, logger)
model_evaluator.save_test_evaluation(args.model_dir)
def train(main_config, model_config, model_name, dataset_name):
main_cfg = MainConfig(main_config)
model = MODELS[model_name]
dataset = DATASETS[dataset_name]()
model_name = '{}_{}'.format(model_name,
main_config['PARAMS']['embedding_size'])
train_data = dataset.train_set_pairs()
vectorizer = DatasetVectorizer(train_data, main_cfg.model_dir)
dataset_helper = Dataset(vectorizer, dataset, main_cfg.batch_size)
max_sentence_len = vectorizer.max_sentence_len
vocabulary_size = vectorizer.vocabulary_size
train_mini_sen1, train_mini_sen2, train_mini_labels = dataset_helper.pick_train_mini_batch(
)
train_mini_labels = train_mini_labels.reshape(-1, 1)
test_sentence1, test_sentence2 = dataset_helper.test_instances()
test_labels = dataset_helper.test_labels()
test_labels = test_labels.reshape(-1, 1)
num_batches = dataset_helper.num_batches
model = model(max_sentence_len, vocabulary_size, main_config, model_config)
model_saver = ModelSaver(main_cfg.model_dir, model_name,
main_cfg.checkpoints_to_keep)
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=main_cfg.log_device_placement)
with tf.Session(config=config) as session:
global_step = 0
init = tf.global_variables_initializer()
session.run(init)
log_saver = LogSaver(main_cfg.logs_path, model_name, dataset_name,
session.graph)
model_evaluator = ModelEvaluator(model, session)
metrics = {'acc': 0.0}
time_per_epoch = []
for epoch in tqdm(range(main_cfg.num_epochs), desc='Epochs'):
start_time = time.time()
train_sentence1, train_sentence2 = dataset_helper.train_instances(
shuffle=True)
train_labels = dataset_helper.train_labels()
train_batch_helper = BatchHelper(train_sentence1, train_sentence2,
train_labels, main_cfg.batch_size)
# small eval set for measuring dev accuracy
dev_sentence1, dev_sentence2, dev_labels = dataset_helper.dev_instances(
)
dev_labels = dev_labels.reshape(-1, 1)
tqdm_iter = tqdm(range(num_batches),
total=num_batches,
desc="Batches",
leave=False,
postfix=metrics)
for batch in tqdm_iter:
global_step += 1
sentence1_batch, sentence2_batch, labels_batch = train_batch_helper.next(
batch)
feed_dict_train = {
model.x1: sentence1_batch,
model.x2: sentence2_batch,
model.is_training: True,
model.labels: labels_batch
}
loss, _ = session.run([model.loss, model.opt],
feed_dict=feed_dict_train)
if batch % main_cfg.eval_every == 0:
feed_dict_train = {
model.x1: train_mini_sen1,
model.x2: train_mini_sen2,
model.is_training: False,
model.labels: train_mini_labels
}
train_accuracy, train_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_train)
log_saver.log_train(train_summary, global_step)
feed_dict_dev = {
model.x1: dev_sentence1,
model.x2: dev_sentence2,
model.is_training: False,
model.labels: dev_labels
}
dev_accuracy, dev_summary = session.run(
[model.accuracy, model.summary_op],
feed_dict=feed_dict_dev)
log_saver.log_dev(dev_summary, global_step)
tqdm_iter.set_postfix(
dev_acc='{:.2f}'.format(float(dev_accuracy)),
train_acc='{:.2f}'.format(float(train_accuracy)),
loss='{:.2f}'.format(float(loss)),
epoch=epoch)
if global_step % main_cfg.save_every == 0:
model_saver.save(session, global_step=global_step)
model_evaluator.evaluate_dev(dev_sentence1, dev_sentence2,
dev_labels)
end_time = time.time()
total_time = timer(start_time, end_time)
time_per_epoch.append(total_time)
model_saver.save(session, global_step=global_step)
feed_dict_train = {
model.x1: test_sentence1,
model.x2: test_sentence2,
model.is_training: False,
model.labels: test_labels
}
#train_accuracy, train_summary, train_e = session.run([model.accuracy, model.summary_op, model.e],
# feed_dict=feed_dict_train)
train_e = session.run([model.e], feed_dict=feed_dict_train)
plt.clf()
f = plt.figure(figsize=(8, 8.5))
ax = f.add_subplot(1, 1, 1)
i = ax.imshow(train_e[0][0], interpolation='nearest', cmap='gray')
cbaxes = f.add_axes([0.2, 0, 0.6, 0.03])
cbar = f.colorbar(i, cax=cbaxes, orientation='horizontal')
cbar.ax.set_xlabel('Probability', labelpad=2)
f.savefig('attention_maps.pdf', bbox_inches='tight')
f.show()
plt.show()
feed_dict_test = {
model.x1: test_sentence1,
model.x2: test_sentence2,
model.is_training: False,
model.labels: test_labels
}
test_accuracy, test_summary = session.run(
[model.accuracy, model.summary_op], feed_dict=feed_dict_test)
print('tst_acc:%.2f loss:%.2f', test_accuracy, loss)
model_evaluator.evaluate_test(test_sentence1, test_sentence2,
test_labels)
model_evaluator.save_evaluation(
'{}/{}'.format(main_cfg.model_dir, model_name), time_per_epoch[-1],
dataset)