def docs(dataset_name):
p = utils.Progbar(target=(utils.lines_in_file(directories.RAW +
dataset_name)))
for i, d in enumerate(utils.load_json_lines(directories.RAW +
dataset_name)):
p.update(i + 1)
yield d
def train(model, training_data, optimizer):
model.train()
num_batches = math.ceil(len(training_data) / args.batch_size)
bar = utils.Progbar(target=num_batches)
train_loss = 0.0
train_total_instances = 0
for batch_id, batch in enumerate(
utils.minibatches(training_data, args.batch_size)):
model.zero_grad()
for sentence, tags in batch:
sentence_in = processor.tensor(sentence)
targets = processor.tensor(tags)
loss = model.neg_log_likelihood(sentence_in, targets)
loss.backward()
train_loss += loss
train_total_instances += 1
optimizer.step()
bar.update(batch_id + 1,
exact=[("train loss", train_loss / train_total_instances)])
if args.save_checkpoint:
save_model(model)
def run(model, sess, dataset, train=False):
j = 0
prog = utils.Progbar(len(dataset))
avg_loss = 0.0
avg_correct = 0.0
for x, masks, y in dataset:
input_feed = {
model.inputs: utils.convert_to_np(x),
model.masks: utils.convert_to_np(masks),
model.labels: utils.convert_to_np(y),
model.train: train
}
if train:
output_feed = [
model.cost, model.accuracy, model.average_guess, model.op
]
cost, num_correct, avg_guess, _ = sess.run(output_feed, input_feed)
else:
output_feed = [model.cost, model.accuracy, model.average_guess]
cost, num_correct, avg_guess = sess.run(output_feed, input_feed)
avg_loss += cost
avg_correct += num_correct
j += 1
prog.update(j,
values=[('num_correct', num_correct)],
exact=[('cost', cost)])
avg_loss /= len(dataset)
avg_correct /= len(dataset)
return avg_correct, avg_loss
def evaluate(model, eval_data, dataset_name):
model.eval()
num_batches = math.ceil(len(eval_data) / args.batch_size)
bar = utils.Progbar(target=num_batches)
eval_score = 0.0
eval_total_instances = 0
eval_total_characters = 0
eval_correct_characters = 0
with torch.no_grad():
for batch_id, batch in enumerate(
utils.minibatches(eval_data, args.batch_size)):
for sentence, tags in batch:
score, tag_out = model(processor.tensor(sentence))
if len(tag_out) != len(tags):
raise IndexError(
'Size of output tag sequence differs from that of reference.'
)
length = len(tags)
correct = [
tag_out[i] == tags[i] for i in range(1, length - 1)
].count(1)
eval_score += score
eval_total_instances += 1
eval_total_characters += length
eval_correct_characters += correct
bar.update(batch_id + 1,
exact=[("eval score", eval_score / eval_total_instances)
])
logger.info('{} dataset accuracy: {}'.format(
dataset_name, eval_correct_characters / eval_total_characters))
def run_model_over_docs(dataset_name, docs, model):
docs_by_id = {doc.did: doc for doc in docs}
prog = utils.Progbar(dataset_name.n_batches)
for i, X in enumerate(dataset_name):
if X['y'].size == 0:
continue
scores = model.predict_on_batch(X)
update_doc(docs_by_id[X['did']], X, scores)
prog.update(i + 1)
def build_dataset(vectors, name, tune_fraction=0.0, reduced=False, columns=None):
doc_vectors = utils.load_pickle(directories.MISC + name.replace("_reduced", "") +
"_document_vectors.pkl")
main_pairs = PairDataBuilder(columns)
tune_pairs = PairDataBuilder(columns)
main_mentions = MentionDataBuilder(columns)
tune_mentions = MentionDataBuilder(columns)
main_docs = DocumentDataBuilder(columns)
tune_docs = DocumentDataBuilder(columns)
print("Building dataset", name + ("/tune" if tune_fraction > 0 else ""))
p = utils.Progbar(target=(2 if reduced else utils.lines_in_file(directories.RAW + name)))
for i, d in enumerate(utils.load_json_lines(directories.RAW + name)):
if reduced and i > 2:
break
p.update(i + 1)
if reduced and tune_fraction != 0:
pairs, mentions, docs = (main_pairs, main_mentions, main_docs) \
if i == 0 else (tune_pairs, tune_mentions, tune_docs)
else:
pairs, mentions, docs = (main_pairs, main_mentions, main_docs) \
if random.random() > tune_fraction else (tune_pairs, tune_mentions, tune_docs)
ms, ps = mentions.size(), pairs.size()
mention_positions = {}
for mention_num in sorted(d["mentions"].keys(), key=int):
mention_positions[mention_num] = mentions.size()
mentions.add_mention(d["mentions"][mention_num], vectors,
doc_vectors[d["mentions"][mention_num]["doc_id"]])
for key in sorted(d["labels"].keys(), key=lambda k: (int(k.split()[1]), int(k.split()[0]))):
k1, k2 = key.split()
pairs.add_pair(d["labels"][key], mention_positions[k1], mention_positions[k2],
int(d["mentions"][k1]["doc_id"]),
int(d["mentions"][k1]["mention_id"]),
int(d["mentions"][k2]["mention_id"]),
d["pair_features"][key])
me, pe = mentions.size(), pairs.size()
docs.add_doc(ms, me, ps, pe, d["document_features"])
suffix = ("_reduced" if reduced else "")
if tune_mentions.size() > 0:
tune_mentions.write(name + "_tune" + suffix)
tune_pairs.write(name + "_tune" + suffix)
tune_docs.write(name + "_tune" + suffix)
main_mentions.write(name + "_train" + suffix)
main_pairs.write(name + "_train" + suffix)
main_docs.write(name + "_train" + suffix)
else:
main_mentions.write(name + suffix)
main_pairs.write(name + suffix)
main_docs.write(name + suffix)
def __init__(self,
trainer,
docs,
data,
message,
replay_memory=None,
beta=0,
docs_per_iteration=10000):
self.trainer = trainer
self.data = data
self.model = trainer.model
self.message = message
self.replay_memory = replay_memory
self.beta = beta
self.loss_aggregator = Aggregator()
self.evaluators = [
evaluation.Evaluator(metric=evaluation.muc),
evaluation.Evaluator(metric=evaluation.b_cubed),
evaluation.Evaluator(metric=evaluation.ceafe)
]
self.merged_pairs = {}
self.training = self.replay_memory is not None
print self.message
random.shuffle(docs)
if self.training:
docs = docs[:docs_per_iteration]
prog = utils.Progbar(len(docs))
for i, (doc, actionstate) in enumerate(docs):
self.trainer.doc = doc
self.trainer.actionstate = actionstate
if len(actionstate.possible_pairs) != 0:
actionstate.load(self.data, self.trainer.pair_model,
self.trainer.anaphoricity_model)
s = State(doc, actionstate)
doc_merged_pairs = self.run_agent(s, beta, i)
for evaluator in self.evaluators:
evaluator.update(doc)
self.merged_pairs[doc.did] = doc_merged_pairs
doc.reset()
actionstate.clear()
muc, b3, ceafe = (self.evaluators[i].get_f1() for i in range(3))
exact = [('muc', 100 * muc), ('b3', 100 * b3),
('ceafe', 100 * ceafe),
('conll', 100 * (muc + b3 + ceafe) / 3),
('loss', self.loss_aggregator.get_avg())]
prog.update(i + 1, exact=exact)
def train_all(self):
timer.start("train")
model_weights = self.model.get_weights()
prog = utils.Progbar(len(self.memory))
random.shuffle(self.memory)
for i, X in enumerate(self.memory):
loss = self.train_on_example(X)
prog.update(i + 1, [("loss", loss)])
self.size = 0
self.memory = []
timer.stop("train")
weight_diffs = [(np.sum(np.abs(new_weight - old_weight)),
new_weight.size) for new_weight, old_weight in zip(
self.model.get_weights(), model_weights)]
summed = np.sum(map(np.array, weight_diffs), axis=0)
print "weight diffs", weight_diffs, summed
def set_costs(dataset, docs):
docs_by_id = {doc.did: doc for doc in docs}
prog = utils.Progbar(dataset.n_batches)
for i, X in enumerate(dataset):
if X['y'].size == 0:
continue
doc = docs_by_id[X['did']]
doc_weight = (len(doc.mention_to_gold) + len(doc.mentions)) / 10.0
for (start, end) in zip(X['starts'][:, 0], X['ends'][:, 0]):
ids = X['ids'][start:end]
ana = ids[0, 1]
old_ant = doc.ana_to_ant[ana]
doc.unlink(ana)
costs = X['cost_ptrs'][start:end]
for ant_ind in range(end - start):
costs[ant_ind] = doc.link(ids[ant_ind, 0],
ana,
hypothetical=True,
beta=1)
doc.link(old_ant, ana)
costs -= costs.max()
costs *= -doc_weight
prog.update(i + 1)
def train(self):
''' train the model '''
def num_steps(bucket):
return int(np.ceil(len(bucket) / float(config.BATCH_SIZE)))
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
self._check_restore_parameters(sess, saver)
# get checkpoint save state
iteration = self.global_step.eval()
# get number of batches per bucket
bucket_sizes = [
num_steps(v['dec_input'])
for k, v in self.train_data.iteritems()
]
cur_epoch = self.epoch.eval()
bucket_index = self.bucket_index.eval()
step_iter = self.bucket_step.eval()
print 'Starting at iter:', iteration, 'epoch:', cur_epoch+1, \
'bucket index:', bucket_index, 'bucket step:', step_iter
for epoch in range(cur_epoch, config.NUM_EPOCHS):
epoch_start = time.time()
total_losses = []
sess.run(tf.assign(self.epoch, epoch))
print '\n', 'Epoch:', epoch + 1
print 'Bucket sizes', bucket_sizes
# print 'Learning rate:', self.lr
# show progress bar if more than one batch
if sum(bucket_sizes) > 1:
prog = utils.Progbar(target=bucket_sizes[bucket_index])
end_while = False
while True:
# get data for iteration
batch_start = time.time()
batch_data = data.get_batch(self.train_data, bucket_index,
config.BUCKETS,
config.BATCH_SIZE, step_iter)
encoder_inputs = batch_data[0]
decoder_inputs = batch_data[1]
decoder_masks = batch_data[2]
next_bucket = batch_data[3]
# run the model
_, step_loss, _ = self.run_step(sess, encoder_inputs,
decoder_inputs,
decoder_masks,
config.BATCH_SIZE,
bucket_index, True)
# reset iteration vars if done with the current bucket
if next_bucket:
step_iter = sess.run(tf.assign(self.bucket_step, 0))
bucket_index = sess.run(
tf.assign(self.bucket_index, bucket_index + 1))
else:
step_iter = sess.run(
tf.assign(self.bucket_step, step_iter + 1))
if bucket_index >= len(config.BUCKETS):
end_while = True
bucket_index = sess.run(tf.assign(
self.bucket_index, 0))
if next_bucket and sum(bucket_sizes) > 1:
print # update progress
prog.update(
bucket_sizes[bucket_index - 1],
[("train loss", step_loss),
('batch runtime', time.time() - batch_start)])
new_bucket = bucket_sizes[bucket_index]
prog = utils.Progbar(target=new_bucket)
total_losses.append(step_loss)
# when to evaluate and save state
if end_while or \
iteration == 200 or \
(iteration > 0 and iteration % 1000 == 0):
saver.save(sess,
os.path.join(self.checkpoint_path,
'summarizer'),
global_step=iteration)
if iteration == 200 or end_while or epoch < 2 \
or iteration % 2000 == 0:
self.evaluate(sess, total_losses, iteration)
iteration += 1
if sum(bucket_sizes) == 1:
print 'Train loss', step_loss
if end_while:
print 'Epoch', epoch + 1, 'took', time.time(
) - epoch_start
break
if sum(bucket_sizes) > 1:
prog.update(
step_iter,
[("train loss", step_loss),
('batch runtime', time.time() - batch_start)])
# eval on test set at the end
self.evaluate(sess, total_losses, iteration, test=True)
saver.save(sess,
os.path.join(self.checkpoint_path, 'summarizer'),
global_step=iteration)
logger.info("Number dev instances: {}".format(len(dev_instances)))
training_total_tokens = 0
best_f1 = 0.
for epoch in range(int(options.num_epochs)):
logger.info("Epoch {} out of {}".format(epoch + 1, options.num_epochs))
random.shuffle(training_instances)
train_loss = 0.0
train_total_instance = 0 # size of trained instances
if options.dropout > 0:
model.set_dropout(options.dropout)
nbatches = (len(training_instances) + options.batch_size -
1) // options.batch_size
bar = utils.Progbar(target=nbatches)
for batch_id, batch in enumerate(
utils.minibatches(training_instances, options.batch_size)):
for idx, instance in enumerate(batch):
if len(instance.sentence) == 0: continue
train_total_instance += 1
loss_expr = model.neg_log_loss(instance.sentence,
instance.tags)
# Forward pass
loss = loss_expr.scalar_value()
# Do backward pass
loss_expr.backward()
# Bail if loss is NaN
if math.isnan(loss):
def main(args):
if not os.path.exists(args.checkpoint):
os.makedirs(args.checkpoint)
c = utils.make_config(args)
d = input_pipeline.DataInputPipeline(args.inputs, args.vocab, args.labels,
c)
os.environ[
'CUDA_VISIBLE_DEVICES'] = args.gpu # Or whichever device you would like to use
# gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(
) #config=tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True))
m = models.Model(c, sess, d)
sess.run(tf.global_variables_initializer())
if args.train:
print 'INFO: starting training...'
prog = utils.Progbar(target=d.get_num_batches())
epoch = 1
for _ in range(c.num_epochs):
epoch_loss = 0
for i, batch in enumerate(d.batch_iter()):
sales_hat, price_hat, shop_hat, category_hat, loss = \
m.train_on_batch(*batch[:-1])
prog.update(i + 1, [('train loss', loss)])
epoch_loss += loss
print '\n INFO: EPOCH ', epoch, ' MEAN LOSS: ', epoch_loss / float(
d.get_num_batches())
print 'INFO: saving checkpoint...'
m.save(args.checkpoint)
print 'INFO: ...done!'
epoch += 1
else:
print 'INFO: loading model from checkpoint...'
m.load(dir=args.checkpoint)
print 'INFO: done!'
if args.output is not None:
print 'INFO: starting inference...'
prog = utils.Progbar(target=d.get_num_batches())
source_out = {'source': [], 'len': [], 'attn': [], 'ids': []}
sales_out = {'label': [], 'pred': []}
price_out = {'label': [], 'pred': []}
shop_out = {'label': [], 'pred': []}
category_out = {'label': [], 'pred': []}
loss_out = []
for i, batch in enumerate(d.batch_iter()):
sales_hat, price_hat, shop_hat, category_hat, loss, attn = \
m.test_on_batch(*batch[:-1])
prog.update(i, [('train loss', loss)])
# record results
source, source_len, log_sales, price, shop, category, ids = batch
source_out['source'] += source
source_out['len'] += source_len
source_out['attn'] += attn.tolist()
source_out['ids'] += ids
sales_out['label'] += log_sales
sales_out['pred'] += sales_hat.tolist()
price_out['label'] += price
price_out['pred'] += price_hat.tolist()
shop_out['label'] += shop
shop_out['pred'] += shop_hat.tolist()
category_out['label'] += category
category_out['pred'] += category_hat.tolist()
loss_out += loss
print '\nINFO: dumping output to ', args.output
utils.write_pickle(source_out, args.output)
print '\n\nINFO: \(^o^)/ done \(^o^)/'
def train_model(dataset, paths, device):
"""The main function for executing network training. It loads the specified
dataset iterator, saliency model, and helper classes. Training is then
performed in a new session by iterating over all batches for a number of
epochs. After validation on an independent set, the model is saved and
the training history is updated.
Args:
dataset (str): Denotes the dataset to be used during training.
paths (dict, str): A dictionary with all path elements.
device (str): Represents either "cpu" or "gpu".
"""
iterator = data.get_dataset_iterator("train", dataset, paths["data"])
next_element, train_init_op, valid_init_op = iterator
input_images, ground_truths = next_element[:2]
input_plhd = tf.placeholder_with_default(input_images,
(None, None, None, 3),
name="input")
#training = tf.placeholder(tf.bool, name="training") ## For BN
msi_net = model_bn.MSINET(is_train=True)
predicted_maps = msi_net.forward(input_plhd)
optimizer, loss = msi_net.train(ground_truths, predicted_maps,
config.PARAMS["learning_rate"])
n_train_data = getattr(data, dataset.upper()).n_train
n_valid_data = getattr(data, dataset.upper()).n_valid
n_train_batches = int(np.ceil(n_train_data / config.PARAMS["batch_size"]))
n_valid_batches = int(np.ceil(n_valid_data / config.PARAMS["batch_size"]))
history = utils.History(n_train_batches,
n_valid_batches,
dataset,
paths["history"],
device)
progbar = utils.Progbar(n_train_data,
n_train_batches,
config.PARAMS["batch_size"],
config.PARAMS["n_epochs"],
history.prior_epochs)
#training = tf.placeholder(tf.bool, name="training") ## For BN
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver = msi_net.restore(sess, dataset, paths, device)
print(">> Start training on %s..." % dataset.upper())
for epoch in range(config.PARAMS["n_epochs"]):
sess.run(train_init_op)
for batch in range(n_train_batches):
#_, error = sess.run([optimizer, loss], feed_dict={training: True})
_, error = sess.run([optimizer, loss])
history.update_train_step(error)
progbar.update_train_step(batch)
sess.run(valid_init_op)
for batch in range(n_valid_batches):
#error = sess.run(loss, feed_dict={training: False})
error = sess.run(loss)
history.update_valid_step(error)
progbar.update_valid_step()
msi_net.save(saver, sess, dataset, paths["latest"], device)
history.save_history()
progbar.write_summary(history.get_mean_train_error(),
history.get_mean_valid_error())
if history.valid_history[-1] == min(history.valid_history):
msi_net.save(saver, sess, dataset, paths["best"], device)
msi_net.optimize(sess, dataset, paths["best"], device)
print("\tBest model!", flush=True)
def main(args):
c = utils.make_config(args)
d = input_pipeline.DataInputPipeline(args.inputs, args.vocab, args.labels,
c)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
# gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(
) #config=tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True))
m = models.Model(c, sess, d)
print 'INFO: loading model from checkpoint...'
m.load(dir=args.checkpoint)
print 'INFO: done!'
sess.run(tf.global_variables_initializer())
print 'INFO: starting inference...'
prog = utils.Progbar(target=d.get_num_batches())
source_out = {'source': [], 'len': [], 'attn': [], 'ids': []}
sales_out = {'label': [], 'pred': []}
price_out = {'label': [], 'pred': []}
shop_out = {'label': [], 'pred': []}
category_out = {'label': [], 'pred': []}
loss_out = []
for i, batch in enumerate(d.batch_iter()):
sales_hat, price_hat, shop_hat, category_hat, loss, attn = \
m.test_on_batch(*batch[:-1])
prog.update(i, [('train loss', loss)])
# record results
source, source_len, log_sales, price, shop, category, ids = batch
source_out['source'] += source
source_out['len'] += source_len
source_out['attn'] += attn.tolist()
source_out['ids'] += ids
sales_out['label'] += log_sales
sales_out['pred'] += sales_hat.tolist()
price_out['label'] += price
price_out['pred'] += price_hat.tolist()
shop_out['label'] += shop
shop_out['pred'] += shop_hat.tolist()
category_out['label'] += category
category_out['pred'] += category_hat.tolist()
loss_out += loss
if args.output is not None:
print '\nINFO: dumping output to ', args.output
write_pickle(source_out, args.output)
print 'INFO: done \(^o^)/'
def __init__(self, list_words, max_distance=3):
self.max_distance = max_distance
self.dictionary = {}
self.max_length = 0
self.progbar = utils.Progbar(len(list_words))
self.build(list_words)
def train(model_props, n_epochs=10000, reduced=False, dev_set_name='dev'):
print "Training", model_props.path
pprint(model_props.__dict__)
model_props.write(model_props.path + 'model_props.pkl')
utils.rmkdir(model_props.path + 'src')
for fname in os.listdir('.'):
if fname.endswith('.py'):
shutil.copyfile(fname, model_props.path + 'src/' + fname)
if model_props.ranking or \
model_props.top_pairs:
write_start = 0
write_every = 10
else:
write_start = 80
write_every = 20
print "Loading data"
vectors = np.load(directories.RELEVANT_VECTORS + 'word_vectors.npy')
train = datasets.DocumentBatchedDataset(
"train_reduced" if reduced else "train", model_props, with_ids=True)
dev = datasets.DocumentBatchedDataset(
dev_set_name + "_reduced" if reduced else dev_set_name,
model_props,
with_ids=True)
print "Building model"
model, _ = pairwise_models.get_model(dev, vectors, model_props)
json_string = model.to_json()
open(model_props.path + 'architecture.json', 'w').write(json_string)
best_val_score = 1000
best_val_score_in_window = 1000
history = []
for epoch in range(n_epochs):
timer.start("train")
print "EPOCH {:}, model = {:}".format((epoch + 1), model_props.path)
epoch_stats = {}
model_weights = model.get_weights()
train_docs = utils.load_pickle(directories.DOCUMENTS +
'train_docs.pkl')
dev_docs = utils.load_pickle(directories.DOCUMENTS + dev_set_name +
'_docs.pkl')
if reduced:
dev_docs = dev_docs[:3]
if model_props.ranking:
print "Running over training set"
run_model_over_docs(train, train_docs, model)
epoch_stats.update(compute_metrics(train_docs, "train"))
if model_props.use_rewards:
print "Setting costs"
set_costs(train, train_docs)
print "Training"
prog = utils.Progbar(train.n_batches)
train.shuffle()
loss_sum, n_examples = 0, 0
for i, X in enumerate(train):
if X['y'].size == 0:
continue
batch_loss = model.train_on_batch(X)
loss_sum += batch_loss * train.scale_factor
n_examples += X['y'].size
prog.update(i + 1, exact=[("train loss", loss_sum / n_examples)])
epoch_stats["train time"] = time.time() - prog.start
for k in prog.unique_values:
epoch_stats[k] = prog.sum_values[k][0] / max(
1, prog.sum_values[k][1])
epoch_stats["weight diffs"] = [
(np.sum(np.abs(new_weight - old_weight)), new_weight.size)
for new_weight, old_weight in zip(model.get_weights(),
model_weights)
]
summed = np.sum(map(np.array, epoch_stats["weight diffs"][1:]), axis=0)
epoch_stats["total weight diff"] = tuple(summed)
print "Testing on dev set"
evaluate_model(dev, dev_docs, model, model_props, epoch_stats)
history.append(epoch_stats)
utils.write_pickle(history, model_props.path + 'history.pkl')
score = -epoch_stats["dev conll"] if model_props.ranking else \
(epoch_stats["dev loss"] if not model_props.anaphoricity_only else
epoch_stats["dev anaphoricity loss"])
if score < best_val_score:
best_val_score = score
print "New best {:}, saving model".format(
"CoNLL F1" if model_props.ranking else "validation loss")
model.save_weights(model_props.path + "best_weights.hdf5",
overwrite=True)
if score < best_val_score_in_window and epoch > write_start:
print "Best in last {:}, saved to weights_{:}".format(
write_every, write_every * (epoch / write_every))
best_val_score_in_window = score
model.save_weights(
model_props.path +
"weights_{:}.hdf5".format(write_every * (epoch / write_every)),
overwrite=True)
if epoch + write_every >= n_epochs:
model.save_weights(model_props.path + "final_weights.hdf5",
overwrite=True)
if epoch % write_every == 0:
best_val_score_in_window = 1000
timer.stop("train")
timer.print_totals()
print
timer.clear()
def evaluate_model(dataset,
docs,
model,
model_props,
stats,
save_output=False,
save_scores=False,
print_table=False):
prog = utils.Progbar(dataset.n_batches)
mt = RankingMetricsTracker(dataset.name, model_props=model_props) \
if model_props.ranking else ClassificationMetricsTracker(dataset.name)
mta = ClassificationMetricsTracker(dataset.name + " anaphoricity",
anaphoricity=True)
docs_by_id = {doc.did: doc for doc in docs} if model_props.ranking else {}
saved_links, saved_scores = (defaultdict(list) if save_output else None,
defaultdict(dict) if save_scores else None)
for i, X in enumerate(dataset):
if X['y'].size == 0:
continue
progress = []
scores = model.predict_on_batch(X)
if model_props.ranking:
update_doc(docs_by_id[X['did']],
X,
scores,
saved_links=saved_links,
saved_scores=saved_scores)
if model_props.anaphoricity and not model_props.ranking:
progress.append(
("anaphoricity loss", mta.update(X, scores[0][:, 0])))
if not model_props.anaphoricity_only:
progress.append(
("loss",
mt.update(
X, scores if model_props.ranking else
scores[1 if model_props.anaphoricity else 0][:, 0])))
prog.update(i + 1, exact=progress)
if save_scores:
print "Writing scores"
utils.write_pickle(saved_scores,
model_props.path + dataset.name + '_scores.pkl')
if save_output:
print "Writing output"
utils.write_pickle(saved_links,
model_props.path + dataset.name + '_links.pkl')
utils.write_pickle(
docs, model_props.path + dataset.name + '_processed_docs.pkl')
timer.start("metrics")
if model_props.ranking:
stats.update(compute_metrics(docs, dataset.name))
stats["validate time"] = time.time() - prog.start
if model_props.anaphoricity and not model_props.ranking:
mta.finish(stats)
if not model_props.anaphoricity_only:
mt.finish(stats)
timer.stop("metrics")
if print_table:
print " & ".join(
map(lambda x: "{:.2f}".format(x * 100), [
stats[dataset.name + " muc precision"],
stats[dataset.name + " muc recall"],
stats[dataset.name + " muc"],
stats[dataset.name + " b3 precision"],
stats[dataset.name + " b3 recall"],
stats[dataset.name + " b3"],
stats[dataset.name + " ceafe precision"],
stats[dataset.name + " ceafe recall"],
stats[dataset.name + " ceafe"],
stats[dataset.name + " conll"],
]))
