def __init__(self,
dir,
filenames,
batch_size,
vocab,
instances_per_epoch=None,
shuffle=True,
dataset_json=None,
types=None,
split_contexts_on_newline=False,
context_token_limit=-1,
include_synonyms=False,
tagger=None,
include_answer_spans=True):
if dataset_json is None:
# load json
with open(os.path.join(dir, filenames[0])) as dataset_file:
dataset_json = json.load(dataset_file)
squad_builder = BioAsqSquadBuilder(
dataset_json,
types=types,
context_token_limit=context_token_limit,
include_synonyms=include_synonyms,
include_answer_spans=include_answer_spans)
squad_json = squad_builder.build().get_result_object()
SQuADSampler.__init__(
self,
None,
None,
batch_size,
vocab,
instances_per_epoch=instances_per_epoch,
shuffle=shuffle,
types=types,
split_contexts_on_newline=split_contexts_on_newline,
dataset_json=squad_json,
tagger=tagger)
def make_sampler(dir, filenames, vocab, types, tagger):
args = {
"dir": dir,
"filenames": filenames,
"batch_size": FLAGS.batch_size,
"vocab": vocab,
"instances_per_epoch": FLAGS.max_instances,
"split_contexts_on_newline": FLAGS.split_contexts,
"types": types,
"tagger": tagger,
}
if FLAGS.is_bioasq:
args.update({
"context_token_limit": FLAGS.bioasq_context_token_limit,
"include_synonyms": FLAGS.bioasq_include_synonyms,
})
return BioAsqSampler(**args)
else:
return SQuADSampler(**args)
def predict():
data = request.data.decode("utf-8")
bioasq_json = json.loads(data)
bioasq_json = clean_bioasq_json(bioasq_json)
squad_json = BioAsqSquadBuilder(bioasq_json, include_answer_spans=False) \
.build().get_result_object()
sampler = SQuADSampler(None,
None,
FLAGS.batch_size,
inferrer.models[0].embedder.vocab,
shuffle=False,
dataset_json=squad_json,
tagger=tagger)
answers = inferrer.get_predictions(sampler)
bioasq_json = insert_answers(bioasq_json, answers,
FLAGS.list_answer_prob_threshold,
FLAGS.preferred_terms, FLAGS.terms_file)
return json.dumps(bioasq_json, indent=2)
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
results = {}
with tf.Session(config=config) as sess:
with open(FLAGS.model_config, "rb") as f:
config = pickle.load(f)
model = model_from_config(config, [FLAGS.device])
sess.run(tf.global_variables_initializer())
model.model_saver.restore(sess, FLAGS.model_path)
model.set_eval(sess)
sampler = SQuADSampler(FLAGS.dir, [FLAGS.file], FLAGS.batch_size,
model.embedder.vocab)
with open(os.path.join(FLAGS.dir, FLAGS.file)) as dataset_file:
squad = json.load(dataset_file)["data"]
question2real_context = {}
for article in squad:
for paragraph in article["paragraphs"]:
context = paragraph["context"]
for qa in paragraph["qas"]:
question2real_context[qa["id"]] = context
counter = 0
while sampler.epoch == 0:
batch = sampler.get_batch()
context_indices, starts, ends = model.run(sess, [
model.predicted_context_indices, model.predicted_answer_starts,
return bioasq_json, squad_json
if __name__ == "__main__":
devices = FLAGS.devices.split(",")
sess = get_session()
models = [get_model(sess, config, devices, scope="model_%d" % i)
for i, config in enumerate(FLAGS.model_config.split(","))]
inferrer = Inferrer(models, sess, FLAGS.beam_size)
# Build sampler from dataset JSON
bioasq_json, squad_json = load_dataset(FLAGS.bioasq_file)
tagger = get_entity_tagger()
sampler = SQuADSampler(None, None, FLAGS.batch_size,
inferrer.models[0].embedder.vocab,
shuffle=False, dataset_json=squad_json,
tagger=tagger)
contexts = {p["qas"][0]["id"] : p["context_original_capitalization"]
for p in squad_json["data"][0]["paragraphs"]}
answers = inferrer.get_predictions(sampler)
bioasq_json = insert_answers(bioasq_json, answers, FLAGS.list_answer_prob_threshold,
FLAGS.preferred_terms, FLAGS.terms_file)
os.makedirs(os.path.dirname(FLAGS.out_file), exist_ok=True)
with open(FLAGS.out_file, "w") as f:
json.dump(bioasq_json, f, indent=2, sort_keys=True)
def main():
devices = FLAGS.devices.split(",")
sess = get_session()
models = [
get_model(sess, config, devices, scope="model_%d" % i)
for i, config in enumerate(FLAGS.model_config.split(","))
]
inferrer = Inferrer(models, sess, FLAGS.beam_size)
print("Initializing Sampler & Trainer...")
data_dir = os.path.dirname(FLAGS.eval_data)
data_filename = os.path.basename(FLAGS.eval_data)
instances = FLAGS.subsample if FLAGS.subsample > 0 else None
tagger = get_entity_tagger()
list_sampler = None
if not FLAGS.is_bioasq:
sampler = SQuADSampler(data_dir, [data_filename],
FLAGS.batch_size,
models[0].embedder.vocab,
instances_per_epoch=instances,
shuffle=False,
split_contexts_on_newline=FLAGS.split_contexts,
tagger=tagger)
else:
sampler = BioAsqSampler(
data_dir, [data_filename],
FLAGS.batch_size,
models[0].embedder.vocab,
instances_per_epoch=instances,
shuffle=False,
split_contexts_on_newline=FLAGS.split_contexts,
context_token_limit=FLAGS.bioasq_context_token_limit,
include_synonyms=FLAGS.bioasq_include_synonyms,
tagger=tagger,
include_answer_spans=False)
list_sampler = BioAsqSampler(
data_dir, [data_filename],
FLAGS.batch_size,
models[0].embedder.vocab,
types=["list"],
instances_per_epoch=instances,
shuffle=False,
split_contexts_on_newline=FLAGS.split_contexts,
context_token_limit=FLAGS.bioasq_context_token_limit,
include_synonyms=FLAGS.bioasq_include_synonyms,
tagger=tagger,
include_answer_spans=False)
if FLAGS.squad_evaluation:
print("Running SQuAD Evaluation...")
trainer = ExtractionGoalDefiner(models[0], devices[0])
trainer.eval(sess, sampler, verbose=True)
list_answer_prob_threshold = FLAGS.list_answer_prob_threshold
list_answer_count = FLAGS.list_answer_count
terms_file = FLAGS.terms_file if FLAGS.preferred_terms else None
if FLAGS.find_optimal_threshold:
evaluator = BioAsqEvaluator(list_sampler, inferrer, terms_file)
list_answer_prob_threshold, _ = evaluator.find_optimal_threshold(
FLAGS.threshold_search_step,
verbosity_level=2 if FLAGS.verbose else 1)
if FLAGS.find_optimal_answer_count:
evaluator = BioAsqEvaluator(list_sampler, inferrer, terms_file)
list_answer_count, _ = evaluator.find_optimal_answer_count(
verbosity_level=2 if FLAGS.verbose else 1)
if FLAGS.bioasq_evaluation:
print("Running BioASQ Evaluation...")
evaluator = BioAsqEvaluator(sampler, inferrer, terms_file)
evaluator.evaluate(
verbosity_level=2 if FLAGS.verbose else 1,
list_answer_count=list_answer_count,
list_answer_prob_threshold=list_answer_prob_threshold)
if FLAGS.find_perfect_cutoff:
evaluator = BioAsqEvaluator(list_sampler, inferrer, terms_file)
evaluator.evaluate(verbosity_level=2 if FLAGS.verbose else 1)