def create_from_config(config, devices, dropout=0.0, reuse=False):
"""
:param config: dictionary of parameters for creating an autoreader
:return:
"""
if "start_output_unit" not in config:
config["start_output_unit"] = "softmax"
from biomedical_qa.models import model_from_config
embedder = model_from_config(config["transfer_model"], devices)
qa_model = QASimplePointerModel(
config["size"],
embedder=embedder,
name=config["name"],
composition=config["composition"],
keep_prob=1.0 - dropout,
devices=devices,
with_features=config["with_features"],
with_inter_fusion=config["with_inter_fusion"],
num_intrafusion_layers=config["num_intrafusion_layers"],
layer_norm=config.get("layer_norm", False),
start_output_unit=config["start_output_unit"],
with_yesno=config.get("with_yesno", False),
with_question_type_features=config.get("with_question_type_features", False),
with_entity_tag_features=config.get("with_entity_tag_features", False))
return qa_model
def create_from_config(config,
devices,
dropout=0.0,
inputs=None,
seq_lengths=None,
reuse=False,
**kwargs):
"""
:param config: dictionary of parameters for creating an embedder
:return:
"""
from biomedical_qa.models import model_from_config
first_embedder = model_from_config(config["embedders"][0], devices,
dropout, inputs, seq_lengths, reuse)
inputs = first_embedder.inputs
seq_lengths = first_embedder.seq_lengths
embedder = ConcatEmbedder([first_embedder] + [
model_from_config(ce, devices, dropout, inputs, seq_lengths, reuse)
for ce in config["embedders"][1:]
])
return embedder
})
return BioAsqSampler(**args)
else:
return SQuADSampler(**args)
with tf.Session(config=config) as sess:
devices = FLAGS.devices.split(",")
train_variable_prefixes = FLAGS.train_variable_prefixes.split(",") \
if FLAGS.train_variable_prefixes else []
if FLAGS.model_config is not None:
with open(FLAGS.model_config, 'rb') as f:
model_config = pickle.load(f)
model = model_from_config(model_config, devices, FLAGS.dropout)
else:
print("Creating transfer model from config %s" %
FLAGS.transfer_model_config)
with open(FLAGS.transfer_model_config, 'rb') as f:
transfer_model_config = pickle.load(f)
transfer_model = model_from_config(transfer_model_config, devices[0:1])
original_transfer_model = transfer_model
if FLAGS.with_chars:
print("Use additional char-based word-embedder")
char_embedder = CharWordEmbedder(FLAGS.size, transfer_model.vocab,
devices[0])
FLAGS.embedder_lr = FLAGS.learning_rate
transfer_model = ConcatEmbedder([transfer_model, char_embedder])
tf.app.flags.DEFINE_string('model_path', None, 'path of model.')
tf.app.flags.DEFINE_string('device', "/cpu:0", 'Device to use.')
tf.app.flags.DEFINE_string('out', "results.json", 'Result file path.')
tf.app.flags.DEFINE_integer('batch_size', 64, 'batch size.')
FLAGS = tf.app.flags.FLAGS
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
results = {}
with tf.Session(config=config) as sess:
with open(FLAGS.model_config, "rb") as f:
config = pickle.load(f)
model = model_from_config(config, [FLAGS.device])
sess.run(tf.global_variables_initializer())
model.model_saver.restore(sess, FLAGS.model_path)
model.set_eval(sess)
sampler = SQuADSampler(FLAGS.dir, [FLAGS.file], FLAGS.batch_size,
model.embedder.vocab)
with open(os.path.join(FLAGS.dir, FLAGS.file)) as dataset_file:
squad = json.load(dataset_file)["data"]
question2real_context = {}
for article in squad:
for paragraph in article["paragraphs"]:
context = paragraph["context"]
def load_embedding_model(sess, config_file, model_file):
with open(config_file, 'rb') as f:
config = pickle.load(f)
embedder = model_from_config(config)
embedder.model_saver.restore(sess, model_file)
return embedder