def create_from_config(config, devices, dropout=0.0, reuse=False): """ :param config: dictionary of parameters for creating an autoreader :return: """ if "start_output_unit" not in config: config["start_output_unit"] = "softmax" from biomedical_qa.models import model_from_config embedder = model_from_config(config["transfer_model"], devices) qa_model = QASimplePointerModel( config["size"], embedder=embedder, name=config["name"], composition=config["composition"], keep_prob=1.0 - dropout, devices=devices, with_features=config["with_features"], with_inter_fusion=config["with_inter_fusion"], num_intrafusion_layers=config["num_intrafusion_layers"], layer_norm=config.get("layer_norm", False), start_output_unit=config["start_output_unit"], with_yesno=config.get("with_yesno", False), with_question_type_features=config.get("with_question_type_features", False), with_entity_tag_features=config.get("with_entity_tag_features", False)) return qa_model
def create_from_config(config, devices, dropout=0.0, inputs=None, seq_lengths=None, reuse=False, **kwargs): """ :param config: dictionary of parameters for creating an embedder :return: """ from biomedical_qa.models import model_from_config first_embedder = model_from_config(config["embedders"][0], devices, dropout, inputs, seq_lengths, reuse) inputs = first_embedder.inputs seq_lengths = first_embedder.seq_lengths embedder = ConcatEmbedder([first_embedder] + [ model_from_config(ce, devices, dropout, inputs, seq_lengths, reuse) for ce in config["embedders"][1:] ]) return embedder
}) return BioAsqSampler(**args) else: return SQuADSampler(**args) with tf.Session(config=config) as sess: devices = FLAGS.devices.split(",") train_variable_prefixes = FLAGS.train_variable_prefixes.split(",") \ if FLAGS.train_variable_prefixes else [] if FLAGS.model_config is not None: with open(FLAGS.model_config, 'rb') as f: model_config = pickle.load(f) model = model_from_config(model_config, devices, FLAGS.dropout) else: print("Creating transfer model from config %s" % FLAGS.transfer_model_config) with open(FLAGS.transfer_model_config, 'rb') as f: transfer_model_config = pickle.load(f) transfer_model = model_from_config(transfer_model_config, devices[0:1]) original_transfer_model = transfer_model if FLAGS.with_chars: print("Use additional char-based word-embedder") char_embedder = CharWordEmbedder(FLAGS.size, transfer_model.vocab, devices[0]) FLAGS.embedder_lr = FLAGS.learning_rate transfer_model = ConcatEmbedder([transfer_model, char_embedder])
tf.app.flags.DEFINE_string('model_path', None, 'path of model.') tf.app.flags.DEFINE_string('device', "/cpu:0", 'Device to use.') tf.app.flags.DEFINE_string('out', "results.json", 'Result file path.') tf.app.flags.DEFINE_integer('batch_size', 64, 'batch size.') FLAGS = tf.app.flags.FLAGS config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True results = {} with tf.Session(config=config) as sess: with open(FLAGS.model_config, "rb") as f: config = pickle.load(f) model = model_from_config(config, [FLAGS.device]) sess.run(tf.global_variables_initializer()) model.model_saver.restore(sess, FLAGS.model_path) model.set_eval(sess) sampler = SQuADSampler(FLAGS.dir, [FLAGS.file], FLAGS.batch_size, model.embedder.vocab) with open(os.path.join(FLAGS.dir, FLAGS.file)) as dataset_file: squad = json.load(dataset_file)["data"] question2real_context = {} for article in squad: for paragraph in article["paragraphs"]: context = paragraph["context"]
def load_embedding_model(sess, config_file, model_file): with open(config_file, 'rb') as f: config = pickle.load(f) embedder = model_from_config(config) embedder.model_saver.restore(sess, model_file) return embedder