def modular_assertion_definition_qa_reader(resources_or_conf=None):
from projects.knowledge_integration.qa.definition_model import XQAAssertionDefinitionInputModule
from projects.knowledge_integration.qa.definition_model import ModularAssertionDefinitionQAModel
from jack.readers.extractive_qa.shared import XQAOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = XQAAssertionDefinitionInputModule(shared_resources)
model_module = ModularAssertionDefinitionQAModel(shared_resources)
output_module = XQAOutputModule()
reader = TFReader(shared_resources, input_module, model_module,
output_module)
input_module.set_reader(reader)
return TFReader(shared_resources, input_module, model_module,
output_module)
def _tf_extractive_qa_reader(model_module_constructor,
resources_or_conf: Union[dict, SharedResources]):
from jack.readers.extractive_qa.shared import XQAInputModule, XQAOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = XQAInputModule(shared_resources)
model_module = model_module_constructor(shared_resources)
output_module = XQAOutputModule()
return TFReader(shared_resources, input_module, model_module,
output_module)
def modelf_reader(resources_or_conf: Union[dict, SharedResources] = None):
"""Creates a knowledge_base_population model F."""
from jack.readers.knowledge_base_population.model_f import ModelFInputModule, ModelFModelModule, ModelFOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = ModelFInputModule(shared_resources)
model_module = ModelFModelModule(shared_resources)
output_module = ModelFOutputModule()
return TFReader(shared_resources, input_module, model_module,
output_module)
def _tf_nli_reader(model_module_constructor,
resources_or_conf: Union[dict, SharedResources] = None):
from jack.readers.classification.shared import ClassificationSingleSupportInputModule
from jack.readers.classification.shared import SimpleClassificationOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = ClassificationSingleSupportInputModule(shared_resources)
model_module = model_module_constructor(shared_resources)
output_module = SimpleClassificationOutputModule(shared_resources)
return TFReader(shared_resources, input_module, model_module,
output_module)
def cbilstm_nli_assertion_reader(resources_or_conf=None):
from projects.knowledge_integration.nli import NLIAssertionModel
from projects.knowledge_integration.nli import MultipleChoiceAssertionInputModule
from jack.readers.classification.shared import SimpleClassificationOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = MultipleChoiceAssertionInputModule(shared_resources)
model_module = NLIAssertionModel(shared_resources)
output_module = SimpleClassificationOutputModule(shared_resources)
return TFReader(shared_resources, input_module, model_module,
output_module)
def modular_qa_reader(resources_or_conf: Union[dict, SharedResources] = None):
"""Creates a FastQA model as described in https://arxiv.org/abs/1703.04816 (extractive qa model)."""
from jack.readers.extractive_qa.shared import XQAInputModule, XQAOutputModule
from jack.readers.extractive_qa.tensorflow.modular_qa_model import ModularQAModel
shared_resources = create_shared_resources(resources_or_conf)
input_module = XQAInputModule(shared_resources)
model_module = ModularQAModel(shared_resources)
output_module = XQAOutputModule()
return TFReader(shared_resources, input_module, model_module,
output_module)
def fastqa_reader(resources_or_conf: Union[dict, SharedResources] = None):
"""Creates a FastQA reader instance (extractive qa model)."""
from jack.readers.extractive_qa.tensorflow.fastqa import FastQAModule
from jack.readers.extractive_qa.shared import XQAInputModule, XQAOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = XQAInputModule(shared_resources)
model_module = FastQAModule(shared_resources)
output_module = XQAOutputModule()
return TFReader(shared_resources, input_module, model_module,
output_module)
def test_fastqa():
tf.reset_default_graph()
data = load_jack('tests/test_data/squad/snippet_jtr.json')
# fast qa must be initialized with existing embeddings, so we create some
embeddings = load_embeddings('./tests/test_data/glove.840B.300d_top256.txt', 'glove')
# we need a vocabulary (with embeddings for our fastqa_reader, but this is not always necessary)
vocab = Vocab(emb=embeddings, init_from_embeddings=True)
# ... and a config
config = {
"batch_size": 1,
"repr_dim": 10,
"repr_dim_input": embeddings.lookup.shape[1],
"with_char_embeddings": True
}
# create/setup reader
shared_resources = SharedResources(vocab, config)
input_module = XQAInputModule(shared_resources)
model_module = FastQAModule(shared_resources)
output_module = XQAOutputModule()
reader = TFReader(shared_resources, input_module, model_module, output_module)
reader.setup_from_data(data, is_training=True)
loss = reader.model_module.tensors[Ports.loss]
optimizer = tf.train.AdagradOptimizer(learning_rate=0.01)
min_op = optimizer.minimize(loss)
session = model_module.tf_session
session.run(tf.global_variables_initializer())
for epoch in range(0, 10):
for batch in reader.input_module.batch_generator(data, 1, False):
feed_dict = reader.model_module.convert_to_feed_dict(batch)
loss_value, _ = session.run((loss, min_op), feed_dict=feed_dict)
print(loss_value)
def complex_reader(resources_or_conf: Union[dict, SharedResources] = None):
""" Creates a knowledge_base_population Complex model."""
from jack.readers.link_prediction.models import KnowledgeGraphEmbeddingInputModule, \
KnowledgeGraphEmbeddingModelModule, \
KnowledgeGraphEmbeddingOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = KnowledgeGraphEmbeddingInputModule(shared_resources)
model_module = KnowledgeGraphEmbeddingModelModule(shared_resources,
model_name='ComplEx')
output_module = KnowledgeGraphEmbeddingOutputModule()
return TFReader(shared_resources, input_module, model_module,
output_module)
def modular_nli_reader(resources_or_conf: Union[dict, SharedResources] = None):
"""Creates a Modular NLI reader instance. Model defined in config."""
from jack.readers.multiple_choice.shared import MultipleChoiceSingleSupportInputModule
from jack.readers.natural_language_inference.modular_nli_model import ModularNLIModel
from jack.readers.multiple_choice.shared import SimpleMCOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = MultipleChoiceSingleSupportInputModule(shared_resources)
model_module = ModularNLIModel(shared_resources)
output_module = SimpleMCOutputModule(shared_resources)
return TFReader(shared_resources, input_module, model_module,
output_module)
def dam_snli_reader(resources_or_conf: Union[dict, SharedResources] = None):
"""Creates a SNLI reader instance (multiple choice qa model).
This particular reader uses a Decomposable Attention Model, as described in [1].
[1] Ankur P. Parikh et al. - A Decomposable Attention Model for Natural Language Inference. EMNLP 2016
"""
from jack.readers.multiple_choice.shared import MultipleChoiceSingleSupportInputModule
from jack.readers.natural_language_inference.decomposable_attention import DecomposableAttentionModel
from jack.readers.multiple_choice.shared import SimpleMCOutputModule
shared_resources = create_shared_resources(resources_or_conf)
input_module = MultipleChoiceSingleSupportInputModule(shared_resources)
model_module = DecomposableAttentionModel(shared_resources)
output_module = SimpleMCOutputModule(shared_resources)
return TFReader(shared_resources, input_module, model_module,
output_module)