def test_works_with_word_and_character_tokenizer(self):
answer_options_simple = ("a<>a sentence<><>")
background_simple = ("a<>a sentence<><>")
line_simple = "\t".join(
str(x) for x in [answer_options_simple, background_simple, "0"])
TextInstance.tokenizer = WordAndCharacterTokenizer(Params({}))
data_indexer = DataIndexer()
a_word_index = data_indexer.add_word_to_index("a", namespace='words')
sentence_index = data_indexer.add_word_to_index("sentence",
namespace='words')
a_index = data_indexer.add_word_to_index("a", namespace='characters')
s_index = data_indexer.add_word_to_index("s", namespace='characters')
e_index = data_indexer.add_word_to_index("e", namespace='characters')
new_instance = TupleInferenceInstance.read_from_line(line_simple)
indexed = new_instance.to_indexed_instance(data_indexer)
padding_lengths = {
'num_question_tuples': 1,
'num_background_tuples': 1,
'num_slots': 2,
'num_sentence_words': 2,
'num_options': 1,
'num_word_characters': 3
}
indexed.pad(padding_lengths)
expected_indexed_tuple = [[[0, 0, 0], [a_word_index, a_index, 0]],
[[a_word_index, a_index, 0],
[sentence_index, s_index, e_index]]]
expected_answers_indexed = numpy.asarray([expected_indexed_tuple])
expected_background_indexed = numpy.asarray(expected_indexed_tuple)
assert numpy.all(indexed.answers_indexed == expected_answers_indexed)
assert numpy.all(
indexed.background_indexed == expected_background_indexed)
TextInstance.tokenizer = tokenizers['words'](Params({}))
def test_trains_and_loads_correctly(self):
self.write_span_prediction_files()
args = Params({
'model_serialization_prefix': self.TEST_DIR + "_bidaf",
'embeddings': {"words": {'dimension': 4}, "characters": {'dimension': 4}},
'save_models': True,
'tokenizer': {'type': 'words and characters'},
'show_summary_with_masking_info': True,
})
bidaf_model = self.get_model(BidirectionalAttentionFlow, args)
bidaf_model.train()
K.clear_session()
bidaf_model_params = self.get_model_params(BidirectionalAttentionFlow, args)
args = Params({
'bidaf_params': bidaf_model_params,
'train_bidaf': False,
'similarity_function': {'type': 'linear', 'combination': 'x,y'},
})
self.write_who_did_what_files()
model, _ = self.ensure_model_trains_and_loads(MultipleChoiceBidaf, args)
# All of the params come from the linear similarity function in the attention layer,
# because we set `train_bidaf` to `False`. 41 comes from 32 + 8 + 1, where 32 is from the
# modeled passage (see the equations in the paper for why it's 32), 8 is from the Bi-LSTM
# operating on the answer options (embedding_dim * 2), and 1 is from the bias.
assert sum([K.count_params(p) for p in model.model.trainable_weights]) == 41
def test_to_indexed_instance_converts_correctly(self):
data_indexer = DataIndexer()
sentence_index = data_indexer.add_word_to_index("sentence",
namespace='words')
capital_a_index = data_indexer.add_word_to_index("A",
namespace='words')
space_index = data_indexer.add_word_to_index(" ", namespace='words')
a_index = data_indexer.add_word_to_index("a", namespace='words')
s_index = data_indexer.add_word_to_index("s", namespace='words')
e_index = data_indexer.add_word_to_index("e", namespace='words')
n_index = data_indexer.add_word_to_index("n", namespace='words')
t_index = data_indexer.add_word_to_index("t", namespace='words')
c_index = data_indexer.add_word_to_index("c", namespace='words')
a_char_index = data_indexer.add_word_to_index("a",
namespace='characters')
s_char_index = data_indexer.add_word_to_index("s",
namespace='characters')
e_char_index = data_indexer.add_word_to_index("e",
namespace='characters')
n_char_index = data_indexer.add_word_to_index("n",
namespace='characters')
t_char_index = data_indexer.add_word_to_index("t",
namespace='characters')
c_char_index = data_indexer.add_word_to_index("c",
namespace='characters')
instance = TextClassificationInstance(
"A sentence", None).to_indexed_instance(data_indexer)
assert instance.word_indices == [a_index, sentence_index]
TextInstance.tokenizer = tokenizers['characters'](Params({}))
instance = TextClassificationInstance(
"A sentence", None).to_indexed_instance(data_indexer)
assert instance.word_indices == [
capital_a_index, space_index, s_index, e_index, n_index, t_index,
e_index, n_index, c_index, e_index
]
TextInstance.tokenizer = tokenizers['words and characters'](Params({}))
instance = TextClassificationInstance(
"A sentence", None).to_indexed_instance(data_indexer)
assert instance.word_indices == [[a_index, a_char_index],
[
sentence_index, s_char_index,
e_char_index, n_char_index,
t_char_index, e_char_index,
n_char_index, c_char_index,
e_char_index
]]
def main():
if len(sys.argv) != 2:
print('USAGE: run_model.py [param_file]')
sys.exit(-1)
log_keras_version_info()
param_file = sys.argv[1]
param_dict = pyhocon.ConfigFactory.parse_file(param_file)
params = Params(replace_none(param_dict))
log_dir = params.get("model_serialization_prefix", None) # pylint: disable=no-member
if log_dir is not None:
sys.stdout = TeeLogger(log_dir + "_stdout.log", sys.stdout)
sys.stderr = TeeLogger(log_dir + "_stderr.log", sys.stderr)
handler = logging.FileHandler(log_dir + "_python_logging.log")
handler.setLevel(logging.INFO)
handler.setFormatter(
logging.Formatter(
'%(asctime)s - %(levelname)s - %(name)s - %(message)s'))
logging.getLogger().addHandler(handler)
shutil.copyfile(param_file, log_dir + "_model_params.json")
model_type = params.pop_choice('model_class', concrete_models.keys())
model_class = concrete_models[model_type]
model = model_class(params)
if model.can_train():
logger.info("Training model")
model.train()
else:
logger.info(
"Not enough training inputs. Assuming you wanted to load a model instead."
)
# TODO(matt): figure out a way to specify which epoch you want to load a model from.
model.load_model()
K.clear_session()
def check_experiment_type_can_train(self, param_file):
param_dict = pyhocon.ConfigFactory.parse_file(param_file)
params = Params(replace_none(param_dict))
model_class = concrete_models[params.pop("model_class")]
# Tests will try to create root directories as we have /net/efs paths,
# so we just remove the serialisation aspect here, alter the train/validation
# paths to the dummy test ones and make sure we only do one epoch to
# speed things up.
params["model_serialization_prefix"] = None
if len(params["train_files"]) > 1:
params["train_files"] = [self.TRAIN_FILE, self.TRAIN_BACKGROUND]
params["validation_files"] = [
self.VALIDATION_FILE, self.VALIDATION_BACKGROUND
]
else:
params["train_files"] = [self.TRAIN_FILE]
params["validation_files"] = [self.TRAIN_FILE]
params["num_epochs"] = 1
try:
if params["embeddings"]["words"]["pretrained_file"]:
params["embeddings"]["words"][
"pretrained_file"] = self.PRETRAINED_VECTORS_GZIP
except KeyError:
# No embedding/words field passed in the parameters,
# so nothing to change.
pass
model = self.get_model(model_class, params)
model.train()
def test_end_to_end_conversion_to_arrays_with_word_and_character_tokenizer(
self):
TextInstance.tokenizer = tokenizers['words and characters'](Params({}))
instance = SentenceInstance("this is a sentence")
indexed_instance = instance.to_indexed_instance(self.data_indexer)
indexed_instance.pad({
'num_sentence_words': 6,
'num_word_characters': 5
})
word_array, label_array = indexed_instance.as_training_data()
assert_array_equal(word_array, [
[0, 0, 0, 0, 0],
[self.start_index, 0, 0, 0, 0],
[
self.this_index, self.t_char_index, self.h_char_index,
self.i_char_index, self.s_char_index
],
[self.is_index, self.i_char_index, self.s_char_index, 0, 0],
[self.a_index, self.a_char_index, 0, 0, 0],
[
self.sentence_index, self.s_char_index, self.e_char_index,
self.n_char_index, self.t_char_index
],
])
assert_array_equal(
label_array,
[[0], [self.this_index], [self.is_index], [self.a_index],
[self.sentence_index], [self.end_index]])
def get_model_params(self, model_class, additional_arguments=None):
params = Params({})
params['save_models'] = False
params['model_serialization_prefix'] = self.TEST_DIR
params['train_files'] = [self.TRAIN_FILE]
params['validation_files'] = [self.VALIDATION_FILE]
params['embeddings'] = {
'words': {
'dimension': 6
},
'characters': {
'dimension': 2
}
}
params['encoder'] = {"default": {'type': 'bow'}}
params['num_epochs'] = 1
params['validation_split'] = 0.0
if self.is_model_with_background(model_class):
# pylint: disable=no-member
params['train_files'].append(self.TRAIN_BACKGROUND)
params['validation_files'].append(self.VALIDATION_BACKGROUND)
# pylint: enable=no-member
if self.is_memory_network(model_class):
params['knowledge_selector'] = {'type': 'dot_product'}
params['memory_updater'] = {'type': 'sum'}
params['entailment_input_combiner'] = {'type': 'memory_only'}
if additional_arguments:
for key, value in additional_arguments.items():
params[key] = deepcopy(value)
return params
def test_cloze_train_does_not_crash(self):
self.write_who_did_what_files()
args = Params({
"qd_common_feature": True,
"gating_function": "+",
"cloze_token": "xxxxx",
"num_gated_attention_layers": 2,
"tokenizer": {
"type": "words and characters"
},
"encoder": {
"word": {
"type": "bi_gru",
"units": 2,
}
},
"seq2seq_encoder": {
"question_0": {
"type": "bi_gru",
"encoder_params": {
"units": 3
},
"wrapper_params": {}
},
"document_0": {
"type": "bi_gru",
"encoder_params": {
"units": 3
},
"wrapper_params": {}
},
"document_final": {
"type": "bi_gru",
"encoder_params": {
"units": 3
},
"wrapper_params": {}
},
"question_final": {
"type": "bi_gru",
"encoder_params": {
"units": 3
},
"wrapper_params": {
"merge_mode": None
}
}
},
})
model, loaded_model = self.ensure_model_trains_and_loads(
GatedAttentionReader, args)
# verify that the gated attention function was set properly
assert model.gating_function == "+"
assert model.gating_function == model.model.get_layer(
"gated_attention_0").gating_function
# verify that the gated attention function was set properly in the loaded model
assert loaded_model.gating_function == "+"
assert loaded_model.gating_function == loaded_model.model.get_layer(
"gated_attention_0").gating_function
def test_loss_function_uses_mask(self):
# We're going to make sure that the loss and accuracy computations are the same for any
# permutation of labels on padded tokens. If not, the loss/accuracy function is paying
# attention to the labels when it shouldn't be. We're not going to test for any particular
# accuracy value, just that all of them are the same - I ran this a few times by hand to be
# sure that we're getting different accuracy values, depending on the initialization.
self.write_sequence_tagging_files()
args = Params({
'show_summary_with_masking_info': True,
'instance_type': 'PreTokenizedTaggingInstance',
'tokenizer': {'processor': {'word_splitter': 'no_op'}},
})
model = self.get_model(SimpleTagger, args)
model.train()
input_indices = [3, 2, 0, 0]
labels = [[[0, 1], [1, 0], [1, 0], [1, 0]],
[[0, 1], [1, 0], [1, 0], [0, 1]],
[[0, 1], [1, 0], [0, 1], [1, 0]],
[[0, 1], [1, 0], [0, 1], [0, 1]]]
results = [model.model.evaluate(numpy.asarray([input_indices]), numpy.asarray([label]))
for label in labels]
loss, accuracy = zip(*results)
assert len(set(loss)) == 1
assert len(set(accuracy)) == 1
def test_tf_and_keras_optimise_identical_variables(self):
self.write_memory_network_files()
# Make sure that the variables designated as trainable by tensorflow
# are the same as those designated as trainable by Keras. These could
# not be equal, for instance, if we manage to use a Keras layer without
# calling layer.build(). This is very hard to do, but because this solver
# mixes in some tensorflow, which does not have a "build" equivalent
# (in that it will happily train a variable which is in a layer which
# hasn't been built) we check this.
# Create a new tf session to avoid variables created in other tests affecting this.
K.clear_session()
# Add in a layer which is within the adaptive memory step which actually has
# parameters.
args = Params({
'recurrence_mode': {'type': 'adaptive'},
'knowledge_selector': {'type': 'parameterized'}
})
solver = self.get_model(MemoryNetwork, args)
solver.training_dataset = solver.load_dataset_from_files(solver.train_files)
solver.set_model_state_from_dataset(solver.training_dataset)
indexed_dataset = solver.training_dataset.to_indexed_dataset(solver.data_indexer)
solver.set_model_state_from_indexed_dataset(indexed_dataset)
model = solver._build_model()
tf_trainable_variables = tf.trainable_variables()
keras_trainable_variables = model.trainable_weights
assert [x.name for x in tf_trainable_variables] == [x.name for x in keras_trainable_variables]
def test_padding_works_correctly(self, _output_debug_info):
self.write_question_answer_memory_network_files()
args = Params({
'num_hidden_layers': 1,
'hidden_layer_width': 2,
'show_summary_with_masking_info': True,
'debug': {
'data': 'training',
'layer_names': ['answer_similarity_softmax']
}
})
model = self.get_model(QuestionAnswerSimilarity, args)
def new_debug(output_dict, epoch): # pylint: disable=unused-argument
# We're going to check in here that the attentions and so on are properly masked. In
# particular, we'll check two things: (1) that the final answer option softmax has
# correctly padded out the extra option, and (2) that the attention weights on all of
# the inputs are properly padded. To see that this test is correct, you have to look
# at the actual file written in `write_multiple_true_false_memory_network_files()`.
print(output_dict)
answer_scores = output_dict['answer_similarity_softmax']
assert answer_scores[0][2] == 0
assert answer_scores[1][2] == 0
assert answer_scores[3][2] == 0
_output_debug_info.side_effect = new_debug
model.train()
def test_trains_and_loads_correctly(self):
self.write_frame_cloze_files()
pre_ = "/Users/nikett/Documents/work/code/thirdparty/deepqa/deep_qa/datasets-pulled/glove.6B.100d.txt.gz"
args = Params({
'save_models': True,
'show_summary_with_masking_info': True,
'instance_type': 'FrameEmbeddedLabelInstance',
'validation_metric': 'val_loss',
# 'model_serialization_prefix': '/Users/nikett/TEMP/',
'loss': 'mean_squared_error',
# 'num_slots': 27, TODO: why? -> "Extra parameters passed to Trainer: {'num_slots': 27}"
"embeddings": {
"words": {
"dimension": 100,
"pretrained_embeddings_file": pre_
},
"characters": {
"dimension": 8
}
},
'tokenizer': {
'processor': {
'word_splitter': 'simple'
}
},
})
self.ensure_model_trains_and_loads(FrameClozeModel, args)
def test_load_model_and_fit(self):
args = Params({
'test_files': [self.TEST_FILE],
'embedding_dim': {
'words': 4,
'characters': 2
},
'save_models': True,
'tokenizer': {
'type': 'words and characters'
},
'show_summary_with_masking_info': True,
})
self.write_true_false_model_files()
model, loaded_model = self.ensure_model_trains_and_loads(
ClassificationModel, args)
# now fit both models on some more data, and ensure that we get the same results.
self.write_additional_true_false_model_files()
_, training_arrays = loaded_model.load_data_arrays(
loaded_model.train_files)
model.model.fit(training_arrays[0],
training_arrays[1],
shuffle=False,
nb_epoch=1)
loaded_model.model.fit(training_arrays[0],
training_arrays[1],
shuffle=False,
nb_epoch=1)
def test_trains_and_loads_correctly(self):
self.write_span_prediction_files()
args = Params({
'embeddings': {
'words': {
'dimension': 8
},
'characters': {
'dimension': 4
}
},
'save_models': True,
'tokenizer': {
'type': 'words and characters'
},
'show_summary_with_masking_info': True,
})
model, _ = self.ensure_model_trains_and_loads(
BidirectionalAttentionFlow, args)
for layer in model.model.layers:
if layer.name == 'characters_embedding':
assert layer.get_output_shape_at(0)[-1] == 4
break
else:
assert False, "couldn't find character embedding layer"
def test_pretrained_embeddings_works_correctly(self):
self.write_true_false_model_files()
self.write_pretrained_vector_files()
args = Params({
'embeddings': {'words': {'dimension': 8, 'pretrained_file': self.PRETRAINED_VECTORS_GZIP},
'characters': {'dimension': 8}},
})
model = self.get_model(ClassificationModel, args)
model.train()
def test_passes_through_correctly(self):
word_processor = WordProcessor(Params({}))
sentence = "this (sentence) has 'crazy' \"punctuation\"."
tokens = word_processor.get_tokens(sentence)
expected_tokens = [
"this", "(", "sentence", ")", "has", "'", "crazy", "'", "\"",
"punctuation", "\"", "."
]
assert tokens == expected_tokens
def test_initialize_lsh_does_not_crash(self):
args = Params({
'corpus_path': self.corpus_path,
'model_serialization_prefix': './',
'num_sentence_words': 3,
})
model = self.get_model(DifferentiableSearchMemoryNetwork, args)
model.encoder_model = FakeEncoder()
model._initialize_lsh()
def test_trains_and_loads_correctly(self):
self.write_span_prediction_files()
args = Params({
'embedding_dim': {'words': 4, 'characters': 4},
'save_models': True,
'tokenizer': {'type': 'words and characters'},
'show_summary_with_masking_info': True,
})
self.ensure_model_trains_and_loads(BidirectionalAttentionFlow, args)
def test_read_from_file(self):
args = Params({"sequence_length": 4})
dataset = LanguageModelingDataset.read_from_file(
self.TRAIN_FILE, SentenceInstance, args)
instances = dataset.instances
assert instances[0].text == "This is a sentence"
assert instances[1].text == "for language modelling. Here's"
assert instances[2].text == "another one for language"
def test_trains_and_loads_correctly(self):
self.write_sequence_tagging_files()
args = Params({
'save_models': True,
'show_summary_with_masking_info': True,
'instance_type': 'PreTokenizedTaggingInstance',
'tokenizer': {'processor': {'word_splitter': 'no_op'}},
})
self.ensure_model_trains_and_loads(SimpleTagger, args)
def test_words_tokenizes_the_sentence_correctly(self):
t = TextClassificationInstance("This is a sentence.", None)
assert t.words() == {'words': ['this', 'is', 'a', 'sentence', '.']}
TextInstance.tokenizer = tokenizers['characters'](Params({}))
assert t.words() == {
'words': [
'T', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 's', 'e',
'n', 't', 'e', 'n', 'c', 'e', '.'
]
}
TextInstance.tokenizer = tokenizers['words and characters'](Params({}))
assert t.words() == {
'words': ['this', 'is', 'a', 'sentence', '.'],
'characters': [
't', 'h', 'i', 's', 'i', 's', 'a', 's', 'e', 'n', 't', 'e',
'n', 'c', 'e', '.'
]
}
def setUp(self):
super(TestPreTokenizedTaggingInstance, self).setUp()
tokens = ["cats", "are", "animals", "."]
tags = ["N", "V", "N", "."]
self.instance = PreTokenizedTaggingInstance(tokens, tags)
TextInstance.tokenizer = tokenizers['words'](Params(
{'processor': {
'word_splitter': 'no_op'
}}))
def test_train_does_not_crash_with_parameterized_heuristic_matching_knowledge_selector(
self):
args = Params({
'knowledge_selector': {
'type': 'parameterized_heuristic_matching'
}
})
model = self.get_model(MemoryNetwork, args)
model.train()
def setUp(self):
super(TestMultiGpu, self).setUp()
self.write_true_false_model_files()
self.args = Params({
'embedding_dim': {'words': 4, 'characters': 2},
'batch_size': 8,
'num_gpus': 2,
'save_models': True,
'show_summary_with_masking_info': True,
})
def test_stems_and_filters_correctly(self):
word_processor = WordProcessor(
Params({
'word_stemmer': 'porter',
'word_filter': 'stopwords'
}))
sentence = "this (sentence) has 'crazy' \"punctuation\"."
expected_tokens = ["sentenc", "ha", "crazi", "punctuat"]
tokens = word_processor.get_tokens(sentence)
assert tokens == expected_tokens
def test_biggest_batch_first(self):
params = Params({
'padding_noise': 0.0,
'dynamic_padding': True,
'biggest_batch_first': True,
})
generator = DataGenerator(self.text_trainer, params)
batches = generator.create_generator(IndexedDataset(self.instances))
biggest_batches = [next(batches) for _ in range(2)]
assert self.as_list(biggest_batches[0][0]) == [3]
assert self.as_list(biggest_batches[1][0]) == [1, 0, 4]
def test_biggest_batch_first(self):
params = Params({
'dynamic_padding': True,
'biggest_batch_first': True,
})
generator = DataGenerator(self.text_trainer, params)
batches = generator.create_generator(IndexedDataset(self.instances))
biggest_batches = [next(batches) for _ in range(2)]
assert_array_equal(biggest_batches[0][0], numpy.asarray([[3]]))
assert_array_equal(biggest_batches[1][0], numpy.asarray([[1], [0],
[4]]))
def test_dynamic_padding_works(self):
args = Params({
'test_files': [self.TEST_FILE],
'embeddings': {'words': {'dimension': 4}, 'characters': {'dimension': 2}},
'save_models': True,
'tokenizer': {'type': 'words and characters'},
'data_generator': {'dynamic_padding': True},
'batch_size': 2,
})
self.write_true_false_model_files()
self.ensure_model_trains_and_loads(ClassificationModel, args)
def test_data_generator_works(self):
args = Params({
'test_files': [self.TEST_FILE],
'embeddings': {'words': {'dimension': 4}, 'characters': {'dimension': 2}},
'save_models': True,
'tokenizer': {'type': 'words and characters'},
'data_generator': {},
'show_summary_with_masking_info': True,
})
self.write_true_false_model_files()
self.ensure_model_trains_and_loads(ClassificationModel, args)
def test_words_and_characters_works_with_matrices(self,
_output_debug_info):
self.write_question_answer_memory_network_files()
args = Params({
'embedding_dim': {
'words': 2,
'characters': 2
},
'tokenizer': {
'type': 'words and characters'
},
'debug': {
'data': 'training',
'layer_names': [
'combined_word_embedding_for_answer_input',
],
'masks': [
'combined_word_embedding_for_answer_input',
],
}
})
model = self.get_model(QuestionAnswerSimilarity, args)
def new_debug(output_dict, epoch): # pylint: disable=unused-argument
# We're going to check two things in here: that the shape of combined word embedding is
# as expected, and that the mask is computed correctly.
# TODO(matt): actually, from this test, it looks like the mask is returned as
# output_dict['combined_word_embedding'][1]. Maybe this means we can simplify the
# logic in Trainer._debug()? I need to look into this more to be sure that's
# consistently happening, though.
word_embeddings = output_dict[
'combined_word_embedding_for_answer_input'][0]
assert len(word_embeddings) == 4
assert word_embeddings[0].shape == (3, 2, 4)
word_masks = output_dict[
'combined_word_embedding_for_answer_input'][1]
# Zeros are added to answer words _from the left_, and to answer options from the
# _right_.
assert numpy.all(word_masks[0, 0, :] == numpy.asarray([1, 1]))
assert numpy.all(word_masks[0, 1, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[0, 2, :] == numpy.asarray([0, 0]))
assert numpy.all(word_masks[1, 0, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[1, 1, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[1, 2, :] == numpy.asarray([0, 0]))
assert numpy.all(word_masks[2, 0, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[2, 1, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[2, 2, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[3, 0, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[3, 1, :] == numpy.asarray([0, 1]))
assert numpy.all(word_masks[3, 2, :] == numpy.asarray([0, 0]))
_output_debug_info.side_effect = new_debug
model.train()