def predict(config,
concept,
positives,
vocab,
entity_model,
concept_model,
original_model,
val_data,
result=None):
entity_examples = examples_evaluation(config, concept, positives, vocab)
c_token_indices = [[
vocab.get(t.lower(), 1) for t in nltk.word_tokenize(neg)
] for neg in concept.names]
concept_examples = pad_sequences(c_token_indices,
padding='post',
maxlen=config.getint(
'embedding', 'length'))
entity_encodings = entity_model.predict_generator(entity_examples,
steps=len(positives))
concept_encodings = concept_model.predict(concept_examples)
###################
from sample import sped_up_format_x
convoluted_input = sped_up_format_x(entity_encodings, concept_encodings)
entity_encodings = Input(shape=(convoluted_input[0].shape[1], ),
dtype='float32',
name='entity_encodings')
concept_encodings = Input(shape=(convoluted_input[1].shape[1], ),
dtype='float32',
name='concept_encodings')
if sysargv5 == 'sem_matrix':
model = _predict_shared_encoder(original_model, entity_encodings,
concept_encodings)
elif sysargv5 == 'cosine_sim':
model = _predict_shared_encoder_dot(original_model, entity_encodings,
concept_encodings)
test_y = model.predict(convoluted_input)
if not result:
evaluation_parameter = callback.evaluate(val_data.mentions, test_y,
val_data.y)
else:
evaluation_parameter = evaluate_w_results(val_data.mentions, test_y,
val_data.y, concept, result)
return evaluation_parameter
def predict(config, concept, positives, vocab, entity_model, concept_model, original_model,val_data,result=None):
entity_examples = examples_evaluation(config, concept, positives, vocab)
#entity_examples_1 = examples_(config, concept, positives, vocab, neg_count = 0)
#import pdb;pdb.set_trace()
c_token_indices = [[vocab.get(t.lower(), 1) for t in nltk.word_tokenize(neg)] for neg in concept.names]
concept_examples = pad_sequences(c_token_indices, padding='post', maxlen=config.getint('embedding','length'))
entity_encodings = entity_model.predict_generator(entity_examples, steps=len(positives))
concept_encodings = concept_model.predict(concept_examples)
###################
from sample import sped_up_format_x
convoluted_input = sped_up_format_x(entity_encodings,concept_encodings)
layerss = ['v_sem','hidden_layer','prediction_layer']
v_sem = original_model.get_layer(layerss[0])
d2 = original_model.get_layer(layerss[2])
if sysargv2 == 'full':
d1 = original_model.get_layer(layerss[1])
entity_encodings = Input(shape=(convoluted_input[0].shape[1],),dtype='float32', name='entity_encodings')
concept_encodings = Input(shape=(convoluted_input[1].shape[1],),dtype='float32', name='concept_encodings')
sem = cnn.semantic_similarity_layer(weights = v_sem.get_weights())([entity_encodings,concept_encodings])
if sysargv2 == 'full':
concatenate_list = [entity_encodings,concept_encodings,sem]
join_layer = Concatenate()(concatenate_list)
hidden_layer = Dense(d1.units, activation=d1.activation,weights=d1.get_weights())(join_layer)
prediction_layer = Dense(d2.units, activation=d2.activation,weights=d2.get_weights())(hidden_layer)
else:
prediction_layer = Dense(d2.units, activation=d2.activation,weights=d2.get_weights())(sem)
model = Model(inputs=[entity_encodings,concept_encodings], outputs=prediction_layer)
test_y = model.predict(convoluted_input)
if not result:
evaluation_parameter = callback.evaluate(val_data.mentions, test_y, val_data.y)
else:
evaluation_parameter = evaluate_w_results(val_data.mentions, test_y, val_data.y, concept, result)
return evaluation_parameter
def on_epoch_end(self, epoch, logs={}):
self.losses.append(logs.get('loss'))
can_dev_examples = candidates_examples(self.conf, self.candidates,
self.positives)
test_y = self.original_model.predict_generator(can_dev_examples,
steps=len(
self.positives))
evaluation_parameter = callback.evaluate(self.val_data.mentions,
test_y, self.val_data.y)
self.accuracy.append(evaluation_parameter)
with open(self.history, 'a', encoding='utf-8') as f:
f.write(
'Epoch: {0}, Training loss: {1}, validation accuracy: {2}\n'.
format(epoch, logs.get('loss'), evaluation_parameter))
if evaluation_parameter > self.best:
logging.info('Intermediate model saved.')
self.best = evaluation_parameter
self.model.save(self.model_path)
self.wait = 0
# something here to print trec_eval doc
else:
self.wait += 1
if self.wait > int(self.conf['training']['patience']):
self.stopped_epoch = epoch
self.model.stop_training = True
if self.save and self.model.stop_training:
logger.info('Saving predictions to {0}'.format(
self.conf['model']['path_saved_predictions']))
model_tools.save_predictions(
self.conf['model']['path_saved_predictions'],
test_y) #(filename,predictions)
logger.info(
'Testing: epoch: {0}, self.model.stop_training: {1}'.format(
epoch, self.model.stop_training))
return