def evaluation_results(self):
# print("Evaluation----------->:: " + self.dataset.name + " "+str(self.dataset.eval()))
print("evaluating dataset:" + self.dataset.name)
print("with: " + str(len(self.dataset.children)))
res = self.dataset.eval(
) #evaluate_classification(self.dataset.documents)
print(str(res))
if self.bestRes == None or self.bestRes["fscore"] < res["fscore"]:
print("new best F-score: " + str(res["fscore"]))
res["best_epoch"] = self.epoch
self.bestRes = res
utility.writeDictAsStringFile(
res, Defaults.output_path + str(Defaults.index) + ".txt")
ltlib.util.save_keras(
self.model, Defaults.saved_mod_path + str(Defaults.index))
return res
def eval_test(modelPath):
global data
# data = MultiLabelDataReader(Defaults.input_path).load(index)
#model = ltlib.util.load_keras(modelPath)
model.load_weights(modelPath + "model.h5")
optimizer = get_optimizer(Defaults)
print("STARTING TEST")
force_oov = set(l.strip()
for l in open(Defaults.oov)) if Defaults.oov else None
w2v = NormEmbeddingFeature.from_file(Defaults.embedding_path,
max_rank=Defaults.max_vocab_size,
vocabulary=data.vocabulary,
force_oov=force_oov,
name='text')
# Add word vector features to tokens
features = [w2v]
data.tokens.add_features(features)
# Summarize word vector featurizer statistics (OOV etc.)
# logging.info(features[0].summary())
# Create inputs at document level
data.documents.add_inputs([
FixedWidthInput(Defaults.doc_size, f['<PADDING>'], f.name)
for f in features
])
# Create keras input and embedding for each feature
# inputs, embeddings = inputs_and_embeddings(features, Defaults)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy', f1, prec, rec])
predictions = model.predict(data.test.documents.inputs,
batch_size=Defaults.batch_size)
# print(str(predictions))
data.test.documents.set_predictions(predictions)
print("TEST RESULTS for: " + str(len(predictions)))
best_sigmoid = utility.readDictFromStringFile(Defaults.output_path +
"out.txt")["best_sigmoid_t"]
res = data.test.eval(sigmoid_t=best_sigmoid)
res["sigmoid_t"] = best_sigmoid
print(str(res))
np.save(Defaults.pred_path + "pred", data.test.get_predictions())
utility.writeDictAsStringFile(res, Defaults.results_path + "res.txt")
def evaluation_results(self):
# print("Evaluation----------->:: " + self.dataset.name + " "+str(self.dataset.eval()))
print("evaluating dataset:" + self.dataset.name)
print("with: " + str(len(self.dataset.children)))
for sigmoid_t in np.arange(Defaults.sigmoid_t_grid_start,
Defaults.sigmoid_t_grid_stop,
Defaults.sigmoid_t_grid_step):
res = self.dataset.eval(
sigmoid_t=sigmoid_t
) #evaluate_classification(self.dataset.documents)
if self.bestRes == None or self.bestRes["fscore"] < res["fscore"]:
res["best_epoch"] = self.epoch
res["best_sigmoid_t"] = sigmoid_t
self.bestRes = res
print("new best F-score: " + str(res["fscore"]))
print("best sigmoid threshold: " + str(res["best_sigmoid_t"]))
utility.writeDictAsStringFile(res,
Defaults.output_path + "out.txt")
ltlib.util.save_keras(self.model, Defaults.saved_mod_path)
return res