def computeF1(self, modelName, sentences):
labelKey = self.labelKeys[modelName]
model = self.models[modelName]
idx2Label = self.idx2Labels[modelName]
correctLabels = [
sentences[idx][labelKey] for idx in range(len(sentences))
]
predLabels = self.predictLabels(model, sentences)
labelKey = self.labelKeys[modelName]
encodingScheme = labelKey[labelKey.index('_') + 1:]
pre, rec, f1 = BIOF1Validation.compute_f1(predLabels, correctLabels,
idx2Label, 'O',
encodingScheme)
pre_b, rec_b, f1_b = BIOF1Validation.compute_f1(
predLabels, correctLabels, idx2Label, 'B', encodingScheme)
if f1_b > f1:
logging.debug(
"Setting wrong tags to B- improves from %.4f to %.4f" %
(f1, f1_b))
pre, rec, f1 = pre_b, rec_b, f1_b
return pre, rec, f1
def computeF1(self, modelName, sentences, tosave=False):
labelKey = self.labelKeys[modelName]
model = self.models[modelName]
idx2Label = self.idx2Labels[modelName]
correctLabels = [sentences[idx][labelKey] for idx in range(len(sentences))]
predLabels = self.predictLabels(model, sentences)
if tosave:
self.correctLabels = correctLabels
self.newpredLabels = predLabels
print([(y,idx2Label[y]) for x in predLabels for y in x][:50])
labelKey = self.labelKeys[modelName]
encodingScheme = labelKey[labelKey.index('_')+1:]
pre, rec, f1 = BIOF1Validation.compute_f1(predLabels, correctLabels, idx2Label, 'O', encodingScheme)
pre_b, rec_b, f1_b = BIOF1Validation.compute_f1(predLabels, correctLabels, idx2Label, 'B', encodingScheme)
if f1_b > f1:
logging.debug("Setting wrong tags to B- improves from %.4f to %.4f" % (f1, f1_b))
pre, rec, f1 = pre_b, rec_b, f1_b
###Added
print(correctLabels[:5],'\n',predLabels[:5])
correctLabels = [idx2Label[x] for y in correctLabels for x in y]
predLabels = [idx2Label[x] for y in predLabels for x in y]
print(correctLabels[:50],'\n',predLabels[:50])
conllEVAL.evaluate(correctLabels, predLabels)
###EndAdded
return pre, rec, f1
def computeF1(self, sentences, name=''):
correctLabels = []
predLabels = []
paddedPredLabels = self.predictLabels(sentences)
for idx in range(len(sentences)):
unpaddedCorrectLabels = []
unpaddedPredLabels = []
for tokenIdx in range(len(sentences[idx]['tokens'])):
if sentences[idx]['tokens'][
tokenIdx] != 0: # Skip padding tokens
unpaddedCorrectLabels.append(
sentences[idx][self.labelKey][tokenIdx])
unpaddedPredLabels.append(paddedPredLabels[idx][tokenIdx])
correctLabels.append(unpaddedCorrectLabels)
predLabels.append(unpaddedPredLabels)
# encodingScheme = self.labelKey[self.labelKey.index('_') + 1:]
encodingScheme = '0'
print("encoding Scheme ", encodingScheme)
pre, rec, f1 = BIOF1Validation.compute_f1(predLabels, correctLabels,
self.idx2Label, 'O',
encodingScheme)
# pre_b, rec_b, f1_b = BIOF1Validation.compute_f1(predLabels, correctLabels, self.idx2Label, 'B', encodingScheme)
#
# if f1_b > f1:
# logging.debug("Setting incorrect tags to B yields improvement from %.4f to %.4f" % (f1, f1_b))
# pre, rec, f1 = pre_b, rec_b, f1_b
if self.writeOutput:
self.writeOutputToFile(sentences, predLabels,
'%.4f_%s' % (f1, name))
return pre, rec, f1
def computeF1(self, modelName, sentences, mode="", epoch=0):
labelKey = self.labelKeys[modelName]
print("Label key : {}".format(labelKey))
model = self.models[modelName]
idx2Label = self.idx2Labels[modelName]
correctLabels = [
sentences[idx][labelKey][:len(sentences[idx]['raw_tokens'])]
for idx in range(len(sentences))
]
#print("Correct labels : {}".format(correctLabels))
#os._exit(2)
predLabels = self.predictLabels(model, sentences)
#print("Prediction labels : {}".format(predLabels))
if mode == "dev":
self.current_dev_prediction = predLabels
if mode == "test":
self.current_test_prediction = predLabels
labelKey = self.labelKeys[modelName]
encodingScheme = labelKey[labelKey.index('_') + 1:]
#pre, rec, f1 = BIOF1Validation.compute_f1(predLabels, correctLabels, idx2Label, 'O', encodingScheme)
#pre_b, rec_b, f1_b = BIOF1Validation.compute_f1(predLabels, correctLabels, idx2Label, 'B', encodingScheme)
pre, rec, f1 = BIOF1Validation.compute_f1_conll(
correctLabels, predLabels, idx2Label)
#logging.info("ConLL version p : {:.2f} r: {:.2f} f1: {:.2f}".format(pre_conll, rec_conll, f1_conll))
#if f1_b > f1:
# logging.info("Setting wrong tags to B- improves from %.4f to %.4f" % (f1, f1_b))
# pre, rec, f1 = pre_b, rec_b, f1_b
return pre, rec, f1
def computeF1(self, task, predLabels, correctLabels):
idx2Label = self.idx2Labels[task]
encodingScheme = task[task.index('_') + 1:]
pre, rec, f1 = BIOF1Validation.compute_f1(predLabels, correctLabels,
idx2Label, 'O',
encodingScheme)
pre_b, rec_b, f1_b = BIOF1Validation.compute_f1(
predLabels, correctLabels, idx2Label, 'B', encodingScheme)
if f1_b > f1:
logging.debug(
"Setting wrong tags to B- improves from %.4f to %.4f" %
(f1, f1_b))
pre, rec, f1 = pre_b, rec_b, f1_b
return pre, rec, f1