def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('traintsv')
parser.add_argument('generalintentscsv')
parser.add_argument('testtsv')
args = parser.parse_args()
data = read_tsv(args.testtsv)
# initialize engine
engine = Engine(args.traintsv, args.generalintentscsv)
correct_entities = 0.
correct_intents = 0.
correct_response = 0.
from fuzzywuzzy import fuzz
# Begin evaluation
for i, line in enumerate(data):
print "Query ", i
query = line[0]
response = line[1]
intent = line[2]
entities = underscore_entities(line[3].split(','))
result = engine.process_message(i, query)
pprint(line)
pprint(result)
if len(result['entities']) >= 1 and set(entities).issubset(
result['entities']):
correct_entities += 1
else:
print "Entities mismatch"
print set(entities)
print result['entities']
if intent == result['intent']:
correct_intents += 1
else:
print "Intent mismatch"
if result['response'] is not None and \
fuzz.partial_ratio(response, result['response']) > 50:
correct_response += 1
else:
print "Response mismatch"
print "Entity score {} Intent score {} Response score {}".format(
correct_entities / len(data), correct_intents / len(data),
correct_response / len(data))
sys.stdout.flush()
