def main(argv):
argv = []
argv.append('--dataset')
argv.append('dstc5_dev')
argv.append('--dataroot')
argv.append('../data')
argv.append('--trackfile')
argv.append('baseline_dev.json')
argv.append('--ontology')
argv.append('config/ontology_dstc5.json')
argv.append('--method')
argv.append('1')
print argv
parser = argparse.ArgumentParser(description='Simple hand-crafted dialog state tracker baseline.')
parser.add_argument('--dataset', dest='dataset', action='store', metavar='DATASET', required=True, help='The dataset to analyze')
parser.add_argument('--dataroot',dest='dataroot',action='store',required=True,metavar='PATH', help='Will look for corpus in <destroot>/<dataset>/...')
parser.add_argument('--trackfile',dest='trackfile',action='store',required=True,metavar='JSON_FILE', help='File to write with tracker output')
parser.add_argument('--ontology',dest='ontology',action='store',metavar='JSON_FILE',required=True,help='JSON Ontology file')
parser.add_argument('--method',dest='method',action='store',choices=['1', '2'],required=True,help='Baseline mode')
args = parser.parse_args(argv)
dataset = dataset_walker.dataset_walker(args.dataset, dataroot=args.dataroot, labels=False, translations = True)
track_file = open(args.trackfile, "wb")
track = {"sessions":[]}
track["dataset"] = args.dataset
start_time = time.time()
if args.method == '1':
tagsets = ontology_reader.OntologyReader(args.ontology).get_tagsets()
tracker = BaselineMethod1(tagsets)
elif args.method == '2':
translated_tagsets = ontology_reader.OntologyReader(args.ontology).get_translated_tagsets()
tracker = BaselineMethod2(translated_tagsets)
for call in dataset:
this_session = {"session_id":call.log["session_id"], "utterances":[]}
tracker.reset()
for (utter, translations, _) in call:
print utter, translations
assert False
sys.stderr.write('%d:%d \r'%(call.log['session_id'], utter['utter_index']))
tracker_result = tracker.addUtter(utter, translations)
if tracker_result is not None:
this_session["utterances"].append(copy.deepcopy(tracker_result))
track["sessions"].append(this_session)
end_time = time.time()
elapsed_time = end_time - start_time
track['wall_time'] = elapsed_time
json.dump(track, track_file, indent=4)
track_file.close()
def main(argv):
install_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
utils_dirname = os.path.join(install_path,'lib')
sys.path.append(utils_dirname)
from dataset_walker import dataset_walker
parser = argparse.ArgumentParser(description='Check the validity of a tracker output object.')
parser.add_argument('--dataset', dest='dataset', action='store', metavar='DATASET', required=True,
help='The dataset to analyze')
parser.add_argument('--dataroot',dest='dataroot',action='store', metavar='PATH', required=True,
help='Will look for corpus in <destroot>/<dataset>/...')
parser.add_argument('--trackfile',dest='scorefile',action='store',metavar='JSON_FILE',required=True,
help='File containing score JSON')
parser.add_argument('--ontology',dest='ontology',action='store',metavar='JSON_FILE',required=True,
help='JSON Ontology file')
args = parser.parse_args()
sessions = dataset_walker(args.dataset,dataroot=args.dataroot,labels=False)
tracker_output = json.load(open(args.scorefile))
tagsets = ontology_reader.OntologyReader(args.ontology).get_tagsets()
checker = TrackChecker(sessions, tracker_output, tagsets)
checker.check()
checker.print_errors()
def main(argv):
parser = argparse.ArgumentParser(
description='Simple hand-crafted dialog state tracker baseline.')
parser.add_argument('--dataset',
dest='dataset',
action='store',
metavar='DATASET',
required=True,
help='The dataset to analyze')
parser.add_argument(
'--dataroot',
dest='dataroot',
action='store',
required=True,
metavar='PATH',
help='Will look for corpus in <destroot>/<dataset>/...')
parser.add_argument('--trackfile',
dest='trackfile',
action='store',
required=True,
metavar='JSON_FILE',
help='File to write with tracker output')
parser.add_argument('--ontology',
dest='ontology',
action='store',
metavar='JSON_FILE',
required=True,
help='JSON Ontology file')
args = parser.parse_args()
dataset = dataset_walker.dataset_walker(args.dataset,
dataroot=args.dataroot,
labels=False)
tagsets = ontology_reader.OntologyReader(args.ontology).get_tagsets()
track_file = open(args.trackfile, "wb")
track = {"sessions": []}
track["dataset"] = args.dataset
start_time = time.time()
tracker = BaselineTracker(tagsets)
for call in dataset:
this_session = {"session_id": call.log["session_id"], "utterances": []}
tracker.reset()
for (utter, _) in call:
sys.stderr.write('%d:%d\n' %
(call.log['session_id'], utter['utter_index']))
tracker_result = tracker.addUtter(utter)
if tracker_result is not None:
this_session["utterances"].append(
copy.deepcopy(tracker_result))
track["sessions"].append(this_session)
end_time = time.time()
elapsed_time = end_time - start_time
track['wall_time'] = elapsed_time
json.dump(track, track_file, indent=4)
track_file.close()
def main(argv):
install_path = os.path.abspath(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
utils_dirname = os.path.join(install_path, 'lib')
sys.path.append(utils_dirname)
from dataset_walker import dataset_walker
parser = argparse.ArgumentParser(
description='Check the validity of a system output for SLU task.')
parser.add_argument('--dataset',
dest='dataset',
action='store',
metavar='DATASET',
required=True,
help='The dataset to analyze')
parser.add_argument(
'--dataroot',
dest='dataroot',
action='store',
metavar='PATH',
required=True,
help='Will look for corpus in <destroot>/<dataset>/...')
parser.add_argument('--jsonfile',
dest='jsonfile',
action='store',
metavar='JSON_FILE',
required=True,
help='File containing JSON output')
parser.add_argument('--ontology',
dest='ontology',
action='store',
metavar='JSON_FILE',
required=True,
help='JSON Ontology file')
parser.add_argument('--roletype',
dest='roletype',
action='store',
choices=['GUIDE', 'TOURIST'],
required=True,
help='Target role')
args = parser.parse_args()
sessions = dataset_walker(args.dataset,
dataroot=args.dataroot,
labels=False)
system_output = json.load(open(args.jsonfile))
tagsets = ontology_reader.OntologyReader(args.ontology).get_pilot_tagsets()
checker = TrackChecker(sessions, system_output, tagsets, args.roletype)
checker.check()
checker.print_errors()
def __init__(self,tagsets=None,nameOfODictPickle=None):
if self.isIgnoreIlligalSlotValueInEstimation:
print "Naive Ensembler ignore following values:"
print self.listIlligalSlotValues
#Load
if nameOfODictPickle is not None:
print "Load output dictionary file from " + nameOfODictPickle
self.dictOut=pickle.load(open(nameOfODictPickle,"r"))
assert self.dictOut is not None, "Failed to read Output dictionary"
if tagsets==None:
self.tagsets = ontology_reader.OntologyReader("scripts/config/ontology_dstc4.json").get_tagsets()
else:
self.tagsets=tagsets
#
self.__initBaseTracker()
#
self.frame = {}
self.reset()
def errorAnalysis(argv):
print "ERROR ANALYSIS OF NAIVEENSEMBLER"
print argv
parser = argparse.ArgumentParser(
description='Simple hand-crafted dialog state tracker baseline.')
parser.add_argument('--dataset',
dest='dataset',
action='store',
metavar='DATASET',
required=True,
help='The dataset to analyze')
parser.add_argument(
'--dataroot',
dest='dataroot',
action='store',
required=True,
metavar='PATH',
help='Will look for corpus in <destroot>/<dataset>/...')
parser.add_argument('--trackfile',
dest='trackfile',
action='store',
required=True,
metavar='JSON_FILE',
help='File to write with tracker output')
parser.add_argument('--ontology',
dest='ontology',
action='store',
metavar='JSON_FILE',
required=True,
help='JSON Ontology file')
#args = parser.parse_args()
args = parser.parse_args(argv)
dataset = dataset_walker.dataset_walker(args.dataset,
dataroot=args.dataroot,
labels=True)
tagsets = ontology_reader.OntologyReader(args.ontology).get_tagsets()
track = {"sessions": []}
track["dataset"] = args.dataset
start_time = time.time()
tracker = NaiveEnsembleBasedTrackerWithNBest(
tagsets, nameOfODictPickle="dictOutput.pic")
for call in dataset:
this_session = {"session_id": call.log["session_id"], "utterances": []}
tracker.reset()
for (utter, label) in call:
#-mae shori2
if utter['segment_info']['target_bio'] == 'B':
print "\n -----New sub-dialogue----------------------------------------------------"
print "s:" + str(call.log['session_id']) + " u:" + str(
utter['utter_index'])
print "Input=" + utter["transcript"]
tracker_result = tracker.addUtter(utter, call)
if tracker_result is not None:
this_session["utterances"].append(tracker_result)
#
print "Tracker's output:"
print tracker_result
if "frame_label" in label:
for slot in label["frame_label"].keys():
if (slot not in tracker_result["frame_label"]):
print "-slot [" + slot + "] is not exsisted in output"
for value in label["frame_label"][slot]:
print "-value [" + value + "] of slot [" + slot + "] is not exsisted in output"
else:
if len(label["frame_label"][slot]) != len(
tracker_result["frame_label"][slot]):
#In case value in output, but repudant
print "-slot [" + slot + "] include repudant values"
for value in label["frame_label"][slot]:
#In case value not in output
if (value not in tracker_result["frame_label"]
[slot]):
print "-value [" + value + "] of slot [" + slot + "] is not exsisted in output"
track["sessions"].append(this_session)
end_time = time.time()
elapsed_time = end_time - start_time
track['wall_time'] = elapsed_time
def main(argv):
#TODO implementation
#Confirmation hypothesis about data
tagsets = ontology_reader.OntologyReader("scripts/config/ontology_dstc4.json").get_tagsets()
datasetTrain = dataset_walker.dataset_walker("dstc4_train",dataroot="data",labels=True)
datasetDev = dataset_walker.dataset_walker("dstc4_dev",dataroot="data",labels=True)
print "Calculate statics of dialog. "
#-Is number of value in each slot is always 1 if it exist? i.e., it does not contain multiple value?
#-There are many multiple value
isEnumerateMultiValueCase=True
isEnumerateMultiSlotCase=True
countMultipleValueInOneSlot=0
#
maxSlotValueTrain={}
countMultipleSlot=0
for call in datasetTrain:
for (uttr,label) in call:
if "frame_label" in label:
if isEnumerateMultiSlotCase:
if len(label["frame_label"].keys()) > 1:
print label["frame_label"].keys()
countMultipleSlot+=1
for slot in label["frame_label"].keys():
if isEnumerateMultiValueCase:
if slot not in maxSlotValueTrain:
maxSlotValueTrain[slot]=len(label["frame_label"][slot])
else:
if maxSlotValueTrain[slot] < len(label["frame_label"][slot]):
maxSlotValueTrain[slot] = len(label["frame_label"][slot])
if len(label["frame_label"][slot]) > 1:
print "slot=" + slot + ":",
print label["frame_label"][slot]
countMultipleValueInOneSlot+=1
for call in datasetDev:
for (uttr,label) in call:
if "frame_label" in label:
if isEnumerateMultiSlotCase:
if len(label["frame_label"].keys()) > 1:
print label["frame_label"].keys()
countMultipleSlot+=1
for slot in label["frame_label"].keys():
if isEnumerateMultiValueCase:
if slot not in maxSlotValueTrain:
maxSlotValueTrain[slot]=len(label["frame_label"][slot])
else:
if maxSlotValueTrain[slot] < len(label["frame_label"][slot]):
maxSlotValueTrain[slot] = len(label["frame_label"][slot])
if len(label["frame_label"][slot]) > 1:
print "slot=" + slot + ":",
print label["frame_label"][slot]
countMultipleValueInOneSlot+=1
if isEnumerateMultiValueCase:
print "Number of multiple value situation = " + str(countMultipleValueInOneSlot)
avr=0.0
for slot in maxSlotValueTrain.keys():
avr+=(float)(maxSlotValueTrain[slot])
avr/=float(len(maxSlotValueTrain.keys()))
maxSlotValueTrain["AverageNumber"]=int(round(avr))
print "Number of max slot value per slot:"
print maxSlotValueTrain
if isEnumerateMultiSlotCase:
print "Number of multiple slot situation = " + str(countMultipleSlot)
#-How many OOV case?
#-Train -> dev: 1195, Dev->Train: 4789
#-With additional text normalizing, Train -> dev: 937, Dev->Train: 3643
#-With additional normalization Train -> dev: 831, Dev->Train: 3237
isCountNumberofOOVCase=False
dictVocabInTrain={}
dictVocabInDev={}
numberOfOOVCaseInTrain2Dev=0
numberOfOOVCaseInDev2Train=0
if isCountNumberofOOVCase:
for call in datasetTrain:
for (uttr,label) in call:
trans=uttr["transcript"]
transt=re.sub("\,","",trans)
transt=re.sub("\?","",transt)
transt=re.sub("\.","",transt)
transt=re.sub("(%.+ )?","",transt)
#Additional normalize
transt=re.sub("(%.+$)?","",transt)
transt=re.sub("%","",transt)
transt=re.sub("(-|~)"," ",transt)
transt=re.sub("\!","",transt)
transt=re.sub("'"," ",transt)
transt=re.sub("\"","",transt)
#
transt=re.sub("/","",transt)
transt=re.sub("[1-9]+","Replacedval",transt)
transt=transt.lower()
words=transt.split(" ")
for word in words:
#Additional normalization
lmtr=nltk.stem.wordnet.WordNetLemmatizer()
word=lmtr.lemmatize(word)
dictVocabInTrain[word]=0
for call in datasetDev:
for (uttr,label) in call:
trans=uttr["transcript"]
transt=re.sub("\,","",trans)
transt=re.sub("\?","",transt)
transt=re.sub("\.","",transt)
transt=re.sub("(%.+ )?","",transt)
#Additional normalize
transt=re.sub("(%.+$)?","",transt)
transt=re.sub("%","",transt)
transt=re.sub("(-|~)"," ",transt)
transt=re.sub("\!","",transt)
transt=re.sub("'"," ",transt)
transt=re.sub("\"","",transt)
#
transt=re.sub("/","",transt)
transt=re.sub("[1-9]+","Replacedval",transt)
transt=transt.lower()
words=transt.split(" ")
for word in words:
#Additional normalization
lmtr=nltk.stem.wordnet.WordNetLemmatizer()
word=lmtr.lemmatize(word)
if word not in dictVocabInTrain:
print word.encode("utf-8")
numberOfOOVCaseInTrain2Dev+=1
print "Number of OOV case in Train -> Dev situation = " + str(numberOfOOVCaseInTrain2Dev)
print "\n\n\n\n\n"
for call in datasetDev:
for (uttr,label) in call:
trans=uttr["transcript"]
transt=re.sub("\,","",trans)
transt=re.sub("\?","",transt)
transt=re.sub("\.","",transt)
transt=re.sub("(%.+ )?","",transt)
#Additional normalize
transt=re.sub("(%.+$)?","",transt)
transt=re.sub("%","",transt)
transt=re.sub("(-|~)"," ",transt)
transt=re.sub("\!","",transt)
transt=re.sub("'"," ",transt)
transt=re.sub("\"","",transt)
#
transt=re.sub("/","",transt)
transt=re.sub("[1-9]+","Replacedval",transt)
transt=transt.lower()
words=transt.split(" ")
for word in words:
#Additional normalization
lmtr=nltk.stem.wordnet.WordNetLemmatizer()
word=lmtr.lemmatize(word)
dictVocabInDev[word]=0
for call in datasetTrain:
for (uttr,label) in call:
trans=uttr["transcript"]
transt=re.sub("\,","",trans)
transt=re.sub("\?","",transt)
transt=re.sub("\.","",transt)
transt=re.sub("(%.+ )?","",transt)
#Additional normalize
transt=re.sub("(%.+$)?","",transt)
transt=re.sub("%","",transt)
transt=re.sub("(-|~)"," ",transt)
transt=re.sub("\!","",transt)
transt=re.sub("'"," ",transt)
transt=re.sub("\"","",transt)
#
transt=re.sub("/","",transt)
transt=re.sub("[1-9]+","Replacedval",transt)
transt=transt.lower()
words=transt.split(" ")
for word in words:
#Additional normalization
lmtr=nltk.stem.wordnet.WordNetLemmatizer()
word=lmtr.lemmatize(word)
if word not in dictVocabInDev:
print word.encode("utf-8")
numberOfOOVCaseInDev2Train+=1
print "Number of OOV case in Dev -> Train situation = " + str(numberOfOOVCaseInDev2Train)
#-How many frame_label are unseen between train and dev data?
#-So many, train -> dev 96/313 (unseen/all in dev), dev -> train 346/563 (unseen/all in train)
isCountUnseenframeLabel=False
dictTopicSlotValueTrain=[]
numUnseenframeLabel=0
alreadychecked=[]
dictTopicSlotValueDev={}
if isCountUnseenframeLabel:
for call in datasetTrain:
for (uttr,label) in call:
if "frame_label" in label:
for slot in label["frame_label"].keys():
for value in label["frame_label"][slot]:
dictTopicSlotValueTrain.append(slot+value)
for call in datasetDev:
for (uttr,label) in call:
if "frame_label" in label:
for slot in label["frame_label"].keys():
for value in label["frame_label"][slot]:
dictTopicSlotValueDev[(slot+value)]=0
if (slot+value) not in dictTopicSlotValueTrain:
if (slot+value) not in alreadychecked:
numUnseenframeLabel+=1
alreadychecked.append((slot+value))
print "Number of Unseen label train -> dev = " + str(numUnseenframeLabel)
print "Ratio (unseen/all in dev) = " + str(numUnseenframeLabel) + "/" + str(len(dictTopicSlotValueDev.keys()))
dictTopicSlotValueDev=[]
numUnseenframeLabel=0
alreadychecked=[]
dictTopicSlotValueTrain={}
for call in datasetDev:
for (uttr,label) in call:
if "frame_label" in label:
for slot in label["frame_label"].keys():
for value in label["frame_label"][slot]:
dictTopicSlotValueDev.append(slot+value)
for call in datasetTrain:
for (uttr,label) in call:
if "frame_label" in label:
for slot in label["frame_label"].keys():
for value in label["frame_label"][slot]:
dictTopicSlotValueTrain[(slot+value)]=0
if (slot+value) not in dictTopicSlotValueDev:
if (slot+value) not in alreadychecked:
numUnseenframeLabel+=1
alreadychecked.append((slot+value))
print "Number of Unseen label dev -> train = " + str(numUnseenframeLabel)
print "Ratio (unseen/all in train) = " + str(numUnseenframeLabel) + "/" + str(len(dictTopicSlotValueTrain.keys()))
def __init__(self,
tagsets=None,
nameOfODictPickle=None,
nameOfIDictPickle=None,
nameOfLSTMFile=None,
NameOfLearnedD2VFile="LearnedDoc2Vec.d2v"):
#Print out Experimenal setup
print "In both of learning and tracking,"
if self.isStoreFrameOutputedByLSTMAtEachTurnInSubDialog:
print "the tracker store output of LSTM at each turn in Sub.dial,"
else:
print "the tracker doesn't store output of LSTM at each turn in Sub.dial,"
if self.isIgnoreUtterancesNotRelatedToMainTask:
print "and tracker ignore the utterance which is not related to main task."
else:
print "and tracker doesn't ignore the utterance which is not related to main task."
if self.isSeparateDialogIntoSubDialog:
print "and tracker consider one subdialog at one sequence."
else:
print "and tracker consider one dialog at one sequence."
if self.isCombineResultWithBaseline:
print "and the output of LSTM is combined with that of baseline."
else:
print "and the output of LSTM is not combined with that of baseline."
if self.isEnableToUseM1sFeature:
print "and features made by M1s are used in input."
else:
print "and features made by M1s are not used in input."
if self.isUseSentenceRepresentationInsteadofBOW:
print "and distributed sentence reprentation is used instead of BOW and meta info."
else:
print "and BOW and meta indo is used for sentense feature, and Distributed sentence reprentation is not used."
#Variables for tracking state
self.LSTM = None
self.dictOut = None
self.dictIn = None
if nameOfLSTMFile is not None:
print "Load LSTM network file from " + nameOfLSTMFile
self.LSTM = NetworkReader.readFrom(nameOfLSTMFile)
assert self.LSTM is not None, "Failed to read LSTM"
if nameOfIDictPickle is not None:
print "Load input dictionary file from " + nameOfIDictPickle
f = open(nameOfIDictPickle, "r")
self.dictIn = pickle.load(f)
f.close()
assert self.dictIn is not None, "Failed to read Input dictionary"
if nameOfODictPickle is not None:
print "Load output dictionary file from " + nameOfODictPickle
f = open(nameOfODictPickle, "r")
self.dictOut = pickle.load(f)
f.close()
assert self.dictOut is not None, "Failed to read Output dictionary"
if tagsets == None:
self.tagsets = ontology_reader.OntologyReader(
"scripts/config/ontology_dstc4.json").get_tagsets()
else:
self.tagsets = tagsets
#Variables for fast processing
#-1
if LSTMWithBOWTracker.dictFuzzyMatchingResult == None:
try:
f = gzip.open(self.FileNameofdictFuzzyMatchingResult, "rb")
except Exception:
print "FuzzyMatchingResult.pic was not found. Dictionary are newly created. "
LSTMWithBOWTracker.dictFuzzyMatchingResult = {}
else:
print "FuzzyMatchingResult.pic Dictionary are loaded."
LSTMWithBOWTracker.dictFuzzyMatchingResult = pickle.load(f)
f.close()
#-2
if self.isEnableToUseM1sFeature and self.isUtilizeM1VectorDictionary:
try:
f = gzip.open(self.FileNameofM1Vector, "rb")
except Exception:
print self.FileNameofM1Vector + "was not found. Dictionary are newly created. "
self.dictM1Vector = {}
else:
print self.FileNameofM1Vector + " Dictionary are loaded."
self.dictM1Vector = pickle.load(f)
f.close()
#
try:
f = open(self.FileNameofNumClassFeature, "rb")
self.TOTALSIZEOFCLASSFeature = pickle.load(f)
f.close()
print "TSizeClasssFeature=" + str(self.TOTALSIZEOFCLASSFeature)
except Exception:
print self.FileNameofNumClassFeature + " was not found. learn() is required before tracking. "
try:
f = open(self.FileNameofNumSentenceFeature, "rb")
self.TOTALSIZEOFSENTENCEFeature = pickle.load(f)
f.close()
print "TSizeSentenceFeature=" + str(
self.TOTALSIZEOFSENTENCEFeature)
except Exception:
print self.FileNameofNumSentenceFeature + " was not found. learn() is required before tracking. "
try:
f = open(self.FileNameofNumM1Feature, "rb")
self.TOTALSIZEOFM1DEFINEDFeature = pickle.load(f)
f.close()
print "TSizeM1Feature=" + str(self.TOTALSIZEOFM1DEFINEDFeature)
except Exception:
print self.FileNameofNumM1Feature + " was not found. learn() is required before tracking. "
#
if self.isUseSentenceRepresentationInsteadofBOW:
self.d2v = LSTMWithBOWTracker.loadDoc2VecAndCheckAppropriateness(
NameOfLearnedD2VFile)
#
self.frame = {}
self.reset()
def main(argv):
install_path = os.path.abspath(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
utils_dirname = os.path.join(install_path, 'lib')
sys.path.append(utils_dirname)
from dataset_walker import dataset_walker
parser = argparse.ArgumentParser(
description='Evaluate output from an SLU system.')
parser.add_argument('--dataset',
dest='dataset',
action='store',
metavar='DATASET',
required=True,
help='The dataset to analyze')
parser.add_argument(
'--dataroot',
dest='dataroot',
action='store',
metavar='PATH',
required=True,
help='Will look for corpus in <destroot>/<dataset>/...')
parser.add_argument('--pilotfile',
dest='pilotfile',
action='store',
metavar='JSON_FILE',
required=True,
help='File containing JSON output')
parser.add_argument('--ontology',
dest='ontology',
action='store',
metavar='JSON_FILE',
required=True,
help='JSON Ontology file')
parser.add_argument('--pilottask',
dest='pilottask',
action='store',
choices=['SLU', 'SAP', 'SLG', 'EES'],
required=True,
help='Target task')
parser.add_argument('--roletype',
dest='roletype',
action='store',
choices=['GUIDE', 'TOURIST'],
required=True,
help='Target role')
parser.add_argument('--scorefile',
dest='scorefile',
action='store',
metavar='JSON_FILE',
required=True,
help='File to write with CSV scoring data')
args = parser.parse_args()
sessions = dataset_walker(args.dataset,
dataroot=args.dataroot,
labels=True)
system_output = json.load(open(args.pilotfile))
tagsets = ontology_reader.OntologyReader(args.ontology).get_tagsets()
stats = {}
if args.pilottask == 'SLU':
stats['semantic_tagged'] = {}
stats['semantic_tagged']['detection'] = Stat_Precision_Recall()
stats['semantic_tagged']['class'] = Stat_Precision_Recall()
stats['semantic_tagged']['all'] = Stat_Precision_Recall()
if args.pilottask == 'SLU' or args.pilottask == 'SAP':
stats['speech_act'] = {}
stats['speech_act']['act'] = Stat_Precision_Recall()
stats['speech_act']['all'] = Stat_Precision_Recall()
if args.pilottask == 'SLG' or args.pilottask == 'EES':
stats['utt_transcriptions'] = {}
stats['utt_transcriptions']['all'] = Stat_BLEU_AM_FM()
for session, track_session in zip(sessions, system_output["sessions"]):
session_id = session.log['session_id']
log_utter_list = []
label_utter_list = []
for log_utter, label_utter in session:
if (args.roletype == 'GUIDE' and log_utter['speaker']
== 'Guide') or (args.roletype == 'TOURIST'
and log_utter['speaker'] == 'Tourist'):
log_utter_list.append(log_utter)
label_utter_list.append(label_utter)
# now iterate through turns
for log_utter, label_utter, track_utter in zip(
log_utter_list, label_utter_list, track_session["utterances"]):
for subtask in stats:
if subtask == 'speech_act':
ref_sa_list = label_utter['speech_act']
pred_sa_list = track_utter['speech_act']
eval_acts(ref_sa_list, pred_sa_list, stats[subtask])
elif subtask == 'semantic_tagged':
ref_tagged = ' '.join(label_utter['semantic_tagged'])
pred_tagged = track_utter['semantic_tagged']
eval_semantics(ref_tagged, pred_tagged, stats[subtask])
elif subtask == 'utt_transcriptions':
ref = log_utter['transcript']
pred = track_utter['generated_sentence']
eval_utt(ref, pred, stats[subtask])
csvfile = open(args.scorefile, 'w')
print >> csvfile, ("task, subtask, schedule, stat, N, result")
for subtask in stats:
for schedule in stats[subtask]:
for measure, N, result in stats[subtask][schedule].results():
print >> csvfile, (
"%s, %s, %s, %s, %i, %s" %
(args.pilottask, subtask, schedule, measure, N, result))
csvfile.close()
