def predict_babi_file(in_model,
vocabs_for_tasks,
dataset,
in_config,
in_session,
target_file_path=None):
if target_file_path:
target_file = open(target_file_path, "w")
# eval tags --> RNN tags
X, ys_for_tasks = make_multitask_dataset(dataset, vocabs_for_tasks[0][0],
vocabs_for_tasks[0][1], in_config)
predictions = predict(in_model, (X, ys_for_tasks),
[vocabs_for_tasks[0][-1]], in_session)
predictions_eval = []
global_word_index = 0
broken_sequences_number = 0
# RNN tags --> eval tags
for utterance in dataset['utterance']:
current_tags = predictions[global_word_index:global_word_index +
len(utterance)]
try:
current_tags_eval = convert_from_inc_disfluency_tags_to_eval_tags(
current_tags, utterance, representation="disf1")
except:
current_tags_eval = current_tags
broken_sequences_number += 1
predictions_eval += current_tags_eval
global_word_index += len(utterance)
print '#broken sequences after RNN --> eval conversion: {} out of {}'.format(
broken_sequences_number, dataset.shape[0])
predictions_eval_iter = iter(predictions_eval)
for speaker, (_, speaker_data) in enumerate(dataset.iterrows()):
if target_file_path:
target_file.write("Speaker: " + str(speaker) + "\n\n")
timing_data, lex_data, pos_data, labels = (create_fake_timings(
len(speaker_data['utterance'])), speaker_data['utterance'],
speaker_data['pos'],
speaker_data['tags'])
for i in range(0, len(timing_data)):
# print i, timing_data[i]
_, end = timing_data[i]
word = lex_data[i]
pos = pos_data[i]
predicted_tags = [next(predictions_eval_iter)]
current_time = end
if target_file_path:
target_file.write("Time: " + str(current_time) + "\n")
new_words = lex_data[i - (len(predicted_tags) - 1):i + 1]
new_pos = pos_data[i - (len(predicted_tags) - 1):i + 1]
new_timings = timing_data[i - (len(predicted_tags) - 1):i + 1]
for t, w, p, tag in zip(new_timings, new_words, new_pos,
predicted_tags):
target_file.write("\t".join(
[str(t[0]), str(t[1]), w, p, tag]))
target_file.write("\n")
target_file.write("\n")
target_file.write("\n")
def eval_babi(in_model,
in_vocabs_for_tasks,
source_file_path,
in_config,
in_session,
verbose=True):
increco_file = 'swbd_disf_heldout_data_output_increco.text'
dataset = pd.read_json(source_file_path)
predict_babi_file(in_model,
in_vocabs_for_tasks,
dataset,
in_config,
in_session,
target_file_path=increco_file)
IDs, timings, words, pos_tags, labels = (map(str, range(
dataset.shape[0])), [None] * dataset.shape[0], dataset['utterance'],
dataset['pos'], dataset['tags'])
gold_data = {} # map from the file name to the data
for dialogue, a, b, c, d in zip(IDs, timings, words, pos_tags, labels):
# if "asr" in division and not dialogue[:4] in good_asr: continue
current_tags_eval = convert_from_inc_disfluency_tags_to_eval_tags(
d, b, representation="disf1")
gold_data[dialogue] = (create_fake_timings(len(b)), b, c,
current_tags_eval)
final_output_name = increco_file.replace("_increco", "_final")
incremental_output_disfluency_eval_from_file(
increco_file,
gold_data,
utt_eval=True,
error_analysis=False,
word=True,
interval=False,
outputfilename=final_output_name)
# hyp_dir = experiment_dir
IDs, timings, words, pos_tags, labels = (map(str, range(
dataset.shape[0])), [None] * dataset.shape[0], dataset['utterance'],
dataset['pos'], dataset['tags'])
gold_data = {} # map from the file name to the data
for dialogue, a, b, c, d in zip(IDs, timings, words, pos_tags, labels):
# if "asr" in division and not dialogue[:4] in good_asr: continue
current_tags_eval = convert_from_inc_disfluency_tags_to_eval_tags(
d, b, representation="disf1")
d = rename_all_repairs_in_line_with_index(current_tags_eval)
gold_data[dialogue] = (create_fake_timings(len(b)), b, c, d)
# the below does just the final output evaluation, assuming a final output file, faster
hyp_file = "swbd_disf_heldout_data_output_final.text"
word = True # world-level analyses
error = True # get an error analysis
results, speaker_rate_dict, error_analysis = final_output_disfluency_eval_from_file(
hyp_file,
gold_data,
utt_eval=False,
error_analysis=error,
word=word,
interval=False,
outputfilename=None)
# the below does incremental and final output in one, also outputting the final outputs
# derivable from the incremental output, takes quite a while
if verbose:
for k, v in results.items():
print k, v
all_results = deepcopy(results)
return {
'f1_<rm_word': all_results['f1_<rm_word'],
'f1_<rps_word': all_results['f1_<rps_word'],
'f1_<e_word': all_results['f1_<e_word']
}
def predict_increco_file(in_model,
vocabs_for_tasks,
source_file_path,
in_config,
in_session,
target_file_path=None,
is_asr_results_file=False):
"""Return the incremental output in an increco style
given the incoming words + POS. E.g.:
Speaker: KB3_1
Time: 1.50
KB3_1:1 0.00 1.12 $unc$yes NNP <f/><tc/>
Time: 2.10
KB3_1:1 0.00 1.12 $unc$yes NNP <rms id="1"/><tc/>
KB3_1:2 1.12 2.00 because IN <rps id="1"/><cc/>
Time: 2.5
KB3_1:2 1.12 2.00 because IN <rps id="1"/><rpndel id="1"/><cc/>
from an ASR increco style input without the POStags:
or a normal style disfluency dectection ground truth corpus:
Speaker: KB3_1
KB3_1:1 0.00 1.12 $unc$yes NNP <rms id="1"/><tc/>
KB3_1:2 1.12 2.00 $because IN <rps id="1"/><cc/>
KB3_1:3 2.00 3.00 because IN <f/><cc/>
KB3_1:4 3.00 4.00 theres EXVBZ <f/><cc/>
KB3_1:6 4.00 5.00 a DT <f/><cc/>
KB3_1:7 6.00 7.10 pause NN <f/><cc/>
:param source_file_path: str, file path to the input file
:param target_file_path: str, file path to output in the above format
:param is_asr_results_file: bool, whether the input is increco style
"""
if target_file_path:
target_file = open(target_file_path, "w")
if 'timings' in source_file_path:
print "input file has timings"
if not is_asr_results_file:
dialogues = []
IDs, timings, words, pos_tags, labels = \
get_tag_data_from_corpus_file(source_file_path)
for dialogue, a, b, c, d in zip(IDs, timings, words, pos_tags,
labels):
dialogues.append((dialogue, (a, b, c, d)))
else:
print "no timings in input file, creating fake timings"
raise NotImplementedError
# collecting a single dataset for the model to predict in batches
utterances, tags, pos = [], [], []
for speaker, speaker_data in dialogues:
timing_data, lex_data, pos_data, labels = speaker_data
# iterate through the utterances
# utt_idx = -1
for i in range(0, len(timing_data)):
# print i, timing_data[i]
_, end = timing_data[i]
if "<t" in labels[i]:
utterances.append([])
tags.append([])
pos.append([])
utterances[-1].append(lex_data[i])
tags[-1].append(labels[i])
pos[-1].append(pos_data[i])
# eval tags --> RNN tags
dataset = pd.DataFrame({
'utterance':
utterances,
'tags': [
convert_from_eval_tags_to_inc_disfluency_tags(
tags_i, words_i, representation="disf1")
for tags_i, words_i in zip(tags, utterances)
],
'pos':
pos
})
X, ys_for_tasks = make_multitask_dataset(dataset, vocabs_for_tasks[0][0],
vocabs_for_tasks[0][1], in_config)
predictions = predict(in_model, (X, ys_for_tasks),
[vocabs_for_tasks[0][-1]], in_session)
predictions_eval = []
global_word_index = 0
broken_sequences_number = 0
# RNN tags --> eval tags
for utterance in utterances:
current_tags = predictions[global_word_index:global_word_index +
len(utterance)]
try:
current_tags_eval = convert_from_inc_disfluency_tags_to_eval_tags(
current_tags, utterance, representation="disf1")
except:
current_tags_eval = current_tags
broken_sequences_number += 1
predictions_eval += current_tags_eval
global_word_index += len(utterance)
print '#broken sequences after RNN --> eval conversion: {} out of {}'.format(
broken_sequences_number, len(utterances))
predictions_eval_iter = iter(predictions_eval)
for speaker, speaker_data in dialogues:
if target_file_path:
target_file.write("Speaker: " + str(speaker) + "\n\n")
timing_data, lex_data, pos_data, labels = speaker_data
for i in range(0, len(timing_data)):
# print i, timing_data[i]
_, end = timing_data[i]
word = lex_data[i]
pos = pos_data[i]
predicted_tags = [next(predictions_eval_iter)]
current_time = end
if target_file_path:
target_file.write("Time: " + str(current_time) + "\n")
new_words = lex_data[i - (len(predicted_tags) - 1):i + 1]
new_pos = pos_data[i - (len(predicted_tags) - 1):i + 1]
new_timings = timing_data[i - (len(predicted_tags) - 1):i + 1]
for t, w, p, tag in zip(new_timings, new_words, new_pos,
predicted_tags):
target_file.write("\t".join(
[str(t[0]), str(t[1]), w, p, tag]))
target_file.write("\n")
target_file.write("\n")
target_file.write("\n")
def tag_new_word(self,
word,
pos=None,
timing=None,
extra=None,
diff_only=True,
rollback=0):
"""Tag new incoming word and update the word and tag graphs.
:param word: the word to consume/tag
:param pos: the POS tag to consume/tag (optional)
:param timing: the duration of the word (optional)
:param diff_only: whether to output only the diffed suffix,
if False, outputs entire output tags
:param rollback: the number of words to rollback
in the case of changed word hypotheses from an ASR
"""
self.rollback(rollback)
if pos is None and self.args.pos:
# if no pos tag provided but there is a pos-tagger, tag word
test_words = [
unicode(x) for x in get_last_n_features(
"words", self.word_graph, len(self.word_graph) - 1, n=4)
] + [unicode(word.lower())]
pos = self.pos_tagger.tag(test_words)[-1][1]
# print "tagging", word, "as", pos
# 0. Add new word to word graph
word, pos = self.standardize_word_and_pos(word, pos)
# print "New word:", word, pos
self.word_graph.append((word, pos, timing))
# 1. load the saved internal rnn state
# TODO these nets aren't (necessarily) trained statefully
# The internal state in training self.args.bs words back
# are the inital ones in training, however here
# They are the actual state reached.
if self.state_history == []:
c0_state = self.initial_c0_state
h0_state = self.initial_h0_state
else:
if self.model_type == "lstm":
c0_state = self.state_history[-1][0][-1]
h0_state = self.state_history[-1][1][-1]
elif self.model_type == "elman":
h0_state = self.state_history[-1][-1]
if self.model_type == "lstm":
self.model.load_weights(c0=c0_state, h0=h0_state)
elif self.model_type == "elman":
self.model.load_weights(h0=h0_state)
else:
raise NotImplementedError("no history loading for\
{0} model".format(self.model_type))
# 2. do the softmax output with converted inputs
word_window = [
self.word_to_index_map[x]
for x in get_last_n_features("words",
self.word_graph,
len(self.word_graph) - 1,
n=self.window_size)
]
pos_window = [
self.pos_to_index_map[x]
for x in get_last_n_features("POS",
self.word_graph,
len(self.word_graph) - 1,
n=self.window_size)
]
# print "word_window, pos_window", word_window, pos_window
if self.model_type == "lstm":
h_t, c_t, s_t = self.model.\
soft_max_return_hidden_layer([word_window], [pos_window])
self.softmax_history.append(s_t)
if len(self.state_history) == 20: # just saving history
self.state_history.pop(0) # pop first one
self.state_history.append((c_t, h_t))
elif self.model_type == "elman":
h_t, s_t = self.model.soft_max_return_hidden_layer([word_window],
[pos_window])
self.softmax_history.append(s_t)
if len(self.state_history) == 20:
self.state_history.pop(0) # pop first one
self.state_history.append(h_t)
else:
raise NotImplementedError("no softmax implemented for\
{0} model".format(self.model_type))
softmax = np.concatenate(self.softmax_history)
# 3. do the decoding on the softmax
if "disf" in self.args.tags:
edit_tag = "<e/><cc>" if "uttseg" in self.args.tags else "<e/>"
# print self.tag_to_index_map[edit_tag]
adjustsoftmax = np.concatenate(
(softmax, softmax[:, self.tag_to_index_map[edit_tag]].reshape(
softmax.shape[0], 1)), 1)
else:
adjustsoftmax = softmax
last_n_timings = None if ((not self.args.use_timing_data) or
not timing) \
else get_last_n_features("timings", self.word_graph,
len(self.word_graph)-1,
n=3)
new_tags = self.decoder.viterbi_incremental(
adjustsoftmax,
a_range=(len(adjustsoftmax) - 1, len(adjustsoftmax)),
changed_suffix_only=True,
timing_data=last_n_timings,
words=[word])
# print "new tags", new_tags
prev_output_tags = deepcopy(self.output_tags)
self.output_tags = self.output_tags[:len(self.output_tags) -
(len(new_tags) - 1)] + new_tags
# 4. convert to standardized output format
if "simple" in self.args.tags:
for p in range(
len(self.output_tags) - (len(new_tags) + 1),
len(self.output_tags)):
rps = self.output_tags[p]
self.output_tags[p] = rps.replace('rm-0',
'rps id="{}"'.format(p))
if "<i" in self.output_tags[p]:
self.output_tags[p] = self.output_tags[p].\
replace("<e/>", "").replace("<i", "<e/><i")
else:
# new_words = [word]
words = get_last_n_features("words",
self.word_graph,
len(self.word_graph) - 1,
n=len(self.word_graph) -
(self.window_size - 1))
self.output_tags = convert_from_inc_disfluency_tags_to_eval_tags(
self.output_tags,
words,
start=len(self.output_tags) - (len(new_tags)),
representation=self.args.tags)
if diff_only:
for i, old_new in enumerate(zip(prev_output_tags,
self.output_tags)):
old, new = old_new
if old != new:
return self.output_tags[i:]
return self.output_tags[len(prev_output_tags):]
return self.output_tags