def train(self, verbose=True):
assert self.train_X is not None and self.train_y_vecBin is not None, 'train_X and train_y_vecBin are required.'
assert self.dev_X is not None and self.dev_y_vecBin is not None, 'dev_X and dev_y_vecBin are required.'
print('\ttraining ...')
self.model = OneVsRestClassifier(SVC(kernel='linear', probability=True, verbose=verbose))
self.model.fit(self.train_X, self.train_y_vecBin)
probs = self.model.predict_proba(self.dev_X)
# evaluation for user intent
precision, recall, fscore, accuracy_frame, self.threshold = eval_intentPredict(probs, self.dev_y_vecBin)
print('\teval_dev: precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}, threshold={:.4f}'.format(precision, recall, fscore, accuracy_frame, self.threshold))
# write prediction results
dev_txt = getActPred(probs, self.threshold, self.id2token)
dev_pred_fname = '{}/{}_{}dev.pred'.format(self.model_folder, self.task_name, self.prefix)
writeTxt(dev_txt, dev_pred_fname, prefix=self.prefix, delimiter=';')
print('\tdev_pred={}'.format(dev_pred_fname))
# write target dev
dev_target_fname = '{}/{}_{}dev.target'.format(self.model_folder, self.task_name, self.prefix)
writeTxt(self.dev_y_txt, dev_target_fname, prefix=self.prefix, delimiter=';')
print('\tdev_target={}'.format(dev_target_fname))
# write utter dev
dev_utter_fname = '{}/utter_dev.txt'.format(self.model_folder)
writeTxt(self.dev_utter_txt, dev_utter_fname, prefix='', delimiter=None)
print('\tdev_utter={}'.format(dev_utter_fname))
# save model
self.model_fname = '{}/{}_{}model_F1={:.4f}_FrameAcc={:.4f}_th={:.4f}.npz'.format(
self.model_folder, self.task_name, self.prefix, fscore, accuracy_frame, self.threshold)
np.savez_compressed(self.model_fname, model=self.model, threshold=self.threshold)
print('\tsaving model: {}'.format(self.model_fname))
def train(self):
print('Training model ...')
# load params
self.window_size = self.train_data.window_size
self.userTagIntent_vocab_size = self.train_data.userTagIntent_vocab_size
self.agentAct_vocab_size = self.train_data.agentAct_vocab_size
self.id2agentAct = self.train_data.id2agentAct
other_npz = '{}/other_vars.npz'.format(self.model_folder)
train_vars = {'window_size': self.window_size,
'userTagIntent_vocab_size': self.userTagIntent_vocab_size,
'agentAct_vocab_size': self.agentAct_vocab_size,
'id2agentAct': self.id2agentAct}
np.savez_compressed(other_npz, **train_vars)
self.params['window_size'] = self.window_size
self.params['userTagIntent_vocab_size'] = self.userTagIntent_vocab_size
self.params['agentAct_vocab_size'] = self.agentAct_vocab_size
print_params(self.params)
# build model graph, save graph and plot graph
self._build()
self._plot_graph()
graph_yaml = '{}/graph-arch.yaml'.format(self.model_folder)
with open(graph_yaml, 'w') as fyaml:
fyaml.write(self.model.to_yaml())
# load train data
X_train = self.train_data.userTagIntent_vecBin
y_train = self.train_data.agentAct_vecBin
train_utter_txt = self.train_data.userUtter_txt
train_act_txt = self.train_data.agentAct_txt
train_fname = '{}/train.target'.format(self.model_folder)
writeUtterActTxt(train_utter_txt, train_act_txt, train_fname)
# load dev data
X_dev = self.dev_data.userTagIntent_vecBin
y_dev = self.dev_data.agentAct_vecBin
dev_utter_txt = self.dev_data.userUtter_txt
dev_act_txt = self.dev_data.agentAct_txt
dev_fname = '{}/dev.target'.format(self.model_folder)
writeUtterActTxt(dev_utter_txt, dev_act_txt, dev_fname)
for ep in xrange(self.epoch_nb):
print('<Epoch {}>'.format(ep))
self.model.fit(x=X_train, y=y_train, batch_size=self.batch_size, nb_epoch=1, verbose=2)
act_probs = self.model.predict(X_dev)
precision, recall, fscore, accuracy_frame, threshold = eval_intentPredict(act_probs, y_dev)
print('ep={}, precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}, threshold={:.4f}'.format(ep, precision, recall, fscore, accuracy_frame, threshold))
dev_pred_txt = getActPred(act_probs, threshold, self.id2agentAct)
dev_results_fname = '{}/dev_results/dev_ep={}.pred'.format(self.model_folder, ep)
writeUtterActTxt(dev_utter_txt, dev_pred_txt, dev_results_fname)
print('Write dev results: {}'.format(dev_results_fname))
weights_fname = '{}/weights/ep={}_f1={:.4f}_frameAcc={:.4f}_th={:.4f}.h5'.format(self.model_folder, ep, fscore, accuracy_frame, threshold)
print('Saving Model: {}'.format(weights_fname))
self.model.save_weights(weights_fname, overwrite=True)
def train(self):
print('Training model ...')
# load params
self.maxlen_userUtter = self.train_data.maxlen_userUtter
self.word_vocab_size = self.train_data.word_vocab_size
self.userIntent_vocab_size = self.train_data.userIntent_vocab_size
self.userTag_vocab_size = self.train_data.userTag_vocab_size
self.id2word = self.train_data.id2word
self.id2userTag = self.train_data.id2userTag
self.id2userIntent = self.train_data.id2userIntent
self.userTag2id = self.train_data.userTag2id
other_npz = '{}/other_vars.npz'.format(self.model_folder)
train_vars = {
'id2userTag': self.id2userTag,
'id2word': self.id2word,
'id2userIntent': self.id2userIntent,
'userTag2id': self.userTag2id,
'userTag_vocab_size': self.userTag_vocab_size,
'userIntent_vocab_size': self.userIntent_vocab_size,
'word_vocab_size': self.word_vocab_size,
'maxlen_userUtter': self.maxlen_userUtter
}
np.savez_compressed(other_npz, **train_vars)
self.params['maxlen_userUtter'] = self.maxlen_userUtter
self.params['word_vocab_size'] = self.word_vocab_size
self.params['userTag_vocab_size'] = self.userTag_vocab_size
self.params['userIntent_vocab_size'] = self.userIntent_vocab_size
print_params(self.params)
# build model graph, save graph and plot graph
self._build()
self._plot_graph()
graph_yaml = '{}/graph-arch.yaml'.format(self.model_folder)
with open(graph_yaml, 'w') as fyaml:
fyaml.write(self.model.to_yaml())
# load train data
X_train = self.train_data.userUtter_encodePad
tag_train = self.train_data.userTag_1hotPad
intent_train = self.train_data.userIntent_vecBin
train_utter_txt = self.train_data.userUtter_txt
train_intent_txt = self.train_data.userIntent_txt
train_tag_txt = self.train_data.userTag_txt
train_target_fname = '{}/train.target'.format(self.model_folder)
writeUtterTagIntentTxt(train_utter_txt, train_tag_txt,
train_intent_txt, train_target_fname)
# load dev data
X_dev = self.dev_data.userUtter_encodePad
tag_dev = self.dev_data.userTag_1hotPad
intent_dev = self.dev_data.userIntent_vecBin
dev_utter_txt = self.dev_data.userUtter_txt
dev_intent_txt = self.dev_data.userIntent_txt
dev_tag_txt = self.dev_data.userTag_txt
dev_target_fname = '{}/dev.target'.format(self.model_folder)
writeUtterTagIntentTxt(dev_utter_txt, dev_tag_txt, dev_intent_txt,
dev_target_fname)
# get mask matrix for train and dev set
mask_array_train = np.zeros_like(X_train)
mask_array_train[X_train != 0] = 1
mask_array_dev = np.zeros_like(X_dev)
mask_array_dev[X_dev != 0] = 1
# jointly training
for ep in xrange(self.epoch_nb):
print('<Epoch {}>'.format(ep))
print '------------------------------------------------------------'
print X_train
self.model.fit(x=X_train,
y={
'slot_output': tag_train,
'intent_output': intent_train
},
sample_weight={
'slot_output': mask_array_train,
'intent_output': None
},
batch_size=self.batch_size,
nb_epoch=1,
verbose=2)
tag_probs, intent_probs = self.model.predict(X_dev)
# calculate token-level scores
precision_tag, recall_tag, fscore_tag, accuracy_frame_tag = eval_slotTagging(
tag_probs, mask_array_dev, tag_dev, self.userTag2id['tag-O'])
print(
'SlotTagging: ep={}, precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}'
.format(ep, precision_tag, recall_tag, fscore_tag,
accuracy_frame_tag))
precision_intent, recall_intent, fscore_intent, accuracy_frame_intent, threshold = eval_intentPredict(
intent_probs, intent_dev)
print(
'Intent Prediction: ep={}, precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}, threshold={:.4f}'
.format(ep, precision_intent, recall_intent, fscore_intent,
accuracy_frame_intent, threshold))
accuracy_frame_both = getNLUframeAccuracy(tag_probs,
mask_array_dev, tag_dev,
intent_probs, intent_dev,
threshold)
print('NLU Frame: ep={}, accuracy={:.4f}'.format(
ep, accuracy_frame_both))
dev_tag_pred_txt, dev_intent_pred_txt = getNLUpred(
tag_probs, mask_array_dev, self.id2userTag, intent_probs,
threshold, self.id2userIntent)
dev_results_fname = '{}/dev_results/dev_ep={}.pred'.format(
self.model_folder, ep)
writeUtterTagIntentTxt(dev_utter_txt, dev_tag_pred_txt,
dev_intent_pred_txt, dev_results_fname)
print('Write dev results: {}'.format(dev_results_fname))
weights_fname = '{}/weights/ep={}_tagF1={:.4f}frameAcc={:.4f}_intentF1={:.4f}frameAcc={:.4f}th={:.4f}.h5'.format(
self.model_folder, ep, fscore_tag, accuracy_frame_tag,
fscore_intent, accuracy_frame_intent, threshold)
print('Saving Model: {}'.format(weights_fname))
self.model.save_weights(weights_fname, overwrite=True)
def train(self):
print('Training model ...')
self.maxlen_userUtter = self.train_data.maxlen_userUtter
self.window_size = self.train_data.window_size
self.word_vocab_size = self.train_data.word_vocab_size
self.agentAct_vocab_size = self.train_data.agentAct_vocab_size
self.userTag_vocab_size = self.train_data.userTag_vocab_size
self.userIntent_vocab_size = self.train_data.userIntent_vocab_size
self.id2agentAct = self.train_data.id2agentAct
self.id2userIntent = self.train_data.id2userIntent
self.id2userTag = self.train_data.id2userTag
self.id2word = self.train_data.id2word
self.userTag2id = self.train_data.userTag2id
if self.context_size is None:
self.context_size = self.train_data.userTagIntent_vocab_size
other_npz = '{}/other_vars.npz'.format(self.model_folder)
train_vars = {
'id2agentAct': self.id2agentAct,
'id2userIntent': self.id2userIntent,
'id2word': self.id2word,
'id2userTag': self.id2userTag,
'agentAct_vocab_size': self.agentAct_vocab_size,
'userIntent_vocab_size': self.userIntent_vocab_size,
'userTag_vocab_size': self.userTag_vocab_size,
'word_vocab_size': self.word_vocab_size,
'maxlen_userUtter': self.maxlen_userUtter,
'window_size': self.window_size,
'userTag2id': self.userTag2id
}
np.savez_compressed(other_npz, **train_vars)
self.params['maxlen_userUtter'] = self.maxlen_userUtter
self.params['window_size'] = self.window_size
self.params['word_vocab_size'] = self.word_vocab_size
self.params['agentAct_vocab_size'] = self.agentAct_vocab_size
self.params['userTag_vocab_size'] = self.userTag_vocab_size
self.params['userIntent_vocab_size'] = self.userIntent_vocab_size
print_params(self.params)
# build model graph, save graph and plot graph
self._build()
self._plot_graph()
graph_yaml = '{}/graph-arch.yaml'.format(self.model_folder)
with open(graph_yaml, 'w') as fyaml:
fyaml.write(self.model.to_yaml())
# load training data
X_train = self.train_data.userUtter_encodePad_window
tag_train = self.train_data.userTag_1hotPad_window
intent_train = self.train_data.userIntent_vecBin_window
act_train = self.train_data.agentAct_vecBin
train_utter_txt = self.train_data.userUtter_txt
train_intent_txt = self.train_data.userIntent_txt
train_tag_txt = self.train_data.userTag_txt
train_act_txt = self.train_data.agentAct_txt
train_utter_fname = '{}/utter_train.target'.format(self.model_folder)
writeTxt(train_utter_txt, train_utter_fname, prefix='', delimiter=None)
train_intent_fname = '{}/intent_train.target'.format(self.model_folder)
writeTxt(train_intent_txt,
train_intent_fname,
prefix='intent-',
delimiter=';')
train_tag_fname = '{}/tag_train.target'.format(self.model_folder)
writeTxt(train_tag_txt, train_tag_fname, prefix='tag-', delimiter=None)
train_act_fname = '{}/act_train.target'.format(self.model_folder)
writeTxt(train_act_txt, train_act_fname, prefix='act-', delimiter=';')
# load dev data
X_dev = self.dev_data.userUtter_encodePad_window
tag_dev = self.dev_data.userTag_1hotPad_window
intent_dev = self.dev_data.userIntent_vecBin_window
act_dev = self.dev_data.agentAct_vecBin
dev_utter_txt = self.dev_data.userUtter_txt
dev_intent_txt = self.dev_data.userIntent_txt
dev_tag_txt = self.dev_data.userTag_txt
dev_act_txt = self.dev_data.agentAct_txt
dev_utter_fname = '{}/utter_dev.target'.format(self.model_folder)
writeTxt(dev_utter_txt, dev_utter_fname, prefix='', delimiter=None)
dev_intent_fname = '{}/intent_dev.target'.format(self.model_folder)
writeTxt(dev_intent_txt,
dev_intent_fname,
prefix='intent-',
delimiter=';')
dev_tag_fname = '{}/tag_dev.target'.format(self.model_folder)
writeTxt(dev_tag_txt, dev_tag_fname, prefix='tag-', delimiter=None)
dev_act_fname = '{}/act_dev.target'.format(self.model_folder)
writeTxt(dev_act_txt, dev_act_fname, prefix='act-', delimiter=';')
# get mask matrix for train and dev data
mask_train = np.zeros((X_train.shape[0], X_train.shape[1]))
mask_train[np.any(X_train != 0, axis=-1)] = 1
mask_dev = np.zeros((X_dev.shape[0], X_dev.shape[1]))
mask_dev[np.any(X_dev != 0, axis=-1)] = 1
mask_dev_maxlen = np.zeros_like(X_dev[:, -1])
mask_dev_maxlen[X_dev[:, -1] != 0] = 1
# joint training
for ep in xrange(self.epoch_nb):
print('<Epoch {}>'.format(ep))
self.model.fit(x=X_train,
y={
'slot_output': tag_train,
'intent_output': intent_train,
'act_output': act_train
},
sample_weight={
'slot_output': mask_train,
'intent_output': mask_train,
'act_output': None
},
batch_size=self.batch_size,
nb_epoch=1,
verbose=2)
tag_probs, intent_probs, act_probs = self.model.predict(X_dev)
# evaluation for agent act
precision_act, recall_act, fscore_act, accuracy_frame_act, threshold_act = eval_intentPredict(
act_probs, act_dev)
print(
'Agent Act Prediction: ep={}, precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}, threshold={:.4f}'
.format(ep, precision_act, recall_act, fscore_act,
accuracy_frame_act, threshold_act))
# evaluation for slot tags
precision_tag, recall_tag, fscore_tag, accuracy_frame_tag = eval_slotTagging(
tag_probs[:, -1], mask_dev_maxlen, tag_dev[:, -1],
self.userTag2id['tag-O'])
print(
'SlotTagging: ep={}, precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}'
.format(ep, precision_tag, recall_tag, fscore_tag,
accuracy_frame_tag))
# evaluation for user intent
precision_intent, recall_intent, fscore_intent, accuracy_frame_intent, threshold_intent = eval_intentPredict(
intent_probs[:, -1], intent_dev[:, -1])
print(
'Intent Prediction: ep={}, precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}, threshold={:.4f}'
.format(ep, precision_intent, recall_intent, fscore_intent,
accuracy_frame_intent, threshold_intent))
# frame-level accuracy of NLU
accuracy_frame_both = getNLUframeAccuracy(
tag_probs[:, -1], mask_dev_maxlen, tag_dev[:, -1],
intent_probs[:, -1], intent_dev[:, -1], threshold_intent)
print('NLU Frame: ep={}, accuracy={:.4f}'.format(
ep, accuracy_frame_both))
# save predicted results
dev_tag_pred_txt, dev_intent_pred_txt = getNLUpred(
tag_probs[:, -1], mask_dev_maxlen, self.id2userTag,
intent_probs[:, -1], threshold_intent, self.id2userIntent)
dev_act_pred_txt = getActPred(act_probs, threshold_act,
self.id2agentAct)
dev_tag_pred_fname = '{}/dev_results/tag_ep={}.pred'.format(
self.model_folder, ep)
writeTxt(dev_tag_pred_txt,
dev_tag_pred_fname,
prefix='tag-',
delimiter=None)
dev_intent_pred_fname = '{}/dev_results/intent_ep={}.pred'.format(
self.model_folder, ep)
writeTxt(dev_intent_pred_txt,
dev_intent_pred_fname,
prefix='intent-',
delimiter=';')
dev_act_pred_fname = '{}/dev_results/act_ep={}.pred'.format(
self.model_folder, ep)
writeTxt(dev_act_pred_txt,
dev_act_pred_fname,
prefix='act-',
delimiter=';')
dev_utter_pred_fname = '{}/dev_results/utter.txt'.format(
self.model_folder)
writeTxt(dev_utter_txt,
dev_utter_pred_fname,
prefix='',
delimiter=None)
print('Write dev results: {}, {}, {}'.format(
dev_utter_pred_fname, dev_act_pred_fname, dev_tag_pred_fname,
dev_intent_pred_fname))
weights_fname = '{}/weights/ep={}_tagF1={:.4f}_intentF1={:.4f}th={:.4f}_NLUframeAcc={:.4f}_actF1={:.4f}frameAcc={:.4f}th={:.4f}.h5'.format(
self.model_folder, ep, fscore_tag, fscore_intent,
threshold_intent, accuracy_frame_both, fscore_act,
accuracy_frame_act, threshold_act)
print('Saving Model: {}'.format(weights_fname))
self.model.save_weights(weights_fname, overwrite=True)
def pipelinePrediction(test_data,
tag_model_weights,
intent_model_weights,
act_model_weights,
result_folder,
tuneTh=True,
threshold=None):
# load slot tagging model, and make prediction
tag_model = load_model_NLU(tag_model_weights, test_data)
tag_model.test_tag_flag = True
tag_model.model_folder = result_folder
tag_model.predict()
tag_pred_txt = '{}/test_result/tag_{}.pred'.format(
tag_model.model_folder,
os.path.basename(tag_model_weights).split('_')[0])
tag_pred_indicator = readTagPredTxt(tag_pred_txt, test_data.userTag2id,
len(test_data.userTag_txt),
test_data.userTag_vocab_size)
# load user intent model and make prediction
intent_model = load_model_NLU(intent_model_weights, test_data)
intent_model.test_intent_flag = True
intent_model.threshold = threshold_intent
intent_model.model_folder = result_folder
intent_model.predict()
intent_pred_txt = '{}/test_result/intent_{}.pred'.format(
intent_model.model_folder,
os.path.basename(intent_model_weights).split('_')[0])
intent_pred_indicator = readIntentPredTxt(intent_pred_txt,
test_data.userIntent2id,
len(test_data.userIntent_txt),
test_data.userIntent_vocab_size)
# merge indicators of slot tagging and user intents, and generate windowed tagIntent matrix
assert len(tag_pred_indicator) == len(
intent_pred_indicator), 'sample_nb is not equal.'
nlu_vecBin = np.hstack((tag_pred_indicator, intent_pred_indicator))
# load agent act model and make prediction
act_model = load_model_Policy(act_model_weights)
act_model.model_folder = result_folder
nlu_vecBin_windowed = get_windowedVec(nlu_vecBin, act_model.window_size)
if tuneTh:
# tune threshold
print('Tuning threshold on Dev ...')
act_probs = act_model.model.predict(nlu_vecBin_windowed)
precision, recall, fscore, accuracy_frame, act_threshold = eval_intentPredict(
act_probs, test_data.agentAct_vecBin)
print(
'AgentActPred on Dev: precision={:.4f}, recall={:.4f}, fscore={:.4f}, accuracy_frame={:.4f}, threshold={:.4f}'
.format(precision, recall, fscore, accuracy_frame, act_threshold))
dev_pred_txt = getActPred(act_probs, act_threshold,
test_data.id2agentAct)
dev_results_fname = '{}/act_dev.pred'.format(act_model.model_folder)
writeUtterActTxt(test_data.userUtter_txt, dev_pred_txt,
dev_results_fname)
print('Write dev results: {}'.format(dev_results_fname))
return act_threshold
else:
# make prediction based on well-tuned threshold
assert threshold is not None, 'Argument required: threshold for agent action prediction.'
act_model.threshold = threshold
act_model.test_data = test_data
act_model.test_data.userTagIntent_vecBin = nlu_vecBin_windowed
act_model.predict()