def eval_local_slot_filling(train_path, test_path):
"""
Evaluate accuracy of the local slot filling classifier.
:param train_path: path to the training data points stored on disk.
:param test_path: path to the test data points stored on disk.
:return:
"""
train_X, train_Y = data_utils.load_slot_filling_data(train_path)
test_X, test_Y = data_utils.load_slot_filling_data(test_path)
# Create model.
model = classifiers.KNearestNeighborModel(FLAGS.num_nn_slot_filling,
train_X, train_Y)
model.eval(train_X, train_Y, verbose=True)
model.eval(test_X, test_Y, verbose=False)
def demo(sess, model, FLAGS):
"""
Simple command line decoding interface.
"""
slot_filling_classifier = None
if FLAGS.fill_argument_slots:
# create slot filling classifier
mapping_param_dir = os.path.join(FLAGS.model_dir,
'train.mappings.X.Y.npz')
train_X, train_Y = \
data_utils.load_slot_filling_data(mapping_param_dir)
slot_filling_classifier = classifiers.KNearestNeighborModel(
FLAGS.num_nn_slot_filling, train_X, train_Y)
print('Slot filling classifier parameters loaded.')
# Decode from standard input.
sys.stdout.write('> ')
sys.stdout.flush()
sentence = sys.stdin.readline()
vocabs = data_utils.load_vocab(FLAGS)
while sentence:
batch_outputs, output_logits = translate_fun(
sentence,
sess,
model,
vocabs,
FLAGS,
slot_filling_classifier=slot_filling_classifier)
if FLAGS.token_decoding_algorithm == 'greedy':
tree, pred_cmd, outputs = batch_outputs[0]
score = output_logits[0]
print('{} ({})'.format(pred_cmd, score))
elif FLAGS.token_decoding_algorithm == 'beam_search':
if batch_outputs:
top_k_predictions = batch_outputs[0]
top_k_scores = output_logits[0]
for j in xrange(min(FLAGS.beam_size, 10,
len(batch_outputs[0]))):
if len(top_k_predictions) <= j:
break
top_k_pred_tree, top_k_pred_cmd = top_k_predictions[j]
print('Prediction {}: {} ({}) '.format(
j + 1, top_k_pred_cmd, top_k_scores[j]))
print()
else:
print(APOLOGY_MSG)
print('> ', end='')
sys.stdout.flush()
sentence = sys.stdin.readline()
FLAGS.tg_token_embedding_size = 400
FLAGS.tg_vocab_size = 3400
FLAGS.dataset = 'bash'
FLAGS.data_dir = os.path.join(learning_module_dir, "data", FLAGS.dataset)
FLAGS.model_root_dir = os.path.join(learning_module_dir, "model", "seq2seq")
# create tensorflow session
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=FLAGS.log_device_placement))
# create model and load nerual model parameters.
model = trans.create_model(sess, forward_only=True, buckets=[(30, 40)])
vocabs = data_utils.load_vocab(FLAGS)
if FLAGS.fill_argument_slots:
# create slot filling classifier
model_param_dir = os.path.join(FLAGS.model_dir, 'train.mappings.X.Y.npz')
train_X, train_Y = data_utils.load_slot_filling_data(model_param_dir)
slot_filling_classifier = classifiers.KNearestNeighborModel(
FLAGS.num_nn_slot_filling, train_X, train_Y)
print('Slot filling classifier parameters loaded.')
else:
slot_filling_classifier = None
def translate_fun(sentence, slot_filling_classifier=slot_filling_classifier):
print('start running translation model')
print(sentence)
return decode_tools.translate_fun(sentence, sess, model, vocabs, FLAGS,
slot_filling_classifier)
def eval_slot_filling(dataset):
"""
Evaluate global slot filling algorithm F1 using ground truth templates.
"""
vocabs = data_utils.load_vocab(FLAGS)
rev_tg_vocab = vocabs.rev_tg_vocab
rev_tg_full_vocab = vocabs.rev_tg_full_vocab
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=FLAGS.log_device_placement)) as sess:
# Create model.
FLAGS.beam_size = 1
FLAGS.token_decoding_algorithm = 'beam_search'
FLAGS.force_reading_input = True
model = graph_utils.create_model(sess,
FLAGS,
Seq2SeqModel,
buckets=_buckets,
forward_only=True)
model_param_dir = os.path.join(FLAGS.model_dir,
'train.mappings.X.Y.npz')
train_X, train_Y = data_utils.load_slot_filling_data(model_param_dir)
slot_filling_classifier = classifiers.KNearestNeighborModel(
FLAGS.num_nn_slot_filling, train_X, train_Y)
print('Slot filling classifier parameters loaded.')
num_correct_argument = 0.0
num_argument = 0.0
num_correct_align = 0.0
num_predict_align = 0.0
num_gt_align = 0.0
for bucket_id in xrange(len(_buckets)):
for data_id in xrange(len(dataset[bucket_id])):
dp = dataset[bucket_id][data_id]
gt_mappings = [tuple(m) for m in dp.mappings]
outputs = dp.tg_ids[1:-1]
full_outputs = dp.tg_full_ids[1:-1]
if gt_mappings:
_, entities = tokenizer.ner_tokenizer(dp.sc_txt)
nl_fillers = entities[0]
encoder_inputs = [dp.sc_ids]
encoder_full_inputs = [dp.sc_copy_ids] \
if FLAGS.use_copy else [dp.sc_full_ids]
decoder_inputs = [dp.tg_ids]
decoder_full_inputs = [dp.tg_full_ids] \
if FLAGS.use_copy else [dp.tg_copy_ids]
pointer_targets = [dp.pointer_targets] \
if FLAGS.use_copy else None
formatted_example = model.format_example(
[encoder_inputs, encoder_full_inputs],
[decoder_inputs, decoder_full_inputs],
pointer_targets=pointer_targets,
bucket_id=bucket_id)
model_outputs = model.step(sess,
formatted_example,
bucket_id,
forward_only=True)
encoder_outputs = model_outputs.encoder_hidden_states
decoder_outputs = model_outputs.decoder_hidden_states
print(decoder_outputs[:, 0, :])
cm_slots = {}
output_tokens = []
for ii in xrange(len(outputs)):
output = outputs[ii]
if output < len(rev_tg_vocab):
token = rev_tg_vocab[output]
if "@@" in token:
token = token.split("@@")[-1]
output_tokens.append(token)
if token.startswith('__ARG__'):
token = token[len('__ARG__'):]
if nl_fillers is not None and \
token in constants._ENTITIES:
if ii > 0 and slot_filling.is_min_flag(
rev_tg_vocab[outputs[ii - 1]]):
token_type = 'Timespan'
else:
token_type = token
cm_slots[ii] = (token, token_type)
else:
output_tokens.append(data_utils._UNK)
if FLAGS.use_copy:
P = pointer_targets[0][0] > 0
pointers = model_outputs.pointers[0]
pointers = np.multiply(
np.sum(P.astype(float)[:pointers.shape[0],
-pointers.shape[1]:],
1,
keepdims=True), pointers)
else:
pointers = None
tree, _, mappings = slot_filling.stable_slot_filling(
output_tokens,
nl_fillers,
cm_slots,
pointers,
encoder_outputs[0],
decoder_outputs[0],
slot_filling_classifier,
verbose=True)
if mappings is not None:
# print(gt_mappings)
for mapping in mappings:
# print(mapping)
if mapping in gt_mappings:
num_correct_align += 1
num_predict_align += len(mappings)
num_gt_align += len(gt_mappings)
tokens = data_tools.ast2tokens(tree)
if not tokens:
continue
for ii in xrange(len(outputs)):
output = outputs[ii]
token = rev_tg_vocab[output]
if token.startswith('__ARG__'):
token = token[len('__ARG__'):]
if token in constants._ENTITIES:
argument = rev_tg_full_vocab[full_outputs[ii]]
if argument.startswith('__ARG__'):
argument = argument[len('__ARG__'):]
pred = tokens[ii]
if constants.remove_quotation(argument) == \
constants.remove_quotation(pred):
num_correct_argument += 1
num_argument += 1
if gt_mappings:
break
precision = num_correct_align / num_predict_align
recall = num_correct_align / num_gt_align
print("Argument Alignment Precision: {}".format(precision))
print("Argument Alignment Recall: {}".format(recall))
print("Argument Alignment F1: {}".format(2 * precision * recall /
(precision + recall)))
print("Argument filling accuracy: {}".format(num_correct_argument /
num_argument))
def decode_set(sess, model, dataset, top_k, FLAGS, verbose=True):
"""
Compute top-k predictions on the dev/test dataset and write the predictions
to disk.
:param sess: A TensorFlow session.
:param model: Prediction model object.
:param top_k: Number of top predictions to compute.
:param FLAGS: Training/testing hyperparameter settings.
:param verbose: If set, also print decoding results to screen.
"""
nl2bash = FLAGS.dataset.startswith('bash') and not FLAGS.explain
tokenizer_selector = 'cm' if FLAGS.explain else 'nl'
grouped_dataset = data_utils.group_data(
dataset,
use_bucket=model.buckets,
use_temp=FLAGS.normalized,
tokenizer_selector=tokenizer_selector)
vocabs = data_utils.load_vocab(FLAGS)
rev_sc_vocab = vocabs.rev_sc_vocab
if FLAGS.fill_argument_slots:
# create slot filling classifier
mapping_param_dir = os.path.join(FLAGS.model_dir,
'train.mappings.X.Y.npz')
train_X, train_Y = data_utils.load_slot_filling_data(mapping_param_dir)
slot_filling_classifier = classifiers.KNearestNeighborModel(
FLAGS.num_nn_slot_filling, train_X, train_Y)
print('Slot filling classifier parameters loaded.')
else:
slot_filling_classifier = None
ts = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d-%H%M%S')
pred_file_path = os.path.join(
model.model_dir, 'predictions.{}.{}'.format(model.decode_sig, ts))
pred_file = open(pred_file_path, 'w')
for example_id in xrange(len(grouped_dataset)):
key, data_group = grouped_dataset[example_id]
sc_txt = data_group[0].sc_txt
sc_temp = ' '.join([rev_sc_vocab[i] for i in data_group[0].sc_ids])
if verbose:
print('Example {}:'.format(example_id))
print('Original Source: {}'.format(sc_txt))
print('Source: {}'.format(sc_temp))
for j in xrange(len(data_group)):
print('GT Target {}: {}'.format(j + 1, data_group[j].tg_txt))
batch_outputs, output_logits = translate_fun(
data_group,
sess,
model,
vocabs,
FLAGS,
slot_filling_classifier=slot_filling_classifier)
if FLAGS.tg_char:
batch_outputs, batch_char_outputs = batch_outputs
if batch_outputs:
if FLAGS.token_decoding_algorithm == 'greedy':
tree, pred_cmd = batch_outputs[0]
if nl2bash:
pred_cmd = data_tools.ast2command(tree,
loose_constraints=True)
score = output_logits[0]
if verbose:
print('Prediction: {} ({})'.format(pred_cmd, score))
pred_file.write('{}\n'.format(pred_cmd))
elif FLAGS.token_decoding_algorithm == 'beam_search':
top_k_predictions = batch_outputs[0]
if FLAGS.tg_char:
top_k_char_predictions = batch_char_outputs[0]
top_k_scores = output_logits[0]
num_preds = min(FLAGS.beam_size, top_k, len(top_k_predictions))
for j in xrange(num_preds):
top_k_pred_tree, top_k_pred_cmd = top_k_predictions[j]
if nl2bash:
pred_cmd = data_tools.ast2command(
top_k_pred_tree, loose_constraints=True)
else:
pred_cmd = top_k_pred_cmd
pred_file.write('{}|||'.format(pred_cmd))
if verbose:
print('Prediction {}: {} ({})'.format(
j + 1, pred_cmd, top_k_scores[j]))
if FLAGS.tg_char:
print('Character-based prediction {}: {}'.format(
j + 1, top_k_char_predictions[j]))
pred_file.write('\n')
else:
print(APOLOGY_MSG)
pred_file.write('\n')
pred_file.close()
shutil.copyfile(
pred_file_path,
os.path.join(FLAGS.model_dir,
'predictions.{}.latest'.format(model.decode_sig)))