def decode_set(model, dataset, rev_nl_vocab, rev_cm_vocab, verbose=True):
grouped_dataset = data_utils.group_data_by_nl(dataset)
with DBConnection() as db:
db.remove_model(model_name)
num_eval = 0
for nl_temp in grouped_dataset:
batch_nl_strs, batch_cm_strs, batch_nls, batch_cmds = \
grouped_dataset[nl_temp]
nl_str = batch_nl_strs[0]
nl = batch_nls[0]
if verbose:
print("Example {}".format(num_eval+1))
print("Original English: " + nl_str.strip())
print("English: " + nl_temp)
for j in xrange(len(batch_cm_strs)):
print("GT Command {}: {}".format(j+1, batch_cm_strs[j].strip()))
top_k_results = model.test(nl, 10)
for i in xrange(len(top_k_results)):
nn, cmd, score = top_k_results[i]
nn_str = ' '.join([rev_nl_vocab[i] for i in nn])
tokens = []
for i in cmd:
pred_token = rev_cm_vocab[i]
if "@@" in pred_token:
pred_token = pred_token.split("@@")[-1]
tokens.append(pred_token)
pred_cmd = ' '.join(tokens)
tree = data_tools.bash_parser(pred_cmd)
if verbose:
print("NN: {}".format(nn_str))
print("Prediction {}: {} ({})".format(i, pred_cmd, score))
print("AST: ")
data_tools.pretty_print(tree, 0)
print
db.add_prediction(model_name, nl_str, pred_cmd, float(score),
update_mode=False)
num_eval += 1
print("train entropy = {}".format(entropy(train_set)))
dev_by_nl = group_data_by_nl(dev_set, use_nl_temp=FLAGS.dataset.startswith("bash"))
print("dev cmd/nl ratio = {}".format(ratio(dev_by_nl, 1)))
print("dev %nl(cmd+) = {}".format(pp(dev_by_nl)))
print("dev nl overlap = {}".format(overlap(train_by_nl, dev_by_nl)))
print("dev entropy = {}".format(entropy(dev_set)))
test_by_nl = group_data_by_nl(test_set, use_nl_temp=FLAGS.dataset.startswith("bash"))
print("test cmd/nl ratio = {}".format(ratio(test_by_nl, 1)))
print("test %nl(cmd+) = {}".format(pp(test_by_nl)))
print("test nl overlap = {}".format(overlap(train_by_nl, test_by_nl)))
print("test entropy = {}".format(entropy(test_set)))
print("total entropy = {}".format(entropy(train_set + dev_set + test_set)))
train_by_cm = group_data_by_cm(train_set, use_cm_temp=FLAGS.dataset.startswith("bash"))
print(len(train_by_cm))
print("train nl/cmd ratio = {}".format(ratio(train_by_cm, 0)))
print("train %cmd(nl+) = {}".format(pp(train_by_cm)))
dev_by_cm = group_data_by_cm(dev_set, use_cm_temp=FLAGS.dataset.startswith("bash"))
print("dev nl/cmd ratio = {}".format(ratio(dev_by_cm, 0)))
print("dev %cmd(nl+) = {}".format(pp(dev_by_cm)))
print("dev cm overlap = {}".format(overlap(train_by_cm, dev_by_cm)))
test_by_cm = group_data_by_cm(test_set, use_cm_temp=FLAGS.dataset.startswith("bash"))
print("test nl/cmd ratio = {}".format(ratio(test_by_cm, 0)))
print("test %cmd(nl+) = {}".format(pp(test_by_cm)))
print("test cm overlap = {}".format(overlap(train_by_cm, test_by_cm)))
if __name__ == "__main__":
ast = data_tools.paren_parser(sys.argv[1])
data_tools.pretty_print(ast)
print(data_tools.ast2template(
ast, loose_constraints=True, arg_type_only=True))
