def _init_training(self, das_file, ttree_file, data_portion):
"""Initialize training.
Store input data, initialize 1-hot feature representations for input and output and
transform training data accordingly, initialize the classification neural network.
"""
# read input
log_info('Reading DAs from ' + das_file + '...')
das = read_das(das_file)
log_info('Reading t-trees from ' + ttree_file + '...')
ttree_doc = read_ttrees(ttree_file)
trees = trees_from_doc(ttree_doc, self.language, self.selector)
# make training data smaller if necessary
train_size = int(round(data_portion * len(trees)))
self.train_trees = trees[:train_size]
self.train_das = das[:train_size]
# add empty tree + empty DA to training data
# (i.e. forbid the network to keep any of its outputs "always-on")
train_size += 1
self.train_trees.append(TreeData())
empty_da = DialogueAct()
empty_da.parse('inform()')
self.train_das.append(empty_da)
self.train_order = range(len(self.train_trees))
log_info('Using %d training instances.' % train_size)
# initialize input features/embeddings
if self.tree_embs:
self.dict_size = self.tree_embs.init_dict(self.train_trees)
self.X = np.array([self.tree_embs.get_embeddings(tree) for tree in self.train_trees])
else:
self.tree_feats = Features(['node: presence t_lemma formeme'])
self.tree_vect = DictVectorizer(sparse=False, binarize_numeric=True)
self.X = [self.tree_feats.get_features(tree, {}) for tree in self.train_trees]
self.X = self.tree_vect.fit_transform(self.X)
# initialize output features
self.da_feats = Features(['dat: dat_presence', 'svp: svp_presence'])
self.da_vect = DictVectorizer(sparse=False, binarize_numeric=True)
self.y = [self.da_feats.get_features(None, {'da': da}) for da in self.train_das]
self.y = self.da_vect.fit_transform(self.y)
# initialize I/O shapes
self.input_shape = [list(self.X[0].shape)]
self.num_outputs = len(self.da_vect.get_feature_names())
# initialize NN classifier
self._init_neural_network()
def evaluate_file(self, das_file, ttree_file):
"""Evaluate the reranking classifier on a given pair of DA/tree files (show the
total Hamming distance and total number of DAIs)
@param das_file: DA file path
@param ttree_file: trees/sentences file path
@return: a tuple (total DAIs, distance)
"""
das = read_das(das_file)
ttree_doc = read_ttrees(ttree_file)
if self.mode == 'tokens':
tokens = tokens_from_doc(ttree_doc, self.language, self.selector)
trees = self._tokens_to_flat_trees(tokens)
elif self.mode == 'tagged_lemmas':
tls = tagged_lemmas_from_doc(ttree_doc, self.language, self.selector)
trees = self._tokens_to_flat_trees(tls)
else:
trees = trees_from_doc(ttree_doc, self.language, self.selector)
da_len = 0
dist = 0
for da, tree in zip(das, trees):
da_len += len(da)
dist += self.dist_to_da(da, [tree])[0]
return da_len, dist
def _init_training(self, das_file, ttree_file, data_portion):
"""Initialize training.
Store input data, initialize 1-hot feature representations for input and output and
transform training data accordingly, initialize the classification neural network.
"""
# read input
log_info('Reading DAs from ' + das_file + '...')
das = read_das(das_file)
log_info('Reading t-trees from ' + ttree_file + '...')
ttree_doc = read_ttrees(ttree_file)
trees = trees_from_doc(ttree_doc, self.language, self.selector)
# make training data smaller if necessary
train_size = int(round(data_portion * len(trees)))
self.train_trees = trees[:train_size]
self.train_das = das[:train_size]
# add empty tree + empty DA to training data
# (i.e. forbid the network to keep any of its outputs "always-on")
train_size += 1
self.train_trees.append(TreeData())
empty_da = DA.parse('inform()')
self.train_das.append(empty_da)
self.train_order = range(len(self.train_trees))
log_info('Using %d training instances.' % train_size)
# initialize input features/embeddings
if self.tree_embs:
self.dict_size = self.tree_embs.init_dict(self.train_trees)
self.X = np.array([
self.tree_embs.get_embeddings(tree)
for tree in self.train_trees
])
else:
self.tree_feats = Features(['node: presence t_lemma formeme'])
self.tree_vect = DictVectorizer(sparse=False,
binarize_numeric=True)
self.X = [
self.tree_feats.get_features(tree, {})
for tree in self.train_trees
]
self.X = self.tree_vect.fit_transform(self.X)
# initialize output features
self.da_feats = Features(['dat: dat_presence', 'svp: svp_presence'])
self.da_vect = DictVectorizer(sparse=False, binarize_numeric=True)
self.y = [
self.da_feats.get_features(None, {'da': da})
for da in self.train_das
]
self.y = self.da_vect.fit_transform(self.y)
# initialize I/O shapes
self.input_shape = [list(self.X[0].shape)]
self.num_outputs = len(self.da_vect.get_feature_names())
# initialize NN classifier
self._init_neural_network()
def evaluate_file(self, das_file, ttree_file):
"""Evaluate the reranking classifier on a given pair of DA/tree files (show the
total Hamming distance and total number of DAIs)
@param das_file: DA file path
@param ttree_file: trees/sentences file path
@return: a tuple (total DAIs, distance)
"""
das = read_das(das_file)
ttree_doc = read_ttrees(ttree_file)
if self.mode == 'tokens':
tokens = tokens_from_doc(ttree_doc, self.language, self.selector)
trees = self._tokens_to_flat_trees(tokens)
elif self.mode == 'tagged_lemmas':
tls = tagged_lemmas_from_doc(ttree_doc, self.language,
self.selector)
trees = self._tokens_to_flat_trees(tls)
else:
trees = trees_from_doc(ttree_doc, self.language, self.selector)
da_len = 0
dist = 0
for da, tree in zip(das, trees):
da_len += len(da)
dist += self.dist_to_da(da, [tree])[0]
return da_len, dist
def sample_gen(args):
from pytreex.core.document import Document
opts, files = getopt(args, 'r:n:o:w:')
num_to_generate = 1
oracle_eval_file = None
fname_ttrees_out = None
for opt, arg in opts:
if opt == '-n':
num_to_generate = int(arg)
elif opt == '-o':
oracle_eval_file = arg
elif opt == '-w':
fname_ttrees_out = arg
if len(files) != 2:
sys.exit(__doc__)
fname_cand_model, fname_da_test = files
# load model
log_info('Initializing...')
candgen = RandomCandidateGenerator.load_from_file(fname_cand_model)
ranker = candgen
tgen = SamplingPlanner({'candgen': candgen, 'ranker': ranker})
# generate
log_info('Generating...')
gen_doc = Document()
das = read_das(fname_da_test)
for da in das:
for _ in xrange(num_to_generate): # repeat generation n times
tgen.generate_tree(da, gen_doc)
# evaluate if needed
if oracle_eval_file is not None:
log_info('Evaluating oracle F1...')
log_info('Loading gold data from ' + oracle_eval_file)
gold_trees = ttrees_from_doc(read_ttrees(oracle_eval_file), tgen.language, tgen.selector)
gen_trees = ttrees_from_doc(gen_doc, tgen.language, tgen.selector)
log_info('Gold data loaded.')
correct, predicted, gold = 0, 0, 0
for gold_tree, gen_trees in zip(gold_trees, chunk_list(gen_trees, num_to_generate)):
# find best of predicted trees (in terms of F1)
_, tc, tp, tg = max([(f1_from_counts(c, p, g), c, p, g) for c, p, g
in map(lambda gen_tree: corr_pred_gold(gold_tree, gen_tree),
gen_trees)],
key=lambda x: x[0])
correct += tc
predicted += tp
gold += tg
# evaluate oracle F1
log_info("Oracle Precision: %.6f, Recall: %.6f, F1: %.6f" % p_r_f1_from_counts(correct, predicted, gold))
# write output
if fname_ttrees_out is not None:
log_info('Writing output...')
write_ttrees(gen_doc, fname_ttrees_out)
def sample_gen(args):
from pytreex.core.document import Document
opts, files = getopt(args, 'r:n:o:w:')
num_to_generate = 1
oracle_eval_file = None
fname_ttrees_out = None
for opt, arg in opts:
if opt == '-n':
num_to_generate = int(arg)
elif opt == '-o':
oracle_eval_file = arg
elif opt == '-w':
fname_ttrees_out = arg
if len(files) != 2:
sys.exit(__doc__)
fname_cand_model, fname_da_test = files
# load model
log_info('Initializing...')
candgen = RandomCandidateGenerator.load_from_file(fname_cand_model)
ranker = candgen
tgen = SamplingPlanner({'candgen': candgen, 'ranker': ranker})
# generate
log_info('Generating...')
gen_doc = Document()
das = read_das(fname_da_test)
for da in das:
for _ in xrange(num_to_generate): # repeat generation n times
tgen.generate_tree(da, gen_doc)
# evaluate if needed
if oracle_eval_file is not None:
log_info('Evaluating oracle F1...')
log_info('Loading gold data from ' + oracle_eval_file)
gold_trees = ttrees_from_doc(read_ttrees(oracle_eval_file), tgen.language, tgen.selector)
gen_trees = ttrees_from_doc(gen_doc, tgen.language, tgen.selector)
log_info('Gold data loaded.')
correct, predicted, gold = 0, 0, 0
for gold_tree, gen_trees in zip(gold_trees, chunk_list(gen_trees, num_to_generate)):
# find best of predicted trees (in terms of F1)
_, tc, tp, tg = max([(f1_from_counts(c, p, g), c, p, g) for c, p, g
in map(lambda gen_tree: corr_pred_gold(gold_tree, gen_tree),
gen_trees)],
key=lambda x: x[0])
correct += tc
predicted += tp
gold += tg
# evaluate oracle F1
log_info("Oracle Precision: %.6f, Recall: %.6f, F1: %.6f" % p_r_f1_from_counts(correct, predicted, gold))
# write output
if fname_ttrees_out is not None:
log_info('Writing output...')
write_ttrees(gen_doc, fname_ttrees_out)
def _load_contexts(self, das, context_file):
"""Load input context utterances from a .yaml.gz/.pickle.gz/.txt file and add them to the
given DAs (each returned item is then a tuple of context + DA)."""
# read contexts, combine them with corresponding DAs for easier handling
if context_file is None:
raise ValueError('Expected context utterances file name!')
log_info('Reading context utterances from %s...' % context_file)
if context_file.endswith('.txt'):
contexts = read_tokens(context_file)
else:
contexts = tokens_from_doc(read_ttrees(context_file), self.language, self.selector)
return [(context, da) for context, da in zip(contexts, das)]
def _load_contexts(self, das, context_file):
"""Load input context utterances from a .yaml.gz/.pickle.gz/.txt file and add them to the
given DAs (each returned item is then a tuple of context + DA)."""
# read contexts, combine them with corresponding DAs for easier handling
if context_file is None:
raise ValueError('Expected context utterances file name!')
log_info('Reading context utterances from %s...' % context_file)
if context_file.endswith('.txt'):
contexts = read_tokens(context_file)
else:
contexts = tokens_from_doc(read_ttrees(context_file), self.language, self.selector)
return [(context, da) for context, da in zip(contexts, das)]
def _load_trees(self, ttree_file, selector=None):
"""Load input trees/sentences from a .yaml.gz/.pickle.gz (trees) or .txt (sentences) file."""
log_info('Reading t-trees/sentences from ' + ttree_file + '...')
if ttree_file.endswith('.txt'):
if not self.use_tokens:
raise ValueError("Cannot read trees from a .txt file (%s)!" % ttree_file)
return read_tokens(ttree_file)
else:
ttree_doc = read_ttrees(ttree_file)
if selector is None:
selector = self.selector
if self.use_tokens:
return tokens_from_doc(ttree_doc, self.language, selector)
else:
return trees_from_doc(ttree_doc, self.language, selector)
def _load_trees(self, ttree_file, selector=None):
"""Load input trees/sentences from a .yaml.gz/.pickle.gz (trees) or .txt (sentences) file."""
log_info('Reading t-trees/sentences from ' + ttree_file + '...')
if ttree_file.endswith('.txt'):
if not self.use_tokens:
raise ValueError("Cannot read trees from a .txt file (%s)!" %
ttree_file)
return read_tokens(ttree_file)
else:
ttree_doc = read_ttrees(ttree_file)
if selector is None:
selector = self.selector
if self.use_tokens:
return tokens_from_doc(ttree_doc, self.language, selector)
else:
return trees_from_doc(ttree_doc, self.language, selector)
def seq2seq_gen(args):
"""Sequence-to-sequence generation"""
ap = ArgumentParser()
ap.add_argument('-e', '--eval-file', type=str, help='A ttree/text file for evaluation')
ap.add_argument('-a', '--abstr-file', type=str,
help='Lexicalization file (a.k.a. abstraction instructions, for postprocessing)')
ap.add_argument('-r', '--ref-selector', type=str, default='',
help='Selector for reference trees in the evaluation file')
ap.add_argument('-t', '--target-selector', type=str, default='',
help='Target selector for generated trees in the output file')
ap.add_argument('-d', '--debug-logfile', type=str, help='Debug output file name')
ap.add_argument('-w', '--output-file', type=str, help='Output tree/text file')
ap.add_argument('-b', '--beam-size', type=int,
help='Override beam size for beam search decoding')
ap.add_argument('-c', '--context-file', type=str,
help='Input ttree/text file with context utterances')
ap.add_argument('seq2seq_model_file', type=str, help='Trained Seq2Seq generator model')
ap.add_argument('da_test_file', type=str, help='Input DAs for generation')
args = ap.parse_args(args)
if args.debug_logfile:
set_debug_stream(file_stream(args.debug_logfile, mode='w'))
# load the generator
tgen = Seq2SeqBase.load_from_file(args.seq2seq_model_file)
if args.beam_size is not None:
tgen.beam_size = args.beam_size
# read input files
das = read_das(args.da_test_file)
if args.context_file:
if not tgen.use_context and not tgen.context_bleu_weight:
log_warn('Generator is not trained to use context, ignoring context input file.')
else:
if args.context_file.endswith('.txt'):
contexts = read_tokens(args.context_file)
else:
contexts = tokens_from_doc(read_ttrees(args.context_file),
tgen.language, tgen.selector)
das = [(context, da) for context, da in zip(contexts, das)]
# generate
log_info('Generating...')
gen_trees = []
for num, da in enumerate(das, start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_trees.append(tgen.generate_tree(da))
log_info(tgen.get_slot_err_stats())
# evaluate the generated trees against golden trees (delexicalized)
eval_doc = None
if args.eval_file and not args.eval_file.endswith('.txt'):
eval_doc = read_ttrees(args.eval_file)
evaler = Evaluator()
evaler.process_eval_doc(eval_doc, gen_trees, tgen.language, args.ref_selector,
args.target_selector or tgen.selector)
# lexicalize, if required
if args.abstr_file and tgen.lexicalizer:
log_info('Lexicalizing...')
tgen.lexicalize(gen_trees, args.abstr_file)
# evaluate the generated & lexicalized tokens (F1 and BLEU scores)
if args.eval_file and args.eval_file.endswith('.txt'):
eval_tokens(das, read_tokens(args.eval_file, ref_mode=True), gen_trees)
# write output .yaml.gz or .txt
if args.output_file is not None:
log_info('Writing output...')
if args.output_file.endswith('.txt'):
write_tokens(gen_trees, args.output_file)
else:
write_ttrees(create_ttree_doc(gen_trees, eval_doc, tgen.language,
args.target_selector or tgen.selector),
args.output_file)
if len(files) != 2:
sys.exit(
'Usage: python inspect_data.py [-l lang] [-s selector] <trees.yaml> <das.txt>'
)
language = 'en'
selector = ''
for opt, arg in opts:
if opt == '-l':
language = arg
elif opt == '-s':
selector = arg
trees = trees_from_doc(read_ttrees(files[0]), language, selector)
das = read_das(files[1])
# TREE SIZES
tree_sizes = defaultdict(int)
for tree in trees:
tree_sizes[len(tree)] += 1
print "Tree sizes:\n==========="
for k, v in sorted(tree_sizes.items()):
print k, "\t", v
# DAS -> NODES
das_for_nodes = {}
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Simple statistics about types of inner nodes and leaves
# in the training data.
#
from tgen.futil import read_ttrees, ttrees_from_doc
import sys
import codecs
if len(sys.argv[1:]) != 1:
sys.exit('Usage: python leaf_stats.py <file.pickle>')
stats = {}
for ttree in ttrees_from_doc(read_ttrees(sys.argv[1]), 'en', ''):
for tnode in ttree.get_descendants():
node_id = (tnode.t_lemma, tnode.formeme)
if node_id not in stats:
stats[node_id] = {'leaf': 0, 'int': 0}
if tnode.get_children():
stats[node_id]['int'] += 1
else:
stats[node_id]['leaf'] += 1
out = codecs.getwriter('UTF-8')(sys.stdout)
for node_id, val in stats.iteritems():
print >> out, node_id, "\t", val['int'], "\t", val['leaf']
opts, files = getopt(sys.argv[1:], 'l:s:')
if len(files) != 2:
sys.exit('Usage: python inspect_data.py [-l lang] [-s selector] <trees.yaml> <das.txt>')
language = 'en'
selector = ''
for opt, arg in opts:
if opt == '-l':
language = arg
elif opt == '-s':
selector = arg
trees = trees_from_doc(read_ttrees(files[0]), language, selector)
das = read_das(files[1])
# TREE SIZES
tree_sizes = defaultdict(int)
for tree in trees:
tree_sizes[len(tree)] += 1
print "Tree sizes:\n==========="
for k, v in sorted(tree_sizes.items()):
print k, "\t", v
# DAS -> NODES
#!/usr/bin/env python
from tgen.futil import read_ttrees
import sys
if len(sys.argv[1:]) != 1:
sys.exit('Usage: python yaml2pickle.py <file.yaml>')
read_ttrees(sys.argv[1])
#!/usr/bin/env python
from flect.config import Config
from tgen.features import Features
from tgen.futil import trees_from_doc, read_ttrees, read_das
import sys
import timeit
import datetime
if len(sys.argv[1:]) != 3:
sys.exit('Usage: ./bench_feats.py features_cfg.py trees.yaml.gz das.txt')
print >> sys.stderr, 'Loading...'
cfg = Config(sys.argv[1])
trees = trees_from_doc(read_ttrees(sys.argv[2]), 'en', '')
das = read_das(sys.argv[3])
feats = Features(cfg['features'])
def test_func():
for tree, da in zip(trees, das):
feats.get_features(tree, {'da': da})
print >> sys.stderr, 'Running test...'
secs = timeit.timeit('test_func()', setup='from __main__ import test_func', number=10)
td = datetime.timedelta(seconds=secs)
print >> sys.stderr, 'Time taken: %s' % str(td)
def asearch_gen(args):
"""A*search generation"""
from pytreex.core.document import Document
opts, files = getopt(args, 'e:d:w:c:s:')
eval_file = None
fname_ttrees_out = None
cfg_file = None
eval_selector = ''
for opt, arg in opts:
if opt == '-e':
eval_file = arg
elif opt == '-s':
eval_selector = arg
elif opt == '-d':
set_debug_stream(file_stream(arg, mode='w'))
elif opt == '-w':
fname_ttrees_out = arg
elif opt == '-c':
cfg_file = arg
if len(files) != 3:
sys.exit('Invalid arguments.\n' + __doc__)
fname_cand_model, fname_rank_model, fname_da_test = files
log_info('Initializing...')
candgen = RandomCandidateGenerator.load_from_file(fname_cand_model)
ranker = PerceptronRanker.load_from_file(fname_rank_model)
cfg = Config(cfg_file) if cfg_file else {}
cfg.update({'candgen': candgen, 'ranker': ranker})
tgen = ASearchPlanner(cfg)
log_info('Generating...')
das = read_das(fname_da_test)
if eval_file is None:
gen_doc = Document()
else:
eval_doc = read_ttrees(eval_file)
if eval_selector == tgen.selector:
gen_doc = Document()
else:
gen_doc = eval_doc
# generate and evaluate
if eval_file is not None:
# generate + analyze open&close lists
lists_analyzer = ASearchListsAnalyzer()
for num, (da, gold_tree) in enumerate(zip(
das, trees_from_doc(eval_doc, tgen.language, eval_selector)),
start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_tree = tgen.generate_tree(da, gen_doc)
lists_analyzer.append(gold_tree, tgen.open_list, tgen.close_list)
if gen_tree != gold_tree:
log_debug("\nDIFFING TREES:\n" +
tgen.ranker.diffing_trees_with_scores(
da, gold_tree, gen_tree) + "\n")
log_info('Gold tree BEST: %.4f, on CLOSE: %.4f, on ANY list: %4f' %
lists_analyzer.stats())
# evaluate the generated trees against golden trees
eval_ttrees = ttrees_from_doc(eval_doc, tgen.language, eval_selector)
gen_ttrees = ttrees_from_doc(gen_doc, tgen.language, tgen.selector)
log_info('Evaluating...')
evaler = Evaluator()
for eval_bundle, eval_ttree, gen_ttree, da in zip(
eval_doc.bundles, eval_ttrees, gen_ttrees, das):
# add some stats about the tree directly into the output file
add_bundle_text(
eval_bundle, tgen.language, tgen.selector + 'Xscore',
"P: %.4f R: %.4f F1: %.4f" %
p_r_f1_from_counts(*corr_pred_gold(eval_ttree, gen_ttree)))
# collect overall stats
evaler.append(eval_ttree, gen_ttree,
ranker.score(TreeData.from_ttree(eval_ttree), da),
ranker.score(TreeData.from_ttree(gen_ttree), da))
# print overall stats
log_info("NODE precision: %.4f, Recall: %.4f, F1: %.4f" %
evaler.p_r_f1())
log_info("DEP precision: %.4f, Recall: %.4f, F1: %.4f" %
evaler.p_r_f1(EvalTypes.DEP))
log_info("Tree size stats:\n * GOLD %s\n * PRED %s\n * DIFF %s" %
evaler.size_stats())
log_info("Score stats:\n * GOLD %s\n * PRED %s\n * DIFF %s" %
evaler.score_stats())
log_info(
"Common subtree stats:\n -- SIZE: %s\n -- ΔGLD: %s\n -- ΔPRD: %s" %
evaler.common_substruct_stats())
# just generate
else:
for da in das:
tgen.generate_tree(da, gen_doc)
# write output
if fname_ttrees_out is not None:
log_info('Writing output...')
write_ttrees(gen_doc, fname_ttrees_out)
def _init_training(self, das, trees, data_portion):
"""Initialize training.
Store input data, initialize 1-hot feature representations for input and output and
transform training data accordingly, initialize the classification neural network.
@param das: name of source file with training DAs, or list of DAs
@param trees: name of source file with corresponding trees/sentences, or list of trees
@param data_portion: portion of the training data to be used (0.0-1.0)
"""
# read input from files or take it directly from parameters
if not isinstance(das, list):
log_info('Reading DAs from ' + das + '...')
das = read_das(das)
if not isinstance(trees, list):
log_info('Reading t-trees from ' + trees + '...')
ttree_doc = read_ttrees(trees)
if self.mode == 'tokens':
tokens = tokens_from_doc(ttree_doc, self.language,
self.selector)
trees = self._tokens_to_flat_trees(tokens)
elif self.mode == 'tagged_lemmas':
tls = tagged_lemmas_from_doc(ttree_doc, self.language,
self.selector)
trees = self._tokens_to_flat_trees(tls, use_tags=True)
else:
trees = trees_from_doc(ttree_doc, self.language, self.selector)
elif self.mode in ['tokens', 'tagged_lemmas']:
trees = self._tokens_to_flat_trees(
trees, use_tags=self.mode == 'tagged_lemmas')
# make training data smaller if necessary
train_size = int(round(data_portion * len(trees)))
self.train_trees = trees[:train_size]
self.train_das = das[:train_size]
# ignore contexts, if they are contained in the DAs
if isinstance(self.train_das[0], tuple):
self.train_das = [da for (context, da) in self.train_das]
# delexicalize if DAs are lexicalized and we don't want that
if self.delex_slots:
self.train_das = [
da.get_delexicalized(self.delex_slots) for da in self.train_das
]
# add empty tree + empty DA to training data
# (i.e. forbid the network to keep any of its outputs "always-on")
train_size += 1
self.train_trees.append(TreeData())
empty_da = DA.parse('inform()')
self.train_das.append(empty_da)
self.train_order = range(len(self.train_trees))
log_info('Using %d training instances.' % train_size)
# initialize input features/embeddings
if self.tree_embs:
self.dict_size = self.tree_embs.init_dict(self.train_trees)
self.X = np.array([
self.tree_embs.get_embeddings(tree)
for tree in self.train_trees
])
else:
self.tree_feats = Features(['node: presence t_lemma formeme'])
self.tree_vect = DictVectorizer(sparse=False,
binarize_numeric=True)
self.X = [
self.tree_feats.get_features(tree, {})
for tree in self.train_trees
]
self.X = self.tree_vect.fit_transform(self.X)
# initialize output features
self.da_feats = Features(['dat: dat_presence', 'svp: svp_presence'])
self.da_vect = DictVectorizer(sparse=False, binarize_numeric=True)
self.y = [
self.da_feats.get_features(None, {'da': da})
for da in self.train_das
]
self.y = self.da_vect.fit_transform(self.y)
log_info('Number of binary classes: %d.' %
len(self.da_vect.get_feature_names()))
# initialize I/O shapes
if not self.tree_embs:
self.input_shape = list(self.X[0].shape)
else:
self.input_shape = self.tree_embs.get_embeddings_shape()
self.num_outputs = len(self.da_vect.get_feature_names())
# initialize NN classifier
self._init_neural_network()
# initialize the NN variables
self.session.run(tf.global_variables_initializer())
#!/usr/bin/env python
from flect.config import Config
from tgen.features import Features
from tgen.futil import trees_from_doc, read_ttrees, read_das
import sys
import timeit
import datetime
if len(sys.argv[1:]) != 3:
sys.exit('Usage: ./bench_feats.py features_cfg.py trees.yaml.gz das.txt')
print >> sys.stderr, 'Loading...'
cfg = Config(sys.argv[1])
trees = trees_from_doc(read_ttrees(sys.argv[2]), 'en', '')
das = read_das(sys.argv[3])
feats = Features(cfg['features'])
def test_func():
for tree, da in zip(trees, das):
feats.get_features(tree, {'da': da})
print >> sys.stderr, 'Running test...'
secs = timeit.timeit('test_func()',
setup='from __main__ import test_func',
number=10)
td = datetime.timedelta(seconds=secs)
def seq2seq_gen(args):
"""Sequence-to-sequence generation"""
ap = ArgumentParser()
ap.add_argument('-e', '--eval-file', type=str, help='A ttree/text file for evaluation')
ap.add_argument('-a', '--abstr-file', type=str,
help='Lexicalization file (a.k.a. abstraction instsructions, for tokens only)')
ap.add_argument('-r', '--ref-selector', type=str, default='',
help='Selector for reference trees in the evaluation file')
ap.add_argument('-t', '--target-selector', type=str, default='',
help='Target selector for generated trees in the output file')
ap.add_argument('-d', '--debug-logfile', type=str, help='Debug output file name')
ap.add_argument('-w', '--output-file', type=str, help='Output tree/text file')
ap.add_argument('-b', '--beam-size', type=int,
help='Override beam size for beam search decoding')
ap.add_argument('-c', '--context-file', type=str,
help='Input ttree/text file with context utterances')
ap.add_argument('seq2seq_model_file', type=str, help='Trained Seq2Seq generator model')
ap.add_argument('da_test_file', type=str, help='Input DAs for generation')
args = ap.parse_args(args)
if args.debug_logfile:
set_debug_stream(file_stream(args.debug_logfile, mode='w'))
# load the generator
tgen = Seq2SeqBase.load_from_file(args.seq2seq_model_file)
if args.beam_size is not None:
tgen.beam_size = args.beam_size
# read input files
das = read_das(args.da_test_file)
if args.context_file:
if not tgen.use_context and not tgen.context_bleu_weight:
log_warn('Generator is not trained to use context, ignoring context input file.')
else:
if args.context_file.endswith('.txt'):
contexts = read_tokens(args.context_file)
else:
contexts = tokens_from_doc(read_ttrees(args.context_file),
tgen.language, tgen.selector)
das = [(context, da) for context, da in zip(contexts, das)]
# prepare evaluation
if args.eval_file is None or args.eval_file.endswith('.txt'): # just tokens
gen_doc = []
else: # Trees: depending on PyTreex
from pytreex.core.document import Document
eval_doc = read_ttrees(args.eval_file)
if args.ref_selector == args.target_selector:
gen_doc = Document()
else:
gen_doc = eval_doc
if args.eval_file:
tgen.init_slot_err_stats()
# generate
log_info('Generating...')
tgen.selector = args.target_selector # override target selector for generation
for num, da in enumerate(das, start=1):
log_debug("\n\nTREE No. %03d" % num)
tgen.generate_tree(da, gen_doc)
# evaluate
if args.eval_file is not None:
log_info(tgen.get_slot_err_stats())
# evaluate the generated tokens (F1 and BLEU scores)
if args.eval_file.endswith('.txt'):
lexicalize_tokens(gen_doc, lexicalization_from_doc(args.abstr_file))
eval_tokens(das, read_tokens(args.eval_file, ref_mode=True), gen_doc)
# evaluate the generated trees against golden trees
else:
eval_trees(das,
ttrees_from_doc(eval_doc, tgen.language, args.ref_selector),
ttrees_from_doc(gen_doc, tgen.language, args.target_selector),
eval_doc, tgen.language, tgen.selector)
# write output .yaml.gz or .txt
if args.output_file is not None:
log_info('Writing output...')
if args.output_file.endswith('.txt'):
write_tokens(gen_doc, args.output_file)
else:
write_ttrees(gen_doc, args.output_file)
#!/usr/bin/env python
from tgen.futil import read_ttrees, write_ttrees
import sys
import re
if len(sys.argv[1:]) != 1:
sys.exit('Usage: python pickle2yaml.py <file.pickle>')
doc = read_ttrees(sys.argv[1])
out_fname = re.sub('\.pickle', '.yaml', sys.argv[1])
write_ttrees(doc, out_fname)
def asearch_gen(args):
"""A*search generation"""
from pytreex.core.document import Document
opts, files = getopt(args, 'e:d:w:c:s:')
eval_file = None
fname_ttrees_out = None
cfg_file = None
eval_selector = ''
for opt, arg in opts:
if opt == '-e':
eval_file = arg
elif opt == '-s':
eval_selector = arg
elif opt == '-d':
set_debug_stream(file_stream(arg, mode='w'))
elif opt == '-w':
fname_ttrees_out = arg
elif opt == '-c':
cfg_file = arg
if len(files) != 3:
sys.exit('Invalid arguments.\n' + __doc__)
fname_cand_model, fname_rank_model, fname_da_test = files
log_info('Initializing...')
candgen = RandomCandidateGenerator.load_from_file(fname_cand_model)
ranker = PerceptronRanker.load_from_file(fname_rank_model)
cfg = Config(cfg_file) if cfg_file else {}
cfg.update({'candgen': candgen, 'ranker': ranker})
tgen = ASearchPlanner(cfg)
log_info('Generating...')
das = read_das(fname_da_test)
if eval_file is None:
gen_doc = Document()
else:
eval_doc = read_ttrees(eval_file)
if eval_selector == tgen.selector:
gen_doc = Document()
else:
gen_doc = eval_doc
# generate and evaluate
if eval_file is not None:
# generate + analyze open&close lists
lists_analyzer = ASearchListsAnalyzer()
for num, (da, gold_tree) in enumerate(zip(das,
trees_from_doc(eval_doc, tgen.language, eval_selector)),
start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_tree = tgen.generate_tree(da, gen_doc)
lists_analyzer.append(gold_tree, tgen.open_list, tgen.close_list)
if gen_tree != gold_tree:
log_debug("\nDIFFING TREES:\n" + tgen.ranker.diffing_trees_with_scores(da, gold_tree, gen_tree) + "\n")
log_info('Gold tree BEST: %.4f, on CLOSE: %.4f, on ANY list: %4f' % lists_analyzer.stats())
# evaluate the generated trees against golden trees
eval_ttrees = ttrees_from_doc(eval_doc, tgen.language, eval_selector)
gen_ttrees = ttrees_from_doc(gen_doc, tgen.language, tgen.selector)
log_info('Evaluating...')
evaler = Evaluator()
for eval_bundle, eval_ttree, gen_ttree, da in zip(eval_doc.bundles, eval_ttrees, gen_ttrees, das):
# add some stats about the tree directly into the output file
add_bundle_text(eval_bundle, tgen.language, tgen.selector + 'Xscore',
"P: %.4f R: %.4f F1: %.4f" % p_r_f1_from_counts(*corr_pred_gold(eval_ttree, gen_ttree)))
# collect overall stats
evaler.append(eval_ttree,
gen_ttree,
ranker.score(TreeData.from_ttree(eval_ttree), da),
ranker.score(TreeData.from_ttree(gen_ttree), da))
# print overall stats
log_info("NODE precision: %.4f, Recall: %.4f, F1: %.4f" % evaler.p_r_f1())
log_info("DEP precision: %.4f, Recall: %.4f, F1: %.4f" % evaler.p_r_f1(EvalTypes.DEP))
log_info("Tree size stats:\n * GOLD %s\n * PRED %s\n * DIFF %s" % evaler.size_stats())
log_info("Score stats:\n * GOLD %s\n * PRED %s\n * DIFF %s" % evaler.score_stats())
log_info("Common subtree stats:\n -- SIZE: %s\n -- ΔGLD: %s\n -- ΔPRD: %s" %
evaler.common_substruct_stats())
# just generate
else:
for da in das:
tgen.generate_tree(da, gen_doc)
# write output
if fname_ttrees_out is not None:
log_info('Writing output...')
write_ttrees(gen_doc, fname_ttrees_out)
def seq2seq_gen(args):
"""Sequence-to-sequence generation"""
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument('-e', '--eval-file', type=str, help='A ttree/text file for evaluation')
ap.add_argument('-a', '--abstr-file', type=str,
help='Lexicalization file (a.k.a. abstraction instructions, for postprocessing)')
ap.add_argument('-r', '--ref-selector', type=str, default='',
help='Selector for reference trees in the evaluation file')
ap.add_argument('-t', '--target-selector', type=str, default='',
help='Target selector for generated trees in the output file')
ap.add_argument('-d', '--debug-logfile', type=str, help='Debug output file name')
ap.add_argument('-w', '--output-file', type=str, help='Output tree/text file')
ap.add_argument('-b', '--beam-size', type=int,
help='Override beam size for beam search decoding')
ap.add_argument('-c', '--context-file', type=str,
help='Input ttree/text file with context utterances')
ap.add_argument('seq2seq_model_file', type=str, help='Trained Seq2Seq generator model')
ap.add_argument('da_test_file', type=str, help='Input DAs for generation')
args = ap.parse_args(args)
if args.debug_logfile:
set_debug_stream(file_stream(args.debug_logfile, mode='w'))
# load the generator
tgen = Seq2SeqBase.load_from_file(args.seq2seq_model_file)
if args.beam_size is not None:
tgen.beam_size = args.beam_size
# read input files (DAs, contexts)
das = read_das(args.da_test_file)
if args.context_file:
if not tgen.use_context and not tgen.context_bleu_weight:
log_warn('Generator is not trained to use context, ignoring context input file.')
else:
if args.context_file.endswith('.txt'):
contexts = read_tokens(args.context_file)
else:
contexts = tokens_from_doc(read_ttrees(args.context_file),
tgen.language, tgen.selector)
das = [(context, da) for context, da in zip(contexts, das)]
elif tgen.use_context or tgen.context_bleu_weight:
log_warn('Generator is trained to use context. ' +
'Using empty contexts, expect lower performance.')
das = [([], da) for da in das]
# generate
log_info('Generating...')
gen_trees = []
for num, da in enumerate(das, start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_trees.append(tgen.generate_tree(da))
if num % 100 == 0:
log_info("Generated tree %d" % num)
log_info(tgen.get_slot_err_stats())
# evaluate the generated trees against golden trees (delexicalized)
eval_doc = None
if args.eval_file and not args.eval_file.endswith('.txt'):
eval_doc = read_ttrees(args.eval_file)
evaler = Evaluator()
evaler.process_eval_doc(eval_doc, gen_trees, tgen.language, args.ref_selector,
args.target_selector or tgen.selector)
# lexicalize, if required
if args.abstr_file and tgen.lexicalizer:
log_info('Lexicalizing...')
tgen.lexicalize(gen_trees, args.abstr_file)
# we won't need contexts anymore, but we do need DAs
if tgen.use_context or tgen.context_bleu_weight:
das = [da for _, da in das]
# evaluate the generated & lexicalized tokens (F1 and BLEU scores)
if args.eval_file and args.eval_file.endswith('.txt'):
eval_tokens(das, read_tokens(args.eval_file, ref_mode=True),
[t.to_tok_list() for t in gen_trees])
# write output .yaml.gz or .txt
if args.output_file is not None:
log_info('Writing output...')
if args.output_file.endswith('.txt'):
gen_toks = [t.to_tok_list() for t in gen_trees]
postprocess_tokens(gen_toks, das)
write_tokens(gen_toks, args.output_file)
else:
write_ttrees(create_ttree_doc(gen_trees, eval_doc, tgen.language,
args.target_selector or tgen.selector),
args.output_file)
def seq2seq_gen(args):
"""Sequence-to-sequence generation"""
def write_trees_or_tokens(output_file, das, gen_trees, base_doc, language,
selector):
"""Decide to write t-trees or tokens based on the output file name."""
if output_file.endswith('.txt'):
gen_toks = [t.to_tok_list() for t in gen_trees]
postprocess_tokens(gen_toks, das)
write_tokens(gen_toks, output_file)
else:
write_ttrees(
create_ttree_doc(gen_trees, base_doc, language, selector),
output_file)
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument('-e',
'--eval-file',
type=str,
help='A ttree/text file for evaluation')
ap.add_argument(
'-a',
'--abstr-file',
type=str,
help=
'Lexicalization file (a.k.a. abstraction instructions, for postprocessing)'
)
ap.add_argument('-r',
'--ref-selector',
type=str,
default='',
help='Selector for reference trees in the evaluation file')
ap.add_argument(
'-t',
'--target-selector',
type=str,
default='',
help='Target selector for generated trees in the output file')
ap.add_argument('-d',
'--debug-logfile',
type=str,
help='Debug output file name')
ap.add_argument('-w',
'--output-file',
type=str,
help='Output tree/text file')
ap.add_argument('-D',
'--delex-output-file',
type=str,
help='Output file for trees/text before lexicalization')
ap.add_argument('-b',
'--beam-size',
type=int,
help='Override beam size for beam search decoding')
ap.add_argument('-c',
'--context-file',
type=str,
help='Input ttree/text file with context utterances')
ap.add_argument('seq2seq_model_file',
type=str,
help='Trained Seq2Seq generator model')
ap.add_argument('da_test_file', type=str, help='Input DAs for generation')
args = ap.parse_args(args)
if args.debug_logfile:
set_debug_stream(file_stream(args.debug_logfile, mode='w'))
# load the generator
tgen = Seq2SeqBase.load_from_file(args.seq2seq_model_file)
if args.beam_size is not None:
tgen.beam_size = args.beam_size
# read input files (DAs, contexts)
das = read_das(args.da_test_file)
if args.context_file:
if not tgen.use_context and not tgen.context_bleu_weight:
log_warn(
'Generator is not trained to use context, ignoring context input file.'
)
else:
if args.context_file.endswith('.txt'):
contexts = read_tokens(args.context_file)
else:
contexts = tokens_from_doc(read_ttrees(args.context_file),
tgen.language, tgen.selector)
das = [(context, da) for context, da in zip(contexts, das)]
elif tgen.use_context or tgen.context_bleu_weight:
log_warn('Generator is trained to use context. ' +
'Using empty contexts, expect lower performance.')
das = [([], da) for da in das]
# generate
log_info('Generating...')
gen_trees = []
for num, da in enumerate(das, start=1):
log_debug("\n\nTREE No. %03d" % num)
gen_trees.append(tgen.generate_tree(da))
if num % 100 == 0:
log_info("Generated tree %d" % num)
log_info(tgen.get_slot_err_stats())
if args.delex_output_file is not None:
log_info('Writing delex output...')
write_trees_or_tokens(args.delex_output_file, das, gen_trees, None,
tgen.language, args.target_selector
or tgen.selector)
# evaluate the generated trees against golden trees (delexicalized)
eval_doc = None
if args.eval_file and not args.eval_file.endswith('.txt'):
eval_doc = read_ttrees(args.eval_file)
evaler = Evaluator()
evaler.process_eval_doc(eval_doc, gen_trees, tgen.language,
args.ref_selector, args.target_selector
or tgen.selector)
# lexicalize, if required
if args.abstr_file and tgen.lexicalizer:
log_info('Lexicalizing...')
tgen.lexicalize(gen_trees, args.abstr_file)
# we won't need contexts anymore, but we do need DAs
if tgen.use_context or tgen.context_bleu_weight:
das = [da for _, da in das]
# evaluate the generated & lexicalized tokens (F1 and BLEU scores)
if args.eval_file and args.eval_file.endswith('.txt'):
eval_tokens(das, read_tokens(args.eval_file, ref_mode=True),
[t.to_tok_list() for t in gen_trees])
# write output .yaml.gz or .txt
if args.output_file is not None:
log_info('Writing output...')
write_trees_or_tokens(args.output_file, das, gen_trees, eval_doc,
tgen.language, args.target_selector
or tgen.selector)
def _init_training(self, das, trees, data_portion):
"""Initialize training.
Store input data, initialize 1-hot feature representations for input and output and
transform training data accordingly, initialize the classification neural network.
@param das: name of source file with training DAs, or list of DAs
@param trees: name of source file with corresponding trees/sentences, or list of trees
@param data_portion: portion of the training data to be used (0.0-1.0)
"""
# read input from files or take it directly from parameters
if not isinstance(das, list):
log_info('Reading DAs from ' + das + '...')
das = read_das(das)
if not isinstance(trees, list):
log_info('Reading t-trees from ' + trees + '...')
ttree_doc = read_ttrees(trees)
if self.mode == 'tokens':
tokens = tokens_from_doc(ttree_doc, self.language, self.selector)
trees = self._tokens_to_flat_trees(tokens)
elif self.mode == 'tagged_lemmas':
tls = tagged_lemmas_from_doc(ttree_doc, self.language, self.selector)
trees = self._tokens_to_flat_trees(tls, use_tags=True)
else:
trees = trees_from_doc(ttree_doc, self.language, self.selector)
elif self.mode in ['tokens', 'tagged_lemmas']:
trees = self._tokens_to_flat_trees(trees, use_tags=self.mode == 'tagged_lemmas')
# make training data smaller if necessary
train_size = int(round(data_portion * len(trees)))
self.train_trees = trees[:train_size]
self.train_das = das[:train_size]
# ignore contexts, if they are contained in the DAs
if isinstance(self.train_das[0], tuple):
self.train_das = [da for (context, da) in self.train_das]
# delexicalize if DAs are lexicalized and we don't want that
if self.delex_slots:
self.train_das = [da.get_delexicalized(self.delex_slots) for da in self.train_das]
# add empty tree + empty DA to training data
# (i.e. forbid the network to keep any of its outputs "always-on")
train_size += 1
self.train_trees.append(TreeData())
empty_da = DA.parse('inform()')
self.train_das.append(empty_da)
self.train_order = range(len(self.train_trees))
log_info('Using %d training instances.' % train_size)
# initialize input features/embeddings
if self.tree_embs:
self.dict_size = self.tree_embs.init_dict(self.train_trees)
self.X = np.array([self.tree_embs.get_embeddings(tree) for tree in self.train_trees])
else:
self.tree_feats = Features(['node: presence t_lemma formeme'])
self.tree_vect = DictVectorizer(sparse=False, binarize_numeric=True)
self.X = [self.tree_feats.get_features(tree, {}) for tree in self.train_trees]
self.X = self.tree_vect.fit_transform(self.X)
# initialize output features
self.da_feats = Features(['dat: dat_presence', 'svp: svp_presence'])
self.da_vect = DictVectorizer(sparse=False, binarize_numeric=True)
self.y = [self.da_feats.get_features(None, {'da': da}) for da in self.train_das]
self.y = self.da_vect.fit_transform(self.y)
log_info('Number of binary classes: %d.' % len(self.da_vect.get_feature_names()))
# initialize I/O shapes
if not self.tree_embs:
self.input_shape = list(self.X[0].shape)
else:
self.input_shape = self.tree_embs.get_embeddings_shape()
self.num_outputs = len(self.da_vect.get_feature_names())
# initialize NN classifier
self._init_neural_network()
# initialize the NN variables
self.session.run(tf.global_variables_initializer())