def __init__(self, cfg):
super(Seq2SeqBase, self).__init__(cfg)
# save the whole configuration for later use (save/load, construction of embedding
# extractors)
self.cfg = cfg
# decoding options
self.beam_size = cfg.get('beam_size', 1)
self.sample_top_k = cfg.get('sample_top_k', 1)
self.length_norm_weight = cfg.get('length_norm_weight', 0.0)
self.context_bleu_weight = cfg.get('context_bleu_weight', 0.0)
self.context_bleu_metric = cfg.get('context_bleu_metric', 'bleu')
self.slot_err_stats = None
self.classif_filter = None
if 'classif_filter' in cfg:
# use the specialized settings for the reranking classifier
rerank_cfg = cfg['classif_filter']
# plus, copy some settings from the main Seq2Seq module (so we're consistent)
for setting in [
'use_tokens', 'embeddings_lowercase',
'embeddings_split_plurals'
]:
if setting in cfg:
rerank_cfg[setting] = cfg[setting]
self.classif_filter = RerankingClassifier(rerank_cfg)
self.misfit_penalty = cfg.get('misfit_penalty', 100)
def rerank_cl_eval(args):
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument(
'-l',
'--language',
type=str,
help='Override classifier language (for t-tree input files)')
ap.add_argument(
'-s',
'--selector',
type=str,
help='Override classifier selector (for t-tree input files)')
ap.add_argument('fname_cl_model',
type=str,
help='Path to trained reranking classifier model')
ap.add_argument('fname_test_da', type=str, help='Path to test DA file')
ap.add_argument('fname_test_sent',
type=str,
help='Path to test trees/sentences file')
args = ap.parse_args(args)
log_info("Loading reranking classifier...")
rerank_cl = RerankingClassifier.load_from_file(args.fname_cl_model)
if args.language is not None:
rerank_cl.language = args.language
if args.selector is not None:
rerank_cl.selector = args.selector
log_info("Evaluating...")
tot_len, dist = rerank_cl.evaluate_file(args.fname_test_da,
args.fname_test_sent)
log_info("Penalty: %d, Total DAIs %d." % (dist, tot_len))
def rerank_cl_eval(args):
opts, files = getopt(args, 's:l:t')
language = None
selector = None
for opt, arg in opts:
if opt == '-l':
language = arg
elif opt == '-s':
selector = arg
if len(files) != 3:
sys.exit("Invalid arguments.\n" + __doc__)
fname_cl_model, fname_test_da, fname_test_sent = files
log_info("Loading reranking classifier...")
rerank_cl = RerankingClassifier.load_from_file(fname_cl_model)
if language is not None:
rerank_cl.language = language
if selector is not None:
rerank_cl.selector = selector
log_info("Evaluating...")
tot_len, dist = rerank_cl.evaluate_file(fname_test_da, fname_test_sent)
log_info("Penalty: %d, Total DAIs %d." % (dist, tot_len))
def load_from_file(model_fname):
"""Load the generator from a file (actually two files, one for configuration and one
for the TensorFlow graph, which must be stored separately).
@param model_fname: file name (for the configuration file); TF graph must be stored with a \
different extension
"""
log_info("Loading generator from %s..." % model_fname)
with file_stream(model_fname, 'rb', encoding=None) as fh:
data = pickle.load(fh)
ret = Seq2SeqGen(cfg=data['cfg'])
ret.load_all_settings(data)
if ret.classif_filter:
classif_filter_fname = re.sub(r'((.pickle)?(.gz)?)$',
r'.tftreecl\1', model_fname)
if os.path.isfile(classif_filter_fname):
ret.classif_filter = RerankingClassifier.load_from_file(
classif_filter_fname)
else:
log_warn("Classification filter data not found, ignoring.")
ret.classif_filter = False
# re-build TF graph and restore the TF session
tf_session_fname = re.sub(r'(.pickle)?(.gz)?$', '.tfsess', model_fname)
ret._init_neural_network()
ret.saver.restore(ret.session, tf_session_fname)
return ret
def load_from_file(model_fname):
"""Load the generator from a file (actually two files, one for configuration and one
for the TensorFlow graph, which must be stored separately).
@param model_fname: file name (for the configuration file); TF graph must be stored with a \
different extension
"""
log_info("Loading generator from %s..." % model_fname)
with file_stream(model_fname, 'rb', encoding=None) as fh:
data = pickle.load(fh)
ret = Seq2SeqGen(cfg=data['cfg'])
ret.load_all_settings(data)
if ret.classif_filter:
classif_filter_fname = re.sub(r'((.pickle)?(.gz)?)$', r'.tftreecl\1', model_fname)
if os.path.isfile(classif_filter_fname):
ret.classif_filter = RerankingClassifier.load_from_file(classif_filter_fname)
else:
log_warn("Classification filter data not found, ignoring.")
ret.classif_filter = False
# re-build TF graph and restore the TF session
tf_session_fname = re.sub(r'(.pickle)?(.gz)?$', '.tfsess', model_fname)
ret._init_neural_network()
ret.saver.restore(ret.session, tf_session_fname)
return ret
def rerank_cl_train(args):
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument(
'-a',
'--add-to-seq2seq',
type=str,
help=
'Replace trained classifier in an existing seq2seq model (path to file)'
)
ap.add_argument('fname_config',
type=str,
help='Reranking classifier configuration file path')
ap.add_argument('fname_da_train', type=str, help='Training DAs file path')
ap.add_argument('fname_trees_train',
type=str,
help='Training trees/sentences file path')
ap.add_argument('fname_cl_model',
type=str,
help='Path for the output trained model')
args = ap.parse_args(args)
if args.add_to_seq2seq:
tgen = Seq2SeqBase.load_from_file(args.add_to_seq2seq)
config = Config(args.fname_config)
rerank_cl = RerankingClassifier(config)
rerank_cl.train(args.fname_da_train, args.fname_trees_train)
if args.add_to_seq2seq:
tgen.classif_filter = rerank_cl
tgen.save_to_file(args.fname_cl_model)
else:
rerank_cl.save_to_file(args.fname_cl_model)
def rerank_cl_train(args):
opts, files = getopt(args, 'a:')
load_seq2seq_model = None
for opt, arg in opts:
if opt == '-a':
load_seq2seq_model = arg
if len(files) != 4:
sys.exit("Invalid arguments.\n" + __doc__)
fname_config, fname_da_train, fname_trees_train, fname_cl_model = files
if load_seq2seq_model:
tgen = Seq2SeqBase.load_from_file(load_seq2seq_model)
config = Config(fname_config)
rerank_cl = RerankingClassifier(config)
rerank_cl.train(fname_da_train, fname_trees_train)
if load_seq2seq_model:
tgen.classif_filter = rerank_cl
tgen.save_to_file(fname_cl_model)
else:
rerank_cl.save_to_file(fname_cl_model)
def build_ensemble(self, models, rerank_settings=None, rerank_params=None):
"""Build the ensemble model (build all networks and load their parameters).
@param models: list of tuples (settings, parameter set) of all models in the ensemble
@param rerank_settings:
"""
for setting, parset in models:
model = Seq2SeqGen(setting['cfg'])
model.load_all_settings(setting)
model._init_neural_network()
model.set_model_params(parset)
self.gens.append(model)
# embedding IDs should be the same for all models, it is safe to use them directly
self.da_embs = self.gens[0].da_embs
self.tree_embs = self.gens[0].tree_embs
if rerank_settings is not None:
self.classif_filter = RerankingClassifier(cfg=rerank_settings['cfg'])
self.classif_filter.load_all_settings(rerank_settings)
self.classif_filter._init_neural_network()
self.classif_filter.set_model_params(rerank_params)
def rerank_cl_train(args):
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument('-a', '--add-to-seq2seq', type=str,
help='Replace trained classifier in an existing seq2seq model (path to file)')
ap.add_argument('fname_config', type=str, help='Reranking classifier configuration file path')
ap.add_argument('fname_da_train', type=str, help='Training DAs file path')
ap.add_argument('fname_trees_train', type=str, help='Training trees file path (must be trees!)')
ap.add_argument('fname_cl_model', type=str, help='Path for the output trained model')
args = ap.parse_args(args)
if args.add_to_seq2seq:
tgen = Seq2SeqBase.load_from_file(args.add_to_seq2seq)
config = Config(args.fname_config)
rerank_cl = RerankingClassifier(config)
rerank_cl.train(args.fname_da_train, args.fname_trees_train)
if args.add_to_seq2seq:
tgen.classif_filter = rerank_cl
tgen.save_to_file(args.fname_cl_model)
else:
rerank_cl.save_to_file(args.fname_cl_model)
def rerank_cl_eval(args):
ap = ArgumentParser(prog=' '.join(sys.argv[0:2]))
ap.add_argument('-l', '--language', type=str,
help='Override classifier language (for t-tree input files)')
ap.add_argument('-s', '--selector', type=str,
help='Override classifier selector (for t-tree input files)')
ap.add_argument('fname_cl_model', type=str, help='Path to trained reranking classifier model')
ap.add_argument('fname_test_da', type=str, help='Path to test DA file')
ap.add_argument('fname_test_sent', type=str, help='Path to test trees file (must be trees!)')
args = ap.parse_args(args)
log_info("Loading reranking classifier...")
rerank_cl = RerankingClassifier.load_from_file(args.fname_cl_model)
if args.language is not None:
rerank_cl.language = args.language
if args.selector is not None:
rerank_cl.selector = args.selector
log_info("Evaluating...")
tot_len, dist = rerank_cl.evaluate_file(args.fname_test_da, args.fname_test_sent)
log_info("Penalty: %d, Total DAIs %d." % (dist, tot_len))
def __init__(self, cfg):
super(Seq2SeqBase, self).__init__(cfg)
# save the whole configuration for later use (save/load, construction of embedding
# extractors)
self.cfg = cfg
# decoding options
self.beam_size = cfg.get('beam_size', 1)
self.sample_top_k = cfg.get('sample_top_k', 1)
self.length_norm_weight = cfg.get('length_norm_weight', 0.0)
self.context_bleu_weight = cfg.get('context_bleu_weight', 0.0)
self.context_bleu_metric = cfg.get('context_bleu_metric', 'bleu')
self.slot_err_stats = None
self.classif_filter = None
if 'classif_filter' in cfg:
# use the specialized settings for the reranking classifier
rerank_cfg = cfg['classif_filter']
# plus, copy some settings from the main Seq2Seq module (so we're consistent)
for setting in ['use_tokens', 'embeddings_lowercase', 'embeddings_split_plurals']:
if setting in cfg:
rerank_cfg[setting] = cfg[setting]
self.classif_filter = RerankingClassifier(rerank_cfg)
self.misfit_penalty = cfg.get('misfit_penalty', 100)
class Seq2SeqBase(SentencePlanner):
"""A common ancestor for the Plain and Ensemble Seq2Seq generators (decoding methods only)."""
def __init__(self, cfg):
super(Seq2SeqBase, self).__init__(cfg)
# save the whole configuration for later use (save/load, construction of embedding
# extractors)
self.cfg = cfg
# decoding options
self.beam_size = cfg.get('beam_size', 1)
self.sample_top_k = cfg.get('sample_top_k', 1)
self.length_norm_weight = cfg.get('length_norm_weight', 0.0)
self.context_bleu_weight = cfg.get('context_bleu_weight', 0.0)
self.context_bleu_metric = cfg.get('context_bleu_metric', 'bleu')
self.slot_err_stats = None
self.classif_filter = None
if 'classif_filter' in cfg:
# use the specialized settings for the reranking classifier
rerank_cfg = cfg['classif_filter']
# plus, copy some settings from the main Seq2Seq module (so we're consistent)
for setting in [
'use_tokens', 'embeddings_lowercase',
'embeddings_split_plurals'
]:
if setting in cfg:
rerank_cfg[setting] = cfg[setting]
self.classif_filter = RerankingClassifier(rerank_cfg)
self.misfit_penalty = cfg.get('misfit_penalty', 100)
def process_das(self, das, gold_trees=None):
"""
Process a list of input DAs, return the corresponding trees (using the generator
network with current parameters).
@param das: input DAs
@param gold_trees: (optional) gold trees against which cost is computed
@return: generated trees as `TreeData` instances, cost if `gold_trees` are given
"""
# encoder inputs
enc_inputs = cut_batch_into_steps(
[self.da_embs.get_embeddings(da) for da in das])
if self.beam_size > 1 and len(das) == 1:
dec_output_ids = self._beam_search(enc_inputs, das[0])
dec_cost = None
else:
dec_output_ids, dec_cost = self._greedy_decoding(
enc_inputs, gold_trees)
dec_trees = [
self.tree_embs.ids_to_tree(ids)
for ids in dec_output_ids.transpose()
]
# return result (trees and optionally cost)
if dec_cost is None:
return dec_trees
return dec_trees, dec_cost
def _greedy_decoding(self, enc_inputs, gold_trees):
"""Run greedy decoding with the given encoder inputs; optionally use given gold trees
as decoder inputs for cost computation."""
# prepare decoder inputs (either fake, or true but used just for cost computation)
if gold_trees is None:
empty_tree_emb = self.tree_embs.get_embeddings(TreeData())
dec_inputs = cut_batch_into_steps(
[empty_tree_emb for _ in enc_inputs[0]])
else:
dec_inputs = cut_batch_into_steps(
[self.tree_embs.get_embeddings(tree) for tree in gold_trees])
# run the decoding per se
dec_output_ids, dec_cost = self._get_greedy_decoder_output(
enc_inputs, dec_inputs, compute_cost=gold_trees is not None)
return dec_output_ids, dec_cost
def _get_greedy_decoder_output(initial_state,
enc_inputs,
dec_inputs,
compute_cost=False):
raise NotImplementedError()
class DecodingPath(object):
"""A decoding path to be used in beam search."""
__slots__ = [
'stop_token_id', 'dec_inputs', 'dec_states', 'logprob', '_length'
]
def __init__(self,
stop_token_id,
dec_inputs=[],
dec_states=[],
logprob=0.0,
length=-1):
self.stop_token_id = stop_token_id
self.dec_inputs = list(dec_inputs)
self.dec_states = list(dec_states)
self.logprob = logprob
self._length = length if length >= 0 else len(dec_inputs)
def expand(self, max_variants, dec_out_probs, dec_state):
"""Expand the path with all possible outputs, updating the log probabilities.
@param max_variants: expand to this number of variants at maximum, discard the less \
probable ones
@param dec_output: the decoder output scores for the current step
@param dec_state: the decoder hidden state for the current step
@return: an array of all possible continuations of this path
"""
ret = []
# select only up to max_variants most probable variants
top_n_idx = np.argpartition(-dec_out_probs,
max_variants)[:max_variants]
for idx in top_n_idx:
expanded = Seq2SeqGen.DecodingPath(self.stop_token_id,
self.dec_inputs,
self.dec_states,
self.logprob, len(self))
if len(self) == len(
self.dec_inputs) and idx != self.stop_token_id:
expanded._length += 1
expanded.logprob += np.log(dec_out_probs[idx])
expanded.dec_inputs.append(np.array(idx, ndmin=1))
expanded.dec_states.append(dec_state)
ret.append(expanded)
return ret
def __len__(self):
"""Return decoding path length (number of decoder input tokens)."""
return self._length
def _beam_search(self, enc_inputs, da):
"""Run beam search decoding."""
# true "batches" not implemented
assert len(enc_inputs[0]) == 1
# run greedy decoder for comparison (debugging purposes)
log_debug("GREEDY DEC WOULD RETURN:\n" + " ".join(
self.tree_embs.ids_to_strings([
out_tok[0]
for out_tok in self._greedy_decoding(enc_inputs, None)[0]
])))
# initialize
self._init_beam_search(enc_inputs)
empty_tree_emb = self.tree_embs.get_embeddings(TreeData())
dec_inputs = cut_batch_into_steps([empty_tree_emb])
paths = [
self.DecodingPath(stop_token_id=self.tree_embs.STOP,
dec_inputs=[dec_inputs[0]])
]
# beam search steps
for step in xrange(len(dec_inputs)):
new_paths = []
for path in paths:
out_probs, st = self._beam_search_step(path.dec_inputs,
path.dec_states)
new_paths.extend(path.expand(self.beam_size, out_probs, st))
def cmp_func(p, q):
"""Length-weighted comparison of two paths' logprobs."""
return cmp(p.logprob / (len(p)**self.length_norm_weight),
q.logprob / (len(q)**self.length_norm_weight))
paths = sorted(new_paths, cmp=cmp_func,
reverse=True)[:self.beam_size]
if all([p.dec_inputs[-1] == self.tree_embs.VOID for p in paths]):
break # stop decoding if we have reached the end in all paths
log_debug(("\nBEAM SEARCH STEP %d\n" % step) +
"\n".join([("%f\t" % p.logprob) + " ".join(
self.tree_embs.ids_to_strings(
[inp[0] for inp in p.dec_inputs]))
for p in paths]) + "\n")
# rerank paths by their distance to the input DA
if self.classif_filter or self.context_bleu_weight:
paths = self._rerank_paths(paths, da)
# measure slot error on the top k paths
if self.slot_err_stats:
for path in paths[:self.sample_top_k]:
self.slot_err_stats.append(
da,
self.tree_embs.ids_to_strings(
[inp[0] for inp in path.dec_inputs]))
# select the "best" path -- either the best, or one in top k
if self.sample_top_k > 1:
best_path = self._sample_path(paths[:self.sample_top_k])
else:
best_path = paths[0]
# return just the best path (as token IDs)
return np.array(best_path.dec_inputs)
def _init_beam_search(self, enc_inputs):
raise NotImplementedError()
def _beam_search_step(self, dec_inputs, dec_states):
raise NotImplementedError()
def _rerank_paths(self, paths, da):
"""Rerank the n-best decoded paths according to the reranking classifier and/or
BLEU against context."""
trees = [
self.tree_embs.ids_to_tree(
np.array(path.dec_inputs).transpose()[0]) for path in paths
]
# rerank using BLEU against context if set to do so
if self.context_bleu_weight:
bm = BLEUMeasure(max_ngram=2)
bleus = []
for path, tree in zip(paths, trees):
bm.reset()
bm.append([(n.t_lemma, None) for n in tree.nodes[1:]], [da[0]])
bleu = (bm.ngram_precision() if self.context_bleu_metric
== 'ngram_prec' else bm.bleu())
bleus.append(bleu)
path.logprob += self.context_bleu_weight * bleu
log_debug(("BLEU for context: %s\n\n" %
" ".join([form for form, _ in da[0]])) +
"\n".join([("%.5f\t" % b) +
" ".join([n.t_lemma for n in t.nodes[1:]])
for b, t in zip(bleus, trees)]))
# add distances to logprob so that non-fitting will be heavily penalized
if self.classif_filter:
self.classif_filter.init_run(da)
fits = self.classif_filter.dist_to_cur_da(trees)
for path, fit in zip(paths, fits):
path.logprob -= self.misfit_penalty * fit
log_debug(("Misfits for DA: %s\n\n" % str(da)) +
"\n".join([("%.5f\t" % fit) + " ".join(
[unicode(n.t_lemma) for n in tree.nodes[1:]])
for fit, tree in zip(fits, trees)]))
# adjust paths for length (if set to do so)
if self.length_norm_weight:
for path in paths:
path.logprob /= len(path)**self.length_norm_weight
return sorted(paths,
cmp=lambda p, q: cmp(p.logprob, q.logprob),
reverse=True)
def _sample_path(self, paths):
"""Sample one path from the top k paths, based on their probabilities."""
# convert the logprobs to a probability distribution, proportionate to their sizes
logprobs = [p.logprob for p in paths]
max_logprob = max(logprobs)
probs = [math.exp(l - max_logprob) for l in logprobs
] # discount to avoid underflow, result is unnormalized
sum_prob = sum(probs)
probs = [p / sum_prob for p in probs] # normalized
# select the path based on a draw from the uniform distribution
draw = rnd.random()
cum = 0.0 # building cumulative distribution function on-the-fly
selected = -1
for idx, prob in enumerate(probs):
high = cum + prob
if cum <= draw and draw < high: # the draw has hit this index in the CDF
selected = idx
break
cum = high
return paths[selected]
def generate_tree(self, da, gen_doc=None):
"""Generate one tree, saving it into the document provided (if applicable).
@param da: the input DA
@param gen_doc: the document where the tree should be saved (defaults to None)
"""
# generate the tree
log_debug("GENERATE TREE FOR DA: " + unicode(da))
tree = self.process_das([da])[0]
log_debug("RESULT: %s" % unicode(tree))
# if requested, append the result to the "document"
# just lists (generated tokens only, disregarding syntax; keep None for POS tags)
if isinstance(gen_doc, list):
# ignore tree technical root, take just "lemmas"
gen_doc.append([(n.t_lemma, None) for n in tree.nodes[1:]])
# full Pytreex documents (full trees)
elif gen_doc:
zone = self.get_target_zone(gen_doc)
zone.ttree = tree.create_ttree()
zone.sentence = unicode(da)
# return the result
return tree
def init_slot_err_stats(self):
"""Initialize slot error statistics accumulator."""
self.slot_err_stats = SlotErrAnalyzer()
def get_slot_err_stats(self):
"""Return current slot error statistics, as a string."""
return ("Slot error: %.2f (M: %d, S: %d, T: %d)" %
(self.slot_err_stats.slot_error(), self.slot_err_stats.missing,
self.slot_err_stats.superfluous, self.slot_err_stats.total))
@staticmethod
def load_from_file(model_fname):
"""Detect correct model type (plain/ensemble) and start loading."""
model_type = Seq2SeqGen # default to plain generator
with file_stream(model_fname, 'rb', encoding=None) as fh:
data = pickle.load(fh)
if isinstance(data, type):
model_type = data
return model_type.load_from_file(model_fname)
class Seq2SeqEnsemble(Seq2SeqBase):
"""Ensemble Sequence-to-Sequence models (averaging outputs of networks with different random
initialization)."""
def __init__(self, cfg):
super(Seq2SeqEnsemble, self).__init__(cfg)
self.gens = []
def build_ensemble(self, models, rerank_settings=None, rerank_params=None):
"""Build the ensemble model (build all networks and load their parameters).
@param models: list of tuples (settings, parameter set) of all models in the ensemble
@param rerank_settings:
"""
for setting, parset in models:
model = Seq2SeqGen(setting['cfg'])
model.load_all_settings(setting)
model._init_neural_network()
model.set_model_params(parset)
self.gens.append(model)
# embedding IDs should be the same for all models, it is safe to use them directly
self.da_embs = self.gens[0].da_embs
self.tree_embs = self.gens[0].tree_embs
if rerank_settings is not None:
self.classif_filter = RerankingClassifier(cfg=rerank_settings['cfg'])
self.classif_filter.load_all_settings(rerank_settings)
self.classif_filter._init_neural_network()
self.classif_filter.set_model_params(rerank_params)
def _get_greedy_decoder_output(self, enc_inputs, dec_inputs, compute_cost=False):
"""Run greedy decoding with the given inputs; return decoder outputs and the cost
(if required). For ensemble decoding, the gready search is implemented as a beam
search with a beam size of 1.
@param enc_inputs: encoder inputs (list of token IDs)
@param dec_inputs: decoder inputs (list of token IDs)
@param compute_cost: if True, decoding cost is computed (the dec_inputs must be valid trees)
@return a tuple of list of decoder outputs + decoding cost (None if not required)
"""
# TODO batches and cost computation not implemented
assert len(enc_inputs[0]) == 1 and not compute_cost
self._init_beam_search(enc_inputs)
# for simplicity, this is implemented exacly like a beam search, but with a path sized one
empty_tree_emb = self.tree_embs.get_embeddings(TreeData())
dec_inputs = cut_batch_into_steps([empty_tree_emb])
path = self.DecodingPath(stop_token_id=self.tree_embs.STOP, dec_inputs=[dec_inputs[0]])
for step in xrange(len(dec_inputs)):
out_probs, st = self._beam_search_step(path.dec_inputs, path.dec_states)
path = path.expand(1, out_probs, st)[0]
if path.dec_inputs[-1] == self.tree_embs.VOID:
break # stop decoding if we have reached the end of path
# return just token IDs, ignore cost computation here
return np.array(path.dec_inputs), None
def _init_beam_search(self, enc_inputs):
"""Initialize beam search for the current DA (with the given encoder inputs)
for all member generators."""
for gen in self.gens:
gen._init_beam_search(enc_inputs)
def _beam_search_step(self, dec_inputs, dec_states):
"""Run one step of beam search decoding with the given decoder inputs and
(previous steps') outputs and states. Outputs are averaged over all member generators,
states are kept separately."""
ensemble_state = []
ensemble_output = None
for gen_no, gen in enumerate(self.gens):
output, state = gen._beam_search_step(dec_inputs,
[state[gen_no] for state in dec_states])
ensemble_state.append(state)
output = np.exp(output) / np.sum(np.exp(output))
if ensemble_output is None:
ensemble_output = output
else:
ensemble_output += output
ensemble_output /= float(len(self.gens))
return ensemble_output, ensemble_state
@staticmethod
def load_from_file(model_fname):
"""Load the whole ensemble from a file (load settings and model parameters, then build the
ensemble network)."""
# TODO support for lexicalizer
log_info("Loading ensemble generator from %s..." % model_fname)
with file_stream(model_fname, 'rb', encoding=None) as fh:
typeid = pickle.load(fh)
if typeid != Seq2SeqEnsemble:
raise ValueError('Wrong type identifier in file %s' % model_fname)
cfg = pickle.load(fh)
ret = Seq2SeqEnsemble(cfg)
gens_dump = pickle.load(fh)
if 'classif_filter' in cfg:
rerank_settings = pickle.load(fh)
rerank_params = pickle.load(fh)
else:
rerank_settings = None
rerank_params = None
ret.build_ensemble(gens_dump, rerank_settings, rerank_params)
return ret
def save_to_file(self, model_fname):
"""Save the whole ensemble into a file (get all settings and parameters, dump them in a
pickle)."""
# TODO support for lexicalizer
log_info("Saving generator to %s..." % model_fname)
with file_stream(model_fname, 'wb', encoding=None) as fh:
pickle.dump(self.__class__, fh, protocol=pickle.HIGHEST_PROTOCOL)
pickle.dump(self.cfg, fh, protocol=pickle.HIGHEST_PROTOCOL)
gens_dump = []
for gen in self.gens:
setting = gen.get_all_settings()
parset = gen.get_model_params()
setting['classif_filter'] = self.classif_filter is not None
gens_dump.append((setting, parset))
pickle.dump(gens_dump, fh, protocol=pickle.HIGHEST_PROTOCOL)
if self.classif_filter:
pickle.dump(self.classif_filter.get_all_settings(), fh,
protocol=pickle.HIGHEST_PROTOCOL)
pickle.dump(self.classif_filter.get_model_params(), fh,
protocol=pickle.HIGHEST_PROTOCOL)
class Seq2SeqBase(SentencePlanner):
"""A common ancestor for the Plain and Ensemble Seq2Seq generators (decoding methods only)."""
def __init__(self, cfg):
super(Seq2SeqBase, self).__init__(cfg)
# save the whole configuration for later use (save/load, construction of embedding
# extractors)
self.cfg = cfg
# decoding options
self.beam_size = cfg.get('beam_size', 1)
self.sample_top_k = cfg.get('sample_top_k', 1)
self.length_norm_weight = cfg.get('length_norm_weight', 0.0)
self.context_bleu_weight = cfg.get('context_bleu_weight', 0.0)
self.context_bleu_metric = cfg.get('context_bleu_metric', 'bleu')
self.slot_err_stats = None
self.classif_filter = None
if 'classif_filter' in cfg:
# use the specialized settings for the reranking classifier
rerank_cfg = cfg['classif_filter']
# plus, copy some settings from the main Seq2Seq module (so we're consistent)
for setting in ['use_tokens', 'embeddings_lowercase', 'embeddings_split_plurals']:
if setting in cfg:
rerank_cfg[setting] = cfg[setting]
self.classif_filter = RerankingClassifier(rerank_cfg)
self.misfit_penalty = cfg.get('misfit_penalty', 100)
def process_das(self, das, gold_trees=None):
"""
Process a list of input DAs, return the corresponding trees (using the generator
network with current parameters).
@param das: input DAs
@param gold_trees: (optional) gold trees against which cost is computed
@return: generated trees as `TreeData` instances, cost if `gold_trees` are given
"""
# encoder inputs
enc_inputs = cut_batch_into_steps([self.da_embs.get_embeddings(da)
for da in das])
if self.beam_size > 1 and len(das) == 1:
dec_output_ids = self._beam_search(enc_inputs, das[0])
dec_cost = None
else:
dec_output_ids, dec_cost = self._greedy_decoding(enc_inputs, gold_trees)
dec_trees = [self.tree_embs.ids_to_tree(ids) for ids in dec_output_ids.transpose()]
# return result (trees and optionally cost)
if dec_cost is None:
return dec_trees
return dec_trees, dec_cost
def _greedy_decoding(self, enc_inputs, gold_trees):
"""Run greedy decoding with the given encoder inputs; optionally use given gold trees
as decoder inputs for cost computation."""
# prepare decoder inputs (either fake, or true but used just for cost computation)
if gold_trees is None:
empty_tree_emb = self.tree_embs.get_embeddings(TreeData())
dec_inputs = cut_batch_into_steps([empty_tree_emb for _ in enc_inputs[0]])
else:
dec_inputs = cut_batch_into_steps([self.tree_embs.get_embeddings(tree)
for tree in gold_trees])
# run the decoding per se
dec_output_ids, dec_cost = self._get_greedy_decoder_output(
enc_inputs, dec_inputs, compute_cost=gold_trees is not None)
return dec_output_ids, dec_cost
def _get_greedy_decoder_output(initial_state, enc_inputs, dec_inputs, compute_cost=False):
raise NotImplementedError()
class DecodingPath(object):
"""A decoding path to be used in beam search."""
__slots__ = ['stop_token_id', 'dec_inputs', 'dec_states', 'logprob', '_length']
def __init__(self, stop_token_id, dec_inputs=[], dec_states=[], logprob=0.0, length=-1):
self.stop_token_id = stop_token_id
self.dec_inputs = list(dec_inputs)
self.dec_states = list(dec_states)
self.logprob = logprob
self._length = length if length >= 0 else len(dec_inputs)
def expand(self, max_variants, dec_out_probs, dec_state):
"""Expand the path with all possible outputs, updating the log probabilities.
@param max_variants: expand to this number of variants at maximum, discard the less \
probable ones
@param dec_output: the decoder output scores for the current step
@param dec_state: the decoder hidden state for the current step
@return: an array of all possible continuations of this path
"""
ret = []
# select only up to max_variants most probable variants
top_n_idx = np.argpartition(-dec_out_probs, max_variants)[:max_variants]
for idx in top_n_idx:
expanded = Seq2SeqGen.DecodingPath(self.stop_token_id,
self.dec_inputs, self.dec_states, self.logprob,
len(self))
if len(self) == len(self.dec_inputs) and idx != self.stop_token_id:
expanded._length += 1
expanded.logprob += np.log(dec_out_probs[idx])
expanded.dec_inputs.append(np.array(idx, ndmin=1))
expanded.dec_states.append(dec_state)
ret.append(expanded)
return ret
def __len__(self):
"""Return decoding path length (number of decoder input tokens)."""
return self._length
def _beam_search(self, enc_inputs, da):
"""Run beam search decoding."""
# true "batches" not implemented
assert len(enc_inputs[0]) == 1
# run greedy decoder for comparison (debugging purposes)
log_debug("GREEDY DEC WOULD RETURN:\n" +
" ".join(self.tree_embs.ids_to_strings(
[out_tok[0] for out_tok in self._greedy_decoding(enc_inputs, None)[0]])))
# initialize
self._init_beam_search(enc_inputs)
empty_tree_emb = self.tree_embs.get_embeddings(TreeData())
dec_inputs = cut_batch_into_steps([empty_tree_emb])
paths = [self.DecodingPath(stop_token_id=self.tree_embs.STOP, dec_inputs=[dec_inputs[0]])]
# beam search steps
for step in xrange(len(dec_inputs)):
new_paths = []
for path in paths:
out_probs, st = self._beam_search_step(path.dec_inputs, path.dec_states)
new_paths.extend(path.expand(self.beam_size, out_probs, st))
def cmp_func(p, q):
"""Length-weighted comparison of two paths' logprobs."""
return cmp(p.logprob / (len(p) ** self.length_norm_weight),
q.logprob / (len(q) ** self.length_norm_weight))
paths = sorted(new_paths, cmp=cmp_func, reverse=True)[:self.beam_size]
if all([p.dec_inputs[-1] == self.tree_embs.VOID for p in paths]):
break # stop decoding if we have reached the end in all paths
log_debug(("\nBEAM SEARCH STEP %d\n" % step) +
"\n".join([("%f\t" % p.logprob) +
" ".join(self.tree_embs.ids_to_strings([inp[0] for inp in p.dec_inputs]))
for p in paths]) + "\n")
# rerank paths by their distance to the input DA
if self.classif_filter or self.context_bleu_weight:
paths = self._rerank_paths(paths, da)
# measure slot error on the top k paths
if self.slot_err_stats:
for path in paths[:self.sample_top_k]:
self.slot_err_stats.append(
da, self.tree_embs.ids_to_strings([inp[0] for inp in path.dec_inputs]))
# select the "best" path -- either the best, or one in top k
if self.sample_top_k > 1:
best_path = self._sample_path(paths[:self.sample_top_k])
else:
best_path = paths[0]
# return just the best path (as token IDs)
return np.array(best_path.dec_inputs)
def _init_beam_search(self, enc_inputs):
raise NotImplementedError()
def _beam_search_step(self, dec_inputs, dec_states):
raise NotImplementedError()
def _rerank_paths(self, paths, da):
"""Rerank the n-best decoded paths according to the reranking classifier and/or
BLEU against context."""
trees = [self.tree_embs.ids_to_tree(np.array(path.dec_inputs).transpose()[0])
for path in paths]
# rerank using BLEU against context if set to do so
if self.context_bleu_weight:
bm = BLEUMeasure(max_ngram=2)
bleus = []
for path, tree in zip(paths, trees):
bm.reset()
bm.append([(n.t_lemma, None) for n in tree.nodes[1:]], [da[0]])
bleu = (bm.ngram_precision()
if self.context_bleu_metric == 'ngram_prec'
else bm.bleu())
bleus.append(bleu)
path.logprob += self.context_bleu_weight * bleu
log_debug(("BLEU for context: %s\n\n" % " ".join([form for form, _ in da[0]])) +
"\n".join([("%.5f\t" % b) + " ".join([n.t_lemma for n in t.nodes[1:]])
for b, t in zip(bleus, trees)]))
# add distances to logprob so that non-fitting will be heavily penalized
if self.classif_filter:
self.classif_filter.init_run(da)
fits = self.classif_filter.dist_to_cur_da(trees)
for path, fit in zip(paths, fits):
path.logprob -= self.misfit_penalty * fit
log_debug(("Misfits for DA: %s\n\n" % str(da)) +
"\n".join([("%.5f\t" % fit) +
" ".join([unicode(n.t_lemma) for n in tree.nodes[1:]])
for fit, tree in zip(fits, trees)]))
# adjust paths for length (if set to do so)
if self.length_norm_weight:
for path in paths:
path.logprob /= len(path) ** self.length_norm_weight
return sorted(paths, cmp=lambda p, q: cmp(p.logprob, q.logprob), reverse=True)
def _sample_path(self, paths):
"""Sample one path from the top k paths, based on their probabilities."""
# convert the logprobs to a probability distribution, proportionate to their sizes
logprobs = [p.logprob for p in paths]
max_logprob = max(logprobs)
probs = [math.exp(l - max_logprob) for l in logprobs] # discount to avoid underflow, result is unnormalized
sum_prob = sum(probs)
probs = [p / sum_prob for p in probs] # normalized
# select the path based on a draw from the uniform distribution
draw = rnd.random()
cum = 0.0 # building cumulative distribution function on-the-fly
selected = -1
for idx, prob in enumerate(probs):
high = cum + prob
if cum <= draw and draw < high: # the draw has hit this index in the CDF
selected = idx
break
cum = high
return paths[selected]
def generate_tree(self, da, gen_doc=None):
"""Generate one tree, saving it into the document provided (if applicable).
@param da: the input DA
@param gen_doc: the document where the tree should be saved (defaults to None)
"""
# generate the tree
log_debug("GENERATE TREE FOR DA: " + unicode(da))
tree = self.process_das([da])[0]
log_debug("RESULT: %s" % unicode(tree))
# if requested, append the result to the "document"
# just lists (generated tokens only, disregarding syntax; keep None for POS tags)
if isinstance(gen_doc, list):
# ignore tree technical root, take just "lemmas"
gen_doc.append([(n.t_lemma, None) for n in tree.nodes[1:]])
# full Pytreex documents (full trees)
elif gen_doc:
zone = self.get_target_zone(gen_doc)
zone.ttree = tree.create_ttree()
zone.sentence = unicode(da)
# return the result
return tree
def init_slot_err_stats(self):
"""Initialize slot error statistics accumulator."""
self.slot_err_stats = SlotErrAnalyzer()
def get_slot_err_stats(self):
"""Return current slot error statistics, as a string."""
return ("Slot error: %.2f (M: %d, S: %d, T: %d)" %
(self.slot_err_stats.slot_error(), self.slot_err_stats.missing,
self.slot_err_stats.superfluous, self.slot_err_stats.total))
@staticmethod
def load_from_file(model_fname):
"""Detect correct model type (plain/ensemble) and start loading."""
model_type = Seq2SeqGen # default to plain generator
with file_stream(model_fname, 'rb', encoding=None) as fh:
data = pickle.load(fh)
if isinstance(data, type):
model_type = data
return model_type.load_from_file(model_fname)