def _read_slf_header(self, fields): """Reads SLF lattice header fields and saves them in member variables. :type fields: list of strs :param fields: fields, such as name="value" """ for field in fields: name, value = self._split_slf_field(field) if (name == 'UTTERANCE') or (name == 'U'): self.utterance_id = value elif (name == 'SUBLAT') or (name == 'S'): raise InputError("Sub-lattices are not supported.") elif name == 'base': value = numpy.float64(value) if value == 0.0: self._log_scale = None else: self._log_scale = logprob_type(numpy.log(value)) elif name == 'lmscale': self.lm_scale = logprob_type(value) elif name == 'wdpenalty': self.wi_penalty = logprob_type(value) elif name == 'start': self._initial_node_id = int(value) elif name == 'end': self._final_node_id = int(value) elif (name == 'NODES') or (name == 'N'): self._num_nodes = int(value) elif (name == 'LINKS') or (name == 'L'): self._num_links = int(value)
def _read_slf_header(self, fields): """Reads SLF lattice header fields and saves them in member variables. :type fields: list of strs :param fields: fields, such as name="value" """ for field in fields: name, value = _split_slf_field(field) if (name == 'UTTERANCE') or (name == 'U'): self.utterance_id = value elif (name == 'SUBLAT') or (name == 'S'): raise InputError("Sub-lattices are not supported.") elif name == 'base': value = numpy.float64(value) if value == 0.0: self._log_scale = None else: self._log_scale = logprob_type(numpy.log(value)) elif name == 'lmscale': self.lm_scale = logprob_type(value) elif name == 'wdpenalty': self.wi_penalty = logprob_type(value) elif name == 'start': self._initial_node_id = int(value) elif name == 'end': self._final_node_id = int(value) elif (name == 'NODES') or (name == 'N'): self._num_nodes = int(value) elif (name == 'LINKS') or (name == 'L'): self._num_links = int(value)
def _read_slf_link(self, link_id, fields): """Reads SLF lattice link fields and creates such link. :type link_id: int :param link_id: ID of the link :type fields: list of strs :param fields: the rest of the link fields after ID """ start_node = None end_node = None word = None ac_logprob = None lm_logprob = None for field in fields: name, value = _split_slf_field(field) if (name == 'START') or (name == 'S'): start_node = self.nodes[int(value)] elif (name == 'END') or (name == 'E'): end_node = self.nodes[int(value)] elif (name == 'WORD') or (name == 'W'): word = value elif (name == 'acoustic') or (name == 'a'): if self._log_scale is None: ac_logprob = logprob_type(numpy.log(numpy.float64(value))) else: ac_logprob = logprob_type(value) * self._log_scale elif (name == 'language') or (name == 'l'): if self._log_scale is None: lm_logprob = logprob_type(numpy.log(numpy.float64(value))) else: lm_logprob = logprob_type(value) * self._log_scale if start_node is None: raise InputError( "Start node is not specified for link {}.".format(link_id)) if end_node is None: raise InputError( "End node is not specified for link {}.".format(link_id)) link = self._add_link(start_node, end_node) link.word = word link.ac_logprob = ac_logprob link.lm_logprob = lm_logprob
def _read_slf_link(self, link_id, fields): """Reads SLF lattice link fields and creates such link. :type link_id: int :param link_id: ID of the link :type fields: list of strs :param fields: the rest of the link fields after ID """ start_node = None end_node = None word = None ac_logprob = None lm_logprob = None for field in fields: name, value = self._split_slf_field(field) if (name == 'START') or (name == 'S'): start_node = self.nodes[int(value)] elif (name == 'END') or (name == 'E'): end_node = self.nodes[int(value)] elif (name == 'WORD') or (name == 'W'): word = value elif (name == 'acoustic') or (name == 'a'): if self._log_scale is None: ac_logprob = logprob_type(numpy.log(numpy.float64(value))) else: ac_logprob = logprob_type(value) * self._log_scale elif (name == 'language') or (name == 'l'): if self._log_scale is None: lm_logprob = logprob_type(numpy.log(numpy.float64(value))) else: lm_logprob = logprob_type(value) * self._log_scale if start_node is None: raise InputError("Start node is not specified for link %d.".format( link_id)) if end_node is None: raise InputError("End node is not specified for link %d.".format( link_id)) link = self._add_link(start_node, end_node) link.word = word link.ac_logprob = ac_logprob link.lm_logprob = lm_logprob
def __init__(self, history=None, state=None, ac_logprob=logprob_type(0.0), lat_lm_logprob=logprob_type(0.0), nn_lm_logprob=logprob_type(0.0)): """Constructs a token with given recurrent state and logprobs. The constructor won't compute the total logprob. The user is responsible for computing it when necessary, to avoid unnecessary overhead. New tokens will not have recombination hash and total log probability set. :type history: list of ints :param history: word IDs that the token has passed :type state: RecurrentState :param state: the state of the recurrent layers for a single sequence :type ac_logprob: logprob_type :param ac_logprob: sum of the acoustic log probabilities of the lattice links :type lat_lm_logprob: logprob_type :param lat_lm_logprob: sum of the LM log probabilities of the lattice links :type nn_lm_logprob: logprob_type :param nn_lm_logprob: sum of the NNLM log probabilities of the lattice links """ self.history = [] if history is None else history self.state = [] if state is None else state self.ac_logprob = ac_logprob self.lat_lm_logprob = lat_lm_logprob self.nn_lm_logprob = nn_lm_logprob self.recombination_hash = None self.lm_logprob = None self.total_logprob = None
def __init__(self, lattice_file): """Reads an SLF lattice file. If ``lattice_file`` is ``None``, creates an empty lattice (useful for testing). :type lattice_file: file object :param lattice_file: a file in SLF lattice format """ super().__init__() # No log conversion by default. "None" means the lattice file uses # linear probabilities. self._log_scale = logprob_type(1.0) self._initial_node_id = None self._final_node_id = None if lattice_file is None: self._num_nodes = 0 self._num_links = 0 return self._num_nodes = None self._num_links = None for line in lattice_file: fields = _split_slf_line(line) self._read_slf_header(fields) if (self._num_nodes is not None) and (self._num_links is not None): break if self.wi_penalty is not None: if self._log_scale is None: self.wi_penalty = numpy.log(self.wi_penalty) else: self.wi_penalty *= self._log_scale self.nodes = [self.Node(node_id) for node_id in range(self._num_nodes)] for line in lattice_file: fields = _split_slf_line(line) if not fields: continue name, value = _split_slf_field(fields[0]) if name == 'I': self._read_slf_node(int(value), fields[1:]) elif name == 'J': self._read_slf_link(int(value), fields[1:]) if len(self.links) != self._num_links: raise InputError( "Number of links in SLF lattice doesn't match the " "LINKS field.") if self._initial_node_id is not None: self.initial_node = self.nodes[self._initial_node_id] else: # Find the node with no incoming links. self.initial_node = None for node in self.nodes: if len(node.in_links) == 0: self.initial_node = node break if self.initial_node is None: raise InputError("Could not find initial node in SLF lattice.") if self._final_node_id is not None: self.final_node = self.nodes[self._final_node_id] else: # Find the node with no outgoing links. self.final_node = None for node in self.nodes: if len(node.out_links) == 0: self.final_node = node break if self.final_node is None: raise InputError("Could not find final node in SLF lattice.") # If word identity information is not present in node definitions then # it must appear in link definitions. self._move_words_to_links() for link in self.links: if link.word is None: raise InputError("SLF lattice does not contain word identity " "in link {} or in the following node.".format( link.id))
def decode(self, lattice): """Propagates tokens through given lattice and returns a list of tokens in the final node. Propagates tokens at a node to every outgoing link by creating a copy of each token and updating the language model scores according to the link. :type lattice: Lattice :param lattice: a word lattice to be decoded :rtype: list of LatticeDecoder.Tokens :returns: the final tokens sorted by total log probability in descending order """ if self._lm_scale is not None: lm_scale = logprob_type(self._lm_scale) elif lattice.lm_scale is not None: lm_scale = logprob_type(lattice.lm_scale) else: lm_scale = logprob_type(1.0) if self._wi_penalty is not None: wi_penalty = logprob_type(self._wi_penalty) if lattice.wi_penalty is not None: wi_penalty = logprob_type(lattice.wi_penalty) else: wi_penalty = logprob_type(0.0) self._tokens = [list() for _ in lattice.nodes] initial_state = RecurrentState(self._network.recurrent_state_size) initial_token = self.Token(history=[self._sos_id], state=initial_state) initial_token.recompute_hash(self._recombination_order) initial_token.recompute_total(self._nnlm_weight, lm_scale, wi_penalty, self._linear_interpolation) self._tokens[lattice.initial_node.id].append(initial_token) lattice.initial_node.best_logprob = initial_token.total_logprob self._sorted_nodes = lattice.sorted_nodes() nodes_processed = 0 for node in self._sorted_nodes: node_tokens = self._tokens[node.id] assert node_tokens num_pruned_tokens = len(node_tokens) self._prune(node) node_tokens = self._tokens[node.id] assert node_tokens num_pruned_tokens -= len(node_tokens) if node.id == lattice.final_node.id: new_tokens = self._propagate(node_tokens, None, lm_scale, wi_penalty) return sorted(new_tokens, key=lambda token: token.total_logprob, reverse=True) num_new_tokens = 0 for link in node.out_links: new_tokens = self._propagate(node_tokens, link, lm_scale, wi_penalty) self._tokens[link.end_node.id].extend(new_tokens) num_new_tokens += len(new_tokens) nodes_processed += 1 if nodes_processed % math.ceil(len(self._sorted_nodes) / 20) == 0: logging.debug("[%d] (%.2f %%) -- tokens = %d +%d -%d", nodes_processed, nodes_processed / len(self._sorted_nodes) * 100, len(node_tokens), num_new_tokens, num_pruned_tokens) raise InputError("Could not reach the final node of word lattice.")
def __init__(self, network, decoding_options, profile=False): """Creates a Theano function that computes the output probabilities for a single time step. Creates the function self._step_function that takes as input a set of word sequences and the current recurrent states. It uses the previous states and word IDs to compute the output distributions, and computes the probabilities of the target words. All invocations of ``decode()`` will use the given NNLM weight and LM scale when computing the total probability. If LM scale is not given, uses the value provided in the lattice files. If it's not provided in a lattice file either, performs no scaling of LM log probabilities. ``decoding_options`` should countain the following elements: nnlm_weight : float weight of the neural network probabilities when interpolating with the lattice probabilities lm_scale : float if other than ``None``, the decoder will scale language model log probabilities by this factor; otherwise the scaling factor will be read from the lattice file wi_penalty : float penalize word insertion by adding this value to the total log probability of a token as many times as there are words ignore_unk : bool if set to ``True``, <unk> tokens are excluded from perplexity computation unk_penalty : float if set to othern than None, used as <unk> token score linear_interpolation : bool if set to ``True``, use linear instead of (pseudo) log-linear interpolation of language model probabilities max_tokens_per_node : int if set to other than None, leave only this many tokens at each node beam : float if set to other than None, prune tokens whose total log probability is further than this from the best token at each point in time recombination_order : int number of words to consider when deciding whether two tokens should be recombined, or ``None`` for the entire word history :type network: Network :param network: the neural network object :type decoding_options: dict :param decoding_options: a dictionary of decoding options (see above) :type profile: bool :param profile: if set to True, creates a Theano profile object """ self._network = network self._vocabulary = network.vocabulary self._nnlm_weight = logprob_type(decoding_options['nnlm_weight']) self._lm_scale = decoding_options['lm_scale'] self._wi_penalty = decoding_options['wi_penalty'] self._ignore_unk = decoding_options['ignore_unk'] self._unk_penalty = decoding_options['unk_penalty'] self._linear_interpolation = decoding_options['linear_interpolation'] self._max_tokens_per_node = decoding_options['max_tokens_per_node'] self._beam = decoding_options['beam'] if self._beam is not None: self._beam = logprob_type(self._beam) self._recombination_order = decoding_options['recombination_order'] self._sos_id = self._vocabulary.word_to_id['<s>'] self._eos_id = self._vocabulary.word_to_id['</s>'] self._unk_id = self._vocabulary.word_to_id['<unk>'] inputs = [ network.input_word_ids, network.input_class_ids, network.target_class_ids ] inputs.extend(network.recurrent_state_input) outputs = [tensor.log(network.target_probs())] outputs.extend(network.recurrent_state_output) # Ignore unused input, because is_training is only used by dropout # layer. self._step_function = theano.function(inputs, outputs, givens=[(network.is_training, numpy.int8(0))], name='step_predictor', profile=profile, on_unused_input='ignore') self._tokens = None self._sorted_nodes = None
def __init__(self, lattice_file): """Reads an SLF lattice file. If ``lattice_file`` is ``None``, creates an empty lattice (useful for testing). :type lattice_file: file object :param lattice_file: a file in SLF lattice format """ super().__init__() # No log conversion by default. "None" means the lattice file uses # linear probabilities. self._log_scale = logprob_type(1.0) self._initial_node_id = None self._final_node_id = None if lattice_file is None: self._num_nodes = 0 self._num_links = 0 return self._num_nodes = None self._num_links = None for line in lattice_file: fields = self._split_slf_line(line) self._read_slf_header(fields) if (not self._num_nodes is None) and (not self._num_links is None): break if not self.wi_penalty is None: if self._log_scale is None: self.wi_penalty = numpy.log(self.wi_penalty) else: self.wi_penalty *= self._log_scale self.nodes = [self.Node(id) for id in range(self._num_nodes)] for line in lattice_file: fields = self._split_slf_line(line) if not fields: continue name, value = self._split_slf_field(fields[0]) if name == 'I': self._read_slf_node(int(value), fields[1:]) elif name == 'J': self._read_slf_link(int(value), fields[1:]) if len(self.links) != self._num_links: raise InputError("Number of links in SLF lattice doesn't match the " "LINKS field.") if not self._initial_node_id is None: self.initial_node = self.nodes[self._initial_node_id] else: # Find the node with no incoming links. self.initial_node = None for node in self.nodes: if len(node.in_links) == 0: self.initial_node = node break if self.initial_node is None: raise InputError("Could not find initial node in SLF lattice.") if not self._final_node_id is None: self.final_node = self.nodes[self._final_node_id] else: # Find the node with no outgoing links. self.final_node = None for node in self.nodes: if len(node.out_links) == 0: self.final_node = node break if self.final_node is None: raise InputError("Could not find final node in SLF lattice.") # If word identity information is not present in node definitions then # it must appear in link definitions. self._move_words_to_links() for link in self.links: if link.word is None: raise InputError("SLF lattice does not contain word identity " "in link %d or in the following node.".format( link.id))