def sample(args): numpy.random.seed(args.random_seed) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(vocabulary, architecture, mode=Network.Mode(minibatch=False)) print("Restoring neural network state.") network.set_state(state) print("Building text sampler.") sys.stdout.flush() sampler = TextSampler(network) sequences = sampler.generate(30, args.num_sentences) for sequence in sequences: try: eos_pos = sequence.index('</s>') sequence = sequence[:eos_pos + 1] except: pass args.output_file.write(' '.join(sequence) + '\n')
def from_file(cls, model_path, mode=None, exclude_unk=False): """Reads a model from an HDF5 file. :type model_path: str :param model_path: path to a HDF5 model file :type mode: Network.Mode :param mode: selects mini-batch or single time step processing :type exclude_unk: bool :param exclude_unk: if set to ``True``, sets ``<unk>`` probability to zero. """ with h5py.File(model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of words in shortlist:", vocabulary.num_shortlist_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) result = cls(architecture, vocabulary, mode=mode, exclude_unk=exclude_unk) print("Restoring neural network state.") sys.stdout.flush() result.set_state(state) return result
def sample(args): numpy.random.seed(args.random_seed) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(vocabulary, architecture, predict_next_distribution=True) print("Restoring neural network state.") network.set_state(state) print("Building text sampler.") sys.stdout.flush() sampler = TextSampler(network) for i in range(args.num_sentences): words = sampler.generate() args.output_file.write('{}: {}\n'.format( i, ' '.join(words)))
def sample(args): numpy.random.seed(args.random_seed) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(architecture, vocabulary, mode=Network.Mode(minibatch=False)) print("Restoring neural network state.") network.set_state(state) print("Building text sampler.") sys.stdout.flush() sampler = TextSampler(network) sequences = sampler.generate(30, args.num_sentences) for sequence in sequences: try: eos_pos = sequence.index('</s>') sequence = sequence[:eos_pos+1] except: pass args.output_file.write(' '.join(sequence) + '\n')
def test_from_state(self): self.classes_file.seek(0) vocabulary1 = Vocabulary.from_file(self.classes_file, 'srilm-classes') f = h5py.File('in-memory.h5', driver='core', backing_store=False) vocabulary1.get_state(f) vocabulary2 = Vocabulary.from_state(f) self.assertTrue(numpy.array_equal(vocabulary1.id_to_word, vocabulary2.id_to_word)) self.assertDictEqual(vocabulary1.word_to_id, vocabulary2.word_to_id) self.assertTrue(numpy.array_equal(vocabulary1.word_id_to_class_id, vocabulary2.word_id_to_class_id)) self.assertListEqual(list(vocabulary1._word_classes), list(vocabulary2._word_classes))
def restoreModel(path): with h5py.File(path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of words in shortlist:", vocabulary.num_shortlist_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(architecture, vocabulary, mode=Network.Mode(minibatch=False)) print("Restoring neural network state.") network.set_state(state) return network
def from_file(cls, model_path, mode=None, exclude_unk=False, default_device=None): """Reads a model from an HDF5 file. :type model_path: str :param model_path: path to a HDF5 model file :type mode: Network.Mode :param mode: selects mini-batch or single time step processing :type exclude_unk: bool :param exclude_unk: if set to ``True``, set ``<unk>`` probability to zero before normalizing the network outputs (required to get exact normalization during inference) :type default_device: str :param default_device: default device where to store the shared variables """ with h5py.File(model_path, 'r') as state: logging.info("Reading vocabulary from network state.") #sys.stdout.flush() vocabulary = Vocabulary.from_state(state) logging.info("Number of words in vocabulary: {}".format( vocabulary.num_words())) logging.info("Number of words in shortlist: {}".format( vocabulary.num_shortlist_words())) logging.info("Number of word classes: {}".format( vocabulary.num_classes())) logging.info("Building neural network.") #sys.stdout.flush() architecture = Architecture.from_state(state) result = cls(architecture, vocabulary, mode=mode, exclude_unk=exclude_unk, default_device=default_device) logging.info("Restoring neural network state.") result.set_training() logging.info("Reseting to the neural network to evaluate.") result.set_state(state) return result
def sample(args): """A function that performs the "theanolm sample" command. :type args: argparse.Namespace :param args: a collection of command line arguments """ numpy.random.seed(args.random_seed) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' with h5py.File(args.model_path, 'r') as state: logging.info("Reading vocabulary from network state.") vocabulary = Vocabulary.from_state(state) logging.info("Number of words in vocabulary: %d", vocabulary.num_words()) logging.info("Number of words in shortlist: %d", vocabulary.num_shortlist_words()) logging.info("Number of word classes: %d", vocabulary.num_classes()) logging.info("Building neural network.") architecture = Architecture.from_state(state) default_device = get_default_device(args.default_device) network = Network(architecture, vocabulary, mode=Network.Mode(minibatch=False), default_device=default_device) logging.info("Restoring neural network state.") network.set_state(state) logging.info("Building text sampler.") sampler = TextSampler(network) sequences = sampler.generate(args.sentence_length, args.num_sentences, seed_sequence=args.seed_sequence) for sequence in sequences: try: eos_pos = sequence.index('</s>') sequence = sequence[:eos_pos + 1] except ValueError: pass args.output_file.write(' '.join(sequence) + '\n')
def score(args): with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(architecture, vocabulary) print("Restoring neural network state.") sys.stdout.flush() network.set_state(state) print("Building text scorer.") sys.stdout.flush() if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty scorer = TextScorer(network, ignore_unk, unk_penalty) print("Scoring text.") if args.output == 'perplexity': _score_text(args.input_file, vocabulary, scorer, args.output_file, args.log_base, False) elif args.output == 'word-scores': _score_text(args.input_file, vocabulary, scorer, args.output_file, args.log_base, True) elif args.output == 'utterance-scores': _score_utterances(args.input_file, vocabulary, scorer, args.output_file, args.log_base) else: print("Invalid output format requested:", args.output) sys.exit(1)
def __init__(self, model_path): self.model_path = model_path numpy.random.seed() theano.config.compute_test_value = 'off' with h5py.File(model_path, 'r') as self.state: print("Reading vocabulary from network state.") #sys.stdout.flush() self.vocabulary = Vocabulary.from_state(self.state) print("Number of words in vocabulary:", self.vocabulary.num_words()) print("Number of words in shortlist:", self.vocabulary.num_shortlist_words()) print("Number of word classes:", self.vocabulary.num_classes()) print("Building neural network.") #sys.stdout.flush() self.architecture = Architecture.from_state(self.state) self.network = Network(self.architecture, self.vocabulary, mode=Network.Mode(minibatch=False)) print("Restoring neural network state.") self.network.set_state(self.state) print("Building text sampler.") #sys.stdout.flush() self.sampler = TextSampler(self.network)
def score(args): with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(vocabulary, architecture) print("Restoring neural network state.") sys.stdout.flush() network.set_state(state) print("Building text scorer.") sys.stdout.flush() if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty scorer = TextScorer(network, ignore_unk, unk_penalty) print("Scoring text.") if args.output == 'perplexity': _score_text(args.input_file, vocabulary, scorer, args.output_file, args.log_base, False) elif args.output == 'word-scores': _score_text(args.input_file, vocabulary, scorer, args.output_file, args.log_base, True) elif args.output == 'utterance-scores': _score_utterances(args.input_file, vocabulary, scorer, args.output_file, args.log_base)
def train(args): numpy.random.seed(args.random_seed) log_file = args.log_file log_level = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level, int): print("Invalid logging level requested:", args.log_level) sys.exit(1) log_format = '%(asctime)s %(funcName)s: %(message)s' if args.log_file == '-': logging.basicConfig(stream=sys.stdout, format=log_format, level=log_level) else: logging.basicConfig(filename=log_file, format=log_format, level=log_level) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' theano.config.profile = args.profile theano.config.profile_memory = args.profile with h5py.File(args.model_path, 'a', driver='core') as state: if state.keys(): print("Reading vocabulary from existing network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) elif args.vocabulary is None: print("Constructing vocabulary from training set.") sys.stdout.flush() vocabulary = Vocabulary.from_corpus(args.training_set, args.num_classes) for training_file in args.training_set: training_file.seek(0) vocabulary.get_state(state) else: print("Reading vocabulary from {}.".format(args.vocabulary)) sys.stdout.flush() with open(args.vocabulary, 'rt', encoding='utf-8') as vocab_file: vocabulary = Vocabulary.from_file(vocab_file, args.vocabulary_format) if args.vocabulary_format == 'classes': print("Computing class membership probabilities from " "unigram word counts.") sys.stdout.flush() vocabulary.compute_probs(args.training_set) vocabulary.get_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() if args.architecture == 'lstm300' or args.architecture == 'lstm1500': architecture = Architecture.from_package(args.architecture) else: with open(args.architecture, 'rt', encoding='utf-8') as arch_file: architecture = Architecture.from_description(arch_file) network = Network(vocabulary, architecture, profile=args.profile) sys.stdout.flush() if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty num_training_files = len(args.training_set) if len(args.weights) > num_training_files: print("You specified more weights than training files.") sys.exit(1) weights = numpy.ones(num_training_files).astype(theano.config.floatX) for index, weight in enumerate(args.weights): weights[index] = weight print("Building text scorer.") scorer = TextScorer(network, ignore_unk, unk_penalty, args.profile) validation_mmap = mmap.mmap(args.validation_file.fileno(), 0, prot=mmap.PROT_READ) validation_iter = \ LinearBatchIterator(validation_mmap, vocabulary, batch_size=args.batch_size, max_sequence_length=None) optimization_options = { 'method': args.optimization_method, 'epsilon': args.numerical_stability_term, 'gradient_decay_rate': args.gradient_decay_rate, 'sqr_gradient_decay_rate': args.sqr_gradient_decay_rate, 'learning_rate': args.learning_rate, 'weights': weights, 'momentum': args.momentum, 'max_gradient_norm': args.gradient_normalization, 'cost_function': args.cost, 'num_noise_samples': args.num_noise_samples, 'ignore_unk': ignore_unk, 'unk_penalty': unk_penalty } logging.debug("OPTIMIZATION OPTIONS") for option_name, option_value in optimization_options.items(): if type(option_value) is list: value_str = ', '.join(str(x) for x in option_value) logging.debug("%s: [%s]", option_name, value_str) else: logging.debug("%s: %s", option_name, str(option_value)) training_options = { 'strategy': args.training_strategy, 'batch_size': args.batch_size, 'sequence_length': args.sequence_length, 'validation_frequency': args.validation_frequency, 'patience': args.patience, 'stopping_criterion': args.stopping_criterion, 'max_epochs': args.max_epochs, 'min_epochs': args.min_epochs, 'max_annealing_count': args.max_annealing_count } logging.debug("TRAINING OPTIONS") for option_name, option_value in training_options.items(): logging.debug("%s: %s", option_name, str(option_value)) print("Building neural network trainer.") sys.stdout.flush() if len(args.sampling) > len(args.training_set): print("You specified more sampling coefficients than training " "files.") sys.exit(1) trainer = create_trainer( training_options, optimization_options, network, vocabulary, scorer, args.training_set, args.sampling, validation_iter, state, args.profile) trainer.set_logging(args.log_interval) print("Training neural network.") sys.stdout.flush() trainer.train() if not 'layers' in state.keys(): print("The model has not been trained. No cross-validations were " "performed or training did not improve the model.") else: network.set_state(state) perplexity = scorer.compute_perplexity(validation_iter) print("Best validation set perplexity:", perplexity)
def _read_vocabulary(args, state): """If ``state`` contains data, reads the vocabulary from the HDF5 state. Otherwise reads a vocabulary file or constructs the vocabulary from the training set and writes it to the HDF5 state. If the state does not contain data and --vocabulary argument is given, reads the vocabulary from the file given after the argument. The rest of the words in the training set will be added as out-of-shortlist words. If the state does not contain data and no vocabulary is given, constructs a vocabulary that contains all the training set words. In that case, --num-classes argument can be used to control the number of classes. :type args: argparse.Namespace :param args: a collection of command line arguments :type state: hdf5.File :param state: HDF5 file where the vocabulary should be saved :rtype: Vocabulary :returns: the created vocabulary """ if state.keys(): print("Reading vocabulary from existing network state.") sys.stdout.flush() result = Vocabulary.from_state(state) if not result.has_unigram_probs(): # This is for backward compatibility. Remove at some point. print("Computing unigram word probabilities from training set.") sys.stdout.flush() word_counts = compute_word_counts(args.training_set) shortlist_words = list(result.id_to_word) shortlist_set = set(shortlist_words) oos_words = [ x for x in word_counts.keys() if x not in shortlist_set ] result.id_to_word = numpy.asarray(shortlist_words + oos_words, dtype=object) result.word_to_id = { word: word_id for word_id, word in enumerate(result.id_to_word) } result.compute_probs(word_counts, update_class_probs=False) result.get_state(state) elif args.vocabulary is None: print("Constructing vocabulary from training set.") sys.stdout.flush() word_counts = compute_word_counts(args.training_set) result = Vocabulary.from_word_counts(word_counts, args.num_classes) result.get_state(state) else: print("Reading vocabulary from {}.".format(args.vocabulary)) sys.stdout.flush() word_counts = compute_word_counts(args.training_set) oos_words = word_counts.keys() with open(args.vocabulary, 'rt', encoding='utf-8') as vocab_file: result = Vocabulary.from_file(vocab_file, args.vocabulary_format, oos_words=oos_words) if args.vocabulary_format == 'classes': print("Computing class membership probabilities and unigram " "probabilities for out-of-shortlist words.") sys.stdout.flush() update_class_probs = True else: print( "Computing unigram probabilities for out-of-shortlist words.") sys.stdout.flush() update_class_probs = False result.compute_probs(word_counts, update_class_probs=update_class_probs) result.get_state(state) print("Number of words in vocabulary:", result.num_words()) print("Number of words in shortlist:", result.num_shortlist_words()) print("Number of word classes:", result.num_classes()) return result
def train(args): numpy.random.seed(args.random_seed) log_file = args.log_file log_level = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level, int): raise ValueError("Invalid logging level requested: " + args.log_level) log_format = '%(asctime)s %(funcName)s: %(message)s' if args.log_file == '-': logging.basicConfig(stream=sys.stdout, format=log_format, level=log_level) else: logging.basicConfig(filename=log_file, format=log_format, level=log_level) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' theano.config.profile = args.profile theano.config.profile_memory = args.profile with h5py.File(args.model_path, 'a', driver='core') as state: if state.keys(): print("Reading vocabulary from existing network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) elif args.vocabulary is None: print("Constructing vocabulary from training set.") sys.stdout.flush() vocabulary = Vocabulary.from_corpus(args.training_set, args.num_classes) for training_file in args.training_set: training_file.seek(0) vocabulary.get_state(state) else: print("Reading vocabulary from {}.".format(args.vocabulary)) sys.stdout.flush() with open(args.vocabulary, 'rt', encoding='utf-8') as vocab_file: vocabulary = Vocabulary.from_file(vocab_file, args.vocabulary_format) if args.vocabulary_format == 'classes': print("Computing class membership probabilities from " "unigram word counts.") sys.stdout.flush() vocabulary.compute_probs(args.training_set) vocabulary.get_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() if args.architecture == 'lstm300' or args.architecture == 'lstm1500': architecture = Architecture.from_package(args.architecture) else: with open(args.architecture, 'rt', encoding='utf-8') as arch_file: architecture = Architecture.from_description(arch_file) network = Network(vocabulary, architecture, profile=args.profile) sys.stdout.flush() if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty num_training_files = len(args.training_set) if len(args.weights) > num_training_files: print("You specified more weights than training files.") sys.exit(1) weights = numpy.ones(num_training_files).astype(theano.config.floatX) for index, weight in enumerate(args.weights): weights[index] = weight print("Building text scorer.") scorer = TextScorer(network, ignore_unk, unk_penalty, args.profile) validation_mmap = mmap.mmap(args.validation_file.fileno(), 0, prot=mmap.PROT_READ) validation_iter = LinearBatchIterator(validation_mmap, vocabulary, batch_size=32) optimization_options = { 'method': args.optimization_method, 'epsilon': args.numerical_stability_term, 'gradient_decay_rate': args.gradient_decay_rate, 'sqr_gradient_decay_rate': args.sqr_gradient_decay_rate, 'learning_rate': args.learning_rate, 'weights': weights, 'momentum': args.momentum, 'max_gradient_norm': args.gradient_normalization, 'ignore_unk': ignore_unk, 'unk_penalty': unk_penalty } logging.debug("OPTIMIZATION OPTIONS") for option_name, option_value in optimization_options.items(): if type(option_value) is list: value_str = ', '.join(str(x) for x in option_value) logging.debug("%s: [%s]", option_name, value_str) else: logging.debug("%s: %s", option_name, str(option_value)) training_options = { 'strategy': args.training_strategy, 'batch_size': args.batch_size, 'sequence_length': args.sequence_length, 'validation_frequency': args.validation_frequency, 'patience': args.patience, 'stopping_criterion': args.stopping_criterion, 'max_epochs': args.max_epochs, 'min_epochs': args.min_epochs, 'max_annealing_count': args.max_annealing_count } logging.debug("TRAINING OPTIONS") for option_name, option_value in training_options.items(): logging.debug("%s: %s", option_name, str(option_value)) print("Building neural network trainer.") sys.stdout.flush() if len(args.sampling) > len(args.training_set): print("You specified more sampling coefficients than training " "files.") sys.exit(1) trainer = create_trainer( training_options, optimization_options, network, vocabulary, scorer, args.training_set, args.sampling, validation_iter, state, args.profile) trainer.set_logging(args.log_interval) print("Training neural network.") sys.stdout.flush() trainer.run() if not state.keys(): print("The model has not been trained.") else: network.set_state(state) perplexity = scorer.compute_perplexity(validation_iter) print("Best validation set perplexity:", perplexity)
def decode(args): log_file = args.log_file log_level = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level, int): print("Invalid logging level requested:", args.log_level) sys.exit(1) log_format = '%(asctime)s %(funcName)s: %(message)s' if args.log_file == '-': logging.basicConfig(stream=sys.stdout, format=log_format, level=log_level) else: logging.basicConfig(filename=log_file, format=log_format, level=log_level) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' theano.config.profile = args.profile theano.config.profile_memory = args.profile with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(architecture, vocabulary, mode=Network.Mode(minibatch=False)) print("Restoring neural network state.") sys.stdout.flush() network.set_state(state) log_scale = 1.0 if args.log_base is None else numpy.log(args.log_base) if args.wi_penalty is None: wi_penalty = None else: wi_penalty = args.wi_penalty * log_scale if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty decoding_options = { 'nnlm_weight': args.nnlm_weight, 'lm_scale': args.lm_scale, 'wi_penalty': wi_penalty, 'ignore_unk': ignore_unk, 'unk_penalty': unk_penalty, 'linear_interpolation': args.linear_interpolation, 'max_tokens_per_node': args.max_tokens_per_node, 'beam': args.beam, 'recombination_order': args.recombination_order } logging.debug("DECODING OPTIONS") for option_name, option_value in decoding_options.items(): logging.debug("%s: %s", option_name, str(option_value)) print("Building word lattice decoder.") sys.stdout.flush() decoder = LatticeDecoder(network, decoding_options) # Combine paths from command line and lattice list. lattices = args.lattices lattices.extend(args.lattice_list.readlines()) lattices = [path.strip() for path in lattices] # Ignore empty lines in the lattice list. lattices = list(filter(None, lattices)) # Pick every Ith lattice, if --num-jobs is specified and > 1. if args.num_jobs < 1: print("Invalid number of jobs specified:", args.num_jobs) sys.exit(1) if (args.job < 0) or (args.job > args.num_jobs - 1): print("Invalid job specified:", args.job) sys.exit(1) lattices = lattices[args.job::args.num_jobs] file_type = TextFileType('r') for index, path in enumerate(lattices): logging.info("Reading word lattice: %s", path) lattice_file = file_type(path) lattice = SLFLattice(lattice_file) if not lattice.utterance_id is None: utterance_id = lattice.utterance_id else: utterance_id = os.path.basename(lattice_file.name) logging.info("Utterance `%s' -- %d/%d of job %d", utterance_id, index + 1, len(lattices), args.job) tokens = decoder.decode(lattice) for index in range(min(args.n_best, len(tokens))): line = format_token(tokens[index], utterance_id, vocabulary, log_scale, args.output) args.output_file.write(line + "\n")
def decode(args): log_file = args.log_file log_level = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level, int): print("Invalid logging level requested:", args.log_level) sys.exit(1) log_format = '%(asctime)s %(funcName)s: %(message)s' if args.log_file == '-': logging.basicConfig(stream=sys.stdout, format=log_format, level=log_level) else: logging.basicConfig(filename=log_file, format=log_format, level=log_level) if args.debug: theano.config.compute_test_value = 'warn' else: theano.config.compute_test_value = 'off' theano.config.profile = args.profile theano.config.profile_memory = args.profile with h5py.File(args.model_path, 'r') as state: print("Reading vocabulary from network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) print("Building neural network.") sys.stdout.flush() architecture = Architecture.from_state(state) network = Network(vocabulary, architecture, mode=Network.Mode.target_words) print("Restoring neural network state.") sys.stdout.flush() network.set_state(state) log_scale = 1.0 if args.log_base is None else numpy.log(args.log_base) if args.wi_penalty is None: wi_penalty = None else: wi_penalty = args.wi_penalty * log_scale if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty decoding_options = { 'nnlm_weight': args.nnlm_weight, 'lm_scale': args.lm_scale, 'wi_penalty': wi_penalty, 'ignore_unk': ignore_unk, 'unk_penalty': unk_penalty, 'linear_interpolation': args.linear_interpolation, 'max_tokens_per_node': args.max_tokens_per_node, 'beam': args.beam, 'recombination_order': args.recombination_order } logging.debug("DECODING OPTIONS") for option_name, option_value in decoding_options.items(): logging.debug("%s: %s", option_name, str(option_value)) print("Building word lattice decoder.") sys.stdout.flush() decoder = LatticeDecoder(network, decoding_options) # Combine paths from command line and lattice list. lattices = args.lattices lattices.extend(args.lattice_list.readlines()) lattices = [path.strip() for path in lattices] # Ignore empty lines in the lattice list. lattices = list(filter(None, lattices)) # Pick every Ith lattice, if --num-jobs is specified and > 1. if args.num_jobs < 1: print("Invalid number of jobs specified:", args.num_jobs) sys.exit(1) if (args.job < 0) or (args.job > args.num_jobs - 1): print("Invalid job specified:", args.job) sys.exit(1) lattices = lattices[args.job::args.num_jobs] file_type = TextFileType('r') for index, path in enumerate(lattices): logging.info("Reading word lattice: %s", path) lattice_file = file_type(path) lattice = SLFLattice(lattice_file) if not lattice.utterance_id is None: utterance_id = lattice.utterance_id else: utterance_id = os.path.basename(lattice_file.name) logging.info("Utterance `%s' -- %d/%d of job %d", utterance_id, index + 1, len(lattices), args.job) tokens = decoder.decode(lattice) for index in range(min(args.n_best, len(tokens))): line = format_token(tokens[index], utterance_id, vocabulary, log_scale, args.output) args.output_file.write(line + "\n")
def train(args): numpy.random.seed(args.random_seed) log_file = args.log_file log_level = getattr(logging, args.log_level.upper(), None) if not isinstance(log_level, int): print("Invalid logging level requested:", args.log_level) sys.exit(1) log_format = "%(asctime)s %(funcName)s: %(message)s" if args.log_file == "-": logging.basicConfig(stream=sys.stdout, format=log_format, level=log_level) else: logging.basicConfig(filename=log_file, format=log_format, level=log_level) if args.debug: theano.config.compute_test_value = "warn" print("Enabled computing test values for tensor variables.") print("Warning: GpuArray backend will fail random number generation!") else: theano.config.compute_test_value = "off" theano.config.profile = args.profile theano.config.profile_memory = args.profile with h5py.File(args.model_path, "a", driver="core") as state: if state.keys(): print("Reading vocabulary from existing network state.") sys.stdout.flush() vocabulary = Vocabulary.from_state(state) elif args.vocabulary is None: print("Constructing vocabulary from training set.") sys.stdout.flush() vocabulary = Vocabulary.from_corpus(args.training_set, args.num_classes) for training_file in args.training_set: training_file.seek(0) vocabulary.get_state(state) else: print("Reading vocabulary from {}.".format(args.vocabulary)) sys.stdout.flush() with open(args.vocabulary, "rt", encoding="utf-8") as vocab_file: vocabulary = Vocabulary.from_file(vocab_file, args.vocabulary_format) if args.vocabulary_format == "classes": print("Computing class membership probabilities from " "unigram word counts.") sys.stdout.flush() vocabulary.compute_probs(args.training_set) vocabulary.get_state(state) print("Number of words in vocabulary:", vocabulary.num_words()) print("Number of word classes:", vocabulary.num_classes()) if args.num_noise_samples > vocabulary.num_classes(): print( "Number of noise samples ({}) is larger than the number of " "classes. This doesn't make sense and would cause sampling " "to fail.".format(args.num_noise_samples) ) sys.exit(1) if args.unk_penalty is None: ignore_unk = False unk_penalty = None elif args.unk_penalty == 0: ignore_unk = True unk_penalty = None else: ignore_unk = False unk_penalty = args.unk_penalty num_training_files = len(args.training_set) if len(args.weights) > num_training_files: print("You specified more weights than training files.") sys.exit(1) weights = numpy.ones(num_training_files).astype(theano.config.floatX) for index, weight in enumerate(args.weights): weights[index] = weight training_options = { "batch_size": args.batch_size, "sequence_length": args.sequence_length, "validation_frequency": args.validation_frequency, "patience": args.patience, "stopping_criterion": args.stopping_criterion, "max_epochs": args.max_epochs, "min_epochs": args.min_epochs, "max_annealing_count": args.max_annealing_count, } logging.debug("TRAINING OPTIONS") for option_name, option_value in training_options.items(): logging.debug("%s: %s", option_name, str(option_value)) optimization_options = { "method": args.optimization_method, "epsilon": args.numerical_stability_term, "gradient_decay_rate": args.gradient_decay_rate, "sqr_gradient_decay_rate": args.sqr_gradient_decay_rate, "learning_rate": args.learning_rate, "weights": weights, "momentum": args.momentum, "max_gradient_norm": args.gradient_normalization, "cost_function": args.cost, "num_noise_samples": args.num_noise_samples, "noise_sharing": args.noise_sharing, "ignore_unk": ignore_unk, "unk_penalty": unk_penalty, } logging.debug("OPTIMIZATION OPTIONS") for option_name, option_value in optimization_options.items(): if type(option_value) is list: value_str = ", ".join(str(x) for x in option_value) logging.debug("%s: [%s]", option_name, value_str) else: logging.debug("%s: %s", option_name, str(option_value)) if len(args.sampling) > len(args.training_set): print("You specified more sampling coefficients than training " "files.") sys.exit(1) print("Creating trainer.") sys.stdout.flush() trainer = Trainer(training_options, vocabulary, args.training_set, args.sampling) trainer.set_logging(args.log_interval) print("Building neural network.") sys.stdout.flush() if args.architecture == "lstm300" or args.architecture == "lstm1500": architecture = Architecture.from_package(args.architecture) else: with open(args.architecture, "rt", encoding="utf-8") as arch_file: architecture = Architecture.from_description(arch_file) network = Network( architecture, vocabulary, trainer.class_prior_probs, args.noise_dampening, default_device=args.default_device, profile=args.profile, ) print("Compiling optimization function.") sys.stdout.flush() optimizer = create_optimizer(optimization_options, network, device=args.default_device, profile=args.profile) if args.print_graph: print("Cost function computation graph:") theano.printing.debugprint(optimizer.gradient_update_function) trainer.initialize(network, state, optimizer) if not args.validation_file is None: print("Building text scorer for cross-validation.") sys.stdout.flush() scorer = TextScorer(network, ignore_unk, unk_penalty, args.profile) print("Validation text:", args.validation_file.name) validation_mmap = mmap.mmap(args.validation_file.fileno(), 0, prot=mmap.PROT_READ) validation_iter = LinearBatchIterator( validation_mmap, vocabulary, batch_size=args.batch_size, max_sequence_length=None ) trainer.set_validation(validation_iter, scorer) else: print("Cross-validation will not be performed.") validation_iter = None print("Training neural network.") sys.stdout.flush() trainer.train() if not "layers" in state.keys(): print( "The model has not been trained. No cross-validations were " "performed or training did not improve the model." ) elif not validation_iter is None: network.set_state(state) perplexity = scorer.compute_perplexity(validation_iter) print("Best validation set perplexity:", perplexity)