def sample(config, params, load_path, part): data = Data(**config['data']) recognizer = SpeechRecognizer(data.recordings_source, data.labels_source, data.eos_label, data.num_features, data.num_labels, character_map=data.character_map, name='recognizer', **config["net"]) recognizer.load_params(load_path) dataset = data.get_dataset(part, add_sources=(data.uttid_source, )) stream = data.get_stream(part, batches=False, shuffle=False, add_sources=(data.uttid_source, )) it = stream.get_epoch_iterator() print_to = sys.stdout for number, data in enumerate(it): print("Utterance {} ({})".format(number, data[2]), file=print_to) groundtruth_text = dataset.pretty_print(data[1]) print("Groundtruth:", groundtruth_text, file=print_to) sample = recognizer.sample(data[0])['outputs'][:, 0] recognized_text = dataset.pretty_print(sample) print("Recognized:", recognized_text, file=print_to)
def create_model(config, data, test_tag): # Build the main brick and initialize all parameters. recognizer = SpeechRecognizer(data.recordings_source, data.labels_source, data.eos_label, data.num_features, data.num_labels, name="recognizer", data_prepend_eos=data.prepend_eos, character_map=data.character_map, **config["net"]) for brick_path, attribute_dict in sorted(config['initialization'].items(), key=lambda (k, v): k.count('/')): for attribute, value in attribute_dict.items(): brick, = Selector(recognizer).select(brick_path).bricks setattr(brick, attribute, value) brick.push_initialization_config() recognizer.initialize() if test_tag: tensor.TensorVariable.__str__ = tensor.TensorVariable.__repr__ __stream = data.get_stream("train") __data = next(__stream.get_epoch_iterator(as_dict=True)) recognizer.recordings.tag.test_value = __data[data.recordings_source] recognizer.recordings_mask.tag.test_value = __data[ data.recordings_source + '_mask'] recognizer.labels.tag.test_value = __data[data.labels_source] recognizer.labels_mask.tag.test_value = __data[data.labels_source + '_mask'] theano.config.compute_test_value = 'warn' return recognizer
def create_model(config, data, load_path=None, test_tag=False): """ Build the main brick and initialize or load all parameters. Parameters ---------- config : dict the configuration dict data : object of class Data the dataset creation object load_path : str or None if given a string, it will be used to load model parameters. Else, the parameters will be randomly initalized by calling recognizer.initialize() test_tag : bool if true, will add tag the input variables with test values """ # First tell the recognizer about required data sources net_config = dict(config["net"]) bottom_class = net_config['bottom']['bottom_class'] input_dims = { source: data.num_features(source) for source in bottom_class.vector_input_sources } input_num_chars = { source: len(data.character_map(source)) for source in bottom_class.discrete_input_sources } recognizer = SpeechRecognizer(input_dims=input_dims, input_num_chars=input_num_chars, eos_label=data.eos_label, num_phonemes=data.num_labels, name="recognizer", data_prepend_eos=data.prepend_eos, character_map=data.character_map('labels'), **net_config) if load_path: recognizer.load_params(load_path) else: for brick_path, attribute_dict in sorted( config['initialization'].items(), key=lambda (k, v): k.count('/')): for attribute, value in attribute_dict.items(): brick, = Selector(recognizer).select(brick_path).bricks setattr(brick, attribute, value) brick.push_initialization_config() recognizer.initialize() if test_tag: # fails with newest theano # tensor.TensorVariable.__str__ = tensor.TensorVariable.__repr__ __stream = data.get_stream("train") __data = next(__stream.get_epoch_iterator(as_dict=True)) for __var in recognizer.inputs.values(): __var.tag.test_value = __data[__var.name] theano.config.compute_test_value = 'warn' return recognizer
def search(config, params, load_path, part, decode_only, report, decoded_save, nll_only, seed): import matplotlib matplotlib.use("Agg") from matplotlib import pyplot from lvsr.notebook import show_alignment data = Data(**config['data']) search_conf = config['monitoring']['search'] logger.info("Recognizer initialization started") recognizer = SpeechRecognizer(data.recordings_source, data.labels_source, data.eos_label, data.num_features, data.num_labels, character_map=data.character_map, name='recognizer', **config["net"]) recognizer.load_params(load_path) recognizer.init_beam_search(search_conf['beam_size']) logger.info("Recognizer is initialized") stream = data.get_stream(part, batches=False, shuffle=part == 'train', add_sources=(data.uttid_source, ), num_examples=500 if part == 'train' else None, seed=seed) it = stream.get_epoch_iterator() if decode_only is not None: decode_only = eval(decode_only) weights = tensor.matrix('weights') weight_statistics = theano.function([weights], [ weights_std(weights.dimshuffle(0, 'x', 1)), monotonicity_penalty(weights.dimshuffle(0, 'x', 1)) ]) print_to = sys.stdout if report: alignments_path = os.path.join(report, "alignments") if not os.path.exists(report): os.mkdir(report) os.mkdir(alignments_path) print_to = open(os.path.join(report, "report.txt"), 'w') decoded_file = None if decoded_save: decoded_file = open(decoded_save, 'w') num_examples = .0 total_nll = .0 total_errors = .0 total_length = .0 total_wer_errors = .0 total_word_length = 0. if config.get('vocabulary'): with open(os.path.expandvars(config['vocabulary'])) as f: vocabulary = dict(line.split() for line in f.readlines()) def to_words(chars): words = chars.split() words = [ vocabulary[word] if word in vocabulary else vocabulary['<UNK>'] for word in words ] return words for number, example in enumerate(it): if decode_only and number not in decode_only: continue print("Utterance {} ({})".format(number, example[2]), file=print_to) groundtruth = data.decode(example[1]) groundtruth_text = data.pretty_print(example[1]) costs_groundtruth, weights_groundtruth = (recognizer.analyze( example[0], example[1], example[1])[:2]) weight_std_groundtruth, mono_penalty_groundtruth = weight_statistics( weights_groundtruth) total_nll += costs_groundtruth.sum() num_examples += 1 print("Groundtruth:", groundtruth_text, file=print_to) print("Groundtruth cost:", costs_groundtruth.sum(), file=print_to) print("Groundtruth weight std:", weight_std_groundtruth, file=print_to) print("Groundtruth monotonicity penalty:", mono_penalty_groundtruth, file=print_to) print("Average groundtruth cost: {}".format(total_nll / num_examples), file=print_to) if nll_only: print_to.flush() continue before = time.time() outputs, search_costs = recognizer.beam_search( example[0], char_discount=search_conf['char_discount'], round_to_inf=search_conf['round_to_inf'], stop_on=search_conf['stop_on']) took = time.time() - before recognized = data.decode(outputs[0]) recognized_text = data.pretty_print(outputs[0]) if recognized: # Theano scan doesn't work with 0 length sequences costs_recognized, weights_recognized = (recognizer.analyze( example[0], example[1], outputs[0])[:2]) weight_std_recognized, mono_penalty_recognized = weight_statistics( weights_recognized) error = min(1, wer(groundtruth, recognized)) else: error = 1 total_errors += len(groundtruth) * error total_length += len(groundtruth) if config.get('vocabulary'): wer_error = min( 1, wer(to_words(groundtruth_text), to_words(recognized_text))) total_wer_errors += len(groundtruth) * wer_error total_word_length += len(groundtruth) if report and recognized: show_alignment(weights_groundtruth, groundtruth, bos_symbol=True) pyplot.savefig( os.path.join(alignments_path, "{}.groundtruth.png".format(number))) show_alignment(weights_recognized, recognized, bos_symbol=True) pyplot.savefig( os.path.join(alignments_path, "{}.recognized.png".format(number))) if decoded_file is not None: print("{} {}".format(example[2], ' '.join(recognized)), file=decoded_file) print("Decoding took:", took, file=print_to) print("Beam search cost:", search_costs[0], file=print_to) print("Recognized:", recognized_text, file=print_to) if recognized: print("Recognized cost:", costs_recognized.sum(), file=print_to) print("Recognized weight std:", weight_std_recognized, file=print_to) print("Recognized monotonicity penalty:", mono_penalty_recognized, file=print_to) print("CER:", error, file=print_to) print("Average CER:", total_errors / total_length, file=print_to) if config.get('vocabulary'): print("WER:", wer_error, file=print_to) print("Average WER:", total_wer_errors / total_word_length, file=print_to) print_to.flush()