def process(b): # create emission graph g_emissions = gtn.linear_graph(T, C, inputs.requires_grad) cpu_data = inputs[b].cpu().contiguous() g_emissions.set_weights(cpu_data.data_ptr()) # create transition graph g_transitions = ASGLossFunction.create_transitions_graph( transitions, calc_trans_grad) # create force align criterion graph g_fal = ASGLossFunction.create_force_align_graph(targets[b]) # compose the graphs g_fal_fwd = gtn.forward_score( gtn.intersect(gtn.intersect(g_fal, g_transitions), g_emissions)) g_fcc_fwd = gtn.forward_score( gtn.intersect(g_emissions, g_transitions)) g_loss = gtn.subtract(g_fcc_fwd, g_fal_fwd) scale = 1.0 if reduction == "mean": L = len(targets[b]) scale = 1.0 / L if L > 0 else scale elif reduction != "none": raise ValueError("invalid value for reduction '" + str(reduction) + "'") # Save for backward: losses[b] = g_loss scales[b] = scale emissions_graphs[b] = g_emissions transitions_graphs[b] = g_transitions
def process(b): # Create emissions graph: emissions = gtn.linear_graph(T, C, inputs.requires_grad) cpu_data = inputs[b].cpu().contiguous() emissions.set_weights(cpu_data.data_ptr()) target = make_chain_graph(targets[b]) target.arc_sort(True) # Create token to grapheme decomposition graph tokens_target = gtn.remove(gtn.project_output(gtn.compose(target, lexicon))) tokens_target.arc_sort() # Create alignment graph: alignments = gtn.project_input( gtn.remove(gtn.compose(tokens, tokens_target)) ) alignments.arc_sort() # Add transition scores: if transitions is not None: alignments = gtn.intersect(transitions, alignments) alignments.arc_sort() loss = gtn.forward_score(gtn.intersect(emissions, alignments)) # Normalize if needed: if transitions is not None: norm = gtn.forward_score(gtn.intersect(emissions, transitions)) loss = gtn.subtract(loss, norm) losses[b] = gtn.negate(loss) # Save for backward: if emissions.calc_grad: emissions_graphs[b] = emissions
def process(b): # create emission graph g_emissions = gtn.linear_graph(T, C, log_probs.requires_grad) cpu_data = log_probs[b].cpu().contiguous() g_emissions.set_weights(cpu_data.data_ptr()) # create criterion graph g_criterion = CTCLossFunction.create_ctc_graph( targets[b], blank_idx) # compose the graphs g_loss = gtn.negate( gtn.forward_score(gtn.intersect(g_emissions, g_criterion))) scale = 1.0 if reduction == "mean": L = len(targets[b]) scale = 1.0 / L if L > 0 else scale elif reduction != "none": raise ValueError("invalid value for reduction '" + str(reduction) + "'") # Save for backward: losses[b] = g_loss scales[b] = scale emissions_graphs[b] = g_emissions
def test_simple_decomposition(self): T = 5 tokens = ["a", "b", "ab", "ba", "aba"] scores = torch.randn((1, T, len(tokens)), requires_grad=True) labels = [[0, 1, 0]] transducer = Transducer(tokens=tokens, graphemes_to_idx={ "a": 0, "b": 1 }) # Hand construct the alignment graph with all of the decompositions alignments = gtn.Graph(False) alignments.add_node(True) # Add the path ['a', 'b', 'a'] alignments.add_node() alignments.add_arc(0, 1, 0) alignments.add_arc(1, 1, 0) alignments.add_node() alignments.add_arc(1, 2, 1) alignments.add_arc(2, 2, 1) alignments.add_node(False, True) alignments.add_arc(2, 3, 0) alignments.add_arc(3, 3, 0) # Add the path ['a', 'ba'] alignments.add_node(False, True) alignments.add_arc(1, 4, 3) alignments.add_arc(4, 4, 3) # Add the path ['ab', 'a'] alignments.add_node() alignments.add_arc(0, 5, 2) alignments.add_arc(5, 5, 2) alignments.add_arc(5, 3, 0) # Add the path ['aba'] alignments.add_node(False, True) alignments.add_arc(0, 6, 4) alignments.add_arc(6, 6, 4) emissions = gtn.linear_graph(T, len(tokens), True) emissions.set_weights(scores.data_ptr()) expected_loss = gtn.subtract( gtn.forward_score(emissions), gtn.forward_score(gtn.intersect(emissions, alignments)), ) loss = transducer(scores, labels) self.assertAlmostEqual(loss.item(), expected_loss.item(), places=5) loss.backward() gtn.backward(expected_loss) expected_grad = torch.tensor(emissions.grad().weights_to_numpy()) expected_grad = expected_grad.view((1, T, len(tokens))) self.assertTrue( torch.allclose(scores.grad, expected_grad, rtol=1e-4, atol=1e-5))
def process(b): for t in range(0, T - kernel_size + 1, stride): input_graph = gtn.linear_graph(kernel_size, C, inputs.requires_grad) window = cpu_inputs[b, t : t + kernel_size, :].contiguous() input_graph.set_weights(window.data_ptr()) if viterbi: window_outputs = [ gtn.viterbi_score(gtn.intersect(input_graph, kernel)) for kernel in kernels ] else: window_outputs = [ gtn.forward_score(gtn.intersect(input_graph, kernel)) for kernel in kernels ] output_graphs[b].append(window_outputs) # Save for backward: if input_graph.calc_grad: input_graphs[b].append(input_graph)
def crf_loss(X, Y, potentials, transitions): feature_graph = gtn.compose(X, potentials) # Compute the unnormalized score of `(X, Y)` target_graph = gtn.compose(feature_graph, gtn.intersect(Y, transitions)) target_score = gtn.forward_score(target_graph) # Compute the partition function norm_graph = gtn.compose(feature_graph, transitions) norm_score = gtn.forward_score(norm_graph) return gtn.subtract(norm_score, target_score)
def forward_single(b): emissions = gtn.linear_graph(T, C, inputs.requires_grad) data = inputs[b].contiguous() emissions.set_weights(data.data_ptr()) target = GTNLossFunction.make_target_graph(targets[b]) # Score the target: target_score = gtn.forward_score(gtn.intersect(target, emissions)) # Normalization term: norm = gtn.forward_score(emissions) # Compute the loss: loss = gtn.subtract(norm, target_score) # Save state for backward: losses[b] = loss emissions_graphs[b] = emissions
def process(b): emissions = gtn.linear_graph(T, C, False) cpu_data = outputs[b].cpu().contiguous() emissions.set_weights(cpu_data.data_ptr()) if self.transitions is not None: full_graph = gtn.intersect(emissions, self.transitions) else: full_graph = emissions # Find the best path and remove back-off arcs: path = gtn.remove(gtn.viterbi_path(full_graph)) # Left compose the viterbi path with the "alignment to token" # transducer to get the outputs: path = gtn.compose(path, self.tokens) # When there are ambiguous paths (allow_repeats is true), we take # the shortest: path = gtn.viterbi_path(path) path = gtn.remove(gtn.project_output(path)) paths[b] = path.labels_to_list()
def process(b): # create emission graph g_emissions = gtn.linear_graph(T, C, False) cpu_data = outputs[b].cpu().contiguous() g_emissions.set_weights(cpu_data.data_ptr()) # create transition graph g_transitions = utils.ASGLossFunction.create_transitions_graph( self.transitions) g_path = gtn.viterbi_path(gtn.intersect(g_emissions, g_transitions)) prediction = g_path.labels_to_list() collapsed_prediction = [p for p, _ in groupby(prediction)] if self.garbage_idx is not None: # remove garbage tokens collapsed_prediction = [ p for p in collapsed_prediction if p != self.garbage_idx ] predictions[b] = utils.unpack_replabels(collapsed_prediction, self.num_replabels)
import kenlm import os import random # bigram LM sent = "wood pittsburgh cindy jean" m = kenlm.Model("lm_small.arpa") counts, vocab = read_counts_from_arpa("lm_small.arpa") # print(vocab, counts) symb = {v: k for k, v in vocab.items()} g_lm = build_lm_graph(counts, vocab) gtn.write_dot(g_lm, "/tmp/g_lm.dot", symb, symb) g_sent = build_setence_graph(sent, vocab) gtn.write_dot(g_sent, "/tmp/g_sent.dot", symb, symb) g_score = gtn.intersect(g_lm, g_sent) gtn.write_dot(g_score, "/tmp/g_score.dot", symb, symb) # compare P(sent </s> | <s>) assert gtn.viterbi_score(g_score).item() == m.score(sent, bos=True, eos=True) # trigram LM lm_file = "/tmp/3-gram.pruned.3e-7.arpa" if not os.path.exists(lm_file): url = "http://www.openslr.org/resources/11/3-gram.pruned.3e-7.arpa.gz" os.system(f"wget {url} -P /tmp/ && gunzip {lm_file}") assert os.path.exists(lm_file) # lower case arpa file with open(lm_file) as f:
tokens = token_graph(word_pieces) gtn.draw(tokens, "tokens.pdf", idx_to_wp, idx_to_wp) # Recognizes "abc": abc = gtn.Graph(False) abc.add_node(True) abc.add_node() abc.add_node() abc.add_node(False, True) abc.add_arc(0, 1, let_to_idx["a"]) abc.add_arc(1, 2, let_to_idx["b"]) abc.add_arc(2, 3, let_to_idx["c"]) gtn.draw(abc, "abc.pdf", idx_to_let) # Compute the decomposition graph for "abc": abc_decomps = gtn.remove(gtn.project_output(gtn.compose(abc, lex))) gtn.draw(abc_decomps, "abc_decomps.pdf", idx_to_wp, idx_to_wp) # Compute the alignment graph for "abc": abc_alignments = gtn.project_input( gtn.remove(gtn.compose(tokens, abc_decomps))) gtn.draw(abc_alignments, "abc_alignments.pdf", idx_to_wp) # From here we can use the alignment graph with an emissions graph and # transitions graphs to compute the sequence level criterion: emissions = gtn.linear_graph(10, len(word_pieces), True) loss = gtn.subtract( gtn.forward_score(emissions), gtn.forward_score(gtn.intersect(emissions, abc_alignments))) print(f"Loss is {loss.item():.2f}")
def test_composition(self): # Compos,ing with an empty graph gives an empty graph g1 = gtn.Graph() g2 = gtn.Graph() self.assertTrue(gtn.equal(gtn.compose(g1, g2), gtn.Graph())) g1.add_node(True) g1.add_node() g1.add_arc(0, 1, 0) g2.add_node(True) g2.add_node(False, True) g2.add_arc(0, 1, 0) g2.add_arc(0, 1, 0) self.assertTrue(gtn.equal(gtn.compose(g1, g2), gtn.Graph())) self.assertTrue(gtn.equal(gtn.compose(g2, g1), gtn.Graph())) self.assertTrue(gtn.equal(gtn.intersect(g2, g1), gtn.Graph())) # Check singly sorted version g1.arc_sort(True) self.assertTrue(gtn.equal(gtn.compose(g1, g2), gtn.Graph())) # Check doubly sorted version g2.arc_sort() self.assertTrue(gtn.equal(gtn.compose(g1, g2), gtn.Graph())) # Self-loop in the composed graph g1 = gtn.Graph() g1.add_node(True) g1.add_node(False, True) g1.add_arc(0, 0, 0) g1.add_arc(0, 1, 1) g1.add_arc(1, 1, 2) g2 = gtn.Graph() g2.add_node(True) g2.add_node() g2.add_node(False, True) g2.add_arc(0, 1, 0) g2.add_arc(1, 1, 0) g2.add_arc(1, 2, 1) g_str = ["0", "2", "0 1 0", "1 1 0", "1 2 1"] expected = create_graph_from_text(g_str) self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) self.assertTrue(gtn.isomorphic(gtn.intersect(g1, g2), expected)) # Check singly sorted version g1.arc_sort(True) self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) # Check doubly sorted version g2.arc_sort() self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) # Loop in the composed graph g1 = gtn.Graph() g1.add_node(True) g1.add_node(False, True) g1.add_arc(0, 1, 0) g1.add_arc(1, 1, 1) g1.add_arc(1, 0, 0) g2 = gtn.Graph() g2.add_node(True) g2.add_node(False, True) g2.add_arc(0, 0, 0) g2.add_arc(0, 1, 1) g2.add_arc(1, 0, 1) g_str = ["0", "2", "0 1 0", "1 0 0", "1 2 1", "2 1 1"] expected = create_graph_from_text(g_str) self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) self.assertTrue(gtn.isomorphic(gtn.intersect(g1, g2), expected)) # Check singly sorted version g1.arc_sort(True) self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) # Check doubly sorted version g2.arc_sort() self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) g1 = gtn.Graph() g1.add_node(True) g1.add_node() g1.add_node() g1.add_node() g1.add_node(False, True) for i in range(g1.num_nodes() - 1): for j in range(3): g1.add_arc(i, i + 1, j, j, j) g2 = gtn.Graph() g2.add_node(True) g2.add_node() g2.add_node(False, True) g2.add_arc(0, 1, 0, 0, 3.5) g2.add_arc(1, 1, 0, 0, 2.5) g2.add_arc(1, 2, 1, 1, 1.5) g2.add_arc(2, 2, 1, 1, 4.5) g_str = [ "0", "6", "0 1 0 0 3.5", "1 2 0 0 2.5", "1 4 1 1 2.5", "2 3 0 0 2.5", "2 5 1 1 2.5", "4 5 1 1 5.5", "3 6 1 1 2.5", "5 6 1 1 5.5", ] expected = create_graph_from_text(g_str) self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) self.assertTrue(gtn.isomorphic(gtn.intersect(g1, g2), expected)) # Check singly sorted version g1.arc_sort(True) self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected)) # Check doubly sorted version g2.arc_sort() self.assertTrue(gtn.isomorphic(gtn.compose(g1, g2), expected))