def marginal_likelihood(e_corpus: Corpus, f_corpus: Corpus, model: ConditionalModel): PL, PM, PAj, PFj = model.components ll = 0.0 for e_snt, f_snt in zip(e_corpus.itersentences(), f_corpus.itersentences()): # observations l = e_snt.shape[0] m = f_snt.shape[0] log_pl = np.log(PL.generate(l)) log_pm = np.log(PM.generate(m)) # P(f|e) = \prod_j P(f_j|e) # = \prod_j \sum_i P(f_j,a_j=i|e) log_pf_e = 0.0 for j, f in enumerate(f_snt): # P(f_j|e) = \sum_i P(f_j,a_j=i|e) pfj_e = 0.0 # contribution of this French word for i, e in enumerate(e_snt): # P(f_j, a_j=i | e) = P(a_j=i) P(f_j|e_i, l, m) pfj_e += PAj.generate((j, i), e_snt, 0, l, m) * PFj.generate((j, f), (j, i), e_snt, 0, l, m) # P(f|z,e) = \prod_j P(f_j|z,e) log_pf_e += np.log(pfj_e) # \sum_{f,e} P(l)P(m)P(f|e,l,m) ll += log_pl + log_pm + log_pf_e return - ll / e_corpus.n_sentences()
def marginal_likelihood(e_corpus: Corpus, f_corpus: Corpus, model: JointModel): PL, PM, PZ, PEi, PAj, PFj = model.components n_clusters = PZ.n_clusters ll = 0.0 for e_snt, f_snt in zip(e_corpus.itersentences(), f_corpus.itersentences()): # observations l = e_snt.shape[0] m = f_snt.shape[0] log_pl = np.log(PL.generate(l)) log_pm = np.log(PM.generate(m)) # 0-order alignments # P(f,e) = \sum_z P(z) P(e|z) P(f|z,e) log_pfe = -np.inf # contribution of this sentence for z in range(n_clusters): # contribution of the cluster log_pz = np.log(PZ.generate(z, l, m)) # compute the contribution of the entire English sentence log_pe_z = 0.0 # P(e|z) = \prod_i P(e_i|z) for i, e in enumerate(e_snt): log_pe_z += np.log(PEi.generate((i, e), z, l, m)) # P(f|z,e) = \prod_j P(f_j|z,e) # = \prod_j \sum_i P(f_j,a_j=i|z,e) log_pf_ze = 0.0 for j, f in enumerate(f_snt): # P(f_j|z,e) = \sum_i P(f_j,a_j=i|z,e) pfj_ze = 0.0 # contribution of this French word for i, e in enumerate(e_snt): pfj_ze += PAj.generate( (j, i), e_snt, z, l, m) * PFj.generate( (j, f), (j, i), e_snt, z, l, m) # P(f|z,e) = \prod_j P(f_j|z,e) log_pf_ze += np.log(pfj_ze) # \sum_z P(z) P(e|z) P(f|z,e) log_pfe = np.logaddexp(log_pfe, log_pz + log_pe_z + log_pf_ze) # \sum_{f,e} P(l)P(m)P(f,e|l,m) ll += log_pl + log_pm + log_pfe return -ll / e_corpus.n_sentences()
def map_decoder(e_corpus: Corpus, f_corpus: Corpus, model: JointModel, callback): """ :param e_corpus: English data :param f_corpus: French data :param model: components :param callback: called for each sentence in the parallel corpus callable(s, z, a, p(z|f,e), p(a|z,f,e)) """ n_clusters = model.PZ.n_clusters ll = 0.0 # E-step for s, (e_snt, f_snt) in enumerate( zip(e_corpus.itersentences(), f_corpus.itersentences())): log_pz_fe, log_post_a = log_posterior(e_snt, f_snt, model) # Here we get the best path for each cluster best_paths_z = log_post_a.argmax(2) # shape: (n_clusters, m) # Now we find out which path is the best one across clusters best_z = 0 best_log_prob = -np.inf for z in range(n_clusters): # p(z,a|f,e) = p(z|f,e) p(a|z,f,e) path_log_prob = log_pz_fe[z] + np.sum( [log_post_a[z, j, i] for j, i in enumerate(best_paths_z[z])]) if path_log_prob > best_log_prob: # update if better best_log_prob = path_log_prob best_z = z # best posterior probabilities: p(a|z,fe) best_log_pa_zfe = np.array( [log_post_a[z, j, i] for j, i in enumerate(best_paths_z[z])]) # communicate the finding callback(s, best_z, best_paths_z[best_z], np.exp(log_pz_fe[best_z]), np.exp(best_log_pa_zfe))
def map_decoder(e_corpus: Corpus, f_corpus: Corpus, model: ConditionalModel, callback): """ :param e_corpus: English data :param f_corpus: French data :param model: components :param callback: called for each sentence in the parallel corpus callable(s, z, a, p(z|f,e), p(a|z,f,e)) """ # E-step for s, (e_snt, f_snt) in enumerate(zip(e_corpus.itersentences(), f_corpus.itersentences())): log_post_a = log_posterior(e_snt, f_snt, model) # Here we get the best path for each cluster best_path = log_post_a.argmax(1) # shape: (m) # best posterior probabilities: p(a|z,fe) best_posterior = np.array([log_post_a[j, i] for j, i in enumerate(best_path)]) # communicate the finding callback(s, best_path, np.exp(best_posterior))
def EM(e_corpus: Corpus, f_corpus: Corpus, model: ConditionalModel, iterations=5): """ Generative story: l ~ P(L) m ~ P(M) a_j ~ P(A_j | l) for j=1..m f_j ~ P(F_j | e_{a_j}) for j=1..m :param e_corpus: English data :param f_corpus: French data :param model: a conditional model :param iterations: EM iterations """ PL, PM, PAj, PFj = model.components logging.info('Iteration %d Likelihood %f', 0, marginal_likelihood(e_corpus, f_corpus, model)) for iteration in range(1, iterations + 1): # E-step for s, (e_snt, f_snt) in enumerate(zip(e_corpus.itersentences(), f_corpus.itersentences())): # get the posterior P(a|f,e) post_a = posterior(e_snt, f_snt, model) l = e_snt.shape[0] m = f_snt.shape[0] # gather expected counts for (f_j, e_j): p(a_j=i|f,e) for j, f in enumerate(f_snt): for i, e in enumerate(e_snt): PAj.observe((j, i), e_snt, 0, l, m, post_a[j, i]) PFj.observe((j, f), (j, i), e_snt, 0, l, m, post_a[j, i]) # M-step model.update() logging.info('Iteration %d Likelihood %f', iteration, marginal_likelihood(e_corpus, f_corpus, model))
def EM(e_corpus: Corpus, f_corpus: Corpus, model: JointModel, iterations=5): """ Generative story: l ~ P(L) m ~ P(M) z ~ P(Z) e_i ~ P(E_i | z) for i=1..l a_j ~ P(A_j | l) for j=1..m f_j ~ P(F_j | e_{a_j}, z) for j=1..m Joint distribution: P(F,E,A,Z,L,M) = P(L)P(M)P(Z)P(E|Z)P(A|L,M)P(F|E,A,Z,L,M) We make the following independence assumptions: P(e|z) = prod_i P(e_i|z) P(f|e,a,z,l,m) = prod_j P(a_j|l,m)P(f_j|e_{a_j},z) The EM algorithm depends on 2 posterior computations: [1] P(z|f,e,l,m) = P(z)P(e|z)P(f|e,z)/P(f,e) where P(e|z) = \prod_i P(e_i|z) P(f|e,z) = \sum_a P(f,a|e,z) = \prod_j \sum_i P(a_j=i)P(f_j|e_i,z) P(f,e) = \sum_z \sum_a P(f,e,z,a|l,m) = \sum_z P(z)P(e|z) P(f|e,z) = \sum_z P(z)P(e|z) \prod_j \sum_i P(a_j=i) P(f_j|e_i,z) and [2] P(a|f,e,z) = P(a,z,f,e,l,m) / P(f,e,z,l,m) = P(z)(e|z)P(a|l,m)P(f|e,a,z) ---------------------------------- \sum_a P(z)(e|z)P(a|l,m)P(f|e,a,z) = P(z)(e|z)P(a|l,m)P(f|e,a,z) ---------------------------------- P(z)(e|z)\sum_a P(a|l,m)P(f|e,a,z) = P(z)(e|z)\prod_j P(a_j|l,m)P(f_j|e_{a_j},z) ----------------------------------------------- P(z)(e|z)\prod_j\sum_i P(a_j=i|l,m)P(f_j|e_i,z) = \prod_j P(a_j|l,m)P(f_j|e_{a_j},z) ------------------------------- \sum_i P(a_j=i|l,m)P(f_j|e_i,z) = \prod_j P(a_j|f, e, z) where P(a_j|f,e,z) = P(a_j|l,m)P(f_j|e_{a_j},z) ------------------------------- \sum_i P(a_j=i|l,m)P(f_j|e_i,z) Note that the choice of parameterisation is indenpendent of the EM algorithm in this method. For example, P(a_j|l,m) can be * uniform (IBM1) * categorical (IBM2) P(f_j|e_{a_j}, z) can be * categorical and independent of z, i.e. P(f_j|e_{a_j}, z) = P(f_j|e_{a_j}) * categorical * PoE: P(f_j|e_{a_j}, z) \propto P(f_j|e_{a_j}) P(f_j|z) * all of the above using MLP or LR instead of categorical distributions we can also have P(a_j|l,m)P(f_j|e_{a_j}, z) modelled by a single LR (with MLP-induced features). :param e_corpus: English data :param f_corpus: French data :param model: all components :param iterations: EM iterations """ PL, PM, PZ, PEi, PAj, PFj = model.components n_clusters = PZ.n_clusters logging.info('Iteration %d Likelihood %f', 0, marginal_likelihood(e_corpus, f_corpus, model)) for iteration in range(1, iterations + 1): # E-step for s, (e_snt, f_snt) in enumerate( zip(e_corpus.itersentences(), f_corpus.itersentences())): # get the factorised posterior: P(z|f,e) and P(a|z,f,e) post_z, post_a = posterior(e_snt, f_snt, model) l = e_snt.shape[0] m = f_snt.shape[0] for z in range(n_clusters): # gather expected count for z: p(z|f, e) PZ.observe(z, l, m, post_z[z]) # gather expected counts for (z, e_i): p(z|f, e) for i, e in enumerate(e_snt): PEi.observe((i, e), z, l, m, post_z[z]) # gather expected counts for (f_j, e_j): p(a_j=i|f,e,z) for j, f in enumerate(f_snt): for i, e in enumerate(e_snt): PAj.observe((j, i), e_snt, z, l, m, post_a[z, j, i]) PFj.observe((j, f), (j, i), e_snt, z, l, m, post_a[z, j, i]) # M-step model.update() logging.info('Iteration %d Likelihood %f', iteration, marginal_likelihood(e_corpus, f_corpus, model))