def one_predictive_var(): Y = RandomVar('Y', 2) X1 = RandomVar('X1', 2) X2 = RandomVar('X2', 2) X3 = RandomVar('X3', 2) f_X1_Y = CPD([X1, Y], [1.0, 0.0, 0.0, 1.0]) f_X2_Y = CPD([X2, Y], [0.5, 0.5, 0.5, 0.5]) f_X3_Y = CPD([X3, Y], [0.5, 0.5, 0.5, 0.5]) f_Y = CPD([Y], [0.5, 0.5]) bn = BayesianNetwork([f_Y, f_X1_Y, f_X2_Y, f_X3_Y]) # Training the model fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() y = X[:, -1] X = X[:, 0:-1] nb = NaiveBayes() nb.fit(X, y) # Evaluating the model fs = ForwardSampler(bn) fs.sample(10) _, X = fs.samples_to_matrix() print(nb.score(X[:, 0:-1], X[:, -1])) print(nb.predict_proba(X[:, 0:-1]))
def init(self, graph): if self.known_cpds is None: self.known_cpds = [] self.alpha = float(self.alpha) known = {cpd.scope[0] for cpd in self.known_cpds} self.unknown = set(self.scope) - known known_cpds = [CPD(cpd.scope, cpd.values) for cpd in self.known_cpds] unknown_cpds = [] self.parents = find_parents(self.scope, graph) for v in self.unknown: pa_v = sorted(self.parents[v]) f = Factor([v] + pa_v) val_pa_v = product(*(range(pa.k) for pa in pa_v)) for assg in val_pa_v: dist = np.random.dirichlet([self.alpha / len(f.values)] * v.k) assg = list(assg) for i in range(v.k): f.values[f.atoi([i] + assg)] = dist[i] unknown_cpds.append(CPD(f.scope, f.values)) self.bn = BayesianNetwork(known_cpds + unknown_cpds)
def build_genetic_network(parents, allele_freqs, prob_trait_genotype): prob_trait_genotype = np.array(prob_trait_genotype) variables = {} for person in parents.keys(): v1 = RandomVar(person + '_allele_1', len(allele_freqs)) v2 = RandomVar(person + '_allele_2', len(allele_freqs)) v3 = RandomVar(person + '_trait', 2) variables[person] = [v1, v2, v3] factors = [] for person in parents.keys(): v1, v2, v3 = variables[person] if parents[person]: p1_vars = variables[parents[person][0]] p2_vars = variables[parents[person][1]] f_allele1 = allele_given_parent_alleles(v1, p1_vars) f_allele2 = allele_given_parent_alleles(v2, p2_vars) else: f_allele1 = CPD([v1], allele_freqs) f_allele2 = CPD([v2], allele_freqs) f_phenotype = phenotype_given_genotype(variables[person], prob_trait_genotype) factors += [f_allele1, f_allele2, f_phenotype] return BayesianNetwork(factors)
def _fit_bn(self): Ms = [self.oc.stats[scope] for scope in self.oc.last_scopes] cpds = [] for M in Ms: cpds.append(M.add_scalar(self.alpha / len(M.values)).to_cpd()) self.bn = BayesianNetwork(cpds)
def _fit_score(self, graph): Ms = [self.oc.stats[scope] for scope in self.oc.last_scopes] cpds = [] self.score = 0 for M in Ms: cpd = M.to_cpd() with np.errstate(divide='ignore'): log_values = np.log(cpd.values) log_values[log_values == -np.inf] = 0 self.score += np.sum(M.values * log_values) cpds.append(cpd) bn = BayesianNetwork(cpds) self.score -= (np.log(self.X.shape[0]) / 2.) * bn.dimension()
def earthquake(): B = RandomVar('B', 2) E = RandomVar('E', 2) A = RandomVar('A', 2) R = RandomVar('R', 2) a_be = CPD([A, B, E], [0.999, 0.01, 0.01, 0.0001, 0.001, 0.99, 0.99, 0.9999]) r_e = CPD([R, E], [1.0, 0.0, 0.0, 1.0]) b = CPD([B], [0.99, 0.01]) e = CPD([E], [0.999, 0.001]) bn = BayesianNetwork([a_be, r_e, b, e]) fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() graph = bn.graph() # graph = {B : set(), E: set(), A: set(), R: set()} score_l = LikelihoodScore(scope).fit(X, graph).score print(score_l) score_bic = BICScore(scope).fit(X, graph).score print(score_bic) score_b = BayesianScore(scope).fit(X, graph).score print(score_b) # scorer = LikelihoodScore(scope) # scorer = BICScore(scope) scorer = BayesianScore(scope) best_graph, best_score = restarting_local_search(X, scope, scorer, restarts=1, iterations=100, epsilon=0.2, verbose=1) print('Best:') print(best_score) print(best_graph)
def traffic(): A = RandomVar('A', 2) T = RandomVar('T', 2) P = RandomVar('P', 2) fP = CPD([P], [0.99, 0.01]) fA = CPD([A], [0.9, 0.1]) fT_AP = CPD([T, P, A], [0.9, 0.5, 0.4, 0.1, 0.1, 0.5, 0.6, 0.9]) bn = BayesianNetwork([fP, fA, fT_AP]) print(bn) fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() mle = MaximumLikelihood(scope) print(mle.fit_predict(X, bn.graph())) ud = UniformDirichlet(scope, alpha=1.0) print(ud.fit_predict(X, bn.graph()))
def simple_chain(): x1 = RandomVar('X1', 2) x2 = RandomVar('X2', 2) x3 = RandomVar('X3', 2) fx1 = CPD([x1], [0.11, 0.89]) fx2_x1 = CPD([x2, x1], [0.59, 0.22, 0.41, 0.78]) fx3_x2 = CPD([x3, x2], [0.39, 0.06, 0.61, 0.94]) bn = BayesianNetwork([fx1, fx2_x1, fx3_x2]) graph = bn.graph() print(bn) fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() mle = MaximumLikelihood(scope) print(mle.fit_predict(X, graph)) ud = UniformDirichlet(scope, alpha=1.0) print(ud.fit_predict(X, graph))
def earthquake(): B = RandomVar('B', 2) E = RandomVar('E', 2) A = RandomVar('A', 2) R = RandomVar('R', 2) a_be = CPD( [A, B, E], [0.999, 0.01, 0.01, 0.0001, 0.001, 0.99, 0.99, 0.9999]) r_e = CPD([R, E], [1.0, 0.0, 0.0, 1.0]) b = CPD([B], [0.99, 0.01]) e = CPD([E], [0.999, 0.001]) bn = BayesianNetwork([a_be, r_e, b, e]) fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() graph = bn.graph() # graph = {B : set(), E: set(), A: set(), R: set()} score_l = LikelihoodScore(scope).fit(X, graph).score print(score_l) score_bic = BICScore(scope).fit(X, graph).score print(score_bic) score_b = BayesianScore(scope).fit(X, graph).score print(score_b) # scorer = LikelihoodScore(scope) # scorer = BICScore(scope) scorer = BayesianScore(scope) best_graph, best_score = restarting_local_search(X, scope, scorer, restarts=1, iterations=100, epsilon=0.2, verbose=1) print('Best:') print(best_score) print(best_graph)
def simple_chain(): x1 = RandomVar('X1', 2) x2 = RandomVar('X2', 2) x3 = RandomVar('X3', 2) fx1 = CPD([x1], [0.11, 0.89]) fx2_x1 = CPD([x2, x1], [0.59, 0.22, 0.41, 0.78]) fx3_x2 = CPD([x3, x2], [0.39, 0.06, 0.61, 0.94]) bn = BayesianNetwork([fx1, fx2_x1, fx3_x2]) fs = ForwardSampler(bn) fs.sample(2000) scope, X = fs.samples_to_matrix() graph = bn.graph() # graph = {x1 : set(), x2: set(), x3: set()} score_l = LikelihoodScore(scope).fit(X, graph).score print(score_l) score_bic = BICScore(scope).fit(X, graph).score print(score_bic) score_b = BayesianScore(scope).fit(X, graph).score print(score_b) # scorer = LikelihoodScore(scope) scorer = BICScore(scope) # scorer = BayesianScore(scope) best_graph, best_score = restarting_local_search(X, scope, scorer, restarts=5, iterations=50, epsilon=0.2, verbose=1) print('Best:') print(best_score) print(best_graph)
def traffic(): A = RandomVar('A', 2) T = RandomVar('T', 2) P = RandomVar('P', 2) fP = CPD([P], [0.99, 0.01]) fA = CPD([A], [0.9, 0.1]) fT_AP = CPD([T, P, A], [0.9, 0.5, 0.4, 0.1, 0.1, 0.5, 0.6, 0.9]) bn = BayesianNetwork([fP, fA, fT_AP]) # print(bn) fs = ForwardSampler(bn) fs.sample(2000) scope, X = fs.samples_to_matrix() graph = bn.graph() score_l = LikelihoodScore(scope).fit(X, graph).score print(score_l) score_bic = BICScore(scope).fit(X, graph).score print(score_bic) score_b = BayesianScore(scope).fit(X, graph).score print(score_b) # scorer = LikelihoodScore(scope) scorer = BICScore(scope) # scorer = BayesianScore(scope) best_graph, best_score = restarting_local_search(X, scope, scorer, restarts=5, iterations=50, epsilon=0.2, verbose=1) print('Best:') print(best_score) print(best_graph)
def earthquake(): B = RandomVar('B', 2) E = RandomVar('E', 2) A = RandomVar('A', 2) R = RandomVar('R', 2) a_be = CPD( [A, B, E], [0.999, 0.01, 0.01, 0.0001, 0.001, 0.99, 0.99, 0.9999]) r_e = CPD([R, E], [1.0, 0.0, 0.0, 1.0]) b = CPD([B], [0.99, 0.01]) e = CPD([E], [0.999, 0.001]) bn = BayesianNetwork([a_be, r_e, b, e]) print(bn) fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() mle = MaximumLikelihood(scope) print(mle.fit_predict(X, bn.graph())) ud = UniformDirichlet(scope, alpha=1.0) print(ud.fit_predict(X, bn.graph()))
def earthquake(): B = RandomVar('B', 2) E = RandomVar('E', 2) A = RandomVar('A', 2) R = RandomVar('R', 2) a_be = CPD([A, B, E], [0.999, 0.01, 0.01, 0.0001, 0.001, 0.99, 0.99, 0.9999]) r_e = CPD([R, E], [1.0, 0.0, 0.0, 1.0]) b = CPD([B], [0.99, 0.01]) e = CPD([E], [0.999, 0.001]) bn = BayesianNetwork([a_be, r_e, b, e]) print(bn) fs = ForwardSampler(bn) fs.sample(1000) scope, X = fs.samples_to_matrix() mle = MaximumLikelihood(scope) print(mle.fit_predict(X, bn.graph())) ud = UniformDirichlet(scope, alpha=1.0) print(ud.fit_predict(X, bn.graph()))
def main(): x1 = RandomVar('X1', 2) x2 = RandomVar('X2', 2) x3 = RandomVar('X3', 2) fx1 = CPD([x1], [0.11, 0.89]) fx2_x1 = CPD([x2, x1], [0.59, 0.22, 0.41, 0.78]) fx3_x2 = CPD([x3, x2], [0.39, 0.06, 0.61, 0.94]) bn = BayesianNetwork([fx1, fx2_x1, fx3_x2]) # mn = MarkovNetwork([fx1, fx2_x1, fx3_x2]) ve = VariableElimination(bn) jm = JointMarginalization(bn) print(ve.posterior([x1, x2], [(x3, 0)])) print(jm.posterior([x1, x2], [(x3, 0)])) print(ve.posterior([x1, x2, x3])) print(jm.posterior([x1, x2, x3])) print(ve.maximum_a_posteriori(evidence=[(x3, 0)])) print(jm.maximum_a_posteriori([x1, x2], [(x3, 0)])) fs = ForwardSampler(bn) fs.sample(10000) for c in itertools.product(range(2), repeat=3): print('{0}: {1}'.format(c, fs.posterior(zip([x1, x2, x3], c)))) px3_0 = fs.posterior([(x3, 0)]) for c in itertools.product(range(2), repeat=2): assg = list(zip([x1, x2], c)) + [(x3, 0)] print('{0}: {1}'.format(c, fs.posterior(assg) / px3_0)) gs = GibbsSampler(bn) gs.sample(burn_in=1000, n=2000) for c in itertools.product(range(2), repeat=3): print('{0}: {1}'.format(c, gs.posterior(zip([x1, x2, x3], c)))) gs.reset() gs.sample(burn_in=1000, n=1000, evidence=[(x3, 0)]) for c in itertools.product(range(2), repeat=2): print('{0}: {1}'.format(c, gs.posterior(zip([x1, x2], c))))
def main(): B = RandomVar('B', 2) E = RandomVar('E', 2) A = RandomVar('A', 2) R = RandomVar('R', 2) a_be = CPD([A, B, E], [0.999, 0.01, 0.01, 0.0001, 0.001, 0.99, 0.99, 0.9999]) r_e = CPD([R, E], [1.0, 0.0, 0.0, 1.0]) b = CPD([B], [0.99, 0.01]) e = CPD([E], [0.999, 0.001]) bn = BayesianNetwork([a_be, r_e, b, e]) ve = VariableElimination(bn) jm = JointMarginalization(bn) print(ve.posterior([B, E, A, R]) == jm.posterior([B, E, A, R])) fs = ForwardSampler(bn) fs.sample(1000)
def main(): A = RandomVar('A', 2) T = RandomVar('T', 2) P = RandomVar('P', 2) fP = CPD([P], [0.99, 0.01]) fA = CPD([A], [0.9, 0.1]) fT_AP = CPD([T, P, A], [0.9, 0.5, 0.4, 0.1, 0.1, 0.5, 0.6, 0.9]) bn = BayesianNetwork([fP, fA, fT_AP]) ve = VariableElimination(bn) jm = JointMarginalization(bn) print(jm.maximum_a_posteriori([A], [(T, 1)])) print(ve.posterior([A], [(T, 1)])) print(jm.posterior([A], [(T, 1)])) print(ve.posterior([A, T, P])) print(jm.posterior([A, T, P]))
def fit(self, X, graph): """Find the parameters for a probabilistic graphical model, given a graph and a data set that possibly contains missing data. After fitting, the model is available as a BayesianNetwork `self.bn`. Parameters ---------- X : two-dimensional np.array or python matrix of integers Matrix representing the observations. The value `X[i, j]` should correspond to the discrete random variable `self.scope[j]` in sample element `i`. The number -1 represents a missing value. graph: dict from RandomVariables to sets of RandomVariables the graph for the probabilistic graphical model """ var_index = {v: i for (i, v) in enumerate(self.scope)} best_ll = float('-inf') best_bn = None for irestart in range(self.n_restarts): if self.verbose > 0: print('Restart {0}.'.format(irestart + 1)) self.init(graph) known_cpds = [ CPD(cpd.scope, cpd.values) for cpd in self.known_cpds ] M_scopes = [] for v in self.unknown: M_scopes.append([v] + sorted(self.parents[v])) for iiteration in range(self.n_iterations): ess = [Factor(M_scope) for M_scope in M_scopes] for x in X: evidence = [] hidden = [] for (i, xi) in enumerate(x): if xi == -1: hidden.append(self.scope[i]) else: evidence.append((self.scope[i], xi)) for M in ess: M_assg = x[[var_index[v] for v in M.scope]] M_h = [] for (i, v) in enumerate(M.scope): if M_assg[i] == -1: M_h.append(v) if M_h: ve = VariableElimination(self.bn) f = ve.posterior(M_h, evidence=evidence) Mh_index = [M.scope.index(v) for v in f.scope] for i in range(len(f.values)): f_assg = f.itoa(i) M_assg[Mh_index] = f_assg M.values[M.atoi(M_assg)] += f.values[i] else: M.values[M.atoi(M_assg)] += 1 self.bn = BayesianNetwork([M.to_cpd() for M in ess] + known_cpds) if self.verbose > 1: print('Iteration {0}. '.format(iiteration + 1)) if self.verbose > 2: ll = self.log_likelihood(X, self.bn) print('Current log-likelihood {0}.'.format(ll)) ll = self.log_likelihood(X, self.bn) print('Final log-likelihood {0}.'.format(ll)) if ll > best_ll: best_ll = ll best_bn = self.bn self.bn = best_bn return self
def die(): # Parameters # d1_ = [0.2, 0.0, 0.5, 0.1, 0.1, 0.1] # d2_ = [0.2, 0.3, 0.1, 0.05, 0.05, 0.3] d1_ = [0.1, 0.9] d2_ = [0.6, 0.4] n_samples = 5000 n_iterations = 10 n_restarts = 2 verbose = 2 # Model creation if len(d1_) != len(d2_): raise Exception('The die should have the same cardinality') h = RandomVar('h', 2) o1 = RandomVar('o1', len(d1_)) o2 = RandomVar('o2', len(d2_)) f_h = CPD([h], [0.5, 0.5]) f_o1_h = Factor([o1, h]) f_o2_h = Factor([o2, h]) for i in range(len(f_o1_h.values)): o_, h_ = f_o1_h.itoa(i) f_o1_h.values[i] = d1_[o_] if h_ == 0 else d2_[o_] f_o2_h.values[i] = d2_[o_] if h_ == 0 else d1_[o_] f_o1_h = CPD(f_o1_h.scope, f_o1_h.values) f_o2_h = CPD(f_o2_h.scope, f_o2_h.values) bn = BayesianNetwork([f_h, f_o1_h, f_o2_h]) # Sampling from true model fs = ForwardSampler(bn) fs.sample(n_samples) scope, X = fs.samples_to_matrix() em = ExpectationMaximization(scope, known_cpds=[f_h], n_iterations=n_iterations, n_restarts=n_restarts, alpha=10.0, verbose=verbose) print('True log-likelihood (no missing variables):') print(em.log_likelihood(X, bn)) print('Maximum log-likelihood (no missing variables):') ls = LikelihoodScore(scope) ls.fit(X, bn.graph()) print(ls.score) # Hiding variable X[:, scope.index(h)] = -1 print('True log-likelihood (missing variables):') print(em.log_likelihood(X, bn)) bn_pred = em.fit_predict(X, bn.graph()) print('Best log-likelihood (missing variables)') print(em.log_likelihood(X, bn_pred)) # Estimation results print('Results:') f_o1_h = [f for f in bn_pred.factors if f.scope[0] == o1][0] f_o2_h = [f for f in bn_pred.factors if f.scope[0] == o2][0] d = np.zeros(o1.k) d1 = np.zeros(o1.k) d2 = np.zeros(o1.k) with printoptions(precision=3): print('d1: {0}'.format(d1_)) for i in range(o1.k): d[i] = f_o1_h.values[f_o1_h.atoi([i, 0])] print('d1 according to o1: {0}'.format(d)) d1 += d for i in range(o2.k): d[i] = f_o2_h.values[f_o2_h.atoi([i, 1])] print('d1 according to o2: {0}'.format(d)) d1 += d print('d2: {0}'.format(d2_)) for i in range(o1.k): d[i] = f_o1_h.values[f_o1_h.atoi([i, 1])] print('d2 according to o1: {0}'.format(d)) d2 += d for i in range(o2.k): d[i] = f_o2_h.values[f_o2_h.atoi([i, 0])] print('d2 according to o2: {0}'.format(d)) d2 += d print('Average estimate:') print('d1: {0}'.format(d1/2.)) print('d2: {0}'.format(d2/2.))
def _fit_bn(self): Ms = [self.oc.stats[scope] for scope in self.oc.last_scopes] self.bn = BayesianNetwork([M.to_cpd() for M in Ms])