def fit(self,X): n_objects = X.shape[0] n_features = X.shape[1] sigma = np.zeros((self.n_clusters, n_features, n_features)) w = np.tile(1.0/self.n_clusters, self.n_clusters) centers_idx = np.random.choice(n_objects, size=self.n_clusters, replace=False) mu = X[centers_idx, :] for cluster in range (self.n_clusters): sigma[cluster :, :] = np.eye(n_features) ll = log_likelihood(X, w, mu, sigma) for i in range(self.max_iter): ll_new = log_likelihood(X, w, mu, sigma) self.save_logs(compute_labels(X, mu), w, mu, sigma, ll) if i > 0 and abs(ll_new - ll) < self.tol: self.cluster_centers_ = mu.copy() self.labels_ = compute_labels(X, mu) self.covars_ = sigma.copy() self.w_ = w.copy() break else: gamma = self.estep(X,w, mu, sigma) w, mu, sigma = self.mstep(X,gamma) ll = ll_new i+=1 if i == self.max_iter: self.convergence = -1
def fit(self, X): n_objects, n_features = X.shape self.covars_ = np.zeros((self.n_clusters, n_features, n_features)) self.w_ = np.tile(1.0 / self.n_clusters, self.n_clusters) centers_idx = np.random.choice(n_objects, size = self.n_clusters, replace = False) self.cluster_centers_ = X[centers_idx, :] for cluster in range(self.n_clusters): self.covars_[cluster :, :] = np.eye(n_features) self.ll = log_likelihood(X, self.w_, self.cluster_centers_, self.covars_) for i in range(self.max_iter): if self.logging: self.logs['log_likelihood'].append(log_likelihood(X, self.w_, self.cluster_centers_, self.covars_)) self.logs['labels'].append(compute_labels(X, self.cluster_centers_)) self.logs['w'].append(self.w_) self.logs['mu'].append(self.cluster_centers_) self.logs['sigma'].append(self.covars_) ll_new = log_likelihood(X, self.w_, self.cluster_centers_, self.covars_) if i > 0 and abs(ll_new - self.ll) < self.tol: break else: g = self.e_step(X) self.m_step(X, g) self.ll = ll_new self.labels_ = compute_labels(X, self.cluster_centers_)
def decode(ciphertext, output_file_name): run_likelihoods, steps, acceptances = [], [], [] f = utils.find_good_start(ciphertext, optimized=True) likelihood = utils.log_likelihood(ciphertext, f) max_likelihood = likelihood i=0 no_recent_acceptance = False acceptance_indices = [] while no_recent_acceptance == False: run_likelihoods.append(likelihood) steps.append(i) i += 1 # construct proposal prop_f = utils.random_transposition(f) prop_likelihood = utils.log_likelihood(ciphertext, prop_f) # make sure proposal even makes sense, i.e. that it has positive # likelihood if prop_likelihood: x = prop_likelihood - likelihood # proposal gets accepted automatically if x >=0: f = prop_f max_likelihood = prop_likelihood likelihood = prop_likelihood acceptances.append(True) acceptance_indices.append(i) print max_likelihood # thresholding to avoid underflow elif x<=0 and x >= math.log(0.00001): a = math.exp(x) u = np.random.random_sample() if u <= a: f = prop_f likelihood = prop_likelihood print max_likelihood acceptances.append(True) acceptance_indices.append(i) else: acceptances.append(False) print "Rejected: {}".format(i) else: acceptances.append(False) print "Rejected: {}".format(i) else: acceptances.append(False) print "Rejected: {}".format(i) ten_percent = max(int(i/10),1000) if len(acceptance_indices) > 0: if acceptance_indices[-1] + ten_percent < i: no_recent_acceptance = True T = 100 acceptance_rate = [np.mean(acceptances[max(0, t-T):t]) for t in xrange(len(acceptances))] res = utils.decipher(f, ciphertext) fo = open(output_file_name, 'w') fo.write(res) fo.close() return f
def ADAM_update(self, iteration): for itr in range(iteration): start_time = time.process_time() self.t += 1 # compute gradient g = self.gradient_w_tau() self.w_tau_g[0] = self.w_tau_g[0] * self.b1 + (1 - self.b1) * g self.w_tau_g[1] = self.w_tau_g[1] * self.b2 + (1 - self.b2) * g * g for rv in self.g.rvs: if rv.value is not None: continue elif rv.domain.continuous: g = self.gradient_mu_var(rv) self.eta_g[0][rv] = self.eta_g[0][rv] * self.b1 + ( 1 - self.b1) * g self.eta_g[1][rv] = self.eta_g[1][rv] * self.b2 + ( 1 - self.b2) * g * g else: g = self.gradient_category_tau(rv) self.eta_tau_g[0][rv] = self.eta_tau_g[0][rv] * self.b1 + ( 1 - self.b1) * g self.eta_tau_g[1][rv] = self.eta_tau_g[1][rv] * self.b2 + ( 1 - self.b2) * g * g # update parameters self.w_tau = self.w_tau - (self.alpha * (self.w_tau_g[0] / (1 - (self.b1 ** self.t)))) \ / (np.sqrt(self.w_tau_g[1] / (1 - (self.b2 ** self.t))) + self.eps) self.w = self.softmax(self.w_tau) for rv in self.g.rvs: if rv.value is not None: continue elif rv.domain.continuous: table = self.eta[rv] - (self.alpha * (self.eta_g[0][rv] / (1 - (self.b1 ** self.t)))) \ / (np.sqrt(self.eta_g[1][rv] / (1 - (self.b2 ** self.t))) + self.eps) table[:, 1] = np.clip(table[:, 1], a_min=self.var_threshold, a_max=np.inf) self.eta[rv] = table else: table = self.eta_tau[rv] - (self.alpha * (self.eta_tau_g[0][rv] / (1 - (self.b1 ** self.t)))) \ / (np.sqrt(self.eta_tau_g[1][rv] / (1 - (self.b2 ** self.t))) + self.eps) self.eta_tau[rv] = table self.eta[rv] = self.softmax(table, 1) if self.is_log: current_time = time.process_time() self.total_time += current_time - start_time if self.log_fe: fe = self.free_energy() else: map_res = dict() for rv in self.g.g.rvs: map_res[rv] = self.map(rv) fe = log_likelihood(self.g.g, map_res) print(fe, self.total_time) self.time_log.append([self.total_time, fe])
def get_log_likelihood(self, X, y): ''' Calculates the log-likelihood of the logistic regression model over the dataset. - Inputs: - X: An ndarray of regressor variable values; i.e., features; i.e., inputs. - y: An ndarray of dependent variable values; i.e., tragets; i.e., labels; i.e., outputs. - Returns: - LL = \sum_{i=1}^n y_i w^{\top} X_i - \text{log} (1 + e^{w^{\top} X_i }). ''' return log_likelihood(X, y, self.get_params())
def score(self, Y, Yhat, Ynull=None, method='circ_corr'): """Score the model. Parameters ---------- Y : array, shape (n_samples, [n_neurons]) The true firing rates. Yhat : array, shape (n_samples, [n_neurons]) The estimated firing rates. Ynull : None | array, shape (n_samples, [n_classes]) The labels for the null model. Must be None if method is not 'pseudo_R2' method : str One of 'pseudo_R2' or 'circ_corr' or 'cosine_dist' """ if method == 'pseudo_R2': if(len(Y.shape) > 1): # There are many neurons, so calculate and return the score for # each neuron score = list() for neuron in range(Y.shape[1]): L1 = log_likelihood(Y[:, neuron], Yhat[:, neuron]) LS = log_likelihood(Y[:, neuron], Y[:, neuron]) L0 = log_likelihood(Y[:, neuron], Ynull[neuron]) score.append(1 - (LS - L1) / (LS - L0)) else: L1 = log_likelihood(Y, Yhat) LS = log_likelihood(Y, Y) L0 = log_likelihood(Y, Ynull) score = 1 - (LS - L1) / (LS - L0) elif method == 'circ_corr': score = circ_corr(np.squeeze(Y), np.squeeze(Yhat)) elif method == 'cosine_dist': score = np.mean(np.cos(np.squeeze(Y) - np.squeeze(Yhat))) return score
data = load_data('Demo/Data/HMLN/0') # for key, rv in rvs_dict.items(): # if key not in data and key not in query and not rv.domain.continuous: # data[key] = 0 # closed world assumption # data = dict() g, rvs_dict = rel_g.add_evidence(data) print(len(rvs_dict)) print(len(g.factors)) # infer = HybridLBP(g, n=10, proposal_approximation='simple') # infer.run(10, c2f=-1, log_enable=False) # infer = HMWS(g) # infer.run(max_tries=1, max_flips=10000, epsilon=0.0, noise_std=0.5) infer = C2FVI(g, num_mixtures=2, num_quadrature_points=3) infer.run(100, lr=0.2) # map_res = infer.rvs_map(rvs_dict.values()) # for key, rv in rvs_dict.items(): # print(key, map_res[rv]) map_res = dict() for key, rv in rvs_dict.items(): map_res[rv] = infer.map(rv) print(key, map_res[rv]) print(log_likelihood(g, map_res))
den += gamma[j, t] mu[:, j] = (np.transpose(weighted_samples) / den).reshape(dim) # Sigma weighted_diff = np.zeros([dim, dim]) for t in range(time_steps): gamma_ = gamma[j, t] diff = data_train[t, :].reshape([-1, 1]) - mu[:, j].reshape( [-1, 1]) # dim x 1 diff_t = np.transpose(diff) weighted_diff += gamma_ * diff.dot(diff_t) sigma_list[j] = weighted_diff / den # Log-likelihood training data llik_new = utils.log_likelihood(n_states, time_steps, gamma, csi, data_train, A, pi, mu, sigma_list) llik_list.append(llik_new / time_steps) # Test data # E-step: # Alpha alpha_log = utils.alpha_log(data_test, pi, mu, sigma_list, A, n_states) # Beta beta_log = utils.beta_log(data_test, pi, mu, sigma_list, A, n_states) # Gamma gamma_log = utils.gamma_log(alpha_log, beta_log)
def run(self, max_tries=100, max_flips=1000, epsilon=0.9, noise_std=1, is_log=True): if is_log: self.time_log = list() total_time = 0 numeric_factors, discrete_factors = self.discrete_and_numeric_factors() numeric_factors = self.prune_factors_without_latent_variables( numeric_factors) discrete_factors = self.prune_factors_without_latent_variables( discrete_factors) self.best_assignment = None self.best_score = -np.Inf for i_try in range(max_tries): assignment = self.random_assignment() for i_flip in range(max_flips): start_time = time.process_time() score = self.score(assignment) if score > self.best_score: self.best_assignment = assignment.copy() self.best_score = score # print(self.best_score, i_try, i_flip) unsatisfied_hard, unsatisfied_soft = self.unsatisfied_factors( assignment, discrete_factors) if len(unsatisfied_hard) > 0: c = np.random.choice(unsatisfied_hard) else: c = self.random_factor(unsatisfied_soft, numeric_factors) if np.random.rand() < epsilon: rv = np.random.choice( list(filter(lambda rv_: rv_.value is None, c.nb))) if rv.value is not None: continue if rv.domain.continuous: assignment[rv] = self.argmax_rv_wrt_factor(c, rv, assignment) + \ np.random.normal(scale=noise_std) else: assignment[rv] = 1 - assignment[rv] else: rv_score_dict = dict() rv_assignment_dict = dict() for rv in c.nb: if rv.value is not None: continue temp_assignment = assignment.copy() if rv.domain.continuous: rv_assignment_dict[rv] = self.argmax_rvs_wrt_score( [rv], assignment)[rv] else: rv_assignment_dict[ rv] = self.argmax_discrete_rv_wrt_score( rv, assignment) temp_assignment[rv] = rv_assignment_dict[rv] rv_score_dict[rv] = self.local_score( c.nb, temp_assignment) if len(rv_score_dict) == 0: continue rv = max(rv_score_dict.keys(), key=lambda k: rv_score_dict[k]) if rv_score_dict[rv] > self.local_score( c.nb, assignment) or c in discrete_factors: assignment[rv] = rv_assignment_dict[rv] else: assignment = self.argmax_numeric_term_wrt_score( c, assignment) if is_log: current_time = time.process_time() total_time += current_time - start_time if self.score(assignment) > self.best_score: map_res = dict() for rv in self.g.rvs: map_res[rv] = assignment[rv] ll = log_likelihood(self.g, map_res) print(ll, total_time, i_flip) self.time_log.append([total_time, ll])
def main(FLAGS): if not os.path.exists(FLAGS.experiment_rootdir_comp_adf): os.makedirs(FLAGS.experiment_rootdir_comp_adf) # Train only if cuda is available if device.type == 'cuda': # Create the experiment rootdir adf if not already there if not os.path.exists(FLAGS.experiment_rootdir_adf): os.makedirs(FLAGS.experiment_rootdir_adf) # Hyperparameters batch_size = FLAGS.batch_size # Default 32 # Loading testing dataset test_steer_dataset = create_dataset(FLAGS.test_dir) test_loader = torch.utils.data.DataLoader(dataset=test_steer_dataset, batch_size=batch_size, shuffle=False) targets = [] for image, target in test_steer_dataset: targets.append(np.asscalar(target.cpu().numpy())) # Cropped image dimensions crop_img_width, crop_img_height = FLAGS.crop_img_width, FLAGS.crop_img_height # Image mode if FLAGS.img_mode == 'rgb': img_channels = 3 elif FLAGS.img_mode == 'grayscale': img_channels = 1 else: raise IOError("Unidentified image mode: use 'grayscale' or 'rgb'") # Output dimension output_dim = 1 # Load standard model model = resnet8_MCDO(img_channels, crop_img_height, crop_img_width, output_dim).to(device) model_ckpt = os.path.join(FLAGS.experiment_rootdir, 'resnet8_MCDO.pt') model.load_state_dict(torch.load(model_ckpt)) # Load heteroscedastic model model_het = resnet8_MCDO_ale(img_channels, crop_img_height, crop_img_width, output_dim).to(device) model_het_ckpt = os.path.join(FLAGS.experiment_rootdir, 'resnet8_MCDO_ale.pt') model_het.load_state_dict(torch.load(model_het_ckpt)) model_adf = Resnet8_MCDO_adf(img_channels, output_dim, FLAGS.noise_var, FLAGS.min_var).to(device) model_adf.load_state_dict(torch.load(model_ckpt)) # Compute epistemic variance FLAGS.is_MCDO = True print("Computing epistemic variances") # Get predictions and ground truth _, pred_steerings_mean_MCDO, real_steerings, epistemic_variances = \ utils.compute_predictions_and_gt(model, test_loader, device, FLAGS) # Compute total variance print("Computing total variances with heteroscedastic") # Get predictions and ground truth _, pred_steerings_mean_het, aleatoric_variances, real_steerings, total_variances = \ utils.compute_predictions_and_gt_het(model_het, test_loader, device, FLAGS) # Compute total variance print("Computing total variances with ADF") # Get predictions and ground truth _, pred_steerings_mean_adf_MCDO, aleatoric_variances_adf, real_steerings, total_variances_adf = \ utils.compute_predictions_and_gt_adf(model_adf, test_loader, device, FLAGS) # Compute log-likelihoods ll_epi = utils.log_likelihood(pred_steerings_mean_MCDO, targets, np.sqrt(epistemic_variances)) ll_ale_het = utils.log_likelihood(pred_steerings_mean_het, targets, np.sqrt(aleatoric_variances)) ll_tot_het = utils.log_likelihood(pred_steerings_mean_het, targets, np.sqrt(total_variances)) ll_ale_adf = utils.log_likelihood(pred_steerings_mean_adf_MCDO, targets, np.sqrt(aleatoric_variances_adf)) ll_tot_adf = utils.log_likelihood(pred_steerings_mean_adf_MCDO, targets, np.sqrt(total_variances_adf)) print( "Log-likelihood considering EPISTEMIC uncertainty is: {}". format(ll_epi)) print( "Log-likelihood considering ALEATORIC_het uncertainty is: {}". format(ll_ale_het)) print( "Log-likelihood considering TOTAL_het uncertainty is: {}". format(ll_tot_het)) print( "Log-likelihood considering ALEATORIC_adf uncertainty is: {}\n" .format(ll_ale_adf)) print( "Log-likelihood considering TOTAL_adf uncertainty is: {}\n" .format(ll_tot_adf)) else: raise IOError('Cuda is not available.')
def main(args): np.random.seed(1982) ts = np.linspace(0., 1200, 2001) heat_on = np.mod((ts / 200).astype('int'), 2) != 1 heat_on[ts >= 800.] = False heat_on = heat_on.astype('bool') heat_on[np.logical_and(ts >= 800, ts <= 820)] = True # heat_on = np.ones(ts.size) p = pot_of_water(0.10, 0.10) print "==============TRUE PARAMETERS=====================" print utils.pretty_params(p) print "==================================================" sig = 0.1 s = np.array(list(utils.compute_heat_source(ts, heat_on, p['k'], p['init_time_off']))) u = np.array(list(utils.compute_temperature(ts, heat_on, **p))) # Add observation noise obs = u + sig * np.random.normal(size=u.size) fig, ax = plt.subplots(1, 2) ax[0].plot(ts, u, color='black') ax[0].plot(ts, obs, alpha=0.5, color='green') ax[1].plot(ts, s, color='red') plt.show() fig, ax = plt.subplots(1, 2) n = 1 best = -np.inf best_params = None for i in range(n): x0 = {'k': 0.1, 'c_v': 3000., 'volume': 1., 'u_0': 300., 'h': 3., 'init_time_off': 100.} fixed = {'wattage': 400, 'u_env': 298.} optimal = utils.optimal_parameters(ts, heat_on, obs, params=x0, sig=sig, **fixed) optimal.update(fixed) print utils.pretty_params(optimal) ll = utils.log_likelihood(ts, heat_on, obs, sig=sig, **optimal) print "optimal_likelihood", ll if ll > best: best = ll best_params = optimal u_samp = np.array(list(utils.compute_temperature(ts, heat_on, **optimal))) ax[0].plot(ts, u, color='black') ax[0].plot(ts, obs, alpha=0.5, color='green') ax[0].plot(ts, u_samp, color='steelblue') plt.show() np.random.seed(1982) sample = utils.sample_parameters(ts, heat_on, obs, n=100, params=optimal, iterations=10) fig, ax = plt.subplots(1, 2) ax[0].plot(ts, u, color='black') a0_ylim = ax[0].get_ylim() ax[1].plot(ts, s, color='black') a1_ylim = ax[1].get_ylim() for k, u_0, u_max, h, r in sample[['k', 'u_0', 'u_max', 'h', 'r']].values: s_samp = np.array(list(utils.compute_heat_source(ts, heat_on, k, u_0, u_max, u_env=25.))) u_samp = np.array(list(utils.compute_temperature(ts, s_samp, h, r, u_0, u_env=25.))) ax[0].plot(ts, u_samp, alpha=0.5, color='steelblue') ax[1].plot(ts, s_samp, alpha=0.5, color='steelblue') ax[0].set_ylim(a0_ylim) ax[1].set_ylim(a1_ylim) plt.show()
# Input stuff if '-infile' in sys.argv: p = sys.argv.index('-infile') data_file = str(sys.argv[p + 1]) data = pd.read_csv(f'./curves/{data_file}') time, flux, err = data['time'].to_numpy(), data['flux'].to_numpy( ), data['flux_error'].to_numpy() # Perform a likelihood fit guesses = [0.001, 0.019, 30] nll = lambda *args: -log_likelihood(*args) soln = minimize(nll, guesses, method="Powell", args=(time, flux, err)) print(soln) # MCMC labels = ['$t_0$', '$R_P/R_*$', '$a/R_*$'] pos = soln.x + 1e-4 * np.random.randn(32, 3) nwalkers, ndim = pos.shape sampler = emcee.EnsembleSampler(nwalkers, ndim,