Beispiel #1
0
    def fit(self,X):
       
        n_objects = X.shape[0]
        n_features = X.shape[1]
        
        sigma = np.zeros((self.n_clusters, n_features, n_features))
        w = np.tile(1.0/self.n_clusters, self.n_clusters)


        centers_idx = np.random.choice(n_objects, size=self.n_clusters, replace=False)
        mu = X[centers_idx, :] 
            
        for cluster in range (self.n_clusters):
            sigma[cluster :, :] = np.eye(n_features)
        
        ll = log_likelihood(X, w, mu, sigma)
        
        for i in range(self.max_iter):
            
            ll_new = log_likelihood(X, w, mu, sigma)
            self.save_logs(compute_labels(X, mu), w, mu, sigma, ll)
            
            if i > 0 and abs(ll_new - ll) < self.tol:
                self.cluster_centers_ = mu.copy()
                self.labels_ = compute_labels(X, mu)
                self.covars_ = sigma.copy()
                self.w_ = w.copy()
                break
            else:
                gamma = self.estep(X,w, mu, sigma)
                w, mu, sigma = self.mstep(X,gamma)
                ll = ll_new
                i+=1
                if i == self.max_iter:
                    self.convergence = -1
Beispiel #2
0
    def fit(self, X):
        n_objects, n_features = X.shape
        
        self.covars_  = np.zeros((self.n_clusters, n_features, n_features))
        self.w_ = np.tile(1.0 / self.n_clusters, self.n_clusters)

        centers_idx = np.random.choice(n_objects, size = self.n_clusters, replace = False)
        self.cluster_centers_ = X[centers_idx, :]
 
        for cluster in range(self.n_clusters):
            self.covars_[cluster :, :] = np.eye(n_features)
            
        self.ll = log_likelihood(X, self.w_, self.cluster_centers_, self.covars_)
        
        for i in range(self.max_iter):
            if self.logging:
                self.logs['log_likelihood'].append(log_likelihood(X, self.w_, self.cluster_centers_, self.covars_))
                self.logs['labels'].append(compute_labels(X, self.cluster_centers_))
                self.logs['w'].append(self.w_)
                self.logs['mu'].append(self.cluster_centers_)
                self.logs['sigma'].append(self.covars_)
                    
            ll_new = log_likelihood(X, self.w_, self.cluster_centers_, self.covars_)
            
            if i > 0 and abs(ll_new - self.ll) < self.tol:
                break
            else:
                g = self.e_step(X)
                self.m_step(X, g)
                self.ll = ll_new
                
        self.labels_ = compute_labels(X, self.cluster_centers_)
Beispiel #3
0
def decode(ciphertext, output_file_name):
    run_likelihoods, steps, acceptances = [], [], []
    f = utils.find_good_start(ciphertext, optimized=True)
    likelihood = utils.log_likelihood(ciphertext, f)
    max_likelihood = likelihood
    i=0
    no_recent_acceptance = False
    acceptance_indices = []
    while no_recent_acceptance == False:
        run_likelihoods.append(likelihood)
        steps.append(i)
        i += 1
        # construct proposal
        prop_f = utils.random_transposition(f)
        prop_likelihood = utils.log_likelihood(ciphertext, prop_f)
        # make sure proposal even makes sense, i.e. that it has positive
        # likelihood
        if prop_likelihood:
            x = prop_likelihood - likelihood
            # proposal gets accepted automatically
            if x >=0:
                f = prop_f
                max_likelihood = prop_likelihood
                likelihood = prop_likelihood
                acceptances.append(True)
                acceptance_indices.append(i)
                print max_likelihood
            # thresholding to avoid underflow
            elif x<=0 and x >= math.log(0.00001):
                a = math.exp(x)
                u = np.random.random_sample()
                if u <= a:
                    f = prop_f
                    likelihood = prop_likelihood
                    print max_likelihood
                    acceptances.append(True)
                    acceptance_indices.append(i)
                else:
                    acceptances.append(False)
                    print "Rejected: {}".format(i)
            else:
                acceptances.append(False)
                print "Rejected: {}".format(i)
        else:
            acceptances.append(False)
            print "Rejected: {}".format(i)
        ten_percent = max(int(i/10),1000)
        if len(acceptance_indices) > 0:
            if acceptance_indices[-1] + ten_percent < i:
                no_recent_acceptance = True
    T = 100
    acceptance_rate = [np.mean(acceptances[max(0, t-T):t]) for t in
                       xrange(len(acceptances))]
    res = utils.decipher(f, ciphertext)
    fo = open(output_file_name, 'w')
    fo.write(res)
    fo.close()
    return f
    def ADAM_update(self, iteration):
        for itr in range(iteration):
            start_time = time.process_time()

            self.t += 1
            # compute gradient
            g = self.gradient_w_tau()
            self.w_tau_g[0] = self.w_tau_g[0] * self.b1 + (1 - self.b1) * g
            self.w_tau_g[1] = self.w_tau_g[1] * self.b2 + (1 - self.b2) * g * g

            for rv in self.g.rvs:
                if rv.value is not None:
                    continue
                elif rv.domain.continuous:
                    g = self.gradient_mu_var(rv)
                    self.eta_g[0][rv] = self.eta_g[0][rv] * self.b1 + (
                        1 - self.b1) * g
                    self.eta_g[1][rv] = self.eta_g[1][rv] * self.b2 + (
                        1 - self.b2) * g * g
                else:
                    g = self.gradient_category_tau(rv)
                    self.eta_tau_g[0][rv] = self.eta_tau_g[0][rv] * self.b1 + (
                        1 - self.b1) * g
                    self.eta_tau_g[1][rv] = self.eta_tau_g[1][rv] * self.b2 + (
                        1 - self.b2) * g * g

            # update parameters
            self.w_tau = self.w_tau - (self.alpha * (self.w_tau_g[0] / (1 - (self.b1 ** self.t)))) \
                         / (np.sqrt(self.w_tau_g[1] / (1 - (self.b2 ** self.t))) + self.eps)
            self.w = self.softmax(self.w_tau)
            for rv in self.g.rvs:
                if rv.value is not None:
                    continue
                elif rv.domain.continuous:
                    table = self.eta[rv] - (self.alpha * (self.eta_g[0][rv] / (1 - (self.b1 ** self.t)))) \
                            / (np.sqrt(self.eta_g[1][rv] / (1 - (self.b2 ** self.t))) + self.eps)
                    table[:, 1] = np.clip(table[:, 1],
                                          a_min=self.var_threshold,
                                          a_max=np.inf)
                    self.eta[rv] = table
                else:
                    table = self.eta_tau[rv] - (self.alpha * (self.eta_tau_g[0][rv] / (1 - (self.b1 ** self.t)))) \
                            / (np.sqrt(self.eta_tau_g[1][rv] / (1 - (self.b2 ** self.t))) + self.eps)
                    self.eta_tau[rv] = table
                    self.eta[rv] = self.softmax(table, 1)

            if self.is_log:
                current_time = time.process_time()
                self.total_time += current_time - start_time
                if self.log_fe:
                    fe = self.free_energy()
                else:
                    map_res = dict()
                    for rv in self.g.g.rvs:
                        map_res[rv] = self.map(rv)
                    fe = log_likelihood(self.g.g, map_res)
                print(fe, self.total_time)
                self.time_log.append([self.total_time, fe])
Beispiel #5
0
    def get_log_likelihood(self, X, y):
        '''
		Calculates the log-likelihood of the logistic regression model over the dataset.

		- Inputs:
			- X: An ndarray of regressor variable values; i.e., features; i.e., inputs.
			- y: An ndarray of dependent variable values; i.e., tragets; i.e., labels; i.e., outputs.
		
		- Returns:
			- LL = \sum_{i=1}^n y_i w^{\top} X_i - \text{log} (1 + e^{w^{\top} X_i }).
		'''

        return log_likelihood(X, y, self.get_params())
Beispiel #6
0
    def score(self, Y, Yhat, Ynull=None, method='circ_corr'):
        """Score the model.
        Parameters
        ----------
        Y : array, shape (n_samples, [n_neurons])
            The true firing rates.
        Yhat : array, shape (n_samples, [n_neurons])
            The estimated firing rates.
        Ynull : None | array, shape (n_samples, [n_classes])
            The labels for the null model. Must be None if method is not
            'pseudo_R2'
        method : str
            One of 'pseudo_R2' or 'circ_corr' or 'cosine_dist'
        """

        if method == 'pseudo_R2':
            if(len(Y.shape) > 1):
                # There are many neurons, so calculate and return the score for
                # each neuron
                score = list()
                for neuron in range(Y.shape[1]):
                    L1 = log_likelihood(Y[:, neuron], Yhat[:, neuron])
                    LS = log_likelihood(Y[:, neuron], Y[:, neuron])
                    L0 = log_likelihood(Y[:, neuron], Ynull[neuron])
                    score.append(1 - (LS - L1) / (LS - L0))
            else:
                L1 = log_likelihood(Y, Yhat)
                LS = log_likelihood(Y, Y)
                L0 = log_likelihood(Y, Ynull)
                score = 1 - (LS - L1) / (LS - L0)

        elif method == 'circ_corr':
            score = circ_corr(np.squeeze(Y), np.squeeze(Yhat))

        elif method == 'cosine_dist':
            score = np.mean(np.cos(np.squeeze(Y) - np.squeeze(Yhat)))

        return score
data = load_data('Demo/Data/HMLN/0')
# for key, rv in rvs_dict.items():
#     if key not in data and key not in query and not rv.domain.continuous:
#         data[key] = 0  # closed world assumption

# data = dict()

g, rvs_dict = rel_g.add_evidence(data)
print(len(rvs_dict))
print(len(g.factors))

# infer = HybridLBP(g, n=10, proposal_approximation='simple')
# infer.run(10, c2f=-1, log_enable=False)

# infer = HMWS(g)
# infer.run(max_tries=1, max_flips=10000, epsilon=0.0, noise_std=0.5)

infer = C2FVI(g, num_mixtures=2, num_quadrature_points=3)
infer.run(100, lr=0.2)

# map_res = infer.rvs_map(rvs_dict.values())
# for key, rv in rvs_dict.items():
#     print(key, map_res[rv])

map_res = dict()
for key, rv in rvs_dict.items():
    map_res[rv] = infer.map(rv)
    print(key, map_res[rv])

print(log_likelihood(g, map_res))
Beispiel #8
0
            den += gamma[j, t]
        mu[:, j] = (np.transpose(weighted_samples) / den).reshape(dim)

        # Sigma
        weighted_diff = np.zeros([dim, dim])
        for t in range(time_steps):
            gamma_ = gamma[j, t]
            diff = data_train[t, :].reshape([-1, 1]) - mu[:, j].reshape(
                [-1, 1])  # dim x 1
            diff_t = np.transpose(diff)
            weighted_diff += gamma_ * diff.dot(diff_t)

        sigma_list[j] = weighted_diff / den

    # Log-likelihood training data
    llik_new = utils.log_likelihood(n_states, time_steps, gamma, csi,
                                    data_train, A, pi, mu, sigma_list)
    llik_list.append(llik_new / time_steps)

    # Test data

    # E-step:

    # Alpha
    alpha_log = utils.alpha_log(data_test, pi, mu, sigma_list, A, n_states)

    # Beta
    beta_log = utils.beta_log(data_test, pi, mu, sigma_list, A, n_states)

    # Gamma
    gamma_log = utils.gamma_log(alpha_log, beta_log)
Beispiel #9
0
    def run(self,
            max_tries=100,
            max_flips=1000,
            epsilon=0.9,
            noise_std=1,
            is_log=True):
        if is_log:
            self.time_log = list()
            total_time = 0

        numeric_factors, discrete_factors = self.discrete_and_numeric_factors()
        numeric_factors = self.prune_factors_without_latent_variables(
            numeric_factors)
        discrete_factors = self.prune_factors_without_latent_variables(
            discrete_factors)

        self.best_assignment = None
        self.best_score = -np.Inf

        for i_try in range(max_tries):
            assignment = self.random_assignment()

            for i_flip in range(max_flips):
                start_time = time.process_time()

                score = self.score(assignment)
                if score > self.best_score:
                    self.best_assignment = assignment.copy()
                    self.best_score = score
                    # print(self.best_score, i_try, i_flip)

                unsatisfied_hard, unsatisfied_soft = self.unsatisfied_factors(
                    assignment, discrete_factors)
                if len(unsatisfied_hard) > 0:
                    c = np.random.choice(unsatisfied_hard)
                else:
                    c = self.random_factor(unsatisfied_soft, numeric_factors)

                if np.random.rand() < epsilon:
                    rv = np.random.choice(
                        list(filter(lambda rv_: rv_.value is None, c.nb)))

                    if rv.value is not None:
                        continue
                    if rv.domain.continuous:
                        assignment[rv] = self.argmax_rv_wrt_factor(c, rv, assignment) + \
                                         np.random.normal(scale=noise_std)
                    else:
                        assignment[rv] = 1 - assignment[rv]
                else:
                    rv_score_dict = dict()
                    rv_assignment_dict = dict()
                    for rv in c.nb:
                        if rv.value is not None:
                            continue

                        temp_assignment = assignment.copy()

                        if rv.domain.continuous:
                            rv_assignment_dict[rv] = self.argmax_rvs_wrt_score(
                                [rv], assignment)[rv]
                        else:
                            rv_assignment_dict[
                                rv] = self.argmax_discrete_rv_wrt_score(
                                    rv, assignment)
                        temp_assignment[rv] = rv_assignment_dict[rv]
                        rv_score_dict[rv] = self.local_score(
                            c.nb, temp_assignment)

                    if len(rv_score_dict) == 0:
                        continue

                    rv = max(rv_score_dict.keys(),
                             key=lambda k: rv_score_dict[k])

                    if rv_score_dict[rv] > self.local_score(
                            c.nb, assignment) or c in discrete_factors:
                        assignment[rv] = rv_assignment_dict[rv]
                    else:
                        assignment = self.argmax_numeric_term_wrt_score(
                            c, assignment)

                if is_log:
                    current_time = time.process_time()
                    total_time += current_time - start_time
                    if self.score(assignment) > self.best_score:
                        map_res = dict()
                        for rv in self.g.rvs:
                            map_res[rv] = assignment[rv]
                        ll = log_likelihood(self.g, map_res)
                        print(ll, total_time, i_flip)
                        self.time_log.append([total_time, ll])
Beispiel #10
0
def main(FLAGS):

    if not os.path.exists(FLAGS.experiment_rootdir_comp_adf):
        os.makedirs(FLAGS.experiment_rootdir_comp_adf)

    # Train only if cuda is available
    if device.type == 'cuda':
        # Create the experiment rootdir adf if not already there
        if not os.path.exists(FLAGS.experiment_rootdir_adf):
            os.makedirs(FLAGS.experiment_rootdir_adf)
        # Hyperparameters
        batch_size = FLAGS.batch_size  # Default 32

        # Loading testing dataset
        test_steer_dataset = create_dataset(FLAGS.test_dir)
        test_loader = torch.utils.data.DataLoader(dataset=test_steer_dataset,
                                                  batch_size=batch_size,
                                                  shuffle=False)

        targets = []
        for image, target in test_steer_dataset:
            targets.append(np.asscalar(target.cpu().numpy()))

        # Cropped image dimensions
        crop_img_width, crop_img_height = FLAGS.crop_img_width, FLAGS.crop_img_height
        # Image mode
        if FLAGS.img_mode == 'rgb':
            img_channels = 3
        elif FLAGS.img_mode == 'grayscale':
            img_channels = 1
        else:
            raise IOError("Unidentified image mode: use 'grayscale' or 'rgb'")

        # Output dimension
        output_dim = 1
        # Load standard model
        model = resnet8_MCDO(img_channels, crop_img_height, crop_img_width,
                             output_dim).to(device)
        model_ckpt = os.path.join(FLAGS.experiment_rootdir, 'resnet8_MCDO.pt')
        model.load_state_dict(torch.load(model_ckpt))
        # Load heteroscedastic model
        model_het = resnet8_MCDO_ale(img_channels, crop_img_height,
                                     crop_img_width, output_dim).to(device)
        model_het_ckpt = os.path.join(FLAGS.experiment_rootdir,
                                      'resnet8_MCDO_ale.pt')
        model_het.load_state_dict(torch.load(model_het_ckpt))

        model_adf = Resnet8_MCDO_adf(img_channels, output_dim, FLAGS.noise_var,
                                     FLAGS.min_var).to(device)

        model_adf.load_state_dict(torch.load(model_ckpt))

        # Compute epistemic variance
        FLAGS.is_MCDO = True
        print("Computing epistemic variances")
        # Get predictions and ground truth
        _, pred_steerings_mean_MCDO, real_steerings, epistemic_variances = \
        utils.compute_predictions_and_gt(model, test_loader, device, FLAGS)

        # Compute total variance
        print("Computing total variances with heteroscedastic")
        # Get predictions and ground truth
        _, pred_steerings_mean_het, aleatoric_variances, real_steerings, total_variances = \
        utils.compute_predictions_and_gt_het(model_het, test_loader, device, FLAGS)

        # Compute total variance
        print("Computing total variances with ADF")
        # Get predictions and ground truth
        _, pred_steerings_mean_adf_MCDO, aleatoric_variances_adf, real_steerings, total_variances_adf = \
        utils.compute_predictions_and_gt_adf(model_adf, test_loader, device, FLAGS)

        # Compute log-likelihoods

        ll_epi = utils.log_likelihood(pred_steerings_mean_MCDO, targets,
                                      np.sqrt(epistemic_variances))
        ll_ale_het = utils.log_likelihood(pred_steerings_mean_het, targets,
                                          np.sqrt(aleatoric_variances))
        ll_tot_het = utils.log_likelihood(pred_steerings_mean_het, targets,
                                          np.sqrt(total_variances))
        ll_ale_adf = utils.log_likelihood(pred_steerings_mean_adf_MCDO,
                                          targets,
                                          np.sqrt(aleatoric_variances_adf))
        ll_tot_adf = utils.log_likelihood(pred_steerings_mean_adf_MCDO,
                                          targets,
                                          np.sqrt(total_variances_adf))

        print(
            "Log-likelihood considering         EPISTEMIC uncertainty is: {}".
            format(ll_epi))
        print(
            "Log-likelihood considering     ALEATORIC_het uncertainty is: {}".
            format(ll_ale_het))
        print(
            "Log-likelihood considering         TOTAL_het uncertainty is: {}".
            format(ll_tot_het))
        print(
            "Log-likelihood considering     ALEATORIC_adf uncertainty is: {}\n"
            .format(ll_ale_adf))
        print(
            "Log-likelihood considering         TOTAL_adf uncertainty is: {}\n"
            .format(ll_tot_adf))

    else:
        raise IOError('Cuda is not available.')
Beispiel #11
0
def main(args):

  np.random.seed(1982)

  ts = np.linspace(0., 1200, 2001)

  heat_on = np.mod((ts / 200).astype('int'), 2) != 1
  heat_on[ts >= 800.] = False
  heat_on = heat_on.astype('bool')

  heat_on[np.logical_and(ts >= 800,
                         ts <= 820)] = True

  # heat_on = np.ones(ts.size)

  p = pot_of_water(0.10, 0.10)

  print "==============TRUE PARAMETERS====================="
  print utils.pretty_params(p)
  print "=================================================="

  sig = 0.1

  s = np.array(list(utils.compute_heat_source(ts, heat_on, p['k'], p['init_time_off'])))
  u = np.array(list(utils.compute_temperature(ts, heat_on, **p)))
  # Add observation noise
  obs = u + sig * np.random.normal(size=u.size)
  
  fig, ax = plt.subplots(1, 2)
  ax[0].plot(ts, u, color='black')
  ax[0].plot(ts, obs, alpha=0.5, color='green')
  ax[1].plot(ts, s, color='red')
  plt.show()
  
  fig, ax = plt.subplots(1, 2)
  n = 1
  best = -np.inf
  best_params = None
  for i in range(n):

    x0 = {'k': 0.1,
          'c_v': 3000.,
          'volume': 1.,
          'u_0': 300., 
          'h': 3.,
          'init_time_off': 100.}

    fixed = {'wattage': 400,
             'u_env': 298.}

    optimal = utils.optimal_parameters(ts, heat_on, obs, params=x0, sig=sig, **fixed)

    optimal.update(fixed)

    print utils.pretty_params(optimal)

    ll = utils.log_likelihood(ts, heat_on, obs, sig=sig, **optimal)
    print "optimal_likelihood", ll
    if ll > best:
      best = ll
      best_params = optimal

    u_samp = np.array(list(utils.compute_temperature(ts, heat_on, **optimal)))

    ax[0].plot(ts, u, color='black')
    ax[0].plot(ts, obs, alpha=0.5, color='green')
    ax[0].plot(ts, u_samp, color='steelblue')

  plt.show()

  np.random.seed(1982)
  sample = utils.sample_parameters(ts, heat_on, obs, n=100, params=optimal, iterations=10)

  fig, ax = plt.subplots(1, 2)

  ax[0].plot(ts, u, color='black')
  a0_ylim = ax[0].get_ylim()

  ax[1].plot(ts, s, color='black')
  a1_ylim = ax[1].get_ylim()

  for k, u_0, u_max, h, r in sample[['k', 'u_0', 'u_max', 'h', 'r']].values:
    s_samp = np.array(list(utils.compute_heat_source(ts, heat_on, k, u_0, u_max, u_env=25.)))
    u_samp = np.array(list(utils.compute_temperature(ts, s_samp, h, r, u_0, u_env=25.)))
    ax[0].plot(ts, u_samp, alpha=0.5, color='steelblue')
    ax[1].plot(ts, s_samp, alpha=0.5, color='steelblue')

  ax[0].set_ylim(a0_ylim)
  ax[1].set_ylim(a1_ylim)

  plt.show()
Beispiel #12
0
    # Input stuff

    if '-infile' in sys.argv:
        p = sys.argv.index('-infile')
        data_file = str(sys.argv[p + 1])

    data = pd.read_csv(f'./curves/{data_file}')

    time, flux, err = data['time'].to_numpy(), data['flux'].to_numpy(
    ), data['flux_error'].to_numpy()

    # Perform a likelihood fit

    guesses = [0.001, 0.019, 30]

    nll = lambda *args: -log_likelihood(*args)

    soln = minimize(nll, guesses, method="Powell", args=(time, flux, err))

    print(soln)

    # MCMC

    labels = ['$t_0$', '$R_P/R_*$', '$a/R_*$']

    pos = soln.x + 1e-4 * np.random.randn(32, 3)

    nwalkers, ndim = pos.shape

    sampler = emcee.EnsembleSampler(nwalkers,
                                    ndim,