def fit(self,X):
n_objects = X.shape[0]
n_features = X.shape[1]
sigma = np.zeros((self.n_clusters, n_features, n_features))
w = np.tile(1.0/self.n_clusters, self.n_clusters)
centers_idx = np.random.choice(n_objects, size=self.n_clusters, replace=False)
mu = X[centers_idx, :]
for cluster in range (self.n_clusters):
sigma[cluster :, :] = np.eye(n_features)
ll = log_likelihood(X, w, mu, sigma)
for i in range(self.max_iter):
ll_new = log_likelihood(X, w, mu, sigma)
self.save_logs(compute_labels(X, mu), w, mu, sigma, ll)
if i > 0 and abs(ll_new - ll) < self.tol:
self.cluster_centers_ = mu.copy()
self.labels_ = compute_labels(X, mu)
self.covars_ = sigma.copy()
self.w_ = w.copy()
break
else:
gamma = self.estep(X,w, mu, sigma)
w, mu, sigma = self.mstep(X,gamma)
ll = ll_new
i+=1
if i == self.max_iter:
self.convergence = -1
def fit(self, X):
n_objects, n_features = X.shape
self.covars_ = np.zeros((self.n_clusters, n_features, n_features))
self.w_ = np.tile(1.0 / self.n_clusters, self.n_clusters)
centers_idx = np.random.choice(n_objects, size = self.n_clusters, replace = False)
self.cluster_centers_ = X[centers_idx, :]
for cluster in range(self.n_clusters):
self.covars_[cluster :, :] = np.eye(n_features)
self.ll = log_likelihood(X, self.w_, self.cluster_centers_, self.covars_)
for i in range(self.max_iter):
if self.logging:
self.logs['log_likelihood'].append(log_likelihood(X, self.w_, self.cluster_centers_, self.covars_))
self.logs['labels'].append(compute_labels(X, self.cluster_centers_))
self.logs['w'].append(self.w_)
self.logs['mu'].append(self.cluster_centers_)
self.logs['sigma'].append(self.covars_)
ll_new = log_likelihood(X, self.w_, self.cluster_centers_, self.covars_)
if i > 0 and abs(ll_new - self.ll) < self.tol:
break
else:
g = self.e_step(X)
self.m_step(X, g)
self.ll = ll_new
self.labels_ = compute_labels(X, self.cluster_centers_)
def decode(ciphertext, output_file_name):
run_likelihoods, steps, acceptances = [], [], []
f = utils.find_good_start(ciphertext, optimized=True)
likelihood = utils.log_likelihood(ciphertext, f)
max_likelihood = likelihood
i=0
no_recent_acceptance = False
acceptance_indices = []
while no_recent_acceptance == False:
run_likelihoods.append(likelihood)
steps.append(i)
i += 1
# construct proposal
prop_f = utils.random_transposition(f)
prop_likelihood = utils.log_likelihood(ciphertext, prop_f)
# make sure proposal even makes sense, i.e. that it has positive
# likelihood
if prop_likelihood:
x = prop_likelihood - likelihood
# proposal gets accepted automatically
if x >=0:
f = prop_f
max_likelihood = prop_likelihood
likelihood = prop_likelihood
acceptances.append(True)
acceptance_indices.append(i)
print max_likelihood
# thresholding to avoid underflow
elif x<=0 and x >= math.log(0.00001):
a = math.exp(x)
u = np.random.random_sample()
if u <= a:
f = prop_f
likelihood = prop_likelihood
print max_likelihood
acceptances.append(True)
acceptance_indices.append(i)
else:
acceptances.append(False)
print "Rejected: {}".format(i)
else:
acceptances.append(False)
print "Rejected: {}".format(i)
else:
acceptances.append(False)
print "Rejected: {}".format(i)
ten_percent = max(int(i/10),1000)
if len(acceptance_indices) > 0:
if acceptance_indices[-1] + ten_percent < i:
no_recent_acceptance = True
T = 100
acceptance_rate = [np.mean(acceptances[max(0, t-T):t]) for t in
xrange(len(acceptances))]
res = utils.decipher(f, ciphertext)
fo = open(output_file_name, 'w')
fo.write(res)
fo.close()
return f
def ADAM_update(self, iteration):
for itr in range(iteration):
start_time = time.process_time()
self.t += 1
# compute gradient
g = self.gradient_w_tau()
self.w_tau_g[0] = self.w_tau_g[0] * self.b1 + (1 - self.b1) * g
self.w_tau_g[1] = self.w_tau_g[1] * self.b2 + (1 - self.b2) * g * g
for rv in self.g.rvs:
if rv.value is not None:
continue
elif rv.domain.continuous:
g = self.gradient_mu_var(rv)
self.eta_g[0][rv] = self.eta_g[0][rv] * self.b1 + (
1 - self.b1) * g
self.eta_g[1][rv] = self.eta_g[1][rv] * self.b2 + (
1 - self.b2) * g * g
else:
g = self.gradient_category_tau(rv)
self.eta_tau_g[0][rv] = self.eta_tau_g[0][rv] * self.b1 + (
1 - self.b1) * g
self.eta_tau_g[1][rv] = self.eta_tau_g[1][rv] * self.b2 + (
1 - self.b2) * g * g
# update parameters
self.w_tau = self.w_tau - (self.alpha * (self.w_tau_g[0] / (1 - (self.b1 ** self.t)))) \
/ (np.sqrt(self.w_tau_g[1] / (1 - (self.b2 ** self.t))) + self.eps)
self.w = self.softmax(self.w_tau)
for rv in self.g.rvs:
if rv.value is not None:
continue
elif rv.domain.continuous:
table = self.eta[rv] - (self.alpha * (self.eta_g[0][rv] / (1 - (self.b1 ** self.t)))) \
/ (np.sqrt(self.eta_g[1][rv] / (1 - (self.b2 ** self.t))) + self.eps)
table[:, 1] = np.clip(table[:, 1],
a_min=self.var_threshold,
a_max=np.inf)
self.eta[rv] = table
else:
table = self.eta_tau[rv] - (self.alpha * (self.eta_tau_g[0][rv] / (1 - (self.b1 ** self.t)))) \
/ (np.sqrt(self.eta_tau_g[1][rv] / (1 - (self.b2 ** self.t))) + self.eps)
self.eta_tau[rv] = table
self.eta[rv] = self.softmax(table, 1)
if self.is_log:
current_time = time.process_time()
self.total_time += current_time - start_time
if self.log_fe:
fe = self.free_energy()
else:
map_res = dict()
for rv in self.g.g.rvs:
map_res[rv] = self.map(rv)
fe = log_likelihood(self.g.g, map_res)
print(fe, self.total_time)
self.time_log.append([self.total_time, fe])
def get_log_likelihood(self, X, y):
'''
Calculates the log-likelihood of the logistic regression model over the dataset.
- Inputs:
- X: An ndarray of regressor variable values; i.e., features; i.e., inputs.
- y: An ndarray of dependent variable values; i.e., tragets; i.e., labels; i.e., outputs.
- Returns:
- LL = \sum_{i=1}^n y_i w^{\top} X_i - \text{log} (1 + e^{w^{\top} X_i }).
'''
return log_likelihood(X, y, self.get_params())
def score(self, Y, Yhat, Ynull=None, method='circ_corr'):
"""Score the model.
Parameters
----------
Y : array, shape (n_samples, [n_neurons])
The true firing rates.
Yhat : array, shape (n_samples, [n_neurons])
The estimated firing rates.
Ynull : None | array, shape (n_samples, [n_classes])
The labels for the null model. Must be None if method is not
'pseudo_R2'
method : str
One of 'pseudo_R2' or 'circ_corr' or 'cosine_dist'
"""
if method == 'pseudo_R2':
if(len(Y.shape) > 1):
# There are many neurons, so calculate and return the score for
# each neuron
score = list()
for neuron in range(Y.shape[1]):
L1 = log_likelihood(Y[:, neuron], Yhat[:, neuron])
LS = log_likelihood(Y[:, neuron], Y[:, neuron])
L0 = log_likelihood(Y[:, neuron], Ynull[neuron])
score.append(1 - (LS - L1) / (LS - L0))
else:
L1 = log_likelihood(Y, Yhat)
LS = log_likelihood(Y, Y)
L0 = log_likelihood(Y, Ynull)
score = 1 - (LS - L1) / (LS - L0)
elif method == 'circ_corr':
score = circ_corr(np.squeeze(Y), np.squeeze(Yhat))
elif method == 'cosine_dist':
score = np.mean(np.cos(np.squeeze(Y) - np.squeeze(Yhat)))
return score
data = load_data('Demo/Data/HMLN/0')
# for key, rv in rvs_dict.items():
# if key not in data and key not in query and not rv.domain.continuous:
# data[key] = 0 # closed world assumption
# data = dict()
g, rvs_dict = rel_g.add_evidence(data)
print(len(rvs_dict))
print(len(g.factors))
# infer = HybridLBP(g, n=10, proposal_approximation='simple')
# infer.run(10, c2f=-1, log_enable=False)
# infer = HMWS(g)
# infer.run(max_tries=1, max_flips=10000, epsilon=0.0, noise_std=0.5)
infer = C2FVI(g, num_mixtures=2, num_quadrature_points=3)
infer.run(100, lr=0.2)
# map_res = infer.rvs_map(rvs_dict.values())
# for key, rv in rvs_dict.items():
# print(key, map_res[rv])
map_res = dict()
for key, rv in rvs_dict.items():
map_res[rv] = infer.map(rv)
print(key, map_res[rv])
print(log_likelihood(g, map_res))
den += gamma[j, t]
mu[:, j] = (np.transpose(weighted_samples) / den).reshape(dim)
# Sigma
weighted_diff = np.zeros([dim, dim])
for t in range(time_steps):
gamma_ = gamma[j, t]
diff = data_train[t, :].reshape([-1, 1]) - mu[:, j].reshape(
[-1, 1]) # dim x 1
diff_t = np.transpose(diff)
weighted_diff += gamma_ * diff.dot(diff_t)
sigma_list[j] = weighted_diff / den
# Log-likelihood training data
llik_new = utils.log_likelihood(n_states, time_steps, gamma, csi,
data_train, A, pi, mu, sigma_list)
llik_list.append(llik_new / time_steps)
# Test data
# E-step:
# Alpha
alpha_log = utils.alpha_log(data_test, pi, mu, sigma_list, A, n_states)
# Beta
beta_log = utils.beta_log(data_test, pi, mu, sigma_list, A, n_states)
# Gamma
gamma_log = utils.gamma_log(alpha_log, beta_log)
def run(self,
max_tries=100,
max_flips=1000,
epsilon=0.9,
noise_std=1,
is_log=True):
if is_log:
self.time_log = list()
total_time = 0
numeric_factors, discrete_factors = self.discrete_and_numeric_factors()
numeric_factors = self.prune_factors_without_latent_variables(
numeric_factors)
discrete_factors = self.prune_factors_without_latent_variables(
discrete_factors)
self.best_assignment = None
self.best_score = -np.Inf
for i_try in range(max_tries):
assignment = self.random_assignment()
for i_flip in range(max_flips):
start_time = time.process_time()
score = self.score(assignment)
if score > self.best_score:
self.best_assignment = assignment.copy()
self.best_score = score
# print(self.best_score, i_try, i_flip)
unsatisfied_hard, unsatisfied_soft = self.unsatisfied_factors(
assignment, discrete_factors)
if len(unsatisfied_hard) > 0:
c = np.random.choice(unsatisfied_hard)
else:
c = self.random_factor(unsatisfied_soft, numeric_factors)
if np.random.rand() < epsilon:
rv = np.random.choice(
list(filter(lambda rv_: rv_.value is None, c.nb)))
if rv.value is not None:
continue
if rv.domain.continuous:
assignment[rv] = self.argmax_rv_wrt_factor(c, rv, assignment) + \
np.random.normal(scale=noise_std)
else:
assignment[rv] = 1 - assignment[rv]
else:
rv_score_dict = dict()
rv_assignment_dict = dict()
for rv in c.nb:
if rv.value is not None:
continue
temp_assignment = assignment.copy()
if rv.domain.continuous:
rv_assignment_dict[rv] = self.argmax_rvs_wrt_score(
[rv], assignment)[rv]
else:
rv_assignment_dict[
rv] = self.argmax_discrete_rv_wrt_score(
rv, assignment)
temp_assignment[rv] = rv_assignment_dict[rv]
rv_score_dict[rv] = self.local_score(
c.nb, temp_assignment)
if len(rv_score_dict) == 0:
continue
rv = max(rv_score_dict.keys(),
key=lambda k: rv_score_dict[k])
if rv_score_dict[rv] > self.local_score(
c.nb, assignment) or c in discrete_factors:
assignment[rv] = rv_assignment_dict[rv]
else:
assignment = self.argmax_numeric_term_wrt_score(
c, assignment)
if is_log:
current_time = time.process_time()
total_time += current_time - start_time
if self.score(assignment) > self.best_score:
map_res = dict()
for rv in self.g.rvs:
map_res[rv] = assignment[rv]
ll = log_likelihood(self.g, map_res)
print(ll, total_time, i_flip)
self.time_log.append([total_time, ll])
def main(FLAGS):
if not os.path.exists(FLAGS.experiment_rootdir_comp_adf):
os.makedirs(FLAGS.experiment_rootdir_comp_adf)
# Train only if cuda is available
if device.type == 'cuda':
# Create the experiment rootdir adf if not already there
if not os.path.exists(FLAGS.experiment_rootdir_adf):
os.makedirs(FLAGS.experiment_rootdir_adf)
# Hyperparameters
batch_size = FLAGS.batch_size # Default 32
# Loading testing dataset
test_steer_dataset = create_dataset(FLAGS.test_dir)
test_loader = torch.utils.data.DataLoader(dataset=test_steer_dataset,
batch_size=batch_size,
shuffle=False)
targets = []
for image, target in test_steer_dataset:
targets.append(np.asscalar(target.cpu().numpy()))
# Cropped image dimensions
crop_img_width, crop_img_height = FLAGS.crop_img_width, FLAGS.crop_img_height
# Image mode
if FLAGS.img_mode == 'rgb':
img_channels = 3
elif FLAGS.img_mode == 'grayscale':
img_channels = 1
else:
raise IOError("Unidentified image mode: use 'grayscale' or 'rgb'")
# Output dimension
output_dim = 1
# Load standard model
model = resnet8_MCDO(img_channels, crop_img_height, crop_img_width,
output_dim).to(device)
model_ckpt = os.path.join(FLAGS.experiment_rootdir, 'resnet8_MCDO.pt')
model.load_state_dict(torch.load(model_ckpt))
# Load heteroscedastic model
model_het = resnet8_MCDO_ale(img_channels, crop_img_height,
crop_img_width, output_dim).to(device)
model_het_ckpt = os.path.join(FLAGS.experiment_rootdir,
'resnet8_MCDO_ale.pt')
model_het.load_state_dict(torch.load(model_het_ckpt))
model_adf = Resnet8_MCDO_adf(img_channels, output_dim, FLAGS.noise_var,
FLAGS.min_var).to(device)
model_adf.load_state_dict(torch.load(model_ckpt))
# Compute epistemic variance
FLAGS.is_MCDO = True
print("Computing epistemic variances")
# Get predictions and ground truth
_, pred_steerings_mean_MCDO, real_steerings, epistemic_variances = \
utils.compute_predictions_and_gt(model, test_loader, device, FLAGS)
# Compute total variance
print("Computing total variances with heteroscedastic")
# Get predictions and ground truth
_, pred_steerings_mean_het, aleatoric_variances, real_steerings, total_variances = \
utils.compute_predictions_and_gt_het(model_het, test_loader, device, FLAGS)
# Compute total variance
print("Computing total variances with ADF")
# Get predictions and ground truth
_, pred_steerings_mean_adf_MCDO, aleatoric_variances_adf, real_steerings, total_variances_adf = \
utils.compute_predictions_and_gt_adf(model_adf, test_loader, device, FLAGS)
# Compute log-likelihoods
ll_epi = utils.log_likelihood(pred_steerings_mean_MCDO, targets,
np.sqrt(epistemic_variances))
ll_ale_het = utils.log_likelihood(pred_steerings_mean_het, targets,
np.sqrt(aleatoric_variances))
ll_tot_het = utils.log_likelihood(pred_steerings_mean_het, targets,
np.sqrt(total_variances))
ll_ale_adf = utils.log_likelihood(pred_steerings_mean_adf_MCDO,
targets,
np.sqrt(aleatoric_variances_adf))
ll_tot_adf = utils.log_likelihood(pred_steerings_mean_adf_MCDO,
targets,
np.sqrt(total_variances_adf))
print(
"Log-likelihood considering EPISTEMIC uncertainty is: {}".
format(ll_epi))
print(
"Log-likelihood considering ALEATORIC_het uncertainty is: {}".
format(ll_ale_het))
print(
"Log-likelihood considering TOTAL_het uncertainty is: {}".
format(ll_tot_het))
print(
"Log-likelihood considering ALEATORIC_adf uncertainty is: {}\n"
.format(ll_ale_adf))
print(
"Log-likelihood considering TOTAL_adf uncertainty is: {}\n"
.format(ll_tot_adf))
else:
raise IOError('Cuda is not available.')
def main(args):
np.random.seed(1982)
ts = np.linspace(0., 1200, 2001)
heat_on = np.mod((ts / 200).astype('int'), 2) != 1
heat_on[ts >= 800.] = False
heat_on = heat_on.astype('bool')
heat_on[np.logical_and(ts >= 800,
ts <= 820)] = True
# heat_on = np.ones(ts.size)
p = pot_of_water(0.10, 0.10)
print "==============TRUE PARAMETERS====================="
print utils.pretty_params(p)
print "=================================================="
sig = 0.1
s = np.array(list(utils.compute_heat_source(ts, heat_on, p['k'], p['init_time_off'])))
u = np.array(list(utils.compute_temperature(ts, heat_on, **p)))
# Add observation noise
obs = u + sig * np.random.normal(size=u.size)
fig, ax = plt.subplots(1, 2)
ax[0].plot(ts, u, color='black')
ax[0].plot(ts, obs, alpha=0.5, color='green')
ax[1].plot(ts, s, color='red')
plt.show()
fig, ax = plt.subplots(1, 2)
n = 1
best = -np.inf
best_params = None
for i in range(n):
x0 = {'k': 0.1,
'c_v': 3000.,
'volume': 1.,
'u_0': 300.,
'h': 3.,
'init_time_off': 100.}
fixed = {'wattage': 400,
'u_env': 298.}
optimal = utils.optimal_parameters(ts, heat_on, obs, params=x0, sig=sig, **fixed)
optimal.update(fixed)
print utils.pretty_params(optimal)
ll = utils.log_likelihood(ts, heat_on, obs, sig=sig, **optimal)
print "optimal_likelihood", ll
if ll > best:
best = ll
best_params = optimal
u_samp = np.array(list(utils.compute_temperature(ts, heat_on, **optimal)))
ax[0].plot(ts, u, color='black')
ax[0].plot(ts, obs, alpha=0.5, color='green')
ax[0].plot(ts, u_samp, color='steelblue')
plt.show()
np.random.seed(1982)
sample = utils.sample_parameters(ts, heat_on, obs, n=100, params=optimal, iterations=10)
fig, ax = plt.subplots(1, 2)
ax[0].plot(ts, u, color='black')
a0_ylim = ax[0].get_ylim()
ax[1].plot(ts, s, color='black')
a1_ylim = ax[1].get_ylim()
for k, u_0, u_max, h, r in sample[['k', 'u_0', 'u_max', 'h', 'r']].values:
s_samp = np.array(list(utils.compute_heat_source(ts, heat_on, k, u_0, u_max, u_env=25.)))
u_samp = np.array(list(utils.compute_temperature(ts, s_samp, h, r, u_0, u_env=25.)))
ax[0].plot(ts, u_samp, alpha=0.5, color='steelblue')
ax[1].plot(ts, s_samp, alpha=0.5, color='steelblue')
ax[0].set_ylim(a0_ylim)
ax[1].set_ylim(a1_ylim)
plt.show()
# Input stuff
if '-infile' in sys.argv:
p = sys.argv.index('-infile')
data_file = str(sys.argv[p + 1])
data = pd.read_csv(f'./curves/{data_file}')
time, flux, err = data['time'].to_numpy(), data['flux'].to_numpy(
), data['flux_error'].to_numpy()
# Perform a likelihood fit
guesses = [0.001, 0.019, 30]
nll = lambda *args: -log_likelihood(*args)
soln = minimize(nll, guesses, method="Powell", args=(time, flux, err))
print(soln)
# MCMC
labels = ['$t_0$', '$R_P/R_*$', '$a/R_*$']
pos = soln.x + 1e-4 * np.random.randn(32, 3)
nwalkers, ndim = pos.shape
sampler = emcee.EnsembleSampler(nwalkers,
ndim,
