def smooth_priors(true_dist_of_total_prob,
true_dist_of_total_bins,
false_dist_of_total_prob,
false_dist_of_total_bins,
kde_mode,
bandwidth="silverman",
kernel="exponential"):
import matplotlib.pyplot as plt
import numpy as np
miss_mass_true = np.diff(true_dist_of_total_bins)[[
0, -1
]] * true_dist_of_total_prob[[0, -1]]
miss_mass_false = np.diff(false_dist_of_total_bins)[[
0, -1
]] * false_dist_of_total_prob[[0, -1]]
import hist_sampler
x_ = hist_sampler.histogram_sampler(true_dist_of_total_prob[1:-1],
true_dist_of_total_bins[1:-1], 400000)
y_ = hist_sampler.histogram_sampler(false_dist_of_total_prob[1:-1],
false_dist_of_total_bins[1:-1], 400000)
import kern_density_est
if kde_mode == "scipy_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
x_, 'g',
np.linspace(true_dist_of_total_bins[1] - 50000,
true_dist_of_total_bins[-2] + 50000, 1000), bandwidth
) # can change the kernel to an expenential kern_density_est.kern_sklearn_expon(x, color, bandwidth, kernel = "exponential", **kwargs)
elif kde_mode == "sklearn_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(
x_,
'g',
np.linspace(true_dist_of_total_bins[1] - 50000,
true_dist_of_total_bins[-2] + 50000, 1000),
bandwidth,
kernel_=kernel)
from scipy.stats import kde
import bisect
insert_index_1 = bisect.bisect_left(
true_dist_of_total_prob_smooth - true_dist_of_total_prob[0], 0.)
insert_index_2 = bisect.bisect_left(
true_dist_of_total_prob_smooth[::-1] - true_dist_of_total_prob[-1], 0.)
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = true_dist_of_total_prob_smooth[
insert_index_1 + 1:-1 * insert_index_2 -
1], true_dist_of_total_bins_smooth[insert_index_1 +
1:-1 * insert_index_2 - 1]
false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
y_, 'y',
np.linspace(false_dist_of_total_bins[1] - 200000,
false_dist_of_total_bins[-2] + 200000, 1000), bandwidth)
plt.clf()
insert_index_3 = bisect.bisect_left(
false_dist_of_total_prob_smooth - false_dist_of_total_prob[0], 0.)
insert_index_4 = bisect.bisect_left(
false_dist_of_total_prob_smooth[::-1] - false_dist_of_total_prob[-1],
0.)
false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = false_dist_of_total_prob_smooth[
insert_index_3 + 1:-1 * insert_index_4 -
1], false_dist_of_total_bins_smooth[insert_index_3 +
1:-1 * insert_index_4 - 1]
prob_true_ = true_dist_of_total_prob_smooth * (1 - np.sum(miss_mass_true))
prob_false_ = false_dist_of_total_prob_smooth * (1 -
np.sum(miss_mass_false))
bins_t_plot = np.r_[true_dist_of_total_bins[0],
true_dist_of_total_bins_smooth[0],
true_dist_of_total_bins_smooth,
true_dist_of_total_bins_smooth[-1],
true_dist_of_total_bins[-1]]
prob_t_plot = np.r_[true_dist_of_total_prob[0], true_dist_of_total_prob[0],
prob_true_, true_dist_of_total_prob[-1],
true_dist_of_total_prob[-1]]
bins_f_plot = np.r_[false_dist_of_total_bins[0],
false_dist_of_total_bins_smooth[0],
false_dist_of_total_bins_smooth,
false_dist_of_total_bins_smooth[-1],
false_dist_of_total_bins[-1]]
prob_f_plot = np.r_[false_dist_of_total_prob[0],
false_dist_of_total_prob[0], prob_false_,
false_dist_of_total_prob[-1],
false_dist_of_total_prob[-1]]
plt.plot(bins_t_plot, prob_t_plot, 'g')
plt.plot(bins_f_plot, prob_f_plot, 'y')
plt.fill_between(bins_t_plot,
prob_t_plot,
where=None,
color='g',
alpha=0.2)
plt.fill_between(bins_f_plot,
prob_f_plot,
where=None,
color='y',
alpha=0.2)
plt.axis([
bins_f_plot[1] - 100000, bins_f_plot[-2] + 100000, 0.,
max(prob_t_plot)
])
mean_diff_true = np.diff(true_dist_of_total_prob_smooth) / 2.
true_smoothed_mean_probabilities = true_dist_of_total_prob_smooth[:-1] + mean_diff_true
mean_diff_false = np.diff(false_dist_of_total_prob_smooth) / 2.
false_smoothed_mean_probabilities = false_dist_of_total_prob_smooth[:-1] + mean_diff_false
miss_mass_true_ = np.r_[true_dist_of_total_prob[0] *
abs(true_dist_of_total_bins[0] -
true_dist_of_total_bins_smooth[0])], [
true_dist_of_total_prob[-1] *
abs(true_dist_of_total_bins[-1] -
true_dist_of_total_bins_smooth[-1])
]
miss_mass_false_ = np.r_[false_dist_of_total_prob[0] *
abs(false_dist_of_total_bins[0] -
false_dist_of_total_bins_smooth[0])], [
false_dist_of_total_prob[-1] *
abs(false_dist_of_total_bins[-1] -
false_dist_of_total_bins_smooth[-1])
]
prob_true_norm = true_smoothed_mean_probabilities / np.sum(
true_smoothed_mean_probabilities *
np.diff(true_dist_of_total_bins_smooth)) * (1. -
np.sum(miss_mass_true_))
prob_false_norm = false_smoothed_mean_probabilities / np.sum(
false_smoothed_mean_probabilities *
np.diff(false_dist_of_total_bins_smooth)) * (1. -
np.sum(miss_mass_false_))
prob_true = np.r_[true_dist_of_total_prob[0], prob_true_norm,
true_dist_of_total_prob[-1]]
bins_true = np.r_[true_dist_of_total_bins[0],
true_dist_of_total_bins_smooth,
true_dist_of_total_bins[-1]]
prob_false = np.r_[false_dist_of_total_prob[0], prob_false_norm,
false_dist_of_total_prob[-1]]
bins_false = np.r_[false_dist_of_total_bins[0],
false_dist_of_total_bins_smooth,
false_dist_of_total_bins[-1]]
#plt.bar(bins_true[1:-2], prob_true[1:-1], np.diff(bins_true[1:-1]), alpha=0.15, color="blue")
return prob_true, bins_true, prob_false, bins_false
def smooth_priors(
true_dist_of_total_prob,
true_dist_of_total_bins,
false_dist_of_total_prob,
false_dist_of_total_bins,
kde_mode,
bandwidth="silverman",
kernel="exponential",
):
import matplotlib.pyplot as plt
import numpy as np
miss_mass_true = np.diff(true_dist_of_total_bins)[[-2, -1]] * true_dist_of_total_prob[[-2, -1]]
miss_mass_false = np.diff(false_dist_of_total_bins)[[-1]] * false_dist_of_total_prob[[-1]]
import hist_sampler
x_ = hist_sampler.histogram_sampler(true_dist_of_total_prob[:-2], true_dist_of_total_bins[:-2], 800000)
y_ = hist_sampler.histogram_sampler(false_dist_of_total_prob[:-2], false_dist_of_total_bins[:-2], 100000)
import kern_density_est
if kde_mode == "scipy_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
x_, "g", np.linspace(true_dist_of_total_bins[0], true_dist_of_total_bins[-3] + 50000, 3000), bandwidth
) # can change the kernel to an expenential kern_density_est.kern_sklearn_expon(x, color, bandwidth, kernel = "exponential", **kwargs)
elif kde_mode == "sklearn_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(
x_,
"g",
np.linspace(true_dist_of_total_bins[0], true_dist_of_total_bins[-3] + 50000, 3000),
bandwidth,
kernel_=kernel,
)
from scipy.stats import kde
import bisect
insert_index_1 = bisect.bisect_left(true_dist_of_total_prob_smooth - true_dist_of_total_prob[0], 0.0)
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = (
true_dist_of_total_prob_smooth[0:insert_index_1],
true_dist_of_total_bins_smooth[0:insert_index_1],
)
false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
y_, "y", np.linspace(false_dist_of_total_bins[0], false_dist_of_total_bins[-2], 2000), bandwidth
)
plt.clf()
prob_true_ = true_dist_of_total_prob_smooth * (1 - np.sum(miss_mass_true))
prob_false_ = false_dist_of_total_prob_smooth * (1 - np.sum(miss_mass_false))
mean_diff_true = np.diff(true_dist_of_total_prob_smooth) / 2.0
true_smoothed_mean_probabilities = true_dist_of_total_prob_smooth[:-1] + mean_diff_true
mean_diff_false = np.diff(false_dist_of_total_prob_smooth) / 2.0
false_smoothed_mean_probabilities = false_dist_of_total_prob_smooth[:-1] + mean_diff_false
miss_mass_true_ = [
true_dist_of_total_prob[-1] * abs(true_dist_of_total_bins[-1] - true_dist_of_total_bins_smooth[-1])
]
miss_mass_false_ = [
false_dist_of_total_prob[-1] * abs(false_dist_of_total_bins[-1] - false_dist_of_total_bins_smooth[-1])
]
prob_true_norm = (
true_smoothed_mean_probabilities
/ np.sum(true_smoothed_mean_probabilities * np.diff(true_dist_of_total_bins_smooth))
* (1.0 - np.sum(miss_mass_true_))
)
prob_false_norm = (
false_smoothed_mean_probabilities
/ np.sum(false_smoothed_mean_probabilities * np.diff(false_dist_of_total_bins_smooth))
* (1.0 - np.sum(miss_mass_false_))
)
prob_true = np.r_[prob_true_norm, true_dist_of_total_prob[-1]]
bins_true = np.r_[true_dist_of_total_bins_smooth, true_dist_of_total_bins[-1]]
prob_false = np.r_[prob_false_norm, false_dist_of_total_prob[-1]]
bins_false = np.r_[false_dist_of_total_bins_smooth, false_dist_of_total_bins[-1]]
return prob_true, bins_true, prob_false, bins_false
def smooth_priors(true_dist_of_total_prob, true_dist_of_total_bins, false_dist_of_total_prob, false_dist_of_total_bins, kde_mode, bandwidth = "silverman", kernel = "exponential"): import matplotlib.pyplot as plt import numpy as np miss_mass_true = np.diff(true_dist_of_total_bins)[[0,-1]]*true_dist_of_total_prob[[0,-1]] miss_mass_false = np.diff(false_dist_of_total_bins)[[0,-1]]*false_dist_of_total_prob[[0,-1]] import hist_sampler x_ = hist_sampler.histogram_sampler(true_dist_of_total_prob[1:-1], true_dist_of_total_bins[1:-1], 400000) y_ = hist_sampler.histogram_sampler(false_dist_of_total_prob[1:-1], false_dist_of_total_bins[1:-1], 400000) import kern_density_est if kde_mode == "scipy_kde": true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(x_, 'g', np.linspace(true_dist_of_total_bins[1]-50000, true_dist_of_total_bins[-2]+50000, 1000), bandwidth) # can change the kernel to an expenential kern_density_est.kern_sklearn_expon(x, color, bandwidth, kernel = "exponential", **kwargs) elif kde_mode == "sklearn_kde": true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(x_, 'g', np.linspace(true_dist_of_total_bins[1]-50000, true_dist_of_total_bins[-2]+50000, 1000), bandwidth, kernel_ = kernel) from scipy.stats import kde import bisect insert_index_1 = bisect.bisect_left(true_dist_of_total_prob_smooth-true_dist_of_total_prob[0], 0.) insert_index_2 = bisect.bisect_left(true_dist_of_total_prob_smooth[::-1]-true_dist_of_total_prob[-1], 0.) true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = true_dist_of_total_prob_smooth[insert_index_1+1:-1*insert_index_2-1], true_dist_of_total_bins_smooth[insert_index_1+1:-1*insert_index_2-1] false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(y_, 'y', np.linspace(false_dist_of_total_bins[1]-200000, false_dist_of_total_bins[-2]+200000, 1000), bandwidth) plt.clf() insert_index_3 = bisect.bisect_left(false_dist_of_total_prob_smooth-false_dist_of_total_prob[0], 0.) insert_index_4 = bisect.bisect_left(false_dist_of_total_prob_smooth[::-1]-false_dist_of_total_prob[-1], 0.) false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = false_dist_of_total_prob_smooth[insert_index_3+1:-1*insert_index_4-1], false_dist_of_total_bins_smooth[insert_index_3+1:-1*insert_index_4-1] prob_true_ = true_dist_of_total_prob_smooth*(1-np.sum(miss_mass_true)) prob_false_ = false_dist_of_total_prob_smooth*(1-np.sum(miss_mass_false)) bins_t_plot = np.r_[true_dist_of_total_bins[0], true_dist_of_total_bins_smooth[0], true_dist_of_total_bins_smooth, true_dist_of_total_bins_smooth[-1], true_dist_of_total_bins[-1]] prob_t_plot = np.r_[true_dist_of_total_prob[0], true_dist_of_total_prob[0], prob_true_, true_dist_of_total_prob[-1], true_dist_of_total_prob[-1]] bins_f_plot = np.r_[false_dist_of_total_bins[0], false_dist_of_total_bins_smooth[0], false_dist_of_total_bins_smooth, false_dist_of_total_bins_smooth[-1], false_dist_of_total_bins[-1]] prob_f_plot = np.r_[false_dist_of_total_prob[0], false_dist_of_total_prob[0], prob_false_, false_dist_of_total_prob[-1], false_dist_of_total_prob[-1]] plt.plot(bins_t_plot, prob_t_plot, 'g') plt.plot(bins_f_plot, prob_f_plot, 'y') plt.fill_between(bins_t_plot, prob_t_plot, where=None, color='g', alpha = 0.2) plt.fill_between(bins_f_plot, prob_f_plot, where=None, color='y', alpha = 0.2) plt.axis([bins_f_plot[1]-100000, bins_f_plot[-2]+100000, 0., max(prob_t_plot)]) mean_diff_true = np.diff(true_dist_of_total_prob_smooth)/2. true_smoothed_mean_probabilities = true_dist_of_total_prob_smooth[:-1] + mean_diff_true mean_diff_false = np.diff(false_dist_of_total_prob_smooth)/2. false_smoothed_mean_probabilities = false_dist_of_total_prob_smooth[:-1] + mean_diff_false miss_mass_true_ = np.r_[true_dist_of_total_prob[0]*abs(true_dist_of_total_bins[0] - true_dist_of_total_bins_smooth[0])], [true_dist_of_total_prob[-1]*abs(true_dist_of_total_bins[-1] - true_dist_of_total_bins_smooth[-1])] miss_mass_false_ = np.r_[false_dist_of_total_prob[0]*abs(false_dist_of_total_bins[0] - false_dist_of_total_bins_smooth[0])], [false_dist_of_total_prob[-1]*abs(false_dist_of_total_bins[-1] - false_dist_of_total_bins_smooth[-1])] prob_true_norm = true_smoothed_mean_probabilities/np.sum(true_smoothed_mean_probabilities*np.diff(true_dist_of_total_bins_smooth))*(1.-np.sum(miss_mass_true_)) prob_false_norm = false_smoothed_mean_probabilities/np.sum(false_smoothed_mean_probabilities*np.diff(false_dist_of_total_bins_smooth))*(1.-np.sum(miss_mass_false_)) prob_true = np.r_[true_dist_of_total_prob[0], prob_true_norm, true_dist_of_total_prob[-1]] bins_true = np.r_[true_dist_of_total_bins[0], true_dist_of_total_bins_smooth, true_dist_of_total_bins[-1]] prob_false = np.r_[false_dist_of_total_prob[0], prob_false_norm, false_dist_of_total_prob[-1]] bins_false = np.r_[false_dist_of_total_bins[0], false_dist_of_total_bins_smooth, false_dist_of_total_bins[-1]] #plt.bar(bins_true[1:-2], prob_true[1:-1], np.diff(bins_true[1:-1]), alpha=0.15, color="blue") return prob_true, bins_true, prob_false, bins_false
def smooth_priors(true_dist_of_total_prob,
true_dist_of_total_bins,
false_dist_of_total_prob,
false_dist_of_total_bins,
kde_mode,
bandwidth="silverman",
kernel="exponential"):
import matplotlib.pyplot as plt
import numpy as np
miss_mass_true = np.diff(true_dist_of_total_bins)[[
-2, -1
]] * true_dist_of_total_prob[[-2, -1]]
miss_mass_false = np.diff(false_dist_of_total_bins)[[
-1
]] * false_dist_of_total_prob[[-1]]
import hist_sampler
x_ = hist_sampler.histogram_sampler(true_dist_of_total_prob[:-2],
true_dist_of_total_bins[:-2], 800000)
y_ = hist_sampler.histogram_sampler(false_dist_of_total_prob[:-2],
false_dist_of_total_bins[:-2], 100000)
import kern_density_est
if kde_mode == "scipy_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
x_, 'g',
np.linspace(true_dist_of_total_bins[0],
true_dist_of_total_bins[-3] + 50000, 3000), bandwidth
) # can change the kernel to an expenential kern_density_est.kern_sklearn_expon(x, color, bandwidth, kernel = "exponential", **kwargs)
elif kde_mode == "sklearn_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(
x_,
'g',
np.linspace(true_dist_of_total_bins[0],
true_dist_of_total_bins[-3] + 50000, 3000),
bandwidth,
kernel_=kernel)
from scipy.stats import kde
import bisect
insert_index_1 = bisect.bisect_left(
true_dist_of_total_prob_smooth - true_dist_of_total_prob[0], 0.)
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = true_dist_of_total_prob_smooth[
0:insert_index_1], true_dist_of_total_bins_smooth[0:insert_index_1]
false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
y_, 'y',
np.linspace(false_dist_of_total_bins[0], false_dist_of_total_bins[-2],
2000), bandwidth)
plt.clf()
prob_true_ = true_dist_of_total_prob_smooth * (1 - np.sum(miss_mass_true))
prob_false_ = false_dist_of_total_prob_smooth * (1 -
np.sum(miss_mass_false))
mean_diff_true = np.diff(true_dist_of_total_prob_smooth) / 2.
true_smoothed_mean_probabilities = true_dist_of_total_prob_smooth[:-1] + mean_diff_true
mean_diff_false = np.diff(false_dist_of_total_prob_smooth) / 2.
false_smoothed_mean_probabilities = false_dist_of_total_prob_smooth[:-1] + mean_diff_false
miss_mass_true_ = [
true_dist_of_total_prob[-1] *
abs(true_dist_of_total_bins[-1] - true_dist_of_total_bins_smooth[-1])
]
miss_mass_false_ = [
false_dist_of_total_prob[-1] *
abs(false_dist_of_total_bins[-1] - false_dist_of_total_bins_smooth[-1])
]
prob_true_norm = true_smoothed_mean_probabilities / np.sum(
true_smoothed_mean_probabilities *
np.diff(true_dist_of_total_bins_smooth)) * (1. -
np.sum(miss_mass_true_))
prob_false_norm = false_smoothed_mean_probabilities / np.sum(
false_smoothed_mean_probabilities *
np.diff(false_dist_of_total_bins_smooth)) * (1. -
np.sum(miss_mass_false_))
prob_true = np.r_[prob_true_norm, true_dist_of_total_prob[-1]]
bins_true = np.r_[true_dist_of_total_bins_smooth,
true_dist_of_total_bins[-1]]
prob_false = np.r_[prob_false_norm, false_dist_of_total_prob[-1]]
bins_false = np.r_[false_dist_of_total_bins_smooth,
false_dist_of_total_bins[-1]]
return prob_true, bins_true, prob_false, bins_false
def smooth_priors(ll, true_dist_of_total_prob, true_dist_of_total_bins, false_dist_of_total_prob, false_dist_of_total_bins, kde_mode, bandwidth = "silverman", kernel = "exponential"): import matplotlib.pyplot as plt import numpy as np miss_mass_true = np.diff(true_dist_of_total_bins)[[0,-1]]*true_dist_of_total_prob[[0,-1]] miss_mass_false = np.diff(false_dist_of_total_bins)[[0,-1]]*false_dist_of_total_prob[[0,-1]] import hist_sampler x_ = hist_sampler.histogram_sampler(true_dist_of_total_prob[1:-1], true_dist_of_total_bins[1:-1], 800000) y_ = hist_sampler.histogram_sampler(false_dist_of_total_prob[1:-1], false_dist_of_total_bins[1:-1], 800000) prob_x_2 ,bins_x_2, patch = plt.hist(abs(x_), 200, normed = True, alpha = 0.15) prob_x_3 ,bins_x_3, patch = plt.hist(abs(y_), bins = 600, normed = True, alpha = 0.15) plt.clf() import kern_density_est if kde_mode == "scipy_kde": true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(abs(x_[abs(x_) < true_dist_of_total_bins[-2]]), 'g', np.linspace(0, true_dist_of_total_bins[-2]+50000, 1000), bandwidth) # can change the kernel to an expenential kern_density_est.kern_sklearn_expon(x, color, bandwidth, kernel = "exponential", **kwargs) elif kde_mode == "sklearn_kde": true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(abs(x_), 'r', np.linspace(0, true_dist_of_total_bins[-2]+50000, 3000), bandwidth, kernel_ = kernel) false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(abs(y_), 'y', np.linspace(0, false_dist_of_total_bins[-2]+50000, 2000), bandwidth = bandwidth) plt.clf() plt.plot(true_dist_of_total_bins_smooth, true_dist_of_total_prob_smooth*(1-sum(miss_mass_true)), lw = 2, color = 'g') plt.clf() plt.plot(false_dist_of_total_bins_smooth, false_dist_of_total_prob_smooth*(1-sum(miss_mass_false)), lw = 2, color = 'y') plt.clf() #plt.show() from scipy.stats import kde import bisect insert_index_2 = bisect.bisect_left(true_dist_of_total_prob_smooth[::-1]-true_dist_of_total_prob[-1], 0.) true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = true_dist_of_total_prob_smooth[0:-1*insert_index_2-1], true_dist_of_total_bins_smooth[0:-1*insert_index_2-1] insert_index_4 = bisect.bisect_left(false_dist_of_total_prob_smooth[::-1]-false_dist_of_total_prob[-1], 0.) false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = false_dist_of_total_prob_smooth[0:-1*insert_index_4-1], false_dist_of_total_bins_smooth[0:-1*insert_index_4-1] prob_true_ = true_dist_of_total_prob_smooth*(1-np.sum(miss_mass_true)) prob_false_ = false_dist_of_total_prob_smooth*(1-np.sum(miss_mass_false)) bins_t_plot = np.r_[0, true_dist_of_total_bins_smooth, true_dist_of_total_bins_smooth[-1], true_dist_of_total_bins[-1]] prob_t_plot = np.r_[0, prob_true_, true_dist_of_total_prob[-1], true_dist_of_total_prob[-1]] bins_f_plot = np.r_[0, false_dist_of_total_bins_smooth, false_dist_of_total_bins_smooth[-1], false_dist_of_total_bins[-1]] prob_f_plot = np.r_[0, prob_false_, false_dist_of_total_prob[-1], false_dist_of_total_prob[-1]] plt.clf() plt.bar(bins_x_2[:-1], prob_x_2*(1-sum(miss_mass_true)), np.diff(bins_x_2), alpha=0.2, color = "green") plt.bar(bins_x_3[:-1], prob_x_3*(1-sum(miss_mass_false)), np.diff(bins_x_3), alpha=0.2, color = "yellow") plt.axis([bins_f_plot[1], bins_f_plot[-2]+100000, 0., max(prob_t_plot)*1.5]) ll.savefig() plt.clf() plt.plot(bins_t_plot, prob_t_plot, 'g') plt.plot(bins_f_plot, prob_f_plot, 'y') plt.fill_between(bins_t_plot, prob_t_plot, where=None, color='g', alpha = 0.2, lw = 2) plt.fill_between(bins_f_plot, prob_f_plot, where=None, color='y', alpha = 0.2, lw = 2) plt.axis([bins_f_plot[1], bins_f_plot[-2]+100000, 0., max(prob_t_plot)*1.5]) ll.savefig() mean_diff_true = np.diff(true_dist_of_total_prob_smooth)/2. true_smoothed_mean_probabilities = true_dist_of_total_prob_smooth[:-1] + mean_diff_true mean_diff_false = np.diff(false_dist_of_total_prob_smooth)/2. false_smoothed_mean_probabilities = false_dist_of_total_prob_smooth[:-1] + mean_diff_false miss_mass_true_ = [true_dist_of_total_prob[-1]*abs(true_dist_of_total_bins[-1] - true_dist_of_total_bins_smooth[-1])] miss_mass_false_ = [false_dist_of_total_prob[-1]*abs(false_dist_of_total_bins[-1] - false_dist_of_total_bins_smooth[-1])] prob_true_norm = true_smoothed_mean_probabilities/np.sum(true_smoothed_mean_probabilities*np.diff(true_dist_of_total_bins_smooth))*(1.-np.sum(miss_mass_true_)) prob_false_norm = false_smoothed_mean_probabilities/np.sum(false_smoothed_mean_probabilities*np.diff(false_dist_of_total_bins_smooth))*(1.-np.sum(miss_mass_false_)) prob_true = np.r_[prob_true_norm, true_dist_of_total_prob[-1]] bins_true = np.r_[true_dist_of_total_bins_smooth, true_dist_of_total_bins[-1]] prob_false = np.r_[prob_false_norm, false_dist_of_total_prob[-1]] bins_false = np.r_[false_dist_of_total_bins_smooth, false_dist_of_total_bins[-1]] plt.clf() return prob_true, bins_true, prob_false, bins_false, ll
def smooth_priors(ll,
true_dist_of_total_prob,
true_dist_of_total_bins,
false_dist_of_total_prob,
false_dist_of_total_bins,
kde_mode,
bandwidth="silverman",
kernel="exponential"):
import matplotlib.pyplot as plt
import numpy as np
miss_mass_true = np.diff(true_dist_of_total_bins)[[
0, -1
]] * true_dist_of_total_prob[[0, -1]]
miss_mass_false = np.diff(false_dist_of_total_bins)[[
0, -1
]] * false_dist_of_total_prob[[0, -1]]
import hist_sampler
x_ = hist_sampler.histogram_sampler(true_dist_of_total_prob[1:-1],
true_dist_of_total_bins[1:-1], 800000)
y_ = hist_sampler.histogram_sampler(false_dist_of_total_prob[1:-1],
false_dist_of_total_bins[1:-1], 800000)
prob_x_2, bins_x_2, patch = plt.hist(abs(x_), 200, normed=True, alpha=0.15)
prob_x_3, bins_x_3, patch = plt.hist(abs(y_),
bins=600,
normed=True,
alpha=0.15)
plt.clf()
import kern_density_est
if kde_mode == "scipy_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_scipy_gaus(
abs(x_[abs(x_) < true_dist_of_total_bins[-2]]), 'g',
np.linspace(0, true_dist_of_total_bins[-2] + 50000,
1000), bandwidth
) # can change the kernel to an expenential kern_density_est.kern_sklearn_expon(x, color, bandwidth, kernel = "exponential", **kwargs)
elif kde_mode == "sklearn_kde":
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(
abs(x_),
'r',
np.linspace(0, true_dist_of_total_bins[-2] + 50000, 3000),
bandwidth,
kernel_=kernel)
false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = kern_density_est.kern_sklearn_expon(
abs(y_),
'y',
np.linspace(0, false_dist_of_total_bins[-2] + 50000, 2000),
bandwidth=bandwidth)
plt.clf()
plt.plot(true_dist_of_total_bins_smooth,
true_dist_of_total_prob_smooth * (1 - sum(miss_mass_true)),
lw=2,
color='g')
plt.clf()
plt.plot(false_dist_of_total_bins_smooth,
false_dist_of_total_prob_smooth * (1 - sum(miss_mass_false)),
lw=2,
color='y')
plt.clf()
#plt.show()
from scipy.stats import kde
import bisect
insert_index_2 = bisect.bisect_left(
true_dist_of_total_prob_smooth[::-1] - true_dist_of_total_prob[-1], 0.)
true_dist_of_total_prob_smooth, true_dist_of_total_bins_smooth = true_dist_of_total_prob_smooth[
0:-1 * insert_index_2 -
1], true_dist_of_total_bins_smooth[0:-1 * insert_index_2 - 1]
insert_index_4 = bisect.bisect_left(
false_dist_of_total_prob_smooth[::-1] - false_dist_of_total_prob[-1],
0.)
false_dist_of_total_prob_smooth, false_dist_of_total_bins_smooth = false_dist_of_total_prob_smooth[
0:-1 * insert_index_4 -
1], false_dist_of_total_bins_smooth[0:-1 * insert_index_4 - 1]
prob_true_ = true_dist_of_total_prob_smooth * (1 - np.sum(miss_mass_true))
prob_false_ = false_dist_of_total_prob_smooth * (1 -
np.sum(miss_mass_false))
bins_t_plot = np.r_[0, true_dist_of_total_bins_smooth,
true_dist_of_total_bins_smooth[-1],
true_dist_of_total_bins[-1]]
prob_t_plot = np.r_[0, prob_true_, true_dist_of_total_prob[-1],
true_dist_of_total_prob[-1]]
bins_f_plot = np.r_[0, false_dist_of_total_bins_smooth,
false_dist_of_total_bins_smooth[-1],
false_dist_of_total_bins[-1]]
prob_f_plot = np.r_[0, prob_false_, false_dist_of_total_prob[-1],
false_dist_of_total_prob[-1]]
plt.clf()
plt.bar(bins_x_2[:-1],
prob_x_2 * (1 - sum(miss_mass_true)),
np.diff(bins_x_2),
alpha=0.2,
color="green")
plt.bar(bins_x_3[:-1],
prob_x_3 * (1 - sum(miss_mass_false)),
np.diff(bins_x_3),
alpha=0.2,
color="yellow")
plt.axis(
[bins_f_plot[1], bins_f_plot[-2] + 100000, 0.,
max(prob_t_plot) * 1.5])
ll.savefig()
plt.clf()
plt.plot(bins_t_plot, prob_t_plot, 'g')
plt.plot(bins_f_plot, prob_f_plot, 'y')
plt.fill_between(bins_t_plot,
prob_t_plot,
where=None,
color='g',
alpha=0.2,
lw=2)
plt.fill_between(bins_f_plot,
prob_f_plot,
where=None,
color='y',
alpha=0.2,
lw=2)
plt.axis(
[bins_f_plot[1], bins_f_plot[-2] + 100000, 0.,
max(prob_t_plot) * 1.5])
ll.savefig()
mean_diff_true = np.diff(true_dist_of_total_prob_smooth) / 2.
true_smoothed_mean_probabilities = true_dist_of_total_prob_smooth[:-1] + mean_diff_true
mean_diff_false = np.diff(false_dist_of_total_prob_smooth) / 2.
false_smoothed_mean_probabilities = false_dist_of_total_prob_smooth[:-1] + mean_diff_false
miss_mass_true_ = [
true_dist_of_total_prob[-1] *
abs(true_dist_of_total_bins[-1] - true_dist_of_total_bins_smooth[-1])
]
miss_mass_false_ = [
false_dist_of_total_prob[-1] *
abs(false_dist_of_total_bins[-1] - false_dist_of_total_bins_smooth[-1])
]
prob_true_norm = true_smoothed_mean_probabilities / np.sum(
true_smoothed_mean_probabilities *
np.diff(true_dist_of_total_bins_smooth)) * (1. -
np.sum(miss_mass_true_))
prob_false_norm = false_smoothed_mean_probabilities / np.sum(
false_smoothed_mean_probabilities *
np.diff(false_dist_of_total_bins_smooth)) * (1. -
np.sum(miss_mass_false_))
prob_true = np.r_[prob_true_norm, true_dist_of_total_prob[-1]]
bins_true = np.r_[true_dist_of_total_bins_smooth,
true_dist_of_total_bins[-1]]
prob_false = np.r_[prob_false_norm, false_dist_of_total_prob[-1]]
bins_false = np.r_[false_dist_of_total_bins_smooth,
false_dist_of_total_bins[-1]]
plt.clf()
return prob_true, bins_true, prob_false, bins_false, ll
