def gamma_distribution(*args):
try:
a, b = float(args[0]), float(args[1])
except (ValueError, TypeError):
return {'is_valid': False}
if not a > 0 or not b > 0:
return {'is_valid': False}
t = Symbol('t')
mean, var = gmm.stats(b, scale=1 / a, moments='mv')
cf = nsimplify((1 - I * t / a)**(-b), tolerance=1e-2)
d_cf = diff(cf, t)
dd_cf = diff(d_cf, t)
mean2 = round(gmm.moment(2, b, scale=1 / a), 2)
return {
'a': a,
'b': b,
'b_minus': round(b - 1, 2),
'b_plus': round(b + 1, 2),
'mean': mean,
'mean2': mean2,
'var': var,
'g': latex(cf),
'g1': latex(d_cf),
'g2': latex(dd_cf),
'type': type,
'is_valid': True,
}
def plotGammaDistributions(a, b, x_array, title):
# initialize loop variables with prior
mu_sum = a/b # Eq. 2.147
var_sum = a/np.square(b) # Eq. 2.148
N = 1
sq_err = 0
a_n = a
b_n = b
for x in np.nditer(x_array):
var_sum += np.square(x - mu) # how far is the noisy observation from the known mean, squared
var_bayes = (1/N)*var_sum # the maximum likelihood estimator of the variance
b_n += var_bayes # Update the hyperparameter per Eq. 2.150
mu_sum += x # offset of noisy observation from the known mean, then squared
mu_bayes = (1/N)*mu_sum # maximum likelihood estimator of the mean
a_n += N/2 # Update the hyperparameter per Eq. 2.151
sq_err += np.square(mu_bayes-mu) # determine the error based on the ground truth mean
mse_ml = sq_err/N # final MSE approximation
plt.figure(4) # accrue into same figure as bayesian versions
plt.plot(N, mse_ml, '.', label=title+" Bayes")
N += 1 # increment the data point counter
# plot 2x2 subplots with the same x-axis and y-axis
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, sharex=True, sharey=True)
gamma.stats(a, moments='mvsk')
x = np.linspace(gamma.ppf(0.01, a), gamma.ppf(0.99, a_n), 1000) # generate x axis values for plotting the PDF
# first subplot is the beta distribution of the initial hyperparameters
ax1.plot(x, gamma.pdf(x, a, loc=(1/b)))
ax1.set_title(title + 'Initial Guess')
# second subplot has 1/3 of the data accounted for
ax2.plot(x, gamma.pdf(x, a_n*(1/3)))
ax2.set_title('Through 1/3 Data')
# third subplot has 2/3 of the data accounted for
ax3.plot(x, gamma.pdf(x, a_n*(2/3)))
ax3.set_title('Through 2/3 Data')
# fourth subplot has all of the data
ax4.plot(x, gamma.pdf(x, a_n, loc=(1/b_n)))
ax4.set_title("Through all Data")
return
def gaussify2(Y, alpha, beta, mu):
# Check: https://stats.stackexchange.com/questions/37461/the-relationship-between-the-gamma-distribution-and-the-normal-distribution
algo = np.divide(np.abs(Y-mu), alpha)
term_num = np.power(algo, beta)
num = beta * np.exp(term_num)
algo2 = gamma.stats(np.divide(1, beta), moments='m')
den = 2 * alpha * algo2
return np.divide(num, den)
def __init__(self, maxmmol, maxgoing, endgoing, typebranch1, typebranch2):
self.xs = range(0, endgoing, 1)
self.ys = []
if typebranch1 == typebranch2:
if typebranch1 == "line":
for x in range(0, endgoing, 1):
if x <= maxgoing:
self.ys.append(x * maxmmol / maxgoing)
else:
self.ys.append(maxmmol - (x - maxgoing) * maxmmol /
(endgoing - maxgoing))
elif typebranch1 == "elrong":
a = 2
mean, var, skew, kurt = gamma.stats(a, moments='mvsk')
xr = np.linspace(0, 1, maxgoing)
xr2 = np.linspace(1.001, 6, endgoing - maxgoing)
xr = xr + xr2
ky = maxmmol / gamma.pdf(1, a)
kx1 = maxgoing
kx2 = (endgoing - maxgoing) / 5
for x in xr:
self.ys.append(gamma.pdf(x, a) * ky)
else:
if typebranch1 == "line":
for x in range(0, maxgoing, 1):
self.ys.append(x * maxmmol / maxgoing)
if typebranch2 == "line":
for x in range(maxgoing + 1, endgoing, 1):
self.ys.append(maxmmol - (x - maxgoing) * maxmmol /
(endgoing - maxgoing))
elif typebranch2 == "norm":
norm = st.norm(loc=1, scale=0.25)
xr = np.linspace(norm.ppf(0.5), norm.ppf(0.9999),
endgoing - maxgoing)
k = maxmmol / norm.pdf(1)
for x in xr:
self.ys.append(norm.pdf(x) * k)
from scipy.stats import gamma import matplotlib.pyplot as plt fig, ax = plt.subplots(1, 1) # Calculate a few first moments: a = 1.99 mean, var, skew, kurt = gamma.stats(a, moments='mvsk') # Display the probability density function (``pdf``): x = np.linspace(gamma.ppf(0.01, a), gamma.ppf(0.99, a), 100) ax.plot(x, gamma.pdf(x, a), 'r-', lw=5, alpha=0.6, label='gamma pdf') # Alternatively, the distribution object can be called (as a function) # to fix the shape, location and scale parameters. This returns a "frozen" # RV object holding the given parameters fixed. # Freeze the distribution and display the frozen ``pdf``: rv = gamma(a) ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf') # Check accuracy of ``cdf`` and ``ppf``: vals = gamma.ppf([0.001, 0.5, 0.999], a) np.allclose([0.001, 0.5, 0.999], gamma.cdf(vals, a)) # True # Generate random numbers:
def image_details(fig1,
fig2,
fig3,
pic_canny,
save=False,
chip_name='',
png='',
dpi=96):
"""A subroutine for debugging contrast adjustment"""
bin_no = 150
nrows, ncols = fig1.shape
fig = plt.figure(figsize=(ncols / dpi / 2, nrows / dpi / 2), dpi=dpi)
ax_img = plt.Axes(fig, [0, 0, 1, 1])
fig.add_axes(ax_img)
ax_img.set_axis_off()
ax_img.imshow(fig3, cmap='gray')
fig3[pic_canny] = fig3.max() * 2
pic_cdf1, cbins1 = cumulative_distribution(fig1, bin_no)
pic_cdf2, cbins2 = cumulative_distribution(fig2, bin_no)
pic_cdf3, cbins3 = cumulative_distribution(fig3, bin_no)
ax_hist1 = plt.axes([.05, .05, .25, .25])
ax_cdf1 = ax_hist1.twinx()
ax_hist2 = plt.axes([.375, .05, .25, .25])
ax_cdf2 = ax_hist2.twinx()
ax_hist3 = plt.axes([.7, .05, .25, .25])
ax_cdf3 = ax_hist3.twinx()
fig1r = fig1.ravel()
fig2r = fig2.ravel()
fig3r = fig3.ravel()
hist1, hbins1, __ = ax_hist1.hist(fig1r,
bin_no,
facecolor='r',
normed=True)
hist2, hbins2, __ = ax_hist2.hist(fig2r,
bin_no,
facecolor='b',
normed=True)
hist3, hbins3, __ = ax_hist3.hist(fig3r,
bin_no,
facecolor='g',
normed=True)
ax_hist1.patch.set_alpha(0)
ax_hist2.patch.set_alpha(0)
ax_hist3.patch.set_alpha(0)
ax_cdf1.plot(cbins1, pic_cdf1, color='w')
ax_cdf2.plot(cbins2, pic_cdf2, color='c')
ax_cdf3.plot(cbins3, pic_cdf3, color='y')
bin_centers2 = 0.5 * (hbins2[1:] + hbins2[:-1])
m2, s2 = norm.fit(fig2r)
pdf2 = norm.pdf(bin_centers2, m2, s2)
ax_hist2.plot(bin_centers2, pdf2, color='m')
mean, var, skew, kurt = gamma.stats(fig2r, moments='mvsk')
# print(mean, var, skew, kurt)
ax_hist1.set_title("Normalized", color='r')
ax_hist2.set_title("CLAHE Equalized", color='b')
ax_hist3.set_title("Contrast Stretched", color='g')
ax_hist1.set_ylim([0, max(hist1)])
ax_hist2.set_ylim([0, max(hist2)])
ax_hist3.set_ylim([0, max(hist3)])
ax_hist1.set_xlim([np.median(fig1) - 0.1, np.median(fig1) + 0.1])
ax_hist2.set_xlim([np.median(fig2) - 0.1, np.median(fig2) + 0.1])
ax_hist3.set_xlim([0, 1])
#ax_cdf1.set_ylim([0,1])
ax_hist1.tick_params(labelcolor='tab:orange')
if save == True:
plt.savefig('../virago_output/' + chip_name + '/processed_images/' +
png + '_image_details.png',
dpi=dpi)
plt.show()
plt.close('all')
plt.clf()
anomalyday=[]
anomalypred=[]
for i in range(2,len(TotalCases)):
new_cases=TotalCases[i]-TotalCases[i-1]
old_new_cases=TotalCases[i-1]-TotalCases[i-2]
# This uses a conjugate prior as a Gamma distribution for b_t, with parameters alpha and beta
alpha =alpha+new_cases
beta=beta +old_new_cases
valpha.append(alpha)
vbeta.append(beta)
#y1 = stats.gamma.pdf(x, a=alpha, scale=1./beta)
mean, var, skew, kurt = gamma.stats(a=alpha, scale=1/beta, moments='mvsk')
#print(mean,var)
RRest=1.+infperiod*ln(mean)
if (RRest<0.): RRest=0.
predR.append(RRest)
testRRM=1.+infperiod*ln( gamma.ppf(0.99, a=alpha, scale=1./beta) )# these are the boundaries of the 99% confidence interval for new cases
if (testRRM <0.): testRRM=0.
pstRRM.append(testRRM)
testRRm=1.+infperiod*ln( gamma.ppf(0.01, a=alpha, scale=1./beta) )
if (testRRm <0.): testRRm=0.
pstRRm.append(testRRm)
#print('estimated RR=',RRest,testRRm,testRRM) # to see the numbers for the evolution of Rt
if (new_cases>0. and old_new_cases>0.):
NewCases.append(new_cases)
pstRRm = []
anomalyday = []
anomalypred = []
for i in range(2, len(TotalCases)):
new_cases = float(TotalCases[i] - TotalCases[i - 1])
old_new_cases = float(TotalCases[i - 1] - TotalCases[i - 2])
# This uses a conjugate prior as a Gamma distribution for b_t, with parameters alpha and beta
alpha = alpha + new_cases
beta = beta + old_new_cases
valpha.append(alpha)
vbeta.append(beta)
mean = gamma.stats(a=alpha, scale=1 / beta, moments='m')
RRest = 1. + infperiod * ln(mean)
if (RRest < 0.): RRest = 0.
predR.append(RRest)
testRRM = 1. + infperiod * ln(
gamma.ppf(0.99, a=alpha, scale=1. / beta)
) # these are the boundaries of the 99% confidence interval for new cases
if (testRRM < 0.): testRRM = 0.
pstRRM.append(testRRM)
testRRm = 1. + infperiod * ln(gamma.ppf(0.01, a=alpha,
scale=1. / beta))
if (testRRm < 0.): testRRm = 0.
pstRRm.append(testRRm)
#print('estimated RR=',RRest,testRRm,testRRM) # to see the numbers for the evolution of Rt
#
# ### ガンマ分布のpython code
# 上記のコードの実行結果です。
#
# - $\displaystyle \left(K,\frac{M}{K} \right) = \left(1,5 \right) , \left(3,\frac{5}{3} \right), \left(15,\frac{5}{15} \right) $の三通りについてプロットしています。教科書のP61の図2-2と同じようなグラフが得られています
#
# In[6]:
from scipy.stats import gamma
x = np.linspace(0, 50, 1000)
a = 1.0
b = 5.0
mean, var, skew, kurt = gamma.stats(a, scale=b, moments='mvsk')
y1 = gamma.pdf(x, a, scale=b)
print('a : {}, b : {:,.3f}, mean : {}'.format(a, b, mean))
a = 3.0
b = 5.0 / 3.0
mean, var, skew, kurt = gamma.stats(a, scale=b, moments='mvsk')
print('a : {}, b : {:,.3f}, mean : {}'.format(a, b, mean))
y2 = gamma.pdf(x, a, scale=b)
a = 15.0
b = 1.0 / 3.0
mean, var, skew, kurt = gamma.stats(a, scale=b, moments='mvsk')
print('a : {}, b : {:,.3f}, mean : {}'.format(a, b, mean))
y3 = gamma.pdf(x, a, scale=b)
_c_mod_adj, dats[dat_name]['dat'] = do_qmap(dats['obs']['dat'],
dats['c_mod']['dat'],
dats['p_mod']['dat'],
proj_adj_type=adj_type)
k, loc, scale = gamma.fit(dats[dat_name]['dat'])
dats[dat_name]['k'] = k.round(1)
dats[dat_name]['loc'] = loc.round(1)
dats[dat_name]['scale'] = scale.round(1)
# endregion process data
# region plot pdf gamma
x = np.linspace(0, 100, 101)
fig, ax = plt.subplots(figsize=(8, 5.5))
for dat_name, dat_info in dats.items():
mu, var = gamma.stats(dat_info['k'],
loc=dat_info['loc'],
scale=dat_info['scale'])
sd = np.sqrt(var)
y = gamma.pdf(x,
dat_info['k'],
loc=dat_info['loc'],
scale=dat_info['scale'])
label = '{}; $\mu$={:2.1f}, sd={:2.1f}'.format(plt_args[dat_name]['name'],
mu, sd)
# ax.plot(
# x, y,
# color=plt_args[dat_name]['color'],
# linestyle=plt_args[dat_name]['linetype'],
# label=label)
sns.distplot(dat_info['dat'],
kde=False,
def run_luis_model(df: pd.DataFrame, filepath: Path) -> None:
infperiod = 4.5 # length of infectious period, adjust as needed
def smooth(y, box_pts):
box = np.ones(box_pts) / box_pts
y_smooth = np.convolve(y, box, mode='same')
return y_smooth
# Loop through states
states = df['state'].unique()
returndf = pd.DataFrame()
for state in states:
from scipy.stats import gamma # not sure why this needs to be recalled after each state, but otherwite get a type exception
import numpy as np
statedf = df[df['state'] == state].sort_values('date')
confirmed = list(statedf['positive'])
dates = list(statedf['date'])
day = list(range(1, len(statedf['date']) + 1))
if (confirmed[-1] < 10.):
continue # this skips the Rt analysis for states for which there are <10 total cases
##### estimation and prediction
dconfirmed = np.diff(confirmed)
for ii in range(len(dconfirmed)):
if dconfirmed[ii] < 0.: dconfirmed[ii] = 0.
xd = dates[1:]
sdays = 15
yy = smooth(
dconfirmed, sdays
) # smoothing over sdays (number of days) moving window, averages large chunking in reporting in consecutive days
yy[-2] = (
dconfirmed[-4] + dconfirmed[-3] + dconfirmed[-2]
) / 3. # these 2 last lines should not be necesary but the data tend to be initially underreported and also the smoother struggles.
yy[-1] = (dconfirmed[-3] + dconfirmed[-2] + dconfirmed[-1]) / 3.
#lyyy=np.cumsum(lwy)
TotalCases = np.cumsum(
yy
) # These are confirmed cases after smoothing: tried also a lowess smoother but was a bit more parameer dependent from place to place.
alpha = 3. # shape parameter of gamma distribution
beta = 2. # rate parameter of gamma distribution see https://en.wikipedia.org/wiki/Gamma_distribution
valpha = []
vbeta = []
pred = []
pstdM = []
pstdm = []
xx = []
NewCases = []
predR = []
pstRRM = []
pstRRm = []
anomalyday = []
anomalypred = []
for i in range(2, len(TotalCases)):
new_cases = float(TotalCases[i] - TotalCases[i - 1])
old_new_cases = float(TotalCases[i - 1] - TotalCases[i - 2])
# This uses a conjugate prior as a Gamma distribution for b_t, with parameters alpha and beta
alpha = alpha + new_cases
beta = beta + old_new_cases
valpha.append(alpha)
vbeta.append(beta)
mean = gamma.stats(a=alpha, scale=1 / beta, moments='m')
RRest = 1. + infperiod * ln(mean)
if (RRest < 0.): RRest = 0.
predR.append(RRest)
testRRM = 1. + infperiod * ln(
gamma.ppf(0.99, a=alpha, scale=1. / beta)
) # these are the boundaries of the 99% confidence interval for new cases
if (testRRM < 0.): testRRM = 0.
pstRRM.append(testRRM)
testRRm = 1. + infperiod * ln(
gamma.ppf(0.01, a=alpha, scale=1. / beta))
if (testRRm < 0.): testRRm = 0.
pstRRm.append(testRRm)
if (new_cases == 0. or old_new_cases == 0.):
pred.append(0.)
pstdM.append(10.)
pstdm.append(0.)
NewCases.append(0.)
if (new_cases > 0. and old_new_cases > 0.):
NewCases.append(new_cases)
# Using a Negative Binomial as the Posterior Predictor of New Cases, given old one
# This takes parameters r,p which are functions of new alpha, beta from Gamma
r, p = alpha, beta / (old_new_cases + beta)
mean, var, skew, kurt = nbinom.stats(r, p, moments='mvsk')
pred.append(mean) # the expected value of new cases
testciM = nbinom.ppf(
0.99, r, p
) # these are the boundaries of the 99% confidence interval for new cases
pstdM.append(testciM)
testcim = nbinom.ppf(0.01, r, p)
pstdm.append(testcim)
np = p
nr = r
flag = 0
while (new_cases > testciM or new_cases < testcim):
if (flag == 0):
anomalypred.append(new_cases)
anomalyday.append(
dates[i + 1]) # the first new cases are at i=2
# annealing: increase variance so as to encompass anomalous observation: allow Bayesian code to recover
# mean of negbinomial=r*(1-p)/p variance= r (1-p)/p**2
# preserve mean, increase variance--> np=0.8*p (smaller), r= r (np/p)*( (1.-p)/(1.-np) )
# test anomaly
nnp = 0.95 * np # this doubles the variance, which tends to be small after many Bayesian steps
nr = nr * (nnp / np) * (
(1. - np) / (1. - nnp)
) # this assignement preserves the mean of expected cases
np = nnp
mean, var, skew, kurt = nbinom.stats(nr,
np,
moments='mvsk')
testciM = nbinom.ppf(0.99, nr, np)
testcim = nbinom.ppf(0.01, nr, np)
flag = 1
else:
if (flag == 1):
alpha = nr # this updates the R distribution with the new parameters that enclose the anomaly
beta = np / (1. - np) * old_new_cases
testciM = nbinom.ppf(0.99, nr, np)
testcim = nbinom.ppf(0.01, nr, np)
# annealing leaves the RR mean unchanged, but we need to adjus its widened CI:
testRRM = 1. + infperiod * ln(
gamma.ppf(0.99, a=alpha, scale=1. / beta)
) # these are the boundaries of the 99% confidence interval for new cases
if (testRRM < 0.): testRRM = 0.
testRRm = 1. + infperiod * ln(
gamma.ppf(0.01, a=alpha, scale=1. / beta))
if (testRRm < 0.): testRRm = 0.
pstRRM = pstRRM[:
-1] # remove last element and replace by expanded CI for RRest
pstRRm = pstRRm[:-1]
pstRRM.append(testRRM)
pstRRm.append(testRRm)
# visualization of the time evolution of R_t with confidence intervals
x = []
for i in range(len(predR)):
x.append(i)
days = dates[3:]
xd = days
dstr = []
for xdd in xd:
dstr.append(xdd.strftime("%Y-%m-%d"))
appenddf = pd.DataFrame({
'state': state,
'date': days,
'RR_pred_luis': predR,
'RR_CI_lower_luis': pstRRm,
'RR_CI_upper_luis': pstRRM
})
returndf = pd.concat([returndf, appenddf], axis=0)
returndf.to_csv(filepath / "luis_code_estimates.csv", index=False)
# default values are loc = 0 and scale = 1.
print('exponential dist')
print(expon.stats())
# for exponential dist scale=1/lambda
print(expon.mean(scale=3))
print('UNIFORM!!!:')
# This distribution is constant between loc and loc + scale.
from scipy.stats import uniform
print(uniform.cdf([0, 1, 2, 34, 5], loc=1, scale=4))
print(np.mean(norm.rvs(5, size=500)))
# shape parameters:
from scipy.stats import gamma
print('GAMMA')
print(gamma(a=1, scale=2).stats(moments="mv"))
print(gamma.stats(a=1, scale=2, moments="mv"))
###################################################
# freezing a distribution:
rv = gamma(a=1, scale=2)
print(rv.mean(), rv.std())
from scipy.stats import t
#help(t.__doc__)
######
#isf gives the critical value of the dist for the given tail
print('T-dsitribution Critical value:')
print('level= 0.05: ', t.isf(0.05, df=20))
print('level = 0.025', t.isf(0.025, df=20))
'''
#specific points for discrete distributions:
#Discrete distribution have mostly the same basic methods as the
# # Gamma Distribution
from scipy import stats
# Read data
dataset1 = yf.download(symbol, start, end)
dataset2 = yf.download(market, start, end)
stock_ret = dataset1['Adj Close'].pct_change().dropna()
mkt_ret = dataset2['Adj Close'].pct_change().dropna()
beta, alpha, r_value, p_value, std_err = stats.linregress(mkt_ret, stock_ret)
print(beta, alpha)
from scipy.stats import gamma
mu, std = gamma.stats(dataset['Returns'])
# Plot the histogram.
plt.hist(dataset['Returns'], bins=25, alpha=0.6, color='g')
# Plot the PDF.
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 1171)
p = gamma.pdf(x, alpha, scale=1 / beta)
plt.plot(x, p, 'k', linewidth=2)
plt.title("Gamma Distribution for Stock")
plt.show()
# ## Binomial Distribution
from scipy.stats import binom
