def create_distributions(listD=[(0.5, 0.5), (5.0, 1.0), (1.0, 3.0), (2.0, 2.0), (2.0, 5.0), (0.8, 7.2), (1.4, 0.6)]):
distributions = {}
listD = [(0.5, 0.5), (5.0, 1.0), (1.0, 3.0), (2.0, 2.0), (2.0, 5.0), (0.8, 7.2), (1.4, 0.6)]
for i in listD:
(a, b, c, d) = create_normalized_beta(i[0], i[1])
dist = beta(a, b, c, d)
s = "beta(" + str(a) + "," + str(b) + "," + str(c) + "," + str(d) + ")"
distributions[s] = dist
print s, "\tmean=\t", beta.stats(a, b, c, d, "m"), "\tstd=\t", beta.std(a, b, c, d)
return distributions
def main():
a = 81
b = 219
stats = beta.stats(a, b, moments='mvsk')
print('Mean: %.3f, Variance: %.3f, Skew: %.3f, Kurtosis: %.3f' % stats)
x = np.linspace(0, 1, 1000)
y = beta.pdf(x, a, b)
plt.plot(x, y, 'r-', lw=2, alpha=0.6, label='beta pdf')
plt.legend()
plt.show()
# -*- coding: utf-8 -*-
"""
Created on Sun May 10 16:00:42 2015
@author: sc1
"""
from scipy.stats import beta
import matplotlib.pyplot as plt
import numpy as np
fig, ax = plt.subplots(1, 1)
a, b = 1, 50
mean, var, skew, kurt = beta.stats(a, b, moments='mvsk')
x = np.linspace(beta.ppf(0.01, a, b),
beta.ppf(0.99, a, b), 100)
rv = beta(a, b)
ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf')
plt.show()
import numpy as np from scipy.stats import beta, gamma, bernoulli import matplotlib.pyplot as plt import matplotlib.animation as animation #======================================================= fig = plt.figure() #setting beta dist graph with mean 0.4 a =2 b = 3 mean, var, skew, kurt = beta.stats(a,b ,moments = 'mvsk') print(mean) rv = beta(a,b) p = np.linspace(0, 1.0, 100) #prob of getting head prior = beta.pdf(p, a, b) posterior = prior#init data = bernoulli.rvs(p = 0.7, size = 160) SIZE = 160 ims = [] # print(bernoulli.pmf(0,[0.4,0.2])) # plt.figure() for i in range(SIZE): if i ==0:
def get_combined_estimate(self):
'''
Get combined payout estimates (sample and XInt combined through inverse variance weighted average)
for given intent I
'''
# totals = self.s[:, self.I] + self.f[:, self.I] # Total samples under this intent
#------------------------#
# Get E_samp and V_samp #
#------------------------#
# 'IVWA*_edit_', 'IVWA*_edit': NO SAMPLING, JUST DATA
# 'IVWA_paper': SAMPLE HERE FROM EXPLORATION ONLY
# 'IVWA*_2': SAMPLE HERE FROM EXPL + OBS DATA
l = ['IVWA - NO TS'] # NO TS!
# if self.algorithm_name in :
if self.algorithm_name.startswith(tuple(l)):
if self.USE_OBSERVATIONAL_DATA:
E_samp, V_samp = beta.stats(
self.s_with_obs[:, self.I],
self.f_with_obs[:, self.I]) # (3) CHANGE
else:
E_samp, V_samp = beta.stats(self.s[:, self.I], self.f[:,
self.I])
# 'ORIGINAL' - NO TS SAMPLE HERE
if self.USE_WEIGHTED_TS:
E_samp, V_samp = beta.stats(self.s[:, self.I], self.f[:, self.I])
# TS SAMPLE from experiment AND obs successes and failures
l = ['IVWA - TS']
if self.algorithm_name.startswith(tuple(l)):
if self.USE_OBSERVATIONAL_DATA:
_, V_samp = beta.stats(self.s_with_obs[:, self.I],
self.f_with_obs[:, self.I])
E_samp = np.random.beta(
self.s_with_obs[:, self.I],
self.f_with_obs[:, self.I]) # TS SAMPLE HERE
else:
_, V_samp = beta.stats(self.s[:, self.I], self.f[:, self.I])
E_samp = np.random.beta(self.s[:, self.I],
self.f[:, self.I]) # TS SAMPLE HERE
# TS sample from experiment successes and failures
l = ['IVWA - paper', 'IVWA - editTS']
if self.algorithm_name.startswith(tuple(l)):
_, V_samp = beta.stats(self.s[:, self.I], self.f[:, self.I])
E_samp = np.random.beta(self.s[:, self.I],
self.f[:, self.I]) # TS SAMPLE HERE
#------------------------#
# Get E_XInt and V_XInt #
#------------------------#
# XINT, XARM ONLY POSSIBLE WITH EXPERIMENTAL DATA??
if self.USE_INTERVENTIONAL_DATA:
E_xint, V_xint = self.get_xint()
#------------------------#
# Get E_Comb #
#------------------------#
if self.USE_INTERVENTIONAL_DATA:
M = np.array([[E_samp, V_samp], [E_xint, V_xint]]) # E_samp etc: K
E_comb, V_comb = self.inverse_variance_weighting(M)
else:
E_comb = E_samp
V_comb = V_samp
return E_comb, V_comb
if __name__ == '__main__':
# a piece of a prolate cycloid, and am going to find
# a, b = 1, 2
# phi = np.linspace(3, 10, 100)
# x1 = a*phi - b*np.sin(phi)
# y1 = a - b*np.cos(phi)
# x2=phi
# y2=np.sin(phi)+2
# x,y=intersection(x1,y1,x2,y2)
# plt.plot(x1,y1,c='r')
# plt.plot(x2,y2,c='g')
# plt.plot(x,y,'*k')
# plt.show()
x = np.linspace(0, 1.0, 100)
hack = beta.pdf(x, 39, 23)
mean1, var, skew, kurt = beta.stats(39, 23, moments='mvsk')
print mean1, var
plt.plot(x, hack)
# week1 normal
week1 = beta.pdf(x, 45, 58)
mean2, var, skew, kurt = beta.stats(7, 58, moments='mvsk')
print mean2, var
a = intersection(x, hack, x, week1)
print a
plt.plot(x, week1)
plt.show()
from scipy.stats import beta
a = 7.2
b = 2.3
m, v = beta.stats(a, b, moments="mv")
mean = beta.mean(a=a, b=b)
var = beta.var(a=a, b=b)
print("The mean is " + str(m))
print("The variance is " + str(v))
prob = 1 - beta.cdf(a=a, b=b, x=.90)
print("The probability of having a variance over 90% is " + str(prob))
if best_p < 0.001:
best_dist, best_D = (min(dist_resultsD, key=lambda item: item[1]))
# store the name of the best fit and its p value
print("Best fitting distribution: " + str(best_dist))
print("Best p value: " + str(best_p))
print("Best KS Statistic value: " + str(best_D))
print("Parameters for the best fit: " + str(params[best_dist]))
#Beta distribution params: (α , β, loc (lower limit), scale (upper limit - lower limit))
# Estimating alpha and beta
# μ=α/α+β
# σ^2=αβ/(α+β)^2(α+β+1)
a, b = 3.54492024676839, 20.292557094400216
mean, var = beta.stats(a, b)
probability = stats.beta.cdf(mean, a, b)
print(
"According to Beta Distribution, we have that the probability of an uber journey being less than 26mins is "
+ str(round(probability * 100, 2)) + "%\n")
# *****************************************************************************
# Hypothesis Test Two:
# *****************************************************************************
# Most uber routes are ones that cross central London
# Given the centre of City of London i.e. the main district for work and economy
# has latitude anbd longitude of 51.51279 -0.09184
#
# H0: μ1 >= μ2 i.e. bus routes in the morning more usage than afternoon
# H1: μ1 < μ2 i.e. bus routes in the morning have less than afternoon
# -----------------------------------------------------------------------------
import numpy
import pandas as pd
import sys
import matplotlib.pyplot as plt
from scipy.stats import beta as beta
if __name__ == "__main__":
a1 = float(sys.argv[1])
b1 = float(sys.argv[2])
a2 = float(sys.argv[3])
b2 = float(sys.argv[4])
data1 = numpy.random.beta(a=a1, b=b1, size=[5000])
data2 = numpy.random.beta(a=a2, b=b2, size=[5000])
maximum = numpy.concatenate((data1, data2)).max()
print("Max", maximum)
print("Moments 1 - ", beta.stats(a1, b1, moments='mvsk'))
numpy.savetxt("Moments1.txt",
beta.stats(a1, b1, moments='mvsk'),
fmt="%.3f")
print("Moments 2 - ", beta.stats(a2, b2, moments='mvsk'))
numpy.savetxt("Moments2.txt",
beta.stats(a2, b2, moments='mvsk'),
fmt="%.3f")
plt.hist(data1, histtype='step', color='r', normed=True)
plt.hist(data2, histtype='step', color='b', normed=True)
numpy.savetxt("d1.txt", data1)
numpy.savetxt("d2.txt", data2)
plt.savefig("distribution.png")
def get_combined_estimate(self):
'''
Get combined payout estimates (sample and XInt combined through inverse variance weighted average)
for given intent I
'''
# totals = self.s[:, self.I] + self.f[:, self.I] # Total samples under this intent
#------------------------#
# Get E_samp and V_samp #
#------------------------#
if self.algorithm_name in [
'RDC_test', 'RDC_paper_NO_TS', 'RDC*_1', 'RDC*_edit',
'RDC*_edit_', 'RDC*_edit_TS', 'RDC*_edit_TS_2',
'RDC*_edit_NO_TS_3', 'RDC*_edit_precision2'
]:
E_samp, V_samp = beta.stats(self.s_with_obs[:, self.I],
self.f_with_obs[:, self.I])
if self.algorithm_name in ['RDC_original++']:
E_samp, V_samp = beta.stats(self.s[:, self.I], self.f[:, self.I])
if self.algorithm_name in [
'RDC_test_2', 'RDC*_2', 'RDC*_edit_TS_4', 'RDC*_edit_precision'
]:
_, V_samp = beta.stats(self.s_with_obs[:, self.I],
self.f_with_obs[:, self.I])
E_samp = np.random.beta(
self.s_with_obs[:, self.I],
self.f_with_obs[:, self.I]) # (6) CHANGE - TS SAMPLE HERE
if self.algorithm_name in ['RDC_paper']:
_, V_samp = beta.stats(self.s[:, self.I], self.f[:, self.I])
E_samp = np.random.beta(
self.s[:, self.I],
self.f[:, self.I]) # (6) CHANGE - TS SAMPLE HERE
#------------------------#
# Get E_XInt and V_XInt #
#------------------------#
E_xint, V_xint = self.get_xint()
# try:
# assert all(E_xint <= 1)
# assert all(E_xint >= 0)
# except:
# if self.algorithm_name in ['RDC*_edit_TS_2']:
# pass
# else:
# pass
#print('WARN: E_xint = {}'.format(E_xint))
#print('E_xint = {} \n p_wins = {} \n I = {}\n'.format(E_xint, self.p_wins, self.I))
#m_xint = [1.,1.,1.,1.]; var_xint = [1.,1.,1.,1.];
# Get E_XArm and V_XArm
#m_xarm, var_xarm = self.Heuristic_XArm()
#m_xarm = [1.,1.,1.,1.]; var_xarm = [1.,1.,1.,1.]
#------------------------#
# Get E_Comb #
#------------------------#
M = np.array([[E_samp, V_samp], [E_xint, V_xint]]) # E_samp etc: K
E_comb, V_comb = self.inverse_variance_weighting(M)
# try:
# assert all(E_comb <= 1)
# assert all(E_comb >= 0)
# except:
#
# # If E_comb is negative, estimate from available data
# #print('\n WARN: E_comb = {}'.format(E_comb))
# if self.algorithm_name in ['RDC*_edit_TS_2', 'RDC*_edit_NO_TS_3']:
# E_comb[np.where(E_comb <= 0)] = self.p_wins[np.where(
# E_comb <= 0), self.I]
# #V_comb[np.where(E_comb <= 0)] = V_samp[np.where(E_comb <= 0)]
# else:
# pass
#
# try:
# assert all(V_comb >= 0)
# except:
# print('\n WARN: V_comb = {}'.format(V_comb))
# Compute weighted s and f values based on above
#weightedS = totals * E_comb
#weightedF = totals * (1 - E_comb)
return E_comb, V_comb
#Reference:
#https://docs.scipy.org/doc/scipy-0.15.1/reference/generated/scipy.stats.beta.html
from scipy.stats import beta
import matplotlib.pyplot as plt
import numpy as np
#Set seed, init plots, get beta parameters
np.random.seed(seed=12345)
fig, ax = plt.subplots(1, 1)
priorA = 13
priorB = 783
#Summary stats of distribution
mean, var, skew, kurt = beta.stats(priorA, priorB, moments='mvsk')
#Plot pdf
x = np.linspace(beta.ppf(0.001, priorA, priorB),
beta.ppf(0.999, priorA, priorB), 1000)
ax.plot(x, beta.pdf(x, priorA, priorB), 'b-', lw=1, alpha=1, label='beta pdf')
#Check pdf vs cdf
vals = beta.ppf([0.001, 0.5, 0.999], priorA, priorB)
print(np.allclose([0.001, 0.5, 0.999], beta.cdf(vals, priorA, priorB))) #True
#Make plot pretty
ax.legend(loc='best', frameon=False)
ax.set_xlim([0.0, 0.1])
ax.set_ylim([0, 99])
plt.title('Posterior - Beta(3,783)')
from scipy.stats import gaussian_kde
from scipy.optimize import minimize
import numpy as np
import matplotlib.pyplot as plt
import numdifftools as nd
import seaborn as sns
# 2.2: Building a model: binomial likelihood with (flat) beta prior
data22 = ['W', 'L', 'W', 'W', 'W', 'L', 'W', 'L', 'W']
aPost = np.cumsum([1 if x == 'W' else 0 for x in data22]) + 1
aPost.shape
a = np.insert(aPost, 0, 1)
bPost = np.cumsum([0 if x == 'W' else 1 for x in data22]) + 1
b = np.insert(bPost, 0, 1)
beta.stats(a[0], b[0], moments='mvsk') # mean, var, skew, kurtosis (mvsk)
# update: {[p ^ x] * [(1 - p) ^ (n - x)]} * {[ p ^ (a - 1)] * [(1-p) ^ (b - 1)]} / B(a,b)
pRange = np.arange(0, 1, 0.01)
yVals = beta.pdf(pRange, a[0], b[0])
for i in range(1, len(a)):
yVals = np.append(yVals, beta.pdf(pRange, a[i], b[i]))
yMax = max(yVals)
plt.figure(1)
for i in range(0, len(a) - 1):
plotloc = '33' + str(i + 1)
plt.subplot(plotloc)
plt.plot(pRange, beta.pdf(pRange, a[i], b[i]), 'b--', pRange,
beta.pdf(pRange, a[i + 1], b[i + 1]), 'r-')
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import beta
np.random.seed(1)
# Beta Distribution
a, b = 3.0, 2.0
mean, var = beta.stats(a, b, moments='mv')
std = beta.std(a, b, loc=0, scale=1)
fig, ax = plt.subplots(1, 1)
x = np.linspace(beta.ppf(0.05, a, b), beta.ppf(0.95, a, b))
ax.plot(x, beta.pdf(x, a, b), 'b-', lw=3, alpha=0.6, label='Beta')
q1 = beta.ppf(.25, a, b)
median = beta.ppf(.5, a, b)
q3 = beta.ppf(.75, a, b)
plt.title(
'Beta Distribution ($\mu$: {:.2f}, $\sigma$: {:.2f}, $\sigma^2$: {:.2f})'.
format(mean, std, var),
size='xx-large')
plt.xlabel('X', size='large')
plt.ylabel('P(X)', size='large')
# Quartile lines
ax.axvline(x=q1, linewidth=3, alpha=0.6, color='black', linestyle='dashed')
ax.axvline(x=median, linewidth=3, alpha=0.6, color='black', linestyle='dashed')
ax.axvline(x=q3, linewidth=3, alpha=0.6, color='black', linestyle='dashed')
