def test_draw_samples_non_mock(self, plot=False):
# Also make sure the non-mock sampler works
dtype = np.float32
num_samples = 1000
low = np.array([0.5])
high = np.array([2])
rv_shape = (1,)
low_mx = mx.nd.array(low, dtype=dtype)
high_mx = mx.nd.array(high, dtype=dtype)
rand_gen = None
var = Uniform.define_variable(shape=rv_shape, rand_gen=rand_gen, dtype=dtype).factor
variables = {var.low.uuid: low_mx, var.high.uuid: high_mx}
rv_samples_rt = var.draw_samples(F=mx.nd, variables=variables, num_samples=num_samples)
assert array_has_samples(mx.nd, rv_samples_rt)
assert get_num_samples(mx.nd, rv_samples_rt) == num_samples
assert rv_samples_rt.dtype == dtype
if plot:
plot_univariate(samples=rv_samples_rt, dist=uniform, loc=low[0], scale=high[0] - low[0], buffer=1)
location_est, scale_est = uniform.fit(rv_samples_rt.asnumpy().ravel())
location_tol = 1e-2
scale_tol = 1e-2
assert np.abs(low[0] - location_est) < location_tol
assert np.abs(high[0] - scale_est - location_est) < scale_tol
def calculate_parameters(self, values):
"""
Calculate parameters of the current distribution
Parameters
-----------
values
Empirical values to work on
"""
if len(values) > 0:
self.loc, self.scale = uniform.fit(values)
def plotNormal(num):
data = uniform.rvs(0, 1.5, num)
plt.hist(data, bins=25, density=True)
mean, std = uniform.fit(data)
xmin, xmax = plt.xlim()
x = np.linspace(xmin, xmax, 100)
p = uniform.pdf(x, mean, std)
plt.plot(x, p, 'k', linewidth=2)
plt.show()
filename = "./P2figures/histogram_" + str(num)
print(filename)
plt.savefig(filename)
def fit(data: FloatIterable,
a: Optional[float] = None,
b: Optional[float] = None) -> 'ContinuousUniform':
"""
Fit a ContinuousUniform distribution to the data.
:param data: Iterable of data to fit to.
:param a: Optional fixed value for lower bound a.
:param b: Optional fixed value for upper bound b.
"""
kwargs = {}
for arg, kw in zip((a, b), ('fa', 'fb')):
if arg is not None:
kwargs[kw] = arg
loc, scale = uniform.fit(data=data, **kwargs)
return ContinuousUniform(a=loc, b=loc + scale)
def test_random_vars(self):
gen = libMHCUDA.minhash_cuda_init(1000, 128, devices=1, verbosity=2)
rs, ln_cs, betas = libMHCUDA.minhash_cuda_retrieve_vars(gen)
libMHCUDA.minhash_cuda_fini(gen)
cs = numpy.exp(ln_cs)
a, loc, scale = gamma.fit(rs)
self.assertTrue(1.97 < a < 2.03)
self.assertTrue(-0.01 < loc < 0.01)
self.assertTrue(0.98 < scale < 1.02)
a, loc, scale = gamma.fit(cs)
self.assertTrue(1.97 < a < 2.03)
self.assertTrue(-0.01 < loc < 0.01)
self.assertTrue(0.98 < scale < 1.02)
bmin, bmax = uniform.fit(betas)
self.assertTrue(0 <= bmin < 0.001)
self.assertTrue(0.999 <= bmax <= 1)
s = np.array(s)
E = s.T @ W @ s
print(s, E)
# ## Problem 9.34
# In[19]:
from scipy.stats import norm, uniform
D = np.array([0.2, 0.5, 0.4, 0.3, 0.9, 0.7, 0.6])
print('MLE for normal model:', norm.fit(D))
print('MLE for uniform model:', uniform.fit(D))
plt.figure(figsize=FIGSIZE)
plt.scatter(D, np.zeros_like(D), label='Data', color='k')
x = np.linspace(0, 1.25, num=2**10)
plt.plot(x, norm(*norm.fit(D)).pdf(x),
label=f'Normal. L = {round(np.log(norm(*norm.fit(D)).pdf(D)).sum(), 3)}')
plt.plot(x, uniform(*uniform.fit(D)).pdf(x),
label=f'Uniform. L = {round(np.log(uniform(*uniform.fit(D)).pdf(D)).sum(), 3)}')
plt.ylabel('$P(x)$')
plt.xlabel('$x$')
plt.legend()
import matplotlib.pyplot as plt
from statsmodels.stats.diagnostic import kstest_normal
import math
# ATERRIZAJES
#Cargamos las muestras de tiempo de las maniobras de aterrizajes
text_file = open("E4.aterrizajes.txt")
datos = np.array(text_file.read().split('\n'))
np.set_printoptions(threshold=np.nan)
datos = datos[:-1].astype(float)
#Ajustamos las muestras de tiempo a las diferentes distribuciones que queremos contrastar
parametros_normal = norm.fit(datos)
parametros_exponencial = expon.fit(datos)
parametros_uniforme = uniform.fit(datos)
#Mostramos los parámetros obtenidos del ajuste de las distintas distribuciones a los datos
print("=============ATERRIZAJES=============")
print("Ajuste de parámetros de una distribución normal: ", parametros_normal)
print("Ajuste de parámetros de una distribución exponencial: ",
parametros_exponencial)
print("Ajuste de parámetros de una distribución uniforme: ",
parametros_uniforme)
#Realizamos el contraste de las distribuciones que hemos obtenido del ajuste anterior con las muestras de tiempo
print(
"KS Test Normal: ",
stats.kstest(datos,
cdf='norm',
def downtime_accepted_models(D=list(), alpha=.05):
params = list()
params.append(uniform.fit(D))
params.append(expon.fit(D))
params.append(rayleigh.fit(D))
params.append(weibull_min.fit(D))
params.append(gamma.fit(D))
params.append(gengamma.fit(D))
params.append(invgamma.fit(D))
params.append(gompertz.fit(D))
params.append(lognorm.fit(D))
params.append(exponweib.fit(D))
llf_value = list()
llf_value.append(log(product(uniform.pdf(D, *params[0]))))
llf_value.append(log(product(expon.pdf(D, *params[1]))))
llf_value.append(log(product(rayleigh.pdf(D, *params[2]))))
llf_value.append(log(product(weibull_min.pdf(D, *params[3]))))
llf_value.append(log(product(gamma.pdf(D, *params[4]))))
llf_value.append(log(product(gengamma.pdf(D, *params[5]))))
llf_value.append(log(product(invgamma.pdf(D, *params[6]))))
llf_value.append(log(product(gompertz.pdf(D, *params[7]))))
llf_value.append(log(product(lognorm.pdf(D, *params[8]))))
llf_value.append(log(product(exponweib.pdf(D, *params[9]))))
AIC = list()
AIC.append(2 * len(params[0]) - 2 * llf_value[0])
AIC.append(2 * len(params[1]) - 2 * llf_value[1])
AIC.append(2 * len(params[2]) - 2 * llf_value[2])
AIC.append(2 * len(params[3]) - 2 * llf_value[3])
AIC.append(2 * len(params[4]) - 2 * llf_value[4])
AIC.append(2 * len(params[5]) - 2 * llf_value[5])
AIC.append(2 * len(params[6]) - 2 * llf_value[6])
AIC.append(2 * len(params[7]) - 2 * llf_value[7])
AIC.append(2 * len(params[8]) - 2 * llf_value[8])
AIC.append(2 * len(params[9]) - 2 * llf_value[9])
model = list()
model.append(
["uniform", params[0],
kstest(D, "uniform", params[0])[1], AIC[0]])
model.append(
["expon", params[1],
kstest(D, "expon", params[1])[1], AIC[1]])
model.append(
["rayleigh", params[2],
kstest(D, "rayleigh", params[2])[1], AIC[2]])
model.append([
"weibull_min", params[3],
kstest(D, "weibull_min", params[3])[1], AIC[3]
])
model.append(
["gamma", params[4],
kstest(D, "gamma", params[4])[1], AIC[4]])
model.append(
["gengamma", params[5],
kstest(D, "gengamma", params[5])[1], AIC[5]])
model.append(
["invgamma", params[6],
kstest(D, "invgamma", params[6])[1], AIC[6]])
model.append(
["gompertz", params[7],
kstest(D, "gompertz", params[7])[1], AIC[7]])
model.append(
["lognorm", params[8],
kstest(D, "lognorm", params[8])[1], AIC[8]])
model.append(
["exponweib", params[9],
kstest(D, "exponweib", params[9])[1], AIC[9]])
accepted_models = [i for i in model if i[2] > alpha]
if accepted_models:
aic_values = [i[3] for i in accepted_models]
final_model = min(range(len(aic_values)), key=aic_values.__getitem__)
return accepted_models, accepted_models[final_model]
elif not accepted_models:
aic_values = [i[3] for i in model]
final_model = min(range(len(aic_values)), key=aic_values.__getitem__)
return model, model[final_model]
plt.hist(DATOS, bins=BINS, alpha=0.7, color="lightblue", normed=True)
xt = plt.xticks()[0]
xmin, xmax = 0, max(xt)
lnspc = np.linspace(xmin, xmax, len(DATOS))
c, d = rayleigh.fit(DATOS)
modelo = rayleigh(c, d)
pdf_g = rayleigh.pdf(lnspc, c, d)
plt.plot(lnspc, pdf_g, 'k-', lw=5, alpha=1, color="blue", label='Rayleigh')
e, f = norm.fit(DATOS)
pdf_g = norm.pdf(lnspc, e, f)
plt.plot(lnspc, pdf_g, 'r-', lw=2, alpha=0.5, color="red", label='Normal')
g, h = uniform.fit(DATOS)
pdf_g = uniform.pdf(lnspc, g, h)
plt.plot(lnspc,
pdf_g,
'r-',
lw=2,
alpha=0.5,
color="black",
label='Uniforme')
i, j = expon.fit(DATOS)
pdf_g = expon.pdf(lnspc, i, j)
plt.plot(lnspc,
pdf_g,
'r-',
lw=2,
def plot_statistical_analysis(R, V, time_step, filename):
# Total kinetic energy and total velocity
V_calc1 = np.linalg.norm(V, axis=1) # Calculating sumations and norms accordingly
V_calc2 = (V_calc1 ** 2) # by the given exercise
E_k = np.sum(V_calc2, axis=1) / 2
V_tot = np.sum(V, axis=2)
V_tot = np.linalg.norm(V_tot, axis=1)
print(R.shape)
dists = R[:, :, np.newaxis, :] - R[:, :, :, np.newaxis] # Creating another axis to be able to compare the values
# And calculate the distances between particles
dists = np.linalg.norm(dists, axis=1) # Calculating norm
dists = np.reshape(dists, (400*400*1000, 1)) # Putting every value in a row
R = np.reshape(R, [400*1000, 2]) # reshape the matrixes so they can be plotted
R_x = R[:, 0] # Creating x and y vectors
R_y = R[:, 1]
V = np.reshape(V, [400*1000, 2])
V_x = V[:, 0] # Creating x and y vectors
V_y = V[:, 1]
V = np.linalg.norm(V, axis=1) # Creating the norm vector needed for plotting
time = "time"
t = np.arange(0, 20, time_step) # Predefinitions
plt.figure(tight_layout=True, figsize=[9., 6.])
# Average time
plt.subplot(421)
plt.plot(t, V_tot) # Everything here is pretty self explanatory
plt.xlabel(time) # plot time with wanted vector, V, dists and E_k
plt.ylabel("Average speed")
# Kinetic energy
plt.subplot(422)
plt.plot(t, E_k)
plt.ylabel("Kinetic energy")
plt.ylim([min(E_k)-0.2, max(E_k)+0.2])
plt.xlabel(time)
# Distance
plt.subplot(423)
plt.xlabel("distance") # Create the histograms with hist
plt.ylabel("Pair distribution\n probability") # and then input the wanted vectors
plt.xlim([-0.5, max(dists)+1])
plt.hist(dists, bins=50, density="True")
# Speed
plt.subplot(424)
plt.xlabel("speed")
plt.ylabel("Velocity norm\n probability")
plt.hist(V, bins="auto", density="True")
x = np.linspace(min(V), max(V), 100) # The vector needed to plot the pdf and reg. plot needs together with
loc_V, scale_V = rayleigh.fit(V) # Creating the values pdf needs
plt.plot(x, rayleigh.pdf(x, loc_V, scale_V))
# All the other pdfs are done
# x position # the same way
plt.subplot(425) # also am creating x.lims and y.lims
plt.xlabel("x position") # where they are needed
plt.ylabel("Probability")
plt.hist(R_x, bins="auto", density="True")
x = np.linspace(min(R_x)-0.5, max(R_x)+0.5, 100)
loc_R_x, scale_R_x = uniform.fit(R_x)
plt.plot(x, uniform.pdf(x, loc_R_x, scale_R_x))
# y position
plt.subplot(426)
plt.xlabel("y position")
plt.ylabel("Probability")
plt.hist(R_y, bins="auto", density="True")
x = np.linspace(min(R_y)-0.5, max(R_y)+0.5, 100)
loc_R_y, scale_R_y = uniform.fit(R_y)
plt.plot(x, uniform.pdf(x, loc_R_y, scale_R_y))
# x velocity
plt.subplot(427)
plt.xlabel("x velocity")
plt.ylabel("Probability")
plt.hist(V_x, bins="auto", density="True")
x = np.linspace(min(V_x)-0.2, max(V_x)+0.2, 100)
loc_v_x, scale_v_x = norm.fit(V_x)
plt.plot(x, norm.pdf(x, loc_v_x, scale_v_x))
# y velocity
plt.subplot(428)
plt.xlabel("y velocity")
plt.ylabel("Probability")
plt.hist(V_y, bins="auto", density="True")
x = np.linspace(min(V_y)-0.2, max(V_y)+0.2, 100)
loc_v_y, scale_v_y = norm.fit(V_y)
plt.plot(x, norm.pdf(x, loc_v_y, scale_v_y))
plt.savefig(filename)
plt.show()
loc = -4577.17
sc = 3357.76
Xjsu = johnsonsu.rvs(a, b, loc, sc, size=points, random_state=None)
print(johnsonsu.fit(Xjsu))
"""
Xjsu = np.array(q1)
#reverse Johnson TRANSFORMATION
Xjsu00 = (Xjsu - loc) / sc #remove scale, location
arcsinhX = np.log(Xjsu00 + (Xjsu00**2 + 1.)**0.5)
z = a + b * arcsinhX #remove a, b shape parameters: ~normally distributed
print(norm.fit(z))
u = norm.cdf(z) #uniformly distributed
print(uniform.fit(u))
#Normal fit:
[mean, var] = norm.fit(z)
print("Normal fit parameters:", mean, var)
#Calculate chi-square test statistic:
k = 32 #number of bins in histogram
#use numpy histogram for the expected value
observed, hist_binedges = np.histogram(z, bins=k)
#use the cumulative density function (c.d.f.)
#of the distribution for the expected value:
cdf = norm.cdf(hist_binedges, mean, var)
expected = len(z) * np.diff(cdf)
#use scipy.stats chisquare function, where
#ddof is the adjustment to the k-1 degrees of freedom,
align='mid',
label='Measured data')
#
# d.sort(reverse=True)
x = np.linspace(d_min, d_max, d_count)
# ax.plot(x, d)
# c = 37.3155
# k = 0.0214833
# param = burr.fit(d)
# y = burr.cdf(x, param[0], param[1])
# ax.plot(x,y,'r--', color='darkred', label='Burr distribution fit')
param = uniform.fit(d)
# y = uniform.pdf(x, param[0], param[1])
y = uniform.cdf(x, param[0], param[1])
ax.plot(x, y, 'r--', color='darkred', label='Uniform distribution fit')
# f = fitter(d)
# f.fit()
#
# print(f.summary())
# plt.gca().get_yaxis().get_major_formatter().set_powerlimits((0, 0))
# plt.gca().get_xaxis().get_major_formatter().set_powerlimits((0, 0))
# plt.xlim([d_min - abs(2*d_min), d_max + abs(2*d_min)])
ax.legend(loc='upper left', frameon=True, prop={'size': 12})
fig.tight_layout(pad=0.5, w_pad=0.5, h_pad=0.5)
def distribution_fitting(bids, agents, key, adv):
#count: number of bids processed
count = 0
## Number of Gaussians in the mixture model
num_gaussian = 5
##################################################
## Distribution characteristics for each agent
## range_phi_min : list of min of range of virtual valuation of all advertisers
## range_phi_max : list of max of range of virtual valuation of all advertisers
## cdf : list of cdf of all advertisers
## pdf : list of pdf of all advertisers
## inv_cdf : list of inverse cdf of all advertisers
## inv_phi : list of inverse pdf of all advertisers
##################################################
cdf = []
pdf = []
inv_cdf = []
inv_phi = []
range_phi_min = []
range_phi_max = []
## b: Hash list of (list of number of bids by top 50 advertisers bidding on the keyword)
## b[str(i)+"count"][0][j]: number of bids on keyword i by jth largest advertiser
## b[str(i)+"agent"][0][j]: id of jth largest advertiser on keyword i
b = sio.loadmat('data/find_agent_keyword')
print("on agent:", adv, flush=True)
# Top agents (in terms of number of bids) selected for keyword
# adv = b[str(key)+'agent'][0][i]
## Form list of bids by agent on the keyword
agent_bids = []
for j in range(len(bids)):
## Only take bids under $100
if agents[j] == adv and bids[j] < 100:
agent_bids.append(bids[j])
#if len(agent_bids)<1000:
# print("Error!!!",flush=True)
# return [cdf,pdf,inv_cdf,inv_phi,range_phi_min,count,-1]
##################################################
## Model fitting here
##################################################
x = np.asarray(agent_bids)
x = np.reshape(x, [len(agent_bids), 1])
# UNIFORM
loc, scale = uniform.fit(x)
scale = np.round(scale)
#
## Fit GaussianMixture with num_gaussian components
# g = mixture.GaussianMixture(n_components=num_gaussian)
# g.fit(x)
## Weights, means and variances of the gaussians
# w = np.reshape(g.weights_,[num_gaussian,1])
# m = g.means_
# var = g.covariances_
## Eliminating low variance agents
## Hard to approximate distributions for these.
## Incrementally reduce number of gausians if variance is low
# if np.min(var) < 0.003:
# num_gaussian = 3
# g = mixture.GaussianMixture(n_components=num_gaussian)
# g.fit(x)
# w = np.reshape(g.weights_,[num_gaussian,1])
# m = g.means_
# var = g.covariances_
# if np.min(var) < 0.003:
# num_gaussian = 1
# g = mixture.GaussianMixture(n_components=num_gaussian)
# g.fit(x)
# w = np.reshape(g.weights_,[num_gaussian,1])
# m = g.means_
# var = g.covariances_
# if np.min(var) < 0.003:
# ## Too low variance cannot include this bidder.
# print("Excluding advertiser: ",adv,"Variance for 1 gaussian is: ",np.min(var),flush=True)
# return [cdf,pdf,inv_cdf,inv_phi,range_phi_min,count,-1]
count += len(agent_bids)
min_x = -2
max_x = 4
samples = 6001
x_range = np.linspace(min_x, max_x, samples)
## Calculating pdf of distribution
# y = []
# for j in range(num_gaussian):
# y.append(np.exp(-(x_range[:-1]-m[j])**2/(2*var[j][0][0]))/(2*np.pi*var[j][0][0])**0.5)
# pdf_fitted = np.dot(np.transpose(w),y)[0]
pdf_fitted = uniform.pdf(x_range, loc, scale)
# # pdf_prime = np.dot(np.transpose(w),yprime)[0]
## Calculates the CDF
cum_values = np.zeros(x_range.shape)
cum_values[1:] = np.cumsum(pdf_fitted[:-1:] * np.diff(x_range))
## Hard code normalization
# cum_values[-1]=1
# cum_old = 0
for j in range(1, len(cum_values)): #lower break the distribution
if cum_values[j] > 0.0001:
cum_values = cum_values[j - 1:]
cum_values[0] = 0
x_range = x_range[j - 1:]
pdf_fitted = pdf_fitted[j - 1:]
break
## Hard code normalisation
# cum_values[-1]=1
## Helpful plotting functions
# import matplotlib.pyplot as plt
# print(cum_values)
# plt.scatter(x_range,pdf_fitted,s=1)
# plt.plot(x_range,pdf_prime)
# plt.scatter(x_range,cum_values,s=1)
# plt.show()
##################################################
## distributions
## x_range: linear range of valuations
## y: values of valuations corresponding to x_range
## y[i] = valuations for x_range[i]
## pdf_fitted: pdf of valuations
## cum_values: cdf of valuations
## cdfh: interpolated cdf of valuations, returns a function
## inv_cdfh: interpolated inverse of cdf of valuations, returns a function
## pdfh: interpolated pdf of valuations, returns a function
## phis: virtual valuations of for fitted pdf and cdf
## inv_phih: interpotaled inverse of phi
##################################################
cdfh = interpolate.interp1d(x_range,
cum_values,
fill_value=(0, 1),
bounds_error=False)
cum_values = sorted(cum_values)
inv_cdfh = interpolate.interp1d(cum_values, x_range)
pdfh = interpolate.interp1d(x_range,
pdf_fitted,
fill_value=(0, 0),
bounds_error=False)
# pdfprimeh = interpolate.interp1d(bin_edges2, pdf_prime)
## Calculate virtual valuation
def vval(i):
if cum_values[i] == 0:
return 0
if cum_values[i] == 1:
return x_range[i]
return x_range[i] - (1 - cum_values[i] / pdf_fitted[i])
phis = [vval(i) for i in range(len(x_range))]
# plt.scatter(x_range, pdfh(x_range), s=1)
# plt.scatter(x_range, phis, s=1)
# plt.title(f"pdf and vval key {key} adv {adv}")
# plt.xlim(0,4)
# plt.ylim(-4,4)
# plt.show()
for j in range(len(phis) - 1):
## Enforce virtual valuation to be strictly monotonic
if phis[j + 1] < phis[j]:
phis[j + 1] = phis[j] + 0.0001
## The functions are not invertible
inv_phih = interpolate.interp1d(phis, x_range)
range_phi_min.append(phis[0])
range_phi_max.append(phis[-1])
cdf.append(cdfh)
pdf.append(pdfh)
inv_cdf.append(inv_cdfh)
inv_phi.append(inv_phih)
return [cdf, pdf, inv_cdf, inv_phi, range_phi_min, count, 1]
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Wed Jun 3 21:10:38 2020 @author: luis """ from scipy.stats import uniform, norm, lognorm, expon import numpy as np x = [73, 18, 20, 60, 99, 81, 40, 33, 27, 0] a, b = uniform.fit(x) print(a,b) print(uniform.pdf(x, a,b)) print(uniform.cdf(x,a,b)) a, b = norm.fit(x) print(a,b) print(norm.pdf(x, a,b)) print(norm.cdf(x,a,b)) #a, b, s = lognorm.fit(x, floc=0) #print(a,b,s) #print(lognorm.pdf(x, a,b,s)) #print(lognorm.cdf(x,a,b,s)) #np.log(a), b # mu, sigma a = expon.fit(x) print(a)