def predband(x, xd, yd, p, func, conf=0.95):
"""
This function estimates the prediction bands for the specified function
https://codereview.stackexchange.com/questions/84414/obtaining-prediction-bands-for-regression-model
"""
# x = requested points
# xd = x data
# yd = y data
# p = parameters
# func = function name
alpha = 1.0 - conf # significance
N = len(xd) # data sample size
var_n = len(p) # number of parameters
# Quantile of Student's t distribution for p=(1-alpha/2)
from scipy.stats.distributions import t
q = t.ppf(1.0 - alpha / 2.0, N - var_n)
# Stdev of an individual measurement
se = np.sqrt(1. / (N - var_n) * \
np.sum((yd - func(xd, *p)) ** 2))
# Auxiliary definitions
sx = (x - xd.mean())**2
sxd = np.sum((xd - xd.mean())**2)
# Predicted values (best-fit model)
yp = func(x, *p)
# Prediction band
dy = q * se * np.sqrt(1.0 + (1.0 / N) + (sx / sxd))
# Upper & lower prediction bands.
lpb, upb = yp - dy, yp + dy
return lpb, upb
def fit_curves(g, fn, arg_names, p0):
vals = g.dropna()
output = None
try:
x = np.tile(timepoints, len(vals))
y = vals.values.flatten()
popt, pcov = curve_fit(fn, x, y, p0=p0, bounds=bounds)
# method of extracting parameter CI values taken from
# http://kitchingroup.cheme.cmu.edu/blog/2013/02/12/Nonlinear-curve-fitting-with-parameter-confidence-intervals/
# other ref:
# https://stackoverflow.com/a/60412600/383744
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(y) # number of data points
p = len(popt) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
tval = t_distribution.ppf(1.0 - alpha / 2., dof)
var = np.diag(pcov)
sigma_k = np.sqrt(var[-1])
ci_k = tval * sigma_k
model_predictions = fn(x, *popt)
abs_error = model_predictions - y
r_squared = 1 - (np.var(abs_error) / np.var(y))
output = [*popt, r_squared, ci_k]
except (ValueError, RuntimeError) as e:
output = [np.nan] * len(arg_names)
return pd.Series(dict(zip(arg_names, output)))
def nlinfit(model, x, y, p0, alpha=0.05):
'''Nonlinear regression with confidence intervals.
x is the independent data
y is the dependent data
model has a signature of f(x, p0, p1, p2, ...)
p0 is the initial guess of the parameters
returns [p, pint, SE]
p is an array of the fitted parameters
pint is an array of confidence intervals
SE is an array of standard errors for the parameters.
'''
pars, pcov = curve_fit(model, x, y, p0=p0)
n = len(y) # number of data points
p = len(pars) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
SE = []
pint = []
for i, p,var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
SE.append(sigma)
pint.append([p - sigma * tval, p + sigma * tval])
return (pars, np.array(pint), np.array(SE))
def conf95(stdev, Ndata, Npar):
"""Calculate (one half) of the symmetric 95% confidence interval.
The symmetric 95% confidence interval is calculated from the standard
deviation and number of degrees of freedom, using Student's t
distribution.
Parameters
----------
stdev : float
Standard deviation.
Ndata : int
Number of data points.
Npar : int
Number of parameters.
Returns
-------
float
The half-width of the 95% confidence interval, such that it
can be reported in the tradtional +/- manner.
based on:
http://kitchingroup.cheme.cmu.edu/blog/2013/02/12/Nonlinear-curve-fitting-with-parameter-confidence-intervals/
"""
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
dof = max(0, Ndata - Npar)
tval = student_t.ppf(1.0 - alpha / 2., dof)
return (stdev * tval)
def compute_coefficient_intervals( y_test, Theta ):
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(y_test) # number of data points
p = len(Theta) # number of parameters
df = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., df)
def stats(n, p_opt, p_cov, p_labels, alpha=0.05):
"""
For explanations on this, see:
https://www.mathworks.com/help/curvefit/confidence-and-prediction-bounds.html
http://kitchingroup.cheme.cmu.edu/blog/2013/02/12/Nonlinear-curve-fitting-with-parameter-confidence-intervals/
"""
# Compute p err -------------------------
try:
p_err = np.sqrt(np.diag(p_cov))
except FloatingPointError:
d = np.diag(p_cov)
d.setflags(write=True)
d[d < 0] = 0
p_err = np.sqrt(d)
# Compute t -----------------------------
p = len(p_opt) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# Inverse of Student's t cumulative distribution function
tval = t.ppf(1.0 - alpha / 2., dof)
# Compute ci ---------------------------
r = {}
for i, (pr, std) in enumerate(zip(p_opt, p_err)):
ci = std * tval
print(f'{PARAM_LABELS[i]}: {pr:.2f} [{pr - ci:.2f} {pr + ci:.2f}]')
r[f"{p_labels[i]}-CI"] = (pr - ci, pr + ci)
return r
def fitSingle(xdata, ydata, guess):
try:
popt, pcov = curve_fit(singleExp, xdata.T, ydata.T, p0=guess)
print('a = ' + str(popt[0]))
print('t1 = '+ str(popt[1]))
print('offset = '+str(popt[2]))
#np.savetxt(f+'_fit.csv', popt, delimiter = ',')
#np.savetxt(f+'_fit_cov.csv', popt, delimiter = ',')
# from http://kitchingroup.cheme.cmu.edu/blog/2013/02/12/Nonlinear-curve-fitting-with-parameter-confidence-intervals/
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(ydata) # number of data points
p = len(popt) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
params = []
for i, p,var in zip(range(n), popt, np.diag(pcov)):
sigma = var**0.5
params.append('p{0}: {1} [{2} {3}]'.format(i, p,p - sigma*tval,p + sigma*tval))
#print(params)
#with open(f+'_fit_param_confidence_int.txt', 'w') as newfile:
# newfile.write('\n'.join(params))
return(popt, pcov, params)
except RuntimeError:
print("Error - curve_fit failed")
return(0,0,0)
def trimmed_mean_ci(data, limits=(0.2,0.2), inclusive=(True,True),
alpha=0.05, axis=None):
"""Returns the selected confidence interval of the trimmed mean along the
given axis.
Parameters
----------
data : sequence
Input data. The data is transformed to a masked array
proportiontocut : float
Proportion of the data to cut from each side of the data .
As a result, (2*proportiontocut*n) values are actually trimmed.
alpha : float
Confidence level of the intervals.
inclusive : tuple of boolean
If relative==False, tuple indicating whether values exactly equal to the
absolute limits are allowed.
If relative==True, tuple indicating whether the number of data being masked
on each side should be rounded (True) or truncated (False).
axis : int
Axis along which to cut. If None, uses a flattened version of the input.
"""
data = ma.array(data, copy=False)
trimmed = mstats.trimr(data, limits=limits, inclusive=inclusive, axis=axis)
tmean = trimmed.mean(axis)
tstde = mstats.trimmed_stde(data,limits=limits,inclusive=inclusive,axis=axis)
df = trimmed.count(axis) - 1
tppf = t.ppf(1-alpha/2.,df)
return np.array((tmean - tppf*tstde, tmean+tppf*tstde))
def trimmed_mean_ci(data, limits=(0.2, 0.2), inclusive=(True, True), alpha=0.05, axis=None):
"""Returns the selected confidence interval of the trimmed mean along the
given axis.
Parameters
----------
data : sequence
Input data. The data is transformed to a masked array
proportiontocut : float
Proportion of the data to cut from each side of the data .
As a result, (2*proportiontocut*n) values are actually trimmed.
alpha : float
Confidence level of the intervals.
inclusive : tuple of boolean
If relative==False, tuple indicating whether values exactly equal to the
absolute limits are allowed.
If relative==True, tuple indicating whether the number of data being masked
on each side should be rounded (True) or truncated (False).
axis : int
Axis along which to cut. If None, uses a flattened version of the input.
"""
data = ma.array(data, copy=False)
trimmed = mstats.trimr(data, limits=limits, inclusive=inclusive, axis=axis)
tmean = trimmed.mean(axis)
tstde = mstats.trimmed_stde(data, limits=limits, inclusive=inclusive, axis=axis)
df = trimmed.count(axis) - 1
tppf = t.ppf(1 - alpha / 2.0, df)
return np.array((tmean - tppf * tstde, tmean + tppf * tstde))
def predict(self, x_eval, alpha=0.05):
"""
0.05 means 95 % confidence interval
"""
self.fit(lower=self.lower, upper=self.upper)
n = self.n
p = self.p
print r('fit')
dof = n - p
tval = t.ppf(1 - alpha / 2., dof)
y_hat = np.empty(x_eval.shape)
y_hat_se = np.empty(x_eval.shape)
for i in xrange(len(x_eval)):
li = self.eq.rsplit('x', 1)
eq_x = ('%f' % x_eval[i]).join(li)
#eq_x = self.eq.replace('x','%f'%x_eval[i])
#foo = np.array(r('delta.method(fit, "%s")'%eq_x))
foo = np.array(r('deltaMethod(fit, "%s")' % eq_x))
y_hat[i] = foo[0]
y_hat_se[i] = foo[1]
y_ul = y_hat + tval * y_hat_se
y_ll = y_hat - tval * y_hat_se
return y_hat, y_ul, y_ll
def interval_confidence(self, x_tot, y_tot):
def retta(x, p0, p1):
return p0*x + p1
par1, par2 = curve_fit(retta, x_tot, y_tot)
y_fit2 = retta(x_tot, par1[0], par1[1])
m = par1[0]
q = par1[1]
# residual = y_tot - y_fit2
# ss_res = np.sum(residual**2)
ss_tot = np.sum((y_tot - np.mean(y_tot))**2)
if ss_tot == 0:
ss_tot = 1
# ss_res = 1
# r2 = 1- (ss_res/ss_tot)
# p = len(par1)
# n = len(x_tot)
alpha = 0.05 #95% confidence interval
dof = max(0, len(x_tot) - len(par1)) #degree of freedom
tval = tstud.ppf(1.0 - alpha/2., dof) #t-student value for the dof and confidence level
sigma = np.diag(par2)**0.5
m_err = sigma[0]*tval
q_err = sigma[1]*tval
y_fit2_up = retta(x_tot, m+(m_err/2), q+(q_err/2))
y_fit2_down = retta(x_tot, m-(m_err/2), q-(q_err/2))
return y_fit2, m, m_err, q, q_err, y_fit2_up, y_fit2_down
def grubbs_test(xvec, sgnf, null='equal'):
"""
The Grubbs' test for outlier detection
Parameters
----------
xvec : ndarray, shape (n,)
Array of sample values.
sgnf : float
Level of significance for the test.
null : str
Options.
Notes
-----
Specification of the null hypothesis can either be 'equal'
(default), 'greater', or 'less'. Only 'equal' is implemented.
Missings (i.e. NaN values) are ignored.
"""
if np.not_equal(np.nanvar(xvec), 0):
return None
n = sum(~np.isnan(xvec))
m = np.nanmean(xvec)
s = np.sqrt(np.nanvar(xvec))
candidates = abs(xvec - m)
gstat = np.nanmax(candidates) / s
tstat = t.ppf(q=1 - (sgnf / (2 * n)), df=n - 2)**2
crval = ((n - 1) / np.sqrt(n)) * np.sqrt(tstat / (n - 2 + tstat))
if (gstat > crval):
return np.nanargmax(candidates)
else:
return None
def GetExpFit(dDistr, infLimit, supLimit, alpha=0.05):
from scipy.stats.distributions import t
from scipy.optimize import curve_fit
dDistr_Log = np.zeros((supLimit - infLimit, 2))
for k in range(supLimit - infLimit):
dDistr_Log[k, 0] = k + infLimit
dDistr_Log[k, 1] = dDistr[k + infLimit]
def exponenial_func(x, a, b, c):
return a * np.power(x, -b) + c
popt, pcov = curve_fit(exponenial_func,
dDistr_Log[:, 0],
dDistr_Log[:, 1],
p0=(1.0, 1.0, 1.0))
perr = np.sqrt(np.diag(pcov))
n = supLimit - infLimit # number of data points
p = 3 # number of parameters
dof = max(0, n - p) # number of degrees of freedom
tval = t.ppf(1.0 - alpha / 2., dof)
Interval = perr[1]
corrInterval = perr[1] * tval
Exp = popt[1]
return Exp, Interval, corrInterval
def errors(estimates, covariance, dof, alpha=0.05):
"""Calculate approximate SD and confidence limits. Default: 95%"""
tval = t.ppf(1.0 - alpha / 2., dof) #student-t value
sigma = np.sqrt(np.diag(covariance))
confidence_limits = np.array(
[estimates - sigma * tval, estimates + sigma * tval]).T
return sigma, confidence_limits
def get_interval(self):
self.degrees_of_freedom = self.trials - 1
half_alpha = (1 - (self.confidence_interval * 10**(-2))) / 2
t_val = tdist.ppf(1 - half_alpha, self.degrees_of_freedom)
error_val = (sqrt((self.p_hat * (1 - self.p_hat)) / self.trials))
interval = (self.p_hat - (t_val * error_val),
self.p_hat + (t_val * error_val))
return interval
def NewState(self):
global START_TIME
power = self.autoscaler.PowerRequestsPerSecond() * 60
power_per_node = (power / len(self.autoscaler.nodes))
# how many requests I got in the last minute
last_requests = sum(self.autoscaler.num_requests)
if self.use_predictions:
x = range(1, 13)
slope, intercept, r_value, p_value, std_err = stats.linregress(
x, self.autoscaler.num_requests)
x = np.arange(1, 13)
y = np.array(self.autoscaler.num_requests)
y_err = y - (intercept + slope * x)
p_x = np.arange(13, 25)
mean_x = np.mean(x)
alpha = 0.05
n = len(x)
t_ = t.ppf(1 - alpha / 2, n - 2)
s_err = np.sum(np.power(y_err, 2))
confs = t_ * np.sqrt((s_err / (n - 2)) * (1.0 / n + (np.power(
(p_x - mean_x), 2) / ((np.sum(np.power(x, 2))) - n *
(np.power(mean_x, 2))))))
p_y = slope * p_x + intercept
lower = p_y - abs(confs)
upper = p_y + abs(confs)
prediction = sum(
[max(0, intercept + slope * x) for x in range(13, 25)])
print 'Last requests:', last_requests
self.out.write('%s Last requests: %s prediction: %s upper: %s\n' %
(time.time() - START_TIME, last_requests,
prediction, sum(upper)))
print 'Prediction: ', prediction, 'Upper:', sum(
upper), 'Lower: ', sum(lower)
# I'm being conservative here, and not decreasing the number of nodes
# before the load goes down. I'm also using the upper bound of 95%
# confidence interval.
last_requests = max(sum(upper), last_requests)
# last_requests = max(prediction, last_requests)
print 'Current power:', power, 'requests to handle:', prediction
min_cost = sys.maxint
n_nodes = 1
nodes = [x for x in self.autoscaler.possible_nodes if not x.IsDown()]
for i in range(1, len(nodes) + 1):
cost = self.ExpectedCost(i, power_per_node, last_requests)
print i, 'nodes, P(SLA): ', self.ProbSLAViolation(
i, power_per_node, last_requests),
print 'cost:', cost
if cost < min_cost:
min_cost, n_nodes = cost, i
nodes = nodes[:n_nodes]
state = {x.name: str(x.AvgPredictionTime()) for x in nodes}
self.autoscaler.nodes = nodes
sys.stdout.flush()
return state
def fit_curve(f, X, Y, alpha=.05):
x, y = map(np.asarray, [X, Y])
pars, pcov = curve_fit(f, x, y)
n = len(y)
p = len(pars)
dof = max(0, n - p)
tval = t.ppf(1.0 - alpha / 2., dof)
conf = [tval * elem**.5 for elem in np.diag(pcov)]
return pars, conf
def regress(A, y, alpha=None):
'''Linear regression with confidence intervals.
A is a matrix of function values in columns, e.g.
A = np.column_stack([T**0, T**1, T**2, T**3, T**4])
y is a vector of values you want to fit
alpha is for the 100*(1 - alpha) confidence level
returns: [b, bint, se]
b is a vector of the fitted parameters
bint is a 2D array of confidence intervals
se is an array of standard error for each parameter.
The confidence intervals account for sample size using a student T
multiplier.
This code is derived from the descriptions at
http://www.weibull.com/DOEWeb/confidence_intervals_in_multiple_linear_regression.htm
and
http://www.weibull.com/DOEWeb/estimating_regression_models_using_least_squares.htm
'''
b, res, rank, s = np.linalg.lstsq(A, y)
bint, se = None, None
if alpha is not None:
# compute the confidence intervals
n = len(y)
k = len(b)
errors = y - np.dot(A, b)
sigma2 = np.sum(errors**2) / (n - k) # RMSE
covariance = np.linalg.inv(np.dot(A.T, A))
C = sigma2 * covariance
dC = np.diag(C)
if (dC < 0.0).any():
warnings.warn('\n{0}\ndetected a negative number in your'
'covariance matrix. Taking the absolute value'
'of the diagonal. something is probably wrong'
'with your data or model'.format(dC))
dC = np.abs(dC)
se = np.sqrt(dC) # standard error
sT = t.ppf(1.0 - alpha/2.0, n - k - 1) # student T multiplier
CI = sT * se
bint = np.array([(beta - ci, beta + ci) for beta, ci in zip(b, CI)])
return (b, bint, se)
def trimmed_mean_ci(data,
limits=(0.2, 0.2),
inclusive=(True, True),
alpha=0.05,
axis=None):
"""
Selected confidence interval of the trimmed mean along the given axis.
Parameters
----------
data : array_like
Input area_data.
limits : {None, tuple}, optional
None or a two item tuple.
Tuple of the percentages to cut on each side of the array, with respect
to the number of unmasked area_data, as floats between 0. and 1. If ``n``
is the number of unmasked area_data before trimming, then
(``n * limits[0]``)th smallest area_data and (``n * limits[1]``)th
largest area_data are masked. The total number of unmasked area_data after
trimming is ``n * (1. - sum(limits))``.
The value of one limit can be set to None to indicate an open interval.
Defaults to (0.2, 0.2).
inclusive : (2,) tuple of boolean, optional
If relative==False, tuple indicating whether values exactly equal to
the absolute limits are allowed.
If relative==True, tuple indicating whether the number of area_data being
masked on each side should be rounded (True) or truncated (False).
Defaults to (True, True).
alpha : float, optional
Confidence level of the intervals.
Defaults to 0.05.
axis : int, optional
Axis along which to cut. If None, uses a flattened version of `area_data`.
Defaults to None.
Returns
-------
trimmed_mean_ci : (2,) ndarray
The lower and upper confidence intervals of the trimmed area_data.
"""
data = ma.array(data, copy=False)
trimmed = mstats.trimr(data, limits=limits, inclusive=inclusive, axis=axis)
tmean = trimmed.mean(axis)
tstde = mstats.trimmed_stde(data,
limits=limits,
inclusive=inclusive,
axis=axis)
df = trimmed.count(axis) - 1
tppf = t.ppf(1 - alpha / 2., df)
return np.array((tmean - tppf * tstde, tmean + tppf * tstde))
def do_curve_fit(func, x, y, alpha=0.05):
x = np.array(x, dtype=np.int64)
y = np.array(y, dtype=np.int64)
coeffs, pcov = curve_fit(func, x, y)
n = len(y)
p = len(coeffs)
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2.0, dof)
cl = np.multiply(np.sqrt(np.diag(pcov)), tval)
return coeffs, cl
def sample(self, x, use_stddev=False):
mu, nu, alpha, beta = self.B[x, :]
scale = np.square(self.w) * max(0, beta * (nu + 1) / (alpha * nu)) # Make sure that the scale is >= 0
df = 2 * alpha
try:
r = t.ppf(q=self.q, df=df, loc=mu, scale=scale)
except:
print(scale)
exit()
b = (r - mu)
return b
def pcov_t_cvft(ydata_to_fit,p_cvft,pcov_cvft,alpha):
# diagonal elements of the covariance matrix gives sigma
delta_p_cvft=np.sqrt(np.diag(pcov_cvft))
# degrees of freedom, should be positive or zero
dof=len(ydata_to_fit)-len(p_cvft)
# get t value from t distribution
from scipy.stats.distributions import t
tval=t.ppf(1.0-alpha/2.0,dof)
# calc t distribution corrected sigma
delta_p_cvft=delta_p_cvft*tval*2
return delta_p_cvft
def confint(n: int,
pars: np.ndarray,
pcov: np.ndarray,
confidence: float = 0.95,
**kwargs):
"""
This function returns the confidence interval for each parameter
Parameters
----------
n : int
The number of data points
pars : [double]
The array with the fitted parameters
pcov : [double]
The covariance matrix
confidence : float
The confidence interval
Returns
-------
np.ndarray
ci: The matrix with the confindence intervals for the parameters
Note:
Adapted from
http://kitchingroup.cheme.cmu.edu/blog/2013/02/12/Nonlinear-curve-fitting-with-parameter-confidence-intervals/
Copyright (C) 2013 by John Kitchin.
https://kite.com/python/examples/702/scipy-compute-a-confidence-interval-from-a-dataset
"""
is_log = kwargs.get('is_log', False)
from scipy.stats.distributions import t
if is_log:
p = np.power(10, pars)
pcov = np.power(10, pcov)
p = len(pars) # number of data points
dof = max(0, n - p) # number of degrees of freedom
# Quantile of Student's t distribution for p=(1 - alpha/2)
# tval = t.ppf((1.0 + confidence)/2.0, dof)
alpha = 1.0 - confidence
tval = t.ppf(1.0 - alpha / 2.0, dof)
ci = np.zeros((p, 2), dtype=np.float64)
for i, p, var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
ci[i, :] = [p - sigma * tval, p + sigma * tval]
return ci
def compute_error(y,pars,pcov,alpha=0.05):
# 95% confidence interval = 100*(1-alpha)
n = len(y) # number of data points
p = len(pars) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
for i, p,var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
print(p - sigma*tval,p + sigma*tval)
def regress(A, y, alpha=None):
'''Linear regression with confidence intervals.
A is a matrix of function values in columns, e.g.
A = np.column_stack([T**0, T**1, T**2, T**3, T**4])
y is a vector of values you want to fit
alpha is for the 100*(1 - alpha) confidence level
returns: [b, bint, se]
b is a vector of the fitted parameters
bint is a 2D array of confidence intervals
se is an array of standard error for each parameter.
The confidence intervals account for sample size using a student T multiplier.
This code is derived from the descriptions at http://www.weibull.com/DOEWeb/confidence_intervals_in_multiple_linear_regression.htm and http://www.weibull.com/DOEWeb/estimating_regression_models_using_least_squares.htm
'''
b, res, rank, s = np.linalg.lstsq(A, y)
bint, se = None, None
if alpha is not None:
# compute the confidence intervals
n = len(y)
k = len(b)
errors = y - np.dot(A, b)
sigma2 = np.sum(errors**2) / (n - k) # RMSE
covariance = np.linalg.inv(np.dot(A.T, A))
C = sigma2 * covariance
dC = np.diag(C)
if (dC < 0.0).any():
warnings.warn('\n{0}\ndetected a negative number in your'
'covariance matrix. Taking the absolute value'
'of the diagonal. something is probably wrong'
'with your data or model'.format(dC))
dC = np.abs(dC)
se = np.sqrt(dC) # standard error
sT = t.ppf(1.0 - alpha/2.0, n - k - 1) # student T multiplier
CI = sT * se
bint = np.array([(beta - ci, beta + ci) for beta,ci in zip(b,CI)])
return (b, bint, se)
def NewState(self):
global START_TIME
power = self.autoscaler.PowerRequestsPerSecond() * 60
power_per_node = (power / len(self.autoscaler.nodes))
# how many requests I got in the last minute
last_requests = sum(self.autoscaler.num_requests)
if self.use_predictions:
x = range(1,13)
slope, intercept, r_value, p_value, std_err = stats.linregress(x, self.autoscaler.num_requests)
x = np.arange(1,13)
y = np.array(self.autoscaler.num_requests)
y_err = y - (intercept + slope * x)
p_x = np.arange(13, 25)
mean_x = np.mean(x)
alpha = 0.05
n = len(x)
t_ = t.ppf(1 - alpha / 2, n - 2)
s_err = np.sum(np.power(y_err,2))
confs = t_ * np.sqrt((s_err/(n-2))*(1.0/n + (np.power((p_x-mean_x),2)/
((np.sum(np.power(x,2)))-n*(np.power(mean_x,2))))))
p_y = slope*p_x+intercept
lower = p_y - abs(confs)
upper = p_y + abs(confs)
prediction = sum([max(0, intercept + slope * x) for x in range(13,25)])
print 'Last requests:', last_requests
self.out.write('%s Last requests: %s prediction: %s upper: %s\n' % (time.time() - START_TIME, last_requests, prediction, sum(upper)))
print 'Prediction: ', prediction, 'Upper:', sum(upper), 'Lower: ', sum(lower)
# I'm being conservative here, and not decreasing the number of nodes
# before the load goes down. I'm also using the upper bound of 95%
# confidence interval.
last_requests = max(sum(upper), last_requests)
# last_requests = max(prediction, last_requests)
print 'Current power:', power, 'requests to handle:', prediction
min_cost = sys.maxint
n_nodes = 1
nodes = [x for x in self.autoscaler.possible_nodes if not x.IsDown()]
for i in range(1, len(nodes) + 1):
cost = self.ExpectedCost(i, power_per_node, last_requests)
print i, 'nodes, P(SLA): ', self.ProbSLAViolation(i, power_per_node, last_requests),
print 'cost:', cost
if cost < min_cost:
min_cost, n_nodes = cost, i
nodes = nodes[:n_nodes]
state = {x.name:str(x.AvgPredictionTime()) for x in nodes}
self.autoscaler.nodes = nodes
sys.stdout.flush()
return state
def predband(x: np.ndarray,
xd: np.ndarray,
yd: np.ndarray,
p: np.ndarray,
func: Callable[[np.ndarray, np.ndarray], np.ndarray],
conf: float = 0.95):
"""
This function estimates the prediction bands for the specified function without using the jacobian of the fit
https://codereview.stackexchange.com/questions/84414/obtaining-prediction-bands-for-regression-model
Parameters
----------
x: np.ndarray
The requested data points for the prediction bands
xd: np.ndarray
The experimental values for x
yd: np.ndarray
The experimental values for y
p: np.ndarray
The fitted parameters
func: Callable[[np.ndarray, np.ndarray], np.ndarray]
The optimized function
conf: float
The confidence level
Returns
-------
np.ndarray:
yp: The value of the function at the requested points (x)
np.ndarray:
lpb: The lower prediction band
np.ndarray:
upb: The upper prediction band
"""
alpha = 1.0 - conf # significance
npoints = len(xd) # data sample size
var_n = len(p) # number of parameters
# Quantile of Student's t distribution for p=(1-alpha/2)
from scipy.stats.distributions import t
q = t.ppf(1.0 - alpha / 2.0, npoints - var_n)
# Stdev of an individual measurement
se = np.sqrt(1. / (npoints - var_n) * np.sum((yd - func(xd, *p))**2))
# Auxiliary definitions
sx = (x - xd.mean())**2
sxd = np.sum((xd - xd.mean())**2)
# Predicted values (best-fit model)
yp = func(x, *p)
# Prediction band
dy = q * se * np.sqrt(1.0 + (1.0 / npoints) + (sx / sxd))
# Upper & lower prediction bands.
lpb, upb = yp - dy, yp + dy
return yp, lpb, upb
def nlinfit(model, x, y, p0, alpha=0.05):
"""Nonlinear regression with confidence intervals.
Parameters
----------
model : function f(x, p0, p1, ...) = y
x : array of the independent data
y : array of the dependent data
p0 : array of the initial guess of the parameters
alpha : 100*(1 - alpha) is the confidence interval
i.e. alpha = 0.05 is 95% confidence
Example
-------
Fit a line \(y = mx + b\) to some data.
>>> import numpy as np
>>> def f(x, m, b):
... return m * x + b
...
>>> X = np.array([0, 1, 2])
>>> y = np.array([0, 2, 4])
>>> nlinfit(f, X, y, [0, 1])
(array([ 2.00000000e+00, -2.18062024e-12]), array([[ 2.00000000e+00, 2.00000000e+00],
[ -2.18315458e-12, -2.17808591e-12]]), array([ 1.21903752e-12, 1.99456367e-16]))
Returns
-------
[p, pint, SE]
p is an array of the fitted parameters
pint is an array of confidence intervals
SE is an array of standard errors for the parameters.
"""
pars, pcov = curve_fit(model, x, y, p0=p0)
n = len(y) # number of data points
p = len(pars) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0 - alpha / 2., dof)
SE = []
pint = []
for i, p, var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
SE.append(sigma)
pint.append([p - sigma * tval, p + sigma * tval])
return (pars, np.array(pint), np.array(SE))
def nlinfit(model, x, y, p0, alpha=0.05):
"""Nonlinear regression with confidence intervals.
Parameters
----------
model : function f(x, p0, p1, ...) = y
x : array of the independent data
y : array of the dependent data
p0 : array of the initial guess of the parameters
alpha : 100*(1 - alpha) is the confidence interval
i.e. alpha = 0.05 is 95% confidence
Example
-------
Fit a line \(y = mx + b\) to some data.
>>> import numpy as np
>>> def f(x, m, b):
... return m * x + b
...
>>> X = np.array([0, 1, 2])
>>> y = np.array([0, 2, 4])
>>> nlinfit(f, X, y, [0, 1])
(array([ 2.00000000e+00, -2.18062024e-12]), array([[ 2.00000000e+00, 2.00000000e+00],
[ -2.18315458e-12, -2.17808591e-12]]), array([ 1.21903752e-12, 1.99456367e-16]))
Returns
-------
[p, pint, SE]
p is an array of the fitted parameters
pint is an array of confidence intervals
SE is an array of standard errors for the parameters.
"""
pars, pcov = curve_fit(model, x, y, p0=p0)
n = len(y) # number of data points
p = len(pars) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
SE = []
pint = []
for i, p, var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
SE.append(sigma)
pint.append([p - sigma * tval, p + sigma * tval])
return (pars, np.array(pint), np.array(SE))
def fit_curve(f, X, Y, alpha=.05):
x, y = list(map(np.asarray, [X, Y]))
try:
# print x, y
pars, pcov = curve_fit(f, x, y)
except:
pars, pcov = curve_fit(f, x, y, method='dogbox')
n = len(y)
p = len(pars)
dof = max(0, n - p)
tval = t.ppf(1.0 - alpha / 2., dof)
conf = [tval * elem**.5 for elem in np.diag(pcov)]
return pars, conf
def trimmed_mean_ci(data, limits=(0.2,0.2), inclusive=(True,True),
alpha=0.05, axis=None):
"""
Selected confidence interval of the trimmed mean along the given axis.
Parameters
----------
data : array_like
Input data.
limits : {None, tuple}, optional
None or a two item tuple.
Tuple of the percentages to cut on each side of the array, with respect
to the number of unmasked data, as floats between 0. and 1. If ``n``
is the number of unmasked data before trimming, then
(``n * limits[0]``)th smallest data and (``n * limits[1]``)th
largest data are masked. The total number of unmasked data after
trimming is ``n * (1. - sum(limits))``.
The value of one limit can be set to None to indicate an open interval.
Defaults to (0.2, 0.2).
inclusive : (2,) tuple of boolean, optional
If relative==False, tuple indicating whether values exactly equal to
the absolute limits are allowed.
If relative==True, tuple indicating whether the number of data being
masked on each side should be rounded (True) or truncated (False).
Defaults to (True, True).
alpha : float, optional
Confidence level of the intervals.
Defaults to 0.05.
axis : int, optional
Axis along which to cut. If None, uses a flattened version of `data`.
Defaults to None.
Returns
-------
trimmed_mean_ci : (2,) ndarray
The lower and upper confidence intervals of the trimmed data.
"""
data = ma.array(data, copy=False)
trimmed = mstats.trimr(data, limits=limits, inclusive=inclusive, axis=axis)
tmean = trimmed.mean(axis)
tstde = mstats.trimmed_stde(data,limits=limits,inclusive=inclusive,axis=axis)
df = trimmed.count(axis) - 1
tppf = t.ppf(1-alpha/2.,df)
return np.array((tmean - tppf*tstde, tmean+tppf*tstde))
def fit_model(x, y):
initial_guess = [1.2, 0.03]
pars, pcov = curve_fit(func, x, y, p0=initial_guess)
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(y) # number of data points
p = len(pars) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
for i, p,var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
print 'p{0}: {1} [{2} {3}]'.format(i, p,p - sigma*tval,p + sigma*tval)
return pars
def pcov_t_lsq(xdata_to_fit, ydata_to_fit, p_lsq, cov_x, alpha):
'''
# calculate covariance of parameters
'''
# calc residual variance/ reduced chi square
res_var=np.sum((Lorentzian_cvft(xdata_to_fit,*p_lsq)-ydata_to_fit)**2)/(len(ydata_to_fit)-len(p_lsq))
pcov_lsq=cov_x*res_var
delta_p_lsq=np.sqrt(np.diag(pcov_lsq))
# degrees of freedom, should be positive or zero
dof=len(ydata_to_fit)-len(p_lsq)
# get t value from t distribution
from scipy.stats.distributions import t
tval=t.ppf(1.0-alpha/2.0,dof)
# calc t distribution corrected sigma
delta_p_lsq=delta_p_lsq*tval*2
return delta_p_lsq
def confidenceInterval(mean, stdev, nSamples, percentConfidence, trueStd = True, printLatex = False):
'''if trueStd, use normal distribution, otherwise, Student
Use otherwise t.interval or norm.interval
ex: norm.interval(0.95, loc = 0., scale = 2.3/sqrt(11))
t.interval(0.95, 10, loc=1.2, scale = 2.3/sqrt(nSamples))
loc is mean, scale is sigma/sqrt(n) (for Student, 10 is df)'''
from math import sqrt
from scipy.stats.distributions import norm, t
if trueStd:
k = round(norm.ppf(0.5+percentConfidence/200., 0, 1)*100)/100. # 1.-(100-percentConfidence)/200.
else: # use Student
k = round(t.ppf(0.5+percentConfidence/200., nSamples-1)*100)/100.
e = k*stdev/sqrt(nSamples)
if printLatex:
print('${0} \pm {1}\\frac{{{2}}}{{\sqrt{{{3}}}}}$'.format(mean, k, stdev, nSamples))
return mean-e, mean+e
def curve_fit_ci(popt,pcov,n):
alpha = 0.05
p = len(popt)
dof = max(0,n-p)
tval = t.ppf(1.0-alpha/2., dof)
lower = []
upper = []
for i,coef,var in zip(range(p),popt, np.diag(pcov)):
sigma = var**0.5
print('p{0}: {1} [{2} {3}]'.format(i, coef,
coef - sigma/np.sqrt(n)*tval,
coef + sigma/np.sqrt(n)*tval))
lower.append(coef - sigma/np.sqrt(n)*tval)
upper.append(coef + sigma/np.sqrt(n)*tval)
return lower,upper
def get_accuracies(pars, pcov, ydata):
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(ydata) # number of data points
p = len(pars) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t_stat.ppf(1.0 - alpha / 2., dof)
sigmas = []
for i, p, var in zip(range(n), pars, np.diag(pcov)):
sigma = np.sqrt(var)
sigmas.append(sigma * tval)
return sigmas, dof
def diffuse_fit(x, y, ti, t0):
dt = ti - t0
popt, pcov = curve_fit(pde_fun, x, y, p0=[0.2, -1, -1])
C1, C2, C3 = popt
n = len(x) # number of data points
p = len(pcov) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
alpha = 0.05
# student-t value for the dof and confidence level
tval = t.ppf(1.0 - 0.5 * alpha, dof)
sigma = np.diag(pcov)**0.5
moe = tval * sigma
#uncertianity analysis
D_2 = -1 / (dt * 4 * C2)
D_2p = 1 / (dt * 4 * (moe[1] + D_2))
D_2n = 1 / (dt * 4 * (moe[1] - D_2))
D_2u = 0.5 * (abs(D_2 - D_2p) + abs(D_2 - D_2n))
return D_2, D_2u, C1, C2, C3
def dict_recur_mean_err(dlist):
"""
Accepts list of nested dictionaries and produces a single
dictionary containing mean values and estimated errors
from these dictionaries. Errors are estimated as confidence
intervals lengths.
"""
if isinstance(dlist[0], dict):
res_dict = {}
for k in dlist[0]:
n_dlist = [d[k] for d in dlist]
res_dict[k] = dict_recur_mean_err(n_dlist)
return res_dict
else:
n = len(dlist)
mean = float(sum(dlist)) / n
variance = float(sum(map(lambda xi: (xi-mean)**2, dlist))) / n
std = math.sqrt(variance)
err = t.ppf(1-alpha/2.,n-1) * std / math.sqrt(n-1)
return (mean, err)
def confidenceFit(func,x,y, initial_guess, alpha=0.1,maxfev=800): # alpha is the degree of confidence interval = 100*(1-alpha), for example for 98% take alpha=0.02 pars, pcov = curve_fit(func, x, y, p0=initial_guess,maxfev=maxfev) n = len(y) # number of data points p = len(pars) # number of parameters dof = max(0, n - p) # number of degrees of freedom #print pcov # student-t value for the dof and confidence level tval = t.ppf(1.0-alpha/2., dof) #std from covariance sigma = np.diag(pcov)**0.5 lower_bound = pars - sigma*tval upper_bound = pars + sigma*tval return pars,lower_bound,upper_bound,sigma,tval
def simple_confint(yy, popt, pcov):
alpha = 0.05
n = len(yy)
p = len(popt)
dof = max(0, n - p)
tval = t.ppf(1.0 - alpha / 2., dof)
cf = np.zeros((2, 2))
for i, p, var in zip(range(n), popt, np.diag(pcov)):
sigma = var**0.5
cf[i, 0] = p - sigma * tval
cf[i, 1] = p + sigma * tval
'''print('p{0}: {1} [{2} {3}]' .format(i, p,
p - sigma*tval,
p + sigma*tval))
'''
return cf
def confidenceInterval(mean,
stdev,
nSamples,
percentConfidence,
trueStd=True,
printLatex=False):
'''if trueStd, use normal distribution, otherwise, Student
Use otherwise t.interval or norm.interval
ex: norm.interval(0.95, loc = 0., scale = 2.3/sqrt(11))
t.interval(0.95, 10, loc=1.2, scale = 2.3/sqrt(nSamples))
loc is mean, scale is sigma/sqrt(n) (for Student, 10 is df)'''
from scipy.stats.distributions import norm, t
if trueStd:
k = round(norm.ppf(0.5 + percentConfidence / 200., 0, 1), 2)
else: # use Student
k = round(t.ppf(0.5 + percentConfidence / 200., nSamples - 1), 2)
e = k * stdev / sqrt(nSamples)
if printLatex:
print('${0} \pm {1}\\frac{{{2}}}{{\sqrt{{{3}}}}}$'.format(
mean, k, stdev, nSamples))
return mean - e, mean + e
def trimmed_mean_ci(data, proportiontocut=0.2, alpha=0.05, axis=None):
"""Returns the selected confidence interval of the trimmed mean along the
given axis.
:Inputs:
data : sequence
Input data. The data is transformed to a masked array
proportiontocut : float *[0.2]*
Proportion of the data to cut from each side of the data .
As a result, (2*proportiontocut*n) values are actually trimmed.
alpha : float *[0.05]*
Confidence level of the intervals
axis : integer *[None]*
Axis along which to cut.
"""
data = masked_array(data, copy=False)
trimmed = trim_both(data, proportiontocut=proportiontocut, axis=axis)
tmean = trimmed.mean(axis)
tstde = trimmed_stde(data, proportiontocut=proportiontocut, axis=axis)
df = trimmed.count(axis) - 1
tppf = t.ppf(1-alpha/2.,df)
return numpy.array((tmean - tppf*tstde, tmean+tppf*tstde))
maxfev=4000,
ftol=1e-6)
print('Fitted parameters=', fitParams)
##--------------t-test
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(y) # number of data points
p = len(fitParams) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
## student-t value for the dof and confidence level
##https://stackoverflow.com/questions/21494141/how-do-i-do-a-f-test-in-python
tval = t.ppf(1.0 - alpha / 2., dof)
#
for i, p, var in zip(range(n), fitParams, np.diag(fitCovariances)):
sigma = var**0.5
print('p{0}: {1} [{2} {3}]'.format(i, p, p - sigma * tval,
p + sigma * tval))
#Mutual information
# http://scikit-learn.org/stable/auto_examples/feature_selection/plot_f_test_vs_mi.html
#F-statistics tutorial
#http://www.statisticshowto.com/f-statistic/
#predicting with fitted function
A = x.T
ypred = fitFunc(A, fitParams[0], fitParams[1], fitParams[2], fitParams[3],
#!/usr/bin/env python
import numpy as np
from scipy.stats.distributions import t
n = 10 # number of measurements
dof = n - 1 # degrees of freedom
avg_x = 16.1 # average measurement
std_x = 0.01 # standard deviation of measurements
# Find 95% prediction interval for next measurement
alpha = 1.0 - 0.95
pred_interval = t.ppf(1 - alpha / 2., dof) * std_x * np.sqrt(1. + 1. / n)
s = ['We are 95% confident the next measurement',
' will be between {0:1.3f} and {1:1.3f}']
print(''.join(s).format(avg_x - pred_interval, avg_x + pred_interval))
x2_big_sample += var(bags_for_big_samples[:number_of_sample], ddof=1)
small_sample_average = x_small_sample / number_of_simulation
medium_sample_average = x_medium_sample / number_of_simulation
big_sample_average = x_big_sample / number_of_simulation
small_sample_variance = x2_small_sample / number_of_simulation / number_of_sample
medium_sample_variance = x2_medium_sample / number_of_simulation / number_of_sample
big_sample_variance = x2_big_sample / number_of_simulation / number_of_sample
print("Small Sample : $\mu$ = " + str(average(small_sample_average)) + ", $\sigma^2$ " + str(average(small_sample_variance)))
print("medium Sample : $\mu$ = " + str(average(medium_sample_average)) + ", $\sigma^2$ " + str(average(medium_sample_variance)))
print("big Sample : $\mu$ = " + str(average(big_sample_average)) + ", $\sigma^2$ " + str(average(big_sample_variance)))
quantile = t.ppf(0.975, number_of_sample-1, loc=0, scale=1)
# print("IC for Small Sample :" + str(quantile*sqrt(small_sample_variance*(1-3/2400))/small_sample_average*100) + "%")
# print("IC for medium Sample :" + str(quantile*sqrt(medium_sample_variance*(1-24/2400))/medium_sample_average*100) + "%")
# print("IC for big Sample :" + str(quantile*sqrt(big_sample_variance*(1-60/2400))/big_sample_average*100) + "%")
total_tank_average_using_small_samples = number_of_small_samples * small_sample_average
total_tank_average_using_medium_samples = number_of_medium_samples * medium_sample_average
total_tank_average_using_big_samples = number_of_big_samples * big_sample_average
total_tank_variance_using_small_samples = number_of_small_samples**2 * small_sample_variance
total_tank_variance_using_medium_samples = number_of_medium_samples**2 * medium_sample_variance
total_tank_variance_using_big_samples = number_of_big_samples**2 * big_sample_variance
print("Total Tank estimation using Small Sample : $\mu$ = " + str(total_tank_average_using_small_samples) + ", $\sigma^2$ " + str(total_tank_variance_using_small_samples))
print("Total Tank estimation using medium Sample : $\mu$ = " + str(total_tank_average_using_medium_samples) + ", $\sigma^2$ " + str(total_tank_variance_using_medium_samples))
b =
4.9671
2.1100
bint =
4.6267 5.3075
1.7671 2.4528
'''
x = np.array([ 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ])
y = np.array([ 4.70192769, 4.46826356, 4.57021389, 4.29240134, 3.88155125,
3.78382253, 3.65454727, 3.86379487, 4.16428541, 4.06079909])
# this is the function we want to fit to our data
def func(x,c0, c1):
return c0 * np.exp(-x) + c1*x
pars, pcov = curve_fit(func, x, y, p0=[4.96, 2.11])
alpha = 0.05 # 95% confidence interval
n = len(y) # number of data points
p = len(pars) # number of parameters
dof = max(0, n-p) # number of degrees of freedom
tval = t.ppf(1.0-alpha/2., dof) # student-t value for the dof and confidence level
for i, p,var in zip(range(n), pars, np.diag(pcov)):
sigma = var**0.5
print('c{0}: {1} [{2} {3}]'.format(i, p,
p - sigma*tval,
p + sigma*tval))
import matplotlib.pyplot as plt
plt.plot(x,y,'bo ')
xfit = np.linspace(0,1)
yfit = func(xfit, pars[0], pars[1])
plt.plot(xfit,yfit,'b-')
plt.legend(['data','fit'],loc='best')
plt.savefig('images/nonlin-fit-ci.png')
print("Small Sample : $\mu$ = " + str(average(small_sample_average)) + ", $\sigma^2$ " + str(average(small_sample_variance)))
print("medium Sample : $\mu$ = " + str(average(medium_sample_average)) + ", $\sigma^2$ " + str(average(medium_sample_variance)))
print("big Sample : $\mu$ = " + str(average(big_sample_average)) + ", $\sigma^2$ " + str(average(big_sample_variance)))
number_of_small_samples = 2400
number_of_medium_samples = 2400/8
number_of_big_samples = 2400/20
total_tank_average_using_small_samples = number_of_small_samples * average(small_sample_average)
total_tank_average_using_medium_samples = number_of_medium_samples * average(medium_sample_average)
total_tank_average_using_big_samples = number_of_big_samples * average(big_sample_average)
total_tank_variance_using_small_samples = number_of_small_samples * average(small_sample_variance)
total_tank_variance_using_medium_samples = number_of_medium_samples * average(medium_sample_variance)
total_tank_variance_using_big_samples = number_of_big_samples * average(big_sample_variance)
print("Total Tank estimation using Small Sample : $\mu$ = " + str(total_tank_average_using_small_samples) + ", $\sigma^2$ " + str(total_tank_variance_using_small_samples))
print("Total Tank estimation using medium Sample : $\mu$ = " + str(total_tank_average_using_medium_samples) + ", $\sigma^2$ " + str(total_tank_variance_using_medium_samples))
print("Total Tank estimation using big Sample : $\mu$ = " + str(total_tank_average_using_big_samples) + ", $\sigma^2$ " + str(total_tank_variance_using_big_samples))
quantile = t.ppf(0.975, 2, loc=0, scale=1)
correction_factor_for_small_samples = 1
print("Total Tank estimation IC using Small Sample : " + str(quantile*sqrt(total_tank_variance_using_small_samples)/total_tank_average_using_small_samples*100) + "%")
print("Total Tank estimation IC using medium Sample : " + str(quantile*sqrt(total_tank_variance_using_medium_samples)/total_tank_average_using_medium_samples*100) + "%")
print("Total Tank estimation IC using big Sample : " + str(quantile*sqrt(total_tank_variance_using_big_samples)/total_tank_average_using_big_samples*100) + "%")
def _calc_ajusted_recomb(dists, recombs, max_recomb, max_zero_dist_recomb,
alpha_recomb_0, plot_fhand=None):
# first rough interpolation
# we remove the physical distances with high recombination rates because
# they're not very informative. e.g. more than 40 cM will not discriminate
# between false recombination due to hidden segregation in the parents and
# true recombination
if plot_fhand:
fig = Figure()
axes = fig.add_subplot(111)
axes.set_axis_bgcolor((1, 0.6, 0.6))
axes.scatter(dists, recombs, c='r', label='For 1st fit')
else:
axes = None
fig = None
dists = numpy.array(dists)
recombs = numpy.array(recombs)
recomb_rate = 1e-7
popt, pcov = _fit_kosambi(dists, recombs, init_params=[recomb_rate, 0])
if popt is None:
_print_figure(axes, fig, plot_fhand)
return None, False, {'kosambi_fit_ok': False,
'reason': '1st fit failed'}
est_dists = dists
est_recombs = _kosambi(est_dists, popt[0], popt[1])
if fig:
axes.plot(est_dists, est_recombs, label='1st fit', c='r')
# now we perform a second fit but only with those markers that are a
# distance that results in a recombination fraction lower than max_recomb
close_markers = est_recombs < max_recomb
close_recombs = recombs[close_markers]
close_dists = dists[close_markers]
if plot_fhand:
axes.scatter(close_dists, close_recombs, c='b', label='For 2nd fit')
if len(close_dists) < 1:
# This marker is so bad that their closest markers are at a large
# distance
_print_figure(axes, fig, plot_fhand)
return None, False, {'kosambi_fit_ok': False,
'reason_no_fit': 'no close region left'}
if len(close_dists) != len(dists):
# If we've removed any points we fit again
popt, pcov = _fit_kosambi(close_dists, close_recombs, init_params=popt)
if popt is None:
_print_figure(axes, fig, plot_fhand)
return None, False, {'kosambi_fit_ok': False,
'reason_no_fit': '2nd fit failed'}
est_close_recombs = _kosambi(close_dists, popt[0], popt[1])
residuals = close_recombs - est_close_recombs
if fig:
axes.plot(close_dists, est_close_recombs, c='b', label='2nd_fit')
# we exclude the markers with a residual outlier
quartile_25, quartile_75 = numpy.percentile(residuals, [25, 75])
iqr = quartile_75 - quartile_25
outlayer_thrld = [quartile_25 - iqr * 1.5, quartile_75 + iqr * 1.5]
ok_markers = [idx for idx, res in enumerate(residuals) if (not isnan(res) and (outlayer_thrld[0] < res < outlayer_thrld[1]))]
ok_recombs = close_recombs[ok_markers]
ok_dists = close_dists[ok_markers]
if fig:
axes.scatter(ok_dists, ok_recombs, c='g', label='For 3rd fit')
if len(ok_dists) != len(close_dists):
# If we've removed any points we fit again
popt, pcov = _fit_kosambi(ok_dists, ok_recombs, init_params=popt)
var_recomb_at_dist_0 = pcov[1, 1]
recomb_at_dist_0 = popt[1]
ok_color = (0.3, 1, 0.6)
if isinf(var_recomb_at_dist_0):
conf_interval = None
if abs(recomb_at_dist_0) < 0.01:
# recomb is 0 for all points and the variance is inf
snp_ok = True
else:
snp_ok = False
else:
if alpha_recomb_0 is None:
conf_interval = None
if abs(recomb_at_dist_0) <= max_zero_dist_recomb:
snp_ok = True
ok_color = (0.3, 1, 0.3)
else:
snp_ok = False
else:
num_data_points = len(ok_dists)
num_params = len(popt)
deg_of_freedom = max(0, num_data_points - num_params)
tval = t.ppf(1.0 - alpha_recomb_0 / 2., deg_of_freedom)
std_dev = var_recomb_at_dist_0 ** 0.5
conf_interval = (recomb_at_dist_0 - std_dev * tval,
recomb_at_dist_0 + std_dev * tval)
if abs(recomb_at_dist_0) <= max_zero_dist_recomb:
snp_ok = True
ok_color = (0.3, 1, 0.3)
elif conf_interval[0] < 0 < conf_interval[1]:
snp_ok = True
else:
snp_ok = False
if plot_fhand:
axes.vlines(0, conf_interval[0], conf_interval[1],
label='conf. interval')
if plot_fhand:
color = ok_color if snp_ok else (1, 0.3, 0.3)
axes.set_axis_bgcolor(color)
if popt is None:
_print_figure(axes, fig, plot_fhand)
return None, False, {'kosambi_fit_ok': False,
'reason_no_fit': '3rd fit failed'}
est2_recombs = _kosambi(ok_dists, popt[0], popt[1])
if fig:
axes.plot(ok_dists, est2_recombs, c='g', label='3rd_fit')
_print_figure(axes, fig, plot_fhand)
return recomb_at_dist_0, snp_ok, {'kosambi_fit_ok': True,
'conf_interval': conf_interval}
def get_eos(self, static=False):
'''calculate the equation of state for the attached atoms.
Returns a dictionary of data for each step. You do not need to
specify any relaxation parameters, only the base parameters for the
calculations. Writes to eos.org with a report of output.
if static is True, then run a final static calculation at high
precision, with ismear=-5.
'''
# this returns if the data exists.
if os.path.exists('eos.json'):
with open('eos.json') as f:
return json.loads(f.read())
# we need an initial calculation to get us going.
self.calculate()
cwd = os.getcwd()
data = {'cwd': os.getcwd()} # dictionary to store results in
org = [] # list of strings to make the org-file report
org += ['#+STARTUP: showeverything']
org += ['* Initial guess']
org += [str(self)]
org += ['',
'[[shell:jaspsum -p {0}][view initial guess]]'.format(cwd)]
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
atoms = self.get_atoms()
original_atoms = atoms.copy() # save for comparison later.
v_init = atoms.get_volume()
# ############################################################
# ## Step 1
# ############################################################
org += ['* step 1 - relax ions and shape']
volumes1, energies1 = [], []
ready = True
factors = [-0.15, -0.07, 0.0, 0.07, 0.15]
for i, f in enumerate(factors):
wd = cwd + '/step-1/f-{0}'.format(i)
self.clone(wd)
with jasp(wd,
isif=4,
ibrion=2,
ediffg=-0.05, ediff=1e-6,
nsw=50,
atoms=atoms) as calc:
try:
# add org-link to calculation
org += ['[[shell:jaspsum {0}][{0}]]'.format(wd)]
atoms.set_volume(v_init * (1 + f))
volumes1.append(atoms.get_volume())
energies1.append(atoms.get_potential_energy())
calc.strip()
except (VaspSubmitted, VaspQueued):
ready = False
if not ready:
log.info('Step 1 is still running')
raise VaspRunning
data['step1'] = {}
data['step1']['volumes'] = volumes1
data['step1']['energies'] = energies1
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
# create an org-table of the data.
org += ['',
'#+tblname: step1',
'| volume (A^3) | Energy (eV) |',
'|-']
for v, e in zip(volumes1, energies1):
org += ['|{0}|{1}|'.format(v, e)]
org += ['']
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
eos1 = EquationOfState(volumes1, energies1)
try:
v1, e1, B1 = eos1.fit()
except:
with open('error', 'w') as f:
f.write('Error fitting the equation of state')
data['step1']['eos'] = (v1, e1, B1)
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
# create a plot
f = eos1.plot(show=False)
f.subplots_adjust(left=0.18, right=0.9, top=0.9, bottom=0.15)
plt.xlabel(u'volume ($\AA^3$)')
plt.ylabel(u'energy (eV)')
plt.title(u'E: %.3f eV, V: %.3f $\AA^3$, B: %.3f GPa' %
(e1, v1, B1 / GPa))
plt.text(eos1.v0, max(eos1.e), 'EOS: %s' % eos1.eos_string)
f.savefig('eos-step1.png')
org += ['[[./eos-step1.png]]',
'']
min_energy_index = np.argmin(energies1)
if min_energy_index in [0, -1]:
log.warn('Your minimum energy is at an endpoint.'
'This indicates something is wrong.')
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
# ########################################################
# # STEP 2
# ########################################################
# step 2 - isif=4, ibrion=1. now we allow the shape of each cell to
# change, and we use the best guess from step 1 for minimum volume.
ready = True
volumes2, energies2 = [], []
factors = [-0.09, -0.06, -0.03, 0.0, 0.03, 0.06, 0.09]
org += ['* step 2 - relax ions and shape with improved minimum estimate']
for i, f in enumerate(factors):
wd = cwd + '/step-2/f-{0}'.format(i)
# clone closest result from above.
with jasp('step-1/f-{0}'.format(min_energy_index)) as calc:
calc.clone(wd)
with jasp(wd,
isif=4,
ibrion=1,
nsw=50) as calc:
try:
atoms = calc.get_atoms()
atoms.set_volume(v1 * (1 + f))
volumes2 += [atoms.get_volume()]
energies2 += [atoms.get_potential_energy()]
calc.strip()
except (VaspSubmitted, VaspQueued):
ready = False
if not ready:
log.info('Step 2 is still running')
raise VaspRunning
# update org and json files.
data['step2'] = {}
data['step2']['volumes'] = volumes2
data['step2']['energies'] = energies2
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
# create an org-table of the data.
org += ['',
'#+tblname: step2',
'| volume (A^3) | Energy (eV) |',
'|-']
for v, e in zip(volumes2, energies2):
org += ['|{0}|{1}|'.format(v, e)]
org += ['']
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
eos2 = EquationOfState(volumes2, energies2)
try:
v2, e2, B2 = eos2.fit()
except:
with open('error', 'w') as f:
f.write('Error fitting the equation of state')
data['step2']['eos'] = (v2, e2, B2)
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
f = eos2.plot(show=False)
f.subplots_adjust(left=0.18, right=0.9, top=0.9, bottom=0.15)
plt.xlabel(u'volume ($\AA^3$)')
plt.ylabel(u'energy (eV)')
plt.title(u'E: %.3f eV, V: %.3f $\AA^3$, B: %.3f GPa' %
(e2, v2, B2 / GPa))
plt.text(eos2.v0, max(eos2.e), 'EOS: %s' % eos2.eos_string)
f.savefig('eos-step2.png')
org += [
'[[./eos-step2.png]]',
'']
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
# statistical analysis of the equation of state
EOS = ['sjeos',
'taylor',
'murnaghan',
'birch',
'birchmurnaghan',
'pouriertarantola',
'vinet']
from ase.units import kJ
Vs, Es, Bs = [], [], []
for label in EOS:
eos = EquationOfState(volumes2, energies2, eos=label)
try:
v, e, B = eos.fit()
Vs += [v]
Es += [e]
Bs += [B / kJ * 1.0e24] # GPa
except:
with open('error', 'w') as f:
f.write('Error fitting the '
'equation of state {0}'.format(label))
avgV = np.mean(Vs)
stdV = np.std(Vs)
avgE = np.mean(Es)
stdE = np.std(Es)
avgB = np.mean(Bs)
stdB = np.std(Bs)
from scipy.stats.distributions import t
n = len(Vs)
dof = n - 1
alpha = 0.05
Vconf = t.ppf(1 - alpha/2., dof) * stdV * np.sqrt(1 + 1./n)
Bconf = t.ppf(1 - alpha/2., dof) * stdB * np.sqrt(1 + 1./n)
data['step2']['avgV'] = avgV
data['step2']['Vconf95'] = Vconf
data['step2']['avgB'] = avgB
data['step2']['Bconf95'] = Bconf
org += ['** Statistical analysis',
'''
Volume = {avgV:1.3f} \pm {Vconf:1.3f} \AA^3 at the 95% confidence level
B = {avgB:1.0f} \pm {Bconf:1.0f} GPa at the 95% confidence level
'''.format(**locals())]
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
# step 3 should be isif = 3 where we let the volume change too
# start from the minimum in step2
org += ['* step 3 - relax volume']
emin_ind = np.argmin(energies2)
log.info('Minimum energy found in factor={0}.'.format(factors[emin_ind]))
with jasp('step-2/f-{0}'.format(emin_ind)) as calc:
calc.clone('step-3')
with jasp('step-3',
isif=3, # vol, shape and internal degrees of freedom
ibrion=1,
prec='high',
nsw=50) as calc:
atoms = calc.get_atoms()
atoms.set_volume(avgV)
calc.calculate()
calc.strip()
org += [str(calc)]
atoms = calc.get_atoms()
data['step3'] = {}
data['step3']['potential_energy'] = atoms.get_potential_energy()
data['step3']['volume'] = atoms.get_volume()
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
# now the final step with ismear=-5 for the accurate energy. This
# is recommended by the VASP manual. We only do this if you
# specify static=True as an argument
if static:
with jasp('step-3') as calc:
calc.clone('step-4')
with jasp('step-4',
isif=None, ibrion=None, nsw=None,
icharg=2, # do not reuse charge
istart=1,
prec='high',
ismear=-5) as calc:
calc.calculate()
atoms = calc.get_atoms()
data['step4'] = {}
data['step4']['potential_energy'] = atoms.get_potential_energy()
org += ['* step-4 - static calculation',
str(calc)]
# final write out
with open('eos.org', 'w') as f:
f.write('\n'.join(org))
# dump data to a json file
with open('eos.json', 'w') as f:
f.write(json.dumps(data))
return data
print(popt[2])
#np.savetxt(f+'_fit.csv', popt, delimiter = ',')
#np.savetxt(f+'_fit_cov.csv', popt, delimiter = ',')
# from http://kitchingroup.cheme.cmu.edu/blog/2013/02/12/Nonlinear-curve-fitting-with-parameter-confidence-intervals/
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
n = len(ydata) # number of data points
p = len(popt) # number of parameters
dof = max(0, n - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
params = []
for i, p,var in zip(range(n), popt, np.diag(pcov)):
sigma = var**0.5
params.append('p{0}: {1} [{2} {3}]'.format(i, p,p - sigma*tval,p + sigma*tval))
print(params)
with open(f+'_fit_param_confidence_int.txt', 'w') as newfile:
newfile.write('\n'.join(params))
fit = func(xdata, popt[0], popt[1], popt[2])
plt.figure(0)
plt.plot(xdata, fit, 'g')
plt.figure(1)
plt.plot(xdata, fit, 'g')
laserpowernum=float(f[f.index(' _')+2:f.index('J')])*0.125167/1e-6#in microJ
laserpower=format(laserpowernum, '.0f')#in microJ
powerlist.append(laserpower)
def processFile(self,fullPath):
fileName, fileExtension = os.path.splitext(fullPath)
fileName = os.path.basename(fullPath)
self.fileNames.append(fileName)
if fileExtension == '.csv':
delimiter = ','
else:
delimiter = None
self.ui.statusbar.showMessage("processing: "+ fileName,2500)
#wait here for the file to be completely written to disk and closed before trying to read it
fi = QFileInfo(fullPath)
while (not fi.isWritable()):
time.sleep(0.001)
fi.refresh()
fp = open(fullPath, mode='r')
fileBuffer = fp.read()
fp.close()
first10 = fileBuffer[0:10]
nMcHeaderLines = 25 #number of header lines in mcgehee IV file format
isMcFile = False #true if this is a McGehee iv file format
if (not first10.__contains__('#')) and (first10.__contains__('/')) and (first10.__contains__('\t')):#the first line is not a comment
#the first 8 chars do not contain comment symbol and do contain / and a tab, it's safe to assume mcgehee iv file format
isMcFile = True
#comment out the first 25 rows here
fileBuffer = '#'+fileBuffer
fileBuffer = fileBuffer.replace('\n', '\n#',nMcHeaderLines-1)
splitBuffer = fileBuffer.splitlines(True)
area = 1
noArea = True
vsTime = False #this is not an i,v vs t data file
#extract header lines and search for area
header = []
for line in splitBuffer:
if line.startswith('#'):
header.append(line)
if line.__contains__('Area'):
area = float(line.split(' ')[3])
noArea = False
if line.__contains__('I&V vs t'):
if float(line.split(' ')[5]) == 1:
vsTime = True
else:
break
outputScaleFactor = np.array(1000/area) #for converstion to [mA/cm^2]
tempFile = QTemporaryFile()
tempFile.open()
tempFile.writeData(fileBuffer)
tempFile.flush()
#read in data
try:
data = np.loadtxt(str(tempFile.fileName()),delimiter=delimiter)
VV = data[:,0]
II = data[:,1]
if vsTime:
time = data[:,2]
except:
self.ui.statusbar.showMessage('Could not read' + fileName +'. Prepend # to all non-data lines and try again',2500)
return
tempFile.close()
tempFile.remove()
if isMcFile: #convert to amps
II = II/1000*area
if not vsTime:
#sort data by ascending voltage
newOrder = VV.argsort()
VV=VV[newOrder]
II=II[newOrder]
#remove duplicate voltage entries
VV, indices = np.unique(VV, return_index =True)
II = II[indices]
else:
#sort data by ascending time
newOrder = time.argsort()
VV=VV[newOrder]
II=II[newOrder]
time=time[newOrder]
time=time-time[0]#start time at t=0
#catch and fix flipped current sign:
if II[0] < II[-1]:
II = II * -1
indexInQuad1 = np.logical_and(VV>0,II>0)
if any(indexInQuad1): #enters statement if there is at least one datapoint in quadrant 1
isDarkCurve = False
else:
self.ui.statusbar.showMessage("Dark curve detected",500)
isDarkCurve = True
#put items in table
self.ui.tableWidget.insertRow(self.rows)
for ii in range(len(self.cols)):
self.ui.tableWidget.setItem(self.rows,ii,QTableWidgetItem())
if not vsTime:
fitParams, fitCovariance, infodict, errmsg, ier = self.bestEffortFit(VV,II)
#print errmsg
I0_fit = fitParams[0]
Iph_fit = fitParams[1]
Rs_fit = fitParams[2]
Rsh_fit = fitParams[3]
n_fit = fitParams[4]
#0 -> LS-straight line
#1 -> cubic spline interpolant
smoothingParameter = 1-2e-6
iFitSpline = SmoothSpline(VV, II, p=smoothingParameter)
def cellModel(voltageIn):
#voltageIn = np.array(voltageIn)
return vectorizedCurrent(voltageIn, I0_fit, Iph_fit, Rs_fit, Rsh_fit, n_fit)
def invCellPowerSpline(voltageIn):
if voltageIn < 0:
return 0
else:
return -1*voltageIn*iFitSpline(voltageIn)
def invCellPowerModel(voltageIn):
if voltageIn < 0:
return 0
else:
return -1*voltageIn*cellModel(voltageIn)
if not isDarkCurve:
VVq1 = VV[indexInQuad1]
IIq1 = II[indexInQuad1]
vMaxGuess = VVq1[np.array(VVq1*IIq1).argmax()]
powerSearchResults = optimize.minimize(invCellPowerSpline,vMaxGuess)
#catch a failed max power search:
if not powerSearchResults.status == 0:
print "power search exit code = " + str(powerSearchResults.status)
print powerSearchResults.message
vMax = nan
iMax = nan
pMax = nan
else:
vMax = powerSearchResults.x[0]
iMax = iFitSpline([vMax])[0]
pMax = vMax*iMax
#only do this stuff if the char eqn fit was good
if ier < 5:
powerSearchResults_charEqn = optimize.minimize(invCellPowerModel,vMaxGuess)
#catch a failed max power search:
if not powerSearchResults_charEqn.status == 0:
print "power search exit code = " + str(powerSearchResults_charEqn.status)
print powerSearchResults_charEqn.message
vMax_charEqn = nan
else:
vMax_charEqn = powerSearchResults_charEqn.x[0]
#dude
try:
Voc_nn_charEqn=optimize.brentq(cellModel, VV[0], VV[-1])
except:
Voc_nn_charEqn = nan
else:
Voc_nn_charEqn = nan
vMax_charEqn = nan
try:
Voc_nn = optimize.brentq(iFitSpline, VV[0], VV[-1])
except:
Voc_nn = nan
else:
Voc_nn = nan
vMax = nan
iMax = nan
pMax = nan
Voc_nn_charEqn = nan
vMax_charEqn = nan
iMax_charEqn = nan
pMax_charEqn = nan
if ier < 5:
dontFindBounds = False
iMax_charEqn = cellModel([vMax_charEqn])[0]
pMax_charEqn = vMax_charEqn*iMax_charEqn
Isc_nn_charEqn = cellModel(0)
FF_charEqn = pMax_charEqn/(Voc_nn_charEqn*Isc_nn_charEqn)
else:
dontFindBounds = True
iMax_charEqn = nan
pMax_charEqn = nan
Isc_nn_charEqn = nan
FF_charEqn = nan
#there is a maddening bug in SmoothingSpline: it can't evaluate 0 alone, so I have to do this:
try:
Isc_nn = iFitSpline([0,1e-55])[0]
except:
Isc_nn = nan
FF = pMax/(Voc_nn*Isc_nn)
if (ier != 7) and (ier != 6) and (not dontFindBounds) and (type(fitCovariance) is not float):
#error estimation:
alpha = 0.05 # 95% confidence interval = 100*(1-alpha)
nn = len(VV) # number of data points
p = len(fitParams) # number of parameters
dof = max(0, nn - p) # number of degrees of freedom
# student-t value for the dof and confidence level
tval = t.ppf(1.0-alpha/2., dof)
lowers = []
uppers = []
#calculate 95% confidence interval
for a, p,var in zip(range(nn), fitParams, np.diag(fitCovariance)):
sigma = var**0.5
lower = p - sigma*tval
upper = p + sigma*tval
lowers.append(lower)
uppers.append(upper)
else:
uppers = [nan,nan,nan,nan,nan]
lowers = [nan,nan,nan,nan,nan]
plotPoints = 1000
fitX = np.linspace(VV[0],VV[-1],plotPoints)
if ier < 5:
modelY = cellModel(fitX)*outputScaleFactor
else:
modelY = np.empty(plotPoints)*nan
splineY = iFitSpline(fitX)*outputScaleFactor
graphData = {'vsTime':vsTime,'origRow':self.rows,'fitX':fitX,'modelY':modelY,'splineY':splineY,'i':II*outputScaleFactor,'v':VV,'Voc':Voc_nn,'Isc':Isc_nn*outputScaleFactor,'Vmax':vMax,'Imax':iMax*outputScaleFactor}
#export button
exportBtn = QPushButton(self.ui.tableWidget)
exportBtn.setText('Export')
exportBtn.clicked.connect(self.handleButton)
self.ui.tableWidget.setCellWidget(self.rows,self.cols.keys().index('exportBtn'), exportBtn)
self.ui.tableWidget.item(self.rows,self.cols.keys().index('pce')).setData(Qt.DisplayRole,round(pMax/area*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('pce')).setToolTip(str(round(pMax_charEqn/area*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('pmax')).setData(Qt.DisplayRole,round(pMax/area*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('pmax')).setToolTip(str(round(pMax_charEqn/area*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('jsc')).setData(Qt.DisplayRole,round(Isc_nn/area*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('jsc')).setToolTip(str(round(Isc_nn_charEqn/area*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('voc')).setData(Qt.DisplayRole,round(Voc_nn*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('voc')).setToolTip(str(round(Voc_nn_charEqn*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('ff')).setData(Qt.DisplayRole,round(FF,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('ff')).setToolTip(str(round(FF_charEqn,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rs')).setData(Qt.DisplayRole,round(Rs_fit*area,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rs')).setToolTip('[{0} {1}]'.format(lowers[2]*area, uppers[2]*area))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rsh')).setData(Qt.DisplayRole,round(Rsh_fit*area,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rsh')).setToolTip('[{0} {1}]'.format(lowers[3]*area, uppers[3]*area))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('jph')).setData(Qt.DisplayRole,round(Iph_fit/area*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('jph')).setToolTip('[{0} {1}]'.format(lowers[1]/area*1e3, uppers[1]/area*1e3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('j0')).setData(Qt.DisplayRole,round(I0_fit/area*1e9,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('j0')).setToolTip('[{0} {1}]'.format(lowers[0]/area*1e9, uppers[0]/area*1e9))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('n')).setData(Qt.DisplayRole,round(n_fit,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('n')).setToolTip('[{0} {1}]'.format(lowers[4], uppers[4]))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('Vmax')).setData(Qt.DisplayRole,round(vMax*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('Vmax')).setToolTip(str(round(vMax_charEqn*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('area')).setData(Qt.DisplayRole,round(area,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('pmax2')).setData(Qt.DisplayRole,round(pMax*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('pmax2')).setToolTip(str(round(pMax_charEqn*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('isc')).setData(Qt.DisplayRole,round(Isc_nn*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('isc')).setToolTip(str(round(Isc_nn_charEqn*1e3,3)))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('iph')).setData(Qt.DisplayRole,round(Iph_fit*1e3,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('iph')).setToolTip('[{0} {1}]'.format(lowers[1]*1e3, uppers[1]*1e3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('i0')).setData(Qt.DisplayRole,round(I0_fit*1e9,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('i0')).setToolTip('[{0} {1}]'.format(lowers[0]*1e9, uppers[0]*1e9))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rs2')).setData(Qt.DisplayRole,round(Rs_fit,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rs2')).setToolTip('[{0} {1}]'.format(lowers[2], uppers[2]))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rsh2')).setData(Qt.DisplayRole,round(Rsh_fit,3))
self.ui.tableWidget.item(self.rows,self.cols.keys().index('rsh2')).setToolTip('[{0} {1}]'.format(lowers[3], uppers[3]))
else:#vs time
graphData = {'vsTime':vsTime,'origRow':self.rows,'time':time,'i':II*outputScaleFactor,'v':VV}
#file name
self.ui.tableWidget.item(self.rows,self.cols.keys().index('file')).setText(fileName)
self.ui.tableWidget.item(self.rows,self.cols.keys().index('file')).setToolTip(''.join(header))
#plot button
plotBtn = QPushButton(self.ui.tableWidget)
plotBtn.setText('Plot')
plotBtn.clicked.connect(self.handleButton)
self.ui.tableWidget.setCellWidget(self.rows,self.cols.keys().index('plotBtn'), plotBtn)
self.ui.tableWidget.item(self.rows,self.cols.keys().index('plotBtn')).setData(Qt.UserRole,graphData)
self.ui.tableWidget.resizeColumnsToContents()
self.rows = self.rows + 1
