def main(dbname, dbtable, dbuser, dbpass):
"""Main script execution"""
# Query database
result = query_database(dbname, dbtable, dbuser, dbpass, 1, 900)
# Break result up into separate columns
frame_counts, image_scales, widths, heights, proc_times = zip(*result)
# Movie area in square pixels and arcseconds
area_px = np.array(widths) * np.array(heights)
area_as = area_px * np.array(image_scales) ** 2
# Limit outlier influence from server downtime, etc
times = np.array(proc_times)
# Determine linear fit for number of frames and area
(m1, b1) = polyfit(frame_counts, times, 1)
(m2, b2) = polyfit(np.sqrt(area_px), times, 1)
print "Linear fit for num frames: y = %fx + %f" % (m1, b1)
print "Linear fit for movie size: y = %fx + %f" % (m2, b2)
# Basic time estimation
w1 = 0.8
w2 = 1 - w1
basic_est = lambda x, y: w1 * (m1 * x + b1) + w2 * (m2 * y + b2)
evaluate_estimation_fn(basic_est, frame_counts, np.sqrt(area_px), times)
def log_regress(f,xs, tol=0.1):
"""find root f(x) = 0 using logistic regression, starting with xs"""
last_log_xp = None
log_xp = None
print "initial seeding for log_regress"
ys = map(f,xs)
log_xs = map(log,xs)
plotting = False
while last_log_xp is None or abs(exp(log_xp) - exp(last_log_xp)) > tol:
print "correlation:",pearsonr(log_xs,ys)
#lin = poly1d(polyfit(log_xs,ys,1))
m, b = (polyfit(log_xs,ys,1))
if plotting:
lin = poly1d([m,b])
plt.scatter(log_xs,ys)
plt.plot(*pl(lin,log_xs))
plt.show()
last_log_xp = log_xp or None
log_xp = -b/m#secant_interval(lin,min(log_xs),max(log_xs))
log_xs.append(log_xp)
yxp = f(exp(log_xp))
ys.append(yxp)
print "x:",log_xp and exp(log_xp),"y:",ys[-1],\
"x last:",last_log_xp and exp(last_log_xp),\
"y last:",ys[-2] if len(ys) >= 2 else None
#lin = poly1d(polyfit(log_xs,ys,1))
m, b = (polyfit(log_xs,ys,1))
log_xp = -b/m#secant_interval(lin,min(log_xs),max(log_xs))
return exp(log_xp)
def log_regress_spec(f,xs, tol=0.01):
"""find root f(x) = 0 using logistic regression, starting with xs"""
#print "initial seeding for log_regress (unweighted)"
ys = map(f,xs)
log_xs = map(log,xs)
plotting = False
honest_guesses = []
while len(honest_guesses) < 2 or abs(honest_guesses[-1] -
honest_guesses[-2]) > tol:
#print "correlation:",pearsonr(log_xs,ys)
#lin = poly1d(polyfit(log_xs,ys,1))
m, b = (polyfit(log_xs,ys,1))
# if plotting:
# lin = poly1d([m,b])
# plt.scatter(log_xs,ys)
# plt.plot(*pl(lin,log_xs))
# plt.show()
honest_guess = -b/m
dx = -(honest_guesses[-1] - honest_guess) if honest_guesses else 0
log_xp = honest_guess + 2*dx
log_xs.append(log_xp)
yxp = f(exp(log_xp))
ys.append(yxp)
honest_guesses.append(honest_guess)
diff = (abs(exp(honest_guesses[-1]) - exp(honest_guesses[-2]))
if len(honest_guesses) > 1 else None)
# print "honest_guess:",(honest_guess),"xp:",(log_xp),\
# "y:",yxp, "diff:",diff
#lin = poly1d(polyfit(log_xs,ys,1))
m, b = (polyfit(log_xs,ys,1))
log_xp = -b/m#secant_interval(lin,min(log_xs),max(log_xs))
print "final guess: log_xp:",log_xp
return exp(log_xp)
def time_fitting(x_fit,y_fit):
"""Fit a linear relation to the x_fit and y_fit parameters
Returns the actual fit and the parameters of the fit,
"""
import numpy as np
x_fit = np.array( x_fit )
y_fit = np.array( y_fit )
###First fit iteration and remove outliers
POLY_FIT_ORDER = 1
slope,intercept = scipy.polyfit(x_fit,y_fit,POLY_FIT_ORDER)
fit = scipy.polyval((slope,intercept),x_fit)
fit_sigma = fit.std()
include_index = np.where(np.abs(fit-y_fit) < 1.5*fit_sigma)[0]
if len(include_index) < 4:
return None,None,False
###Final Fit
x_fit_clipped = x_fit[include_index]
y_fit_clipped = y_fit[include_index]
parameters = scipy.polyfit(x_fit_clipped,y_fit_clipped,POLY_FIT_ORDER)
fit = scipy.polyval(parameters,x_fit)
return fit,parameters,True
def trainingAndTesting(self):
global train
global x, y, xa, xb, ya, yb
# separating training from testing data
frac = 0.3
split_idx = int(frac * len(xb))
rangeX = range(len(xb))
listX = list(rangeX)
logging.info("delta : %i", len(set(rangeX).difference(listX)))
shuffled = sp.random.permutation(list(range(len(xb))))
test = sorted(shuffled[:split_idx])
train = sorted(shuffled[split_idx:])
fbt1 = sp.poly1d(sp.polyfit(xb[train], yb[train], 1))
fbt2 = sp.poly1d(sp.polyfit(xb[train], yb[train], 2))
print("fbt2(x)= \n%s" % fbt2)
print("fbt2(x)-100,000= \n%s" % (fbt2 - 100000))
print("fbt2(x)= \n%s" % (fbt2))
fbt3 = sp.poly1d(sp.polyfit(xb[train], yb[train], 3))
fbt10 = sp.poly1d(sp.polyfit(xb[train], yb[train], 10))
fbt100 = sp.poly1d(sp.polyfit(xb[train], yb[train], 100))
print("Test errors for only the time after inflection point")
for f in [fbt1, fbt2, fbt3, fbt10, fbt100]:
print("Error d=%i: %f" % (f.order, self.error(f, xb[test], yb[test])))
def fitKvsPower(self, filename="PowerVGain.dat", zeroed="_0"):
if not hasattr(self, "k_avg"):
raise RuntimeError, "Must load all data and run calculate() before fitting"
gain, self.power = loadtxt(filename, skiprows=1, unpack=True)
if zeroed != None:
gain_0, power_0 = loadtxt(filename.replace(".", zeroed + ".", 1), skiprows=1, unpack=True)
self.power = self.power - power_0
savetxt(filename.replace(".", "_subtract."), self.power, fmt="%.5f")
if self.gain.tolist() != gain.tolist():
raise ValueError, "Laser power was not measured over the same gains as calibration measurements:\n\
Laser gain: " + str(
gain
) + "\nMeasurement gains: " + str(
self.gain
)
self.kfitX = polyfit(self.power, self.k_avg[:, 0], 1)
self.kfitY = polyfit(self.power, self.k_avg[:, 1], 1)
print "Fit parameters for X axis:"
print poly1d(self.kfitX, variable="power")
print "\nFit parameters for Y axis:"
print poly1d(self.kfitY, variable="power")
def main():
data = sp.genfromtxt('./data/web_traffic.tsv', delimiter='\t')
x = data[:, 0]
y = data[:, 1]
x = x[~sp.isnan(y)]
y = y[~sp.isnan(y)]
fp1 = sp.polyfit(x, y, 1)
print('Model parameters for fp1 %s' % fp1)
f1 = sp.poly1d(fp1)
print('This is the error rate for fp1 %f' % error(f1, x, y))
fp2 = sp.polyfit(x, y, 2)
print('Model parameters for fp2 %s' % fp2)
f2 = sp.poly1d(fp2)
print('This is the error rate for fp2 %f' % error(f2, x, y))
plt.scatter(x, y,color= 'pink')
plt.title('My first impression')
plt.xlabel('Time')
plt.ylabel('#Hits')
plt.xticks([w * 7 * 24 for w in range(10)], ['week %i' % w for w in range(10)])
fx = sp.linspace(0, x[-1], 1000)
plt.plot(fx, f1(fx), linewidth=3,color='cyan')
plt.plot(fx, f2(fx), linewidth=3, linestyle='--',color= 'red')
plt.legend(['d = %i' %f1.order, 'd = %i' %f2.order], loc='upper left')
plt.autoscale(tight=True)
plt.grid()
plt.show()
def dsi_adc(q_axis, data, mid_pos):
""" TODO
Returns
-------
ADC4, ADC8 : (I,J,K) volumes
Returns TODO
"""
# Fit with a polynomial the signals S and derive the ADC and the kurtosis
# initialization of the output maps
I, J, K, N = data.shape
ADC4 = np.zeros((I,J,K))
#Ku4 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2]))
#P04 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2]))
ADC8 = np.zeros(((I,J,K)))
#Ku8 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2]))
#P08 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2]))
pc = -1
count = 0
n = I * J * K
print "Polynomial fitting for each voxel signal over q-axis..."
# loop throughout all the voxels of the volume
for i in range(b0.shape[0]):
for j in range(b0.shape[1]):
for k in range(b0.shape[2]):
# Percent counter
count = count + 1
pcN = int(round( float(100*count)/n ))
if pcN > pc and pcN%1 == 0:
pc = pcN
print str(pc) + " %"
# for the current voxel, extract the signal to be fitted
S = data[i,j,k,:]
# normalization respect to the b0
S = S / S[mid_pos]
coeff = sp.polyfit(q_axis,S,8)
ADC8[i,j,k] = (-coeff[-3] / (2 * math.pi * math.pi))
# Ku8[i,j,k] = (6 * coeff[-5] / (coeff[-3] * coeff[-3])) - 3
# temp = np.polyval(coeff, q_axis)
# P08[i,j,k] = np.sum(temp)
coeff = sp.polyfit(q_axis,S,4)
ADC4[i,j,k] = (-coeff[-3] / (2 * math.pi * math.pi))
# Ku4[i,j,k] = (6 * coeff[-5] / (coeff[-3] * coeff[-3])) - 3
# temp = np.polyval(coeff, q_axis)
# P04[i,j,k] = np.sum(temp)
print "[ OK ]"
return ADC4, ADC8
def go(
mags=(15, 30.1, 0.5), redshifts=(0.01, 12, 0.01), coeff_file="prior_K_zmax7_coeff.dat", outfile="prior_K_extend.dat"
):
fp = open(coeff_file)
lines = fp.readlines()
fp.close()
mag_list = np.cast[float](lines[0].split()[2:])
z0 = np.cast[float](lines[1].split()[1:])
gamma = np.cast[float](lines[2].split()[1:])
z_grid = np.arange(redshifts[0], redshifts[1], redshifts[2])
NZ = z_grid.shape[0]
mag_grid = np.arange(mags[0], mags[1], mags[2])
NM = mag_grid.shape[0]
#### Polynomial extrapolation not reliable
# p_z0 = scipy.polyfit(mag_list, z0, order)
# z0_grid = np.maximum(scipy.polyval(p_z0, mag_grid), 0.05)
# p_gamma = scipy.polyfit(mag_list, gamma, order)
# gamma_grid = np.maximum(scipy.polyval(p_gamma, mag_grid), 0.05)
#### Interpolations on the defined grid
z0_grid = np.interp(mag_grid, mag_list, z0)
gamma_grid = np.interp(mag_grid, mag_list, gamma)
#### Linear extrapolations of fit coefficients
p_z0 = scipy.polyfit(mag_list[:3], z0[:3], 1)
z0_grid[mag_grid < mag_list[0]] = np.maximum(scipy.polyval(p_z0, mag_grid[mag_grid < mag_list[0]]), 0.05)
p_z0 = scipy.polyfit(mag_list[-3:], z0[-3:], 1)
z0_grid[mag_grid > mag_list[-1]] = np.maximum(scipy.polyval(p_z0, mag_grid[mag_grid > mag_list[-1]]), 0.05)
p_gamma = scipy.polyfit(mag_list[:3], gamma[:3], 1)
gamma_grid[mag_grid < mag_list[0]] = np.maximum(scipy.polyval(p_gamma, mag_grid[mag_grid < mag_list[0]]), 0.05)
p_gamma = scipy.polyfit(mag_list[-3:], gamma[-3:], 1)
gamma_grid[mag_grid > mag_list[-1]] = np.maximum(scipy.polyval(p_gamma, mag_grid[mag_grid > mag_list[-1]]), 0.05)
out_matrix = np.zeros((NZ, NM + 1))
out_matrix[:, 0] = z_grid
for i in range(NM):
pz = z_grid * np.exp(-(z_grid / z0_grid[i]) ** gamma_grid[i])
pz /= np.trapz(pz, z_grid)
plt.plot(z_grid, pz, label=mag_grid[i])
out_matrix[:, i + 1] = pz
plt.legend(ncol=3, loc="upper right")
header = "# z "
for m in mag_grid:
header += "%6.1f" % (m)
fp = open(outfile, "w")
fp.write(header + "\n")
np.savetxt(fp, out_matrix, fmt="%6.3e")
fp.close()
def generate_report():
"""See title"""
word_file = open("word_file.txt", 'ab+')
bigram_file = open("bigram_file.txt", 'ab+')
sorted_words = sorted(words.items(), key = lambda x: x[1])
sorted_bigrams = sorted(bigrams.items(), key = lambda x: x[1])
print(sorted_words, file=word_file)
print(sorted_bigrams, file=bigram_file)
word_names, word_values = zip(*reversed(sorted_words))
bigram_names, bigram_values = zip(*reversed(sorted_bigrams))
#Create word plot
xdata, ydata = range(1,len(word_values)+1), word_values
xd, yd = log10(xdata), log10(ydata)
polycoef = polyfit(xd,yd,1)
yfit = 10** (polycoef[0]*xd+polycoef[1])
plt.figure()
plt.loglog(xdata, ydata, 'ro' , xdata, yfit, 'g--')
plt.title("Word frequency distribution")
plt.ylabel("Word Frequency")
plt.xlabel("Words")
plt.grid()
plt.savefig('Question1_wordplot.png')
total_words = sum(word_values)
word_percent = [(float(wv)/total_words) for wv in word_values] #word_percent
c_values = [(lambda x: x[0]*x[1])(x) for x in zip(xdata, word_percent)]
c = sum(c_values)/len(c_values)
print("Word c: ", c)
#Create bigram plot
xdata, ydata = range(1,len(bigram_values)+1), bigram_values
xd, yd = log10(xdata), log10(ydata)
polycoef = polyfit(xd,yd,1)
yfit = 10** (polycoef[0]*xd+polycoef[1])
plt.figure()
plt.loglog(xdata, ydata, 'ro' , xdata, yfit, 'g--')
plt.title("Bigram frequency distribution")
plt.ylabel("Bigram Frequency")
plt.xlabel("Bigrams")
plt.grid()
plt.savefig('Question1_bigramplot.png')
total_bigrams = sum(bigram_values)
bigram_percent = [(float(bv)/total_bigrams) for bv in bigram_values] #word_percent
c_values = [(lambda x: x[0]*x[1])(x) for x in zip(xdata, bigram_percent)]
c = sum(c_values)/len(c_values)
print("Bigram c: ", c)
return
def estimateBeta(priceY,priceX,algo = 'standard'):
'''
estimate stock Y vs stock X beta using iterative linear
regression. Outliers outside 3 sigma boundary are filtered out
Parameters
--------
priceX : price series of x (usually market)
priceY : price series of y (estimate beta of this price)
Returns
--------
beta : stockY beta relative to stock X
'''
X = pd.DataFrame({'x':priceX,'y':priceY})
if algo=='returns':
ret = (X/X.shift(1)-1).dropna().values
x = ret[:,0]
y = ret[:,1]
# filter high values
low = np.percentile(x,20)
high = np.percentile(x,80)
iValid = (x>low) & (x<high)
x = x[iValid]
y = y[iValid]
iteration = 1
nrOutliers = 1
while iteration < 10 and nrOutliers > 0 :
(a,b) = polyfit(x,y,1)
yf = polyval([a,b],x)
#plot(x,y,'x',x,yf,'r-')
err = yf-y
idxOutlier = abs(err) > 3*np.std(err)
nrOutliers =sum(idxOutlier)
beta = a
#print 'Iteration: %i beta: %.2f outliers: %i' % (iteration,beta, nrOutliers)
x = x[~idxOutlier]
y = y[~idxOutlier]
iteration += 1
elif algo=='log':
x = np.log(X['x'])
y = np.log(X['y'])
(a,b) = polyfit(x,y,1)
beta = a
elif algo=='standard':
ret =np.log(X).diff().dropna()
beta = ret['x'].cov(ret['y'])/ret['x'].var()
else:
raise TypeError("unknown Beta algorithm type, use 'standard', 'log' or 'returns'")
return beta
def concat_extrap_ends(x, npts, polyorder=1, lowside=True, highside=True):
i=numpy.arange(npts, dtype='float64')
if lowside:
ans=scipy.polyfit(-1*(i+1.), x[:npts], polyorder)
x=numpy.concatenate([scipy.polyval(list(ans), i[::-1]), x])
if highside:
ans=scipy.polyfit(-1*(i[::-1]-1.), x[-1*npts:], polyorder)
x=numpy.concatenate([x, scipy.polyval(list(ans), i)])
return x
def plotFullModel(dm, sacc):
"""
"""
fig = plt.figure(figsize = (15, 5))
plt.suptitle("Sacc = %s" % sacc)
ax1 = plt.subplot(141)
lCols = ["blue", "red", "orange", "yellow", "green", "pink"]
xVar = "sacc%s_ex" % sacc
yVar = "sacc%s_ey" % sacc
dm = dm.select("%s != ''" % xVar)
dm = dm.select("%s != ''" % yVar)
dm = dm.select("%s != -1000" % xVar)
dm = dm.select("%s != -1000" % yVar)
for a in dm.unique("realAngle"):
_dm = dm.select("realAngle == %s" % a)
col = lCols.pop()
plt.scatter(_dm[xVar], _dm[yVar], color = col, marker = ".", label = a)
plt.axvline(constants.xCen, color = gray[3], linestyle = '--')
plt.axhline(constants.yCen, color = gray[3], linestyle = '--')
ax2 = plt.subplot(142)
pm = PivotMatrix(dm, ["stim_type", "realAngle"], ["file"], "xNorm1", colsWithin =True)
pm.linePlot(fig = fig)
pm.save("PM.csv")
pm._print()
plt.axhline(0, color = gray[3], linestyle = "--")
ax3 = plt.subplot(143)
plt.title("object")
dmObj= dm.select("stim_type == 'object'")
#slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(stimObj["ecc"], stimObj["xNorm1"])
x = dmObj["ecc"]
y = dmObj["xNorm%s" % sacc]
fit = scipy.polyfit(x,y,1)
fit_fn = scipy.poly1d(fit)
plt.plot(x,y, 'yo', x, fit_fn(x), '--k')
ax4 = plt.subplot(144)
plt.title("non-object")
dmNo= dm.select("stim_type == 'non-object'")
#slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(stimObj["ecc"], stimObj["xNorm1"])
x = dmNo["ecc"]
y = dmNo["xNorm%s" % sacc]
fit = scipy.polyfit(x,y,1)
fit_fn = scipy.poly1d(fit)
plt.plot(x,y, 'yo', x, fit_fn(x), '--k')
plt.savefig("./plots/Full_model_004C_sacc%s.png" % sacc)
plt.show()
def logaod_polyfit(wvl,aod,polynum=False):
"""
Purpose:
Take in an array of aod spectra and calculate the polynomials associated with each log of the spectrum
takes in the wvl
Input:
aod: numpy array of time,wvl
wvl: numpy array of wavelengths in nm
Output:
array of polynomial coefficients for each time point in the log(aod)
Keywords:
polynum: (optional, defaults to N-2 where N is the number of points in the spectrum)
number of polynomials coefficients to return (order of the polynomial), if set to False, uses default N-2
Dependencies:
- numpy
- scipy
Needed Files:
- None
Modification History:
Written: Samuel LeBlanc, NASA Ames Research Center, 2017-11-28
Modified:
"""
import numpy as np
from scipy import polyfit
# sanitize inputs
shape = aod.shape
wvl = np.array(wvl)
if not len(wvl.shape)==1:
raise ValueError('wvl is not a array with one dimension')
if not len(wvl) in shape:
raise ValueError('No size of aod is the same as wvl')
if not polynum:
polynum = len(wvl)-2
if len(shape)>1:
ni,n = [(i,j) for (i,j) in enumerate(shape) if not j==len(wvl)][0]
cc = np.zeros((polynum,n))
for i in range(n):
if ni==0:
cc[:,i] = polyfit(np.log(wvl),np.log(aod[i,:]),polynum)
else:
cc[:,i] = polyfit(np.log(wvl),np.log(aod[:,i]),polynum)
else:
cc = polyfit(np.log(wvl),np.log(aod),polynum)
return cc
def estimateBeta(priceY, priceX, algo="standard"):
"""
estimate stock Y vs stock X beta using iterative linear
regression. Outliers outside 3 sigma boundary are filtered out
Parameters
--------
priceX : price series of x (usually market)
priceY : price series of y (estimate beta of this price)
Returns
--------
beta : stockY beta relative to stock X
"""
X = DataFrame({"x": priceX, "y": priceY})
if algo == "returns":
ret = (X / X.shift(1) - 1).dropna().values
# print len(ret)
x = ret[:, 0]
y = ret[:, 1]
iteration = 1
nrOutliers = 1
while iteration < 10 and nrOutliers > 0:
(a, b) = polyfit(x, y, 1)
yf = polyval([a, b], x)
# plot(x,y,'x',x,yf,'r-')
err = yf - y
idxOutlier = abs(err) > 3 * np.std(err)
nrOutliers = sum(idxOutlier)
beta = a
# print 'Iteration: %i beta: %.2f outliers: %i' % (iteration,beta, nrOutliers)
x = x[~idxOutlier]
y = y[~idxOutlier]
iteration += 1
elif algo == "log":
x = np.log(X["x"])
y = np.log(X["y"])
(a, b) = polyfit(x, y, 1)
beta = a
elif algo == "standard":
ret = np.log(X).diff().dropna()
beta = ret["x"].cov(ret["y"]) / ret["x"].var()
else:
raise TypeError("unknown algorithm type, use 'standard', 'log' or 'returns'")
return beta
def intersect2ary(ary1, ary2):
"""
Takes the two 2-d arrays I{ary1} and I{ary2}
and finds their intersection. The arrays are organized
[X, Y], where X is a 1-d array for the abscissa
and Y is the corresponding 1-d array for the ordinate.
"""
X1, Y1 = ary1
X2, Y2 = ary2
m, c = polyfit(X1, Y1, 1)
n, d = polyfit(X2, Y2, 1)
return intersect2(m, c, n, d)
def estimate_rate_func(t, T, N, plot_flag=False, method='central diff'):
t_est_pts = scipy.linspace(t.min(), t.max(), N+2)
interp_func = scipy.interpolate.interp1d(t,T,'linear')
T_est_pts = interp_func(t_est_pts)
if plot_flag == True:
pylab.figure()
pylab.subplot(211)
pylab.plot(t_est_pts, T_est_pts,'or')
# Estimate slopes
slope_pts = scipy.zeros((N,))
T_slope_pts = scipy.zeros((N,))
if method == 'local fit':
for i in range(1,(N+1)):
mask0 = t > 0.5*(t_est_pts[i-1] + t_est_pts[i])
mask1 = t < 0.5*(t_est_pts[i+1] + t_est_pts[i])
mask = scipy.logical_and(mask0, mask1)
t_slope_est = t[mask]
T_slope_est = T[mask]
local_fit = scipy.polyfit(t_slope_est,T_slope_est,2)
dlocal_fit = scipy.polyder(local_fit)
slope_pts[i-1] = scipy.polyval(dlocal_fit,t_est_pts[i])
T_slope_pts[i-1] = scipy.polyval(local_fit,t_est_pts[i])
if plot_flag == True:
t_slope_fit = scipy.linspace(t_slope_est[0], t_slope_est[-1], 100)
T_slope_fit = scipy.polyval(local_fit,t_slope_fit)
pylab.plot(t_slope_fit, T_slope_fit,'g')
elif method == 'central diff':
dt = t_est_pts[1] - t_est_pts[0]
slope_pts = (T_est_pts[2:] - T_est_pts[:-2])/(2.0*dt)
T_slope_pts = T_est_pts[1:-1]
else:
raise ValueError, 'unkown method %s'%(method,)
# Fit line to slope estimates
fit = scipy.polyfit(T_slope_pts, slope_pts,1)
if plot_flag == True:
T_slope_fit = scipy.linspace(T_slope_pts.min(), T_slope_pts.max(),100)
slope_fit = scipy.polyval(fit,T_slope_fit)
pylab.subplot(212)
pylab.plot(T_slope_fit, slope_fit,'r')
pylab.plot(T_slope_pts, slope_pts,'o')
pylab.show()
rate_slope = fit[0]
rate_offset = fit[1]
return rate_slope, rate_offset
def pre_edge(energy, mu, group=None, e0=None, step=None,
nnorm=3, nvict=0, pre1=None, pre2=-50,
norm1=100, norm2=None, _larch=None, **kws):
"""pre edge, normalization for XAFS
"""
if _larch is None:
raise Warning("cannot remove pre_edge -- larch broken?")
if e0 is None or e0 < energy[0] or e0 > energy[-1]:
e0 = find_e0(energy, mu, group=group, _larch=_larch)
p1 = min(np.where(energy >= e0-10.0)[0])
p2 = max(np.where(energy <= e0+10.0)[0])
ie0 = np.where(energy-e0 == min(abs(energy[p1:p2] - e0)))[0][0]
if pre1 is None: pre1 = min(energy) - e0
if norm2 is None: norm2 = max(energy) - e0
p1 = min(np.where(energy >= pre1+e0)[0])
p2 = max(np.where(energy <= pre2+e0)[0])
if p2-p1 < 2:
p2 = min(len(energy), p1 + 2)
omu = mu*energy**nvict
coefs = polyfit(energy[p1:p2], omu[p1:p2], 1)
pre_edge = (coefs[0] * energy + coefs[1]) * energy**(-nvict)
# normalization
p1 = min(np.where(energy >= norm1+e0)[0])
p2 = max(np.where(energy <= norm2+e0)[0])
if p2-p1 < 2:
p2 = min(len(energy), p1 + 2)
coefs = polyfit(energy[p1:p2], omu[p1:p2], nnorm)
post_edge = 0
for n, c in enumerate(reversed(list(coefs))):
post_edge += c * energy**(n-nvict)
edge_step = post_edge[ie0] - pre_edge[ie0]
norm = (mu - pre_edge)/edge_step
if _larch.symtable.isgroup(group):
setattr(group, 'e0', e0)
setattr(group, 'edge_step', edge_step)
setattr(group, 'norm', norm)
setattr(group, 'pre_edge', pre_edge)
setattr(group, 'post_edge', post_edge)
else:
return edge_step, e0
def main():
data = sp.genfromtxt("web_traffic.tsv", delimiter="\t")
plt.xkcd()
x = data[:, 0]
y = data[:, 1]
x = x[~sp.isnan(y)]
y = y[~sp.isnan(y)]
fp1, _, _, _, _ = sp.polyfit(x, y, 1, full=True)
# Here we try 3 degrees of freedom
fp2, _, _, _, _ = sp.polyfit(x, y, 3, full=True)
f1 = sp.poly1d(fp1)
f2 = sp.poly1d(fp2)
# We have an obvious inflection point between 3rd and 4th week
inflection_in_hours = int(3.5 * 7 * 24)
x_before_inflection = x[:inflection_in_hours]
x_after_inflection = x[inflection_in_hours:]
y_after_inflection = y[inflection_in_hours:]
f_after = sp.poly1d(sp.polyfit(x_after_inflection, y_after_inflection, 1))
fx = sp.linspace(0, x[-1], 1000)
fx_after = sp.linspace(len(x_before_inflection)+1, x[-1], 1000)
plt.scatter(x, y, s=5)
plt.title("Web traffic over the last month.")
plt.xlabel("Time")
plt.ylabel("Hits/hour")
plt.xticks([w * 7 * 24 for w in range(10)],
['week {}'.format(w) for w in range(10)])
plt.autoscale(tight=True)
plt.plot(fx, f1(fx), linewidth=2)
plt.plot(fx, f2(fx), linewidth=2)
plt.plot(fx_after, f_after(fx_after), linewidth=3)
plt.legend(["d={}".format(f1.order),
"d={}".format(f2.order),
"d after inflection"],
loc="upper left")
# plt.grid(True, linestyle="-", color='0.75')
plt.show()
def ar1fit(ts):
'''
Fits an AR(1) model to the time series data ts. AR(1) is a
linear model of the form
x_t = beta * x_{t-1} + c + e_{t-1}
where beta is the coefficient of term x_{t-1}, c is a constant
and x_{t-1} is an i.i.d. noise term. Here we assume that e_{t-1}
is normally distributed.
Returns the tuple (beta, c, sigma).
'''
# Fitting AR(1) entails finding beta, c, and the noise term.
# Beta is well approximated by the coefficient of OLS regression
# on the lag of the data with itself. Since the noise term is
# assumed to be i.i.d. and normal, we must only estimate sigma,
# the standard deviation.
# Estimate beta - auto regressive parameter
x = ts[0:-1]
y = ts[1:]
p = sp.polyfit(x, y, 1)
beta = p[0]
# Estimate c
c = sp.mean(ts) * (1 - beta)
# Estimate the variance from the residuals of the OLS regression.
yhat = sp.polyval(p, x)
variance = sp.var(y - yhat)
sigma = sp.sqrt(variance)
return beta, c, sigma
def Gradient(self,Results='',TestSet=False):
G = GBR(loss='ls', learning_rate=0.1, n_estimators=100, subsample=1.0, criterion='friedman_mse', min_samples_split=2,
min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_depth=3, min_impurity_split=None, init=None, random_state=None,
max_features=None, alpha=0.9, verbose=0, max_leaf_nodes=None, warm_start=False, presort='auto')
if TestSet==False:
GResult = G.fit(self.X,np.ravel(self.y,1))
if Results==True:
print(str(GResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(GResult.get_params()))
plt.plot(G.fit(self.X,np.ravel(self.y,1)).predict(self.X))
y = np.array(self.y[self.DVCols])
plt.plot(y,'ro')
plt.show()
else:
x_train = self.X[:len(self.X)//2]
y_train = np.ravel(self.y,1)[:len(self.y)//2]
x_test = self.X[len(self.X)//2:]
y_test = np.ravel(self.y,1)[len(self.y)//2:]
GResult = G.fit(x_train,y_train)
if Results==True:
print(str(GResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(GResult.get_params()))
GRPredict = GResult.predict(x_test)
plt.plot(GRPredict,polyval(polyfit(GRPredict,y_test.reshape(-1),1),GRPredict),'r-',label='predicted')
plt.plot(GRPredict,y_test.reshape(-1),'bo')
plt.legend()
plt.show()
def Neural(self,Results=True,TestSet=False):
NN = MLPR(hidden_layer_sizes=(100, ), activation='relu', solver='adam', alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001,
power_t=0.5, max_iter=200, shuffle=True, random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True,
early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if TestSet==False:
NNResult = NN.fit(self.X,np.ravel(self.y,1))
if Results==True:
print(str(NNResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(NNResult.get_params()))
plt.plot(NN.fit(self.X,np.ravel(self.y,1)).predict(self.X))
y = np.array(self.y[self.DVCols])
plt.plot(y,'ro')
plt.show()
else:
x_train = self.X[:len(self.X)//2]
y_train = np.ravel(self.y,1)[:len(self.y)//2]
x_test = self.X[len(self.X)//2:]
y_test = np.ravel(self.y,1)[len(self.y)//2:]
NNResult = NN.fit(x_train,y_train)
if Results==True:
print(str(NNResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(NNResult.get_params()))
NNRPredict = NNResult.predict(x_test)
plt.plot(NNRPredict,polyval(polyfit(NNRPredict,y_test.reshape(-1),1),NNRPredict),'r-',label='predicted')
plt.plot(NNRPredict,y_test.reshape(-1),'bo')
plt.legend()
plt.show()
def MNK(x, y, st):
n = len(x)
s = sum(y)
fp, residuals, rank, sv, rcond = scipy.polyfit(x, y, st, full=True)
f = scipy.poly1d
print('a =', round(fp[0], 4))
print('b =', round(fp[1], 4))
print('c =', round(fp[2], 4))
y1 = [fp[0] * x[i]**2 + fp[1] * x[i] + fp[2] for i in range(n)]
s1 = sum([1 / x[i] for i in range(n)]) # сумма 1/x
s2 = sum([(1 / x[i])**2 for i in range(n)]) # сумма (1/x)**2
s3 = sum([y[i] / x[i] for i in range(n)]) # сумма y/x
a = round((s * s2 - s1 * s3) / (n * s2 - s1**2),
3) # коэфициент а с тремя дробными цифрами
b = round((n * s3 - s1 * s) / (n * s2 - s1**2),
3) # коэфициент b с тремя дробными цифрами
s4 = [a + b / x[i]
for i in range(n)] # список значений гиперболической функции
so = round(
sum([abs(y[i] - y1[i])
for i in range(0, len(x))]) / (len(x) * sum(y)) * 100,
4) # средняя ошибка аппроксимации
plt.plot(x, s4, color='black', linewidth=2)
plt.legend(loc='best')
plt.xlabel('x')
plt.ylabel('y')
plt.plot(x, y, color='deeppink', linestyle=' ', marker='o')
plt.grid(True)
plt.show()
def linear_interpolate(self, x, y):
new_x, new_y = [], []
for x_i, y_i in zip(x, y):
if y_i > 0.0:
new_x.append(x_i)
new_y.append(y_i)
x, y = new_x, new_y
half_x = x[len(x) / 4:-len(x) / 4]
half_y = y[len(y) / 4:-len(y) / 4]
arrayx = np.ndarray(len(half_x))
for val, i in zip(half_x, range(len(half_x))):
arrayx[i] = math.log10(val)
all_arrayx = np.ndarray(len(x))
for val, i in zip(x, range(len(x))):
all_arrayx[i] = math.log10(val)
arrayy = np.ndarray(len(half_y))
for val, i in zip(half_y, range(len(half_y))):
arrayy[i] = math.log10(val)
(ar, br) = polyfit(arrayx, arrayy, 1)
xr = polyval([ar, br], all_arrayx)
return map(lambda val: 10.0**val, xr), map(lambda val: 10.0**val,
all_arrayx), ar, br
def skew_model(self):
'''
a model for skewness of theta_1 | theta_0, y_0
skew model is a(y)*theta_0 + b(y)
'''
theta_array = self.mid_theta[self.theta_range]
query_y = self.query_y
skew_slope_dict, skew_intercept_dict = {}, {}
x_agg, y_agg = [], []
for y_class in query_y:
y = self.mid_y[y_class]
theta_trans_mat = trans_prob_given_y(self.marginal, y_class,
self.mapping)
moments_mat = np.zeros((theta_trans_mat.shape[1], 4))
for j in self.theta_range:
prob = theta_trans_mat[:, j]
moments_mat[j, :] = moments_given_pdf(self.mid_theta, prob)
xp, yp = theta_array, moments_mat[self.theta_range, 2]
nan_filter = ~np.isnan(yp)
xp, yp = xp[nan_filter], yp[nan_filter]
x_agg.extend(xp)
y_agg.extend(yp)
skew_slope_dict[y], skew_intercept_dict[y] = sci.polyfit(xp, yp, 1)
line = lambda x, a: a * x
popt, pcov = curve_fit(line, x_agg, y_agg)
self.avg_skew_slope = popt[0]
return skew_slope_dict, skew_intercept_dict
def test_LGST_1overSqrtN_dependence(self):
my_datagen_gateset = self.gateset.depolarize(gate_noise=0.05,
spam_noise=0)
# !!don't depolarize spam or 1/sqrt(N) dependence saturates!!
nSamplesList = np.array([16, 128, 1024, 8192])
diffs = []
for nSamples in nSamplesList:
ds = pygsti.construction.generate_fake_data(my_datagen_gateset,
self.lgstStrings,
nSamples,
sampleError='binomial',
seed=100)
gs_lgst = pygsti.do_lgst(ds,
self.specs,
self.gateset,
svdTruncateTo=4,
verbosity=0)
gs_lgst_go = pygsti.optimize_gauge(
gs_lgst,
"target",
targetGateset=my_datagen_gateset,
spamWeight=1.0,
gateWeight=1.0)
diffs.append(my_datagen_gateset.frobeniusdist(gs_lgst_go))
diffs = np.array(diffs, 'd')
a, b = polyfit(np.log10(nSamplesList), np.log10(diffs), deg=1)
#print "\n",nSamplesList; print diffs; print a #DEBUG
self.assertLess(a + 0.5, 0.05)
def ivFitSuperconduct(self,
vbias,
vfb,
vbias_offset,
show_plot=False,
ignore_nans=True):
'''Fit the superconducting branch of the IV and return the slope and intercept. '''
fb_diff = np.diff(vfb)
nan_indicies = np.where(np.isnan(vfb))
if len(nan_indicies[0]) > 0 and ignore_nans == False:
# transition data is at values larger than the NaNs, set ignore_nans to False to turn this on
print('not igonring nans')
end_sc = int(min(nan_indicies[0]) - 1)
else:
neg_slope_array = np.where(fb_diff < 0)
end_sc = min(neg_slope_array[0]
) - 2 # Go down two point to get away from the edge
vbias_sc = vbias[:end_sc] - vbias_offset
vfb_sc = vfb[:end_sc]
sc_m, sc_b = scipy.polyfit(vbias_sc, vfb_sc, 1)
vfb_sc_fit = scipy.polyval([sc_m, sc_b], vbias_sc)
sc_fit = [sc_m, sc_b]
if show_plot is True:
pylab.plot(vbias_sc, vfb_sc_fit, label='SC Branch Fit')
pylab.legend()
return sc_fit
def mean_var_model(self):
'''
a model for the mean of theta_1 | theta_0, y_0
mean model is a(theta)*y + b(theta). For theta in mid_theta[query_theta]
we will save a, b in mean_slopes[theta], mean_intercepts[theta]
'''
y_array = self.mid_y[self.y_range]
query_theta = self.query_theta
mean_slope_dict, mean_intercept_dict, var_dict = {}, {}, {}
for theta_class in query_theta:
theta = self.mid_theta[theta_class]
theta_trans_mat = trans_prob_given_theta(self.marginal,
theta_class, self.mapping)
moments_mat = np.zeros((theta_trans_mat.shape[1], 4))
for j in self.y_range:
prob = theta_trans_mat[:, j]
moments_mat[j, :] = moments_given_pdf(self.mid_theta, prob)
xp, yp = y_array, moments_mat[self.y_range, 0]
nan_filter = ~np.isnan(yp)
xp, yp = xp[nan_filter], yp[nan_filter]
mean_slope_dict[theta], mean_intercept_dict[theta] = sci.polyfit(
xp, yp, 1)
var_vector = moments_mat[self.y_range, 1]
var_vector = var_vector[~np.isnan(var_vector)]
var_dict[theta] = np.mean(var_vector)
return mean_slope_dict, mean_intercept_dict, var_dict
def line_of_best_fit(degree):
plt.plot(dates, prices, 'o')
p1 = sp.polyfit(dates, prices, degree)
plt.plot(dates, sp.polyval(p1, dates), 'r-')
prices_predict = p1[0] * dates + p1[1] # y = mx + b
# print("Predicted:\n" , prices_predict, "Actual:\n", prices)
plt.show()
def calculate_polyfit(x, y):
""" Calculate a linear regression using polyfit instead """
# FIXME(pica) This doesn't work for large ranges (tens of orders of
# magnitude). No idea why as the fixRange() function above should make
# all values greater than one. The trouble mainly seems to happen when
# log10(min(x)) negative.
# Check the smallest value in x or y isn't below 2x10-7 otherwise
# we hit numerical instabilities.
xFactorFix, xFactor = False, 0
yFactorFix, yFactor = False, 0
minX = min(x)
minY = min(y)
if minX < 2e-7:
x, xFactor, xFactorFix = fixRange(x)
if minY < 2e-7:
y, yFactor, yFactorFix = fixRange(y)
(ar, br) = polyfit(x, y, 1)
yf = polyval([ar, br], x)
xf = x
if xFactorFix:
xf = xf / 10**xFactor
if yFactorFix:
yf = yf / 10**yFactor
return xf, yf
def test_scipy():
#Sample data creation
#number of points
n=50
t=linspace(-5,5,n)
#parameters
a=0.8; b=-4
x=polyval([a,b],t)
#add some noise
xn=x+randn(n)
#Linear regressison -polyfit - polyfit can be used other orders polys
(ar,br)=polyfit(t,xn,1)
xr=polyval([ar,br],t)
#compute the mean square error
err=sqrt(sum((xr-xn)**2)/n)
print('Linear regression using polyfit')
print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, ms error= %.3f' % (a,b,ar,br,err))
#matplotlib ploting
title('Linear Regression Example')
plot(t,x,'g.--')
plot(t,xn,'k.')
plot(t,xr,'r.-')
legend(['original','plus noise', 'regression'])
show()
#Linear regression using stats.linregress
(a_s,b_s,r,tt,stderr)=stats.linregress(t,xn)
print('Linear regression using stats.linregress')
print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, std error= %.3f' % (a,b,a_s,b_s,stderr))
def interpolate_rects(self, uid):
rects_for_interpolate = self.cars[uid]['rects'].copy()
cur_rect = self.cars[uid]['cur_rect'].copy()
x, y, w, h = zip(*rects_for_interpolate)
fp, residuals, rank, sv, rcond = sp.polyfit(x, y, 3, full=True)
f = sp.poly1d(fp)
fx = sp.linspace(x[0], x[-1] + 30, 100).astype(int)
y_interp = f(fx).astype(int)
fp, residuals, rank, sv, rcond = sp.polyfit(w, h, 3, full=True)
f = sp.poly1d(fp)
fw = sp.linspace(w[0], w[-1] + 30, 100).astype(int)
h_interp = f(fw).astype(int)
rects = list(zip(fx, y_interp, fw, h_interp))
self.cars[uid]['rects'] = rects
cur_rect = rects[-1]
self.cars[uid]['cur_rect'] = cur_rect
def linearRegression(histogram):
x=[]
y=[]
for i in range(257):
for k in range(257):
if histogram.GetBinContent(i,k)!=0:
x.append(i)
y.append(k)
(ar,br)=polyfit(x,y,1)
xr=polyval([ar,br],x)
err=sqrt(sum((xr-y)**2)/len(y))
if abs(ar)>1.0:
(ar,br)=polyfit(y,x,1)
xr=polyval([ar,br],y)
err=sqrt(sum((xr-x)**2)/len(y))
return err
def linearRegression(list):
x = np.linspace(1, len(list), len(list))
# Fit the data to y = mx + c
linearModel = polyfit(x, list, 1, full=True)
m = linearModel[0][0]
res = linearModel[1]
return m, res
def test_lagrange_cheby_product(self):
import pylab
import itertools
import scipy
test_function = lambda x,y: numpy.cos(CONST_2PI*x*2)*numpy.sin(CONST_2PI*x)
#numpy.cos(CONST_2PI*x*4)*(float(y <= 0.5)*2-1)*abs(y-0.2)
sample_points=range(10,30)
errors=[]
n_test_points = 30
for n_sample_points in sample_points:
Q = chebyshev_grid(n_sample_points)*0.5+0.5
U = itertools.product(Q,Q)
X = [ float(i)/n_test_points for i in range(n_test_points) ]
W = itertools.product(X,X)
L = lagrange_matrix(Q, X)
F = numpy.array( [ [test_function(x,y) for x in Q ] for y in Q ] )
H = numpy.dot( numpy.dot(L, F), numpy.transpose(L) )
G = numpy.array( [ [test_function(x,y) for x in X ] for y in X ] )
errors.append( numpy.log10(numpy_norm(G-H)/numpy_norm(G)+1E-17) )
(slope,offset)=scipy.polyfit(sample_points,errors,1)
mean = scipy.mean(errors)
#pylab.plot(sample_points,errors)
#pylab.show()
#print slope,mean
self.assertTrue( slope <= -0.4 and mean <= -4 )
def test_LGST_1overSqrtN_dependence(self):
my_datagen_gateset = self.model.depolarize(op_noise=0.05, spam_noise=0)
# !!don't depolarize spam or 1/sqrt(N) dependence saturates!!
nSamplesList = np.array([16, 128, 1024, 8192])
diffs = []
for nSamples in nSamplesList:
ds = pygsti.data.simulate_data(my_datagen_gateset,
self.lgstStrings,
nSamples,
sample_error='binomial',
seed=100)
mdl_lgst = pygsti.run_lgst(ds,
self.fiducials,
self.fiducials,
self.model,
svd_truncate_to=4,
verbosity=0)
mdl_lgst_go = pygsti.gaugeopt_to_target(mdl_lgst,
my_datagen_gateset, {
'spam': 1.0,
'gate': 1.0
},
check_jac=True)
diffs.append(my_datagen_gateset.frobeniusdist(mdl_lgst_go))
diffs = np.array(diffs, 'd')
a, b = polyfit(np.log10(nSamplesList), np.log10(diffs), deg=1)
#print "\n",nSamplesList; print diffs; print a #DEBUG
self.assertLess(a + 0.5, 0.05)
def h_estimation(p_dict, npoints, alpha=1., Ns=None):
"""Compute an estimation of the negative entropy of a distribution.
The estimation is done using a Bayesian estimator, followed by an
extrapolation step to reduce biases in the estimation.
Input arguments:
p_dict -- a dictionary containing the distribution of states, as created
by the function `states2dict`
npoints -- total number of points used to estimate the distribution
alpha -- paramters of the Dirichlet prior (usually set to 1)
"""
ns = array([4,2,1], dtype='int64')
h_estimate = zeros((len(ns),))
for j, d in enumerate(ns):
h_est = zeros((d,))
for i in range(d):
p = p_dict[d][i].flatten() # for joint distributions
h_est[i] = mean_H_estimate(p + alpha)
h_estimate[j] = h_est.mean()
# extrapolate
if Ns is None:
Ns = npoints/ns
Ns = Ns.astype('d')
h_extr = scipy.polyfit(Ns, Ns*Ns * h_estimate, 2)
return h_extr[0]
def predict(A):
if len(A) > min(3, n):
xtrain = np.linspace(1,len(A),len(A))
ytrain = A
xtest = len(A) + 1
f = poly1d(polyfit(xtrain, ytrain, n))
return f(xtest).round() if all(f(xtrain).round() == ytrain) else None
def plot_ci_bootstrap(xs, ys, resid, nboot=500, ax=None):
"""Return an axes of confidence bands using a bootstrap approach.
Notes
-----
The bootstrap approach iteratively resampling residuals.
It plots `nboot` number of straight lines and outlines the shape of a band.
The density of overlapping lines indicates improved confidence.
Returns
-------
ax : axes
- Cluster of lines
- Upper and Lower bounds (high and low) (optional) Note: sensitive to outliers
References
----------
.. [1] J. Stults. "Visualizing Confidence Intervals", Various Consequences.
http://www.variousconsequences.com/2010/02/visualizing-confidence-intervals.html
"""
if ax is None:
ax = plt.gca()
bootindex = sp.random.randint
for _ in range(nboot):
resamp_resid = resid[bootindex(0, len(resid) - 1, len(resid))]
# Make coeffs of for polys
pc = sp.polyfit(xs, ys + resamp_resid, 1)
# Plot bootstrap cluster
ax.plot(xs, sp.polyval(pc, xs), "b-", linewidth=2, alpha=3.0 / float(nboot))
return ax
def GetBrineDensityLinearFit(h, T0, gradT, P0, Xs0, gradXs=0, ReturnAll = False, discretization = 100):
'''Compute (rho0, xi), the brine density linear fit rho(z) = rho0 + xi*(z-z0), in kg.m-3, for a given
temperature T(z) = T0 + gradT+(z-z0) in Celcius, Pressure P(z0) = P0 in Pa and salinity Xs(z) = Xs0 + gradXs*(z-z0) as mass fraction.
T0 = T(z0), P0 = P(z0), Xs0 = Xs(z0)'''
# variables definitions
dz = h/discretization
z = arange(0., h*1.00001, dz)
rho = array([]) # density
# initialization at z = 0
T = T0; P = P0; Xs = Xs0;
# incrementation. We move towards the surface.
for zz in z:
rho = append(rho, GetBrineDensity(T, P, Xs))
T += -gradT*dz
Xs += -gradXs*dz
P += -rho[-1]*9.81*dz
# Linear fit
[xi, rho0] = polyfit(z, rho, 1)
if ReturnAll == True:
return rho0, xi, rho, z
else:
return rho0, xi
def Asaoka_fit(T0,S0,interval,start_date=0, end_date=0):
if end_date ==0:
end_date = T0.max()
if start_date==0:
start_date =T0.min()
#now convert T0 to days starting from the origin
T0 =(T0-T0[0])/np.timedelta64(1,'D')
#user does not specify the end date for fitting, take the maximum
start_date = np.datetime64(start_date)
end_date = np.datetime64(end_date)
n = int(np.floor((end_date-start_date)/np.timedelta64(1,'D')/interval))
T_bar = np.zeros(n)
origin = T0[0]
for i in np.arange(n):
T_bar[i] = origin + i*interval
interpolate_1d = interp1d(T0,S0)
S1=interpolate_1d(T_bar)
#fit the linear curve
beta1, beta0 = polyfit(S1[0:-1], S1[1:],1)
return beta0, beta1
#we do the interpolation here
# fig = plt.figure(figsize=(11.69,8.27),dpi=100)
# ax = fig.add_subplot(2,1,1)
# ax_Asaoka = fig.add_subplot(2,1,2)
# ax_Asaoka.plot(S1[0:-1],S1[1:],'ko',label=r'$s_i$ vs $s_{i-1}$',markersize=3)
# ax_Asaoka.set_xlabel(r'$S_{i-1}$')
ax_Asaoka.set_ylabel(r'$S_i$')
def get_state_data(state0):
state = state0.replace(" ", "-")
print(state0)
state_data = requests.get(
f"https://www.worldometers.info/coronavirus/usa/{state}/")
text = bs4.BeautifulSoup(state_data.text, "lxml")
data = text.find(
"h3", text=f"Daily New Cases in {state0}"
).next_sibling.next_sibling.next_sibling.next_sibling.string
dates = json.loads(data[data.find("categories:") + 12:data.find("yAxis:") -
14])
cases = json.loads(data[data.find("data:") +
6:data.find("3-day moving average") - 44])
data_d = text.find(
"h3", text=f"Daily New Deaths in {state0}"
).next_sibling.next_sibling.next_sibling.next_sibling.string
deaths = json.loads(data_d[data_d.find("data:") +
6:data_d.find("3-day moving average") - 44])
df = pd.DataFrame({"cases": cases, "deaths": deaths}, index=dates)
df.index = pd.DatetimeIndex(df.index + ", 2020")
tail = df.resample("1W").mean().iloc[-5:-1]["cases"]
poly = scipy.polyfit(range(tail.shape[0]), tail.values, 1)
return poly
def getDiameterQuantilesAlongSinglePath(self,path,G,counter=None):
G=self.filterPathDiameters(path, G,self.__scale)
x=[]
y=[]
length=0
vprop=G.vertex_properties["vp"]
for i in path:
length+=1
if vprop[i]['diameter'] > 0:
x.append(length)
y.append(vprop[i]['diameter'])
coeffs=polyfit(x,y,1)
besty = polyval ( coeffs , x)
self.__io.saveArray(x,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterX')
self.__io.saveArray(y,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterY')
l=len(y)-1
l25=int(l*0.25)
l50=int(l*0.5)
l75=int(l*0.75)
l90=int(l*0.90)
d25=np.average(y[:l25])
d50=np.average(y[l25:l50])
d75=np.average(y[l50:l75])
d90=np.average(y[l90:])
self.__io.saveArray(y,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterHistoTP')
return d25,d50,d75,d90
def interpMaskedAreas(x, dtfunc):
idx = num.where(dtfunc == -1)[0]
# find consecutive points of dtfunc set at -1
diffs = idx - num.roll(idx, 1)
setIdx1 = num.where(diffs > 1)[0]
setIdx2 = num.where(diffs > 1)[0] - 1
setIdx1 = num.hstack( (num.array([0]), setIdx1) )
setIdx2 = num.hstack( (setIdx2, num.array([-1])) )
# each int in setIdx1 is beginning of a set of points
# each int in setIdx2 is end of a set of points
for i in range(len(setIdx1)):
i1 = setIdx1[i]
i2 = setIdx2[i]
vals_to_interp = dtfunc[ idx[i1]:idx[i2]+1]
# construct interpolation from points on either side of vals
# first ydata
nVals = len(vals_to_interp)
left = num.array( \
[ dtfunc[idx[i1]-1-nVals], dtfunc[idx[i1]-1] ])
right = num.array( \
[ dtfunc[idx[i2]+1], dtfunc[idx[i2]+1+nVals] ])
vals_to_construct_interp = num.hstack( (left, right) )
vtci = vals_to_construct_interp
# now xdata
leftx = num.array([ x[idx[i1]-1-nVals], x[idx[i1]-1] ])
rightx = num.array([ x[idx[i2]+1], x[idx[i2]+1+nVals] ])
x_vtci = num.hstack( (leftx, rightx) )
# conduct interpolation
intpCoeffs = scipy.polyfit(x_vtci, vtci, 3)
dtfunc[ idx[i1]:idx[i2]+1 ] = \
scipy.polyval(intpCoeffs, x[ idx[i1]:idx[i2]+1 ])
return dtfunc
def plot_model_order_n(n, x, y):
fpn = sp.polyfit(x, y, n)
fn = sp.poly1d(fpn)
print("Order %d model error: %f" % (fn.order, error(fn, x, y)))
fx = sp.linspace(0, x[-1], 1000)
if n == 1:
linecolor = "red"
elif n == 2:
linecolor = "black"
elif n == 3:
linecolor = "green"
elif n == 10:
linecolor = "violet"
elif n == 100:
linecolor = "cyan"
else:
linecolor = "orange"
plt.plot(fx, fn(fx), color=linecolor, linewidth=4, label=str(fn.order))
plt.legend(loc="upper left")
plt.autoscale(tight=True)
plt.ylim(ymin=0)
plt.xlim(xmin=0)
plt.grid(True)
def main_df(sim_num, rho):
d1 = random.Random()
d2 = random.Random()
d3 = random.Random()
pd_limits = [0.0, 0.1]
lgds = [ 1.0, 1.0]
vol_lgd = [0.2, 0.2]
alpha =0.999
extra_corr = correl_function(rho)
results = []
asympt_results = []
x_values = []
cdis = []
for i in range(sim_num):
pds = np.array([ d1.uniform(*pd_limits), d2.uniform(*pd_limits) ])
weight = d3.uniform(0,1)
weights = np.array([weight, 1-weight])
r_intra = np.array([correl_f(pds[0]), correl_f(pds[1])])
exp_loss = sum(weights*lgds*pds)
results.append((loss(extra_corr, pds, lgds, weights, r_intra, alpha)-exp_loss)/2)
asympt_results.append((sum([w*lgd*loss_var(alpha, pd, corr**2)
for (w, lgd, pd, corr) in zip(weights, lgds, pds, r_intra)])-exp_loss)/2)
cdis.append(cdi(alpha,weights, lgds, pds, r_intra))
x_values.append(avg(pds) )
#pylab.plot( x_values, results, 'b+')
#pylab.plot( x_values, asympt_results, 'go')
pylab.ylim(ymin=0.0, ymax=1.0)
dfs = [i/j for (i,j) in zip(results, asympt_results)]
pylab.plot( cdis, dfs, 'b+')
(a,b) = polyfit(cdis, dfs,1)
print 'coefficient a = ', a , ' and b = ' , b
def regress_col(m, pto, cols, selector, allow_missing=False):
# Discard the constants, we will pick a reference point later
slopes = []
for col in cols:
'''
For each row find an y position
y = row * c0 + c1
'''
rows = []
deps = []
for row in range(m.height()):
fn = m.get_image(col, row)
if fn is None:
if allow_missing:
continue
raise Exception('c%d r%d not in map' % (col, row))
il = pto.get_image_by_fn(fn)
if il is None:
raise Exception('Could not find %s in map' % fn)
rows.append(row)
deps.append(selector(il))
if len(rows) == 0:
if not allow_missing:
raise Exception('No matches')
continue
(c0, c1) = polyfit(rows, deps, 1)
slopes.append(c0)
if len(slopes) == 0:
if not allow_missing:
raise Exception('No matches')
# No dependence
return 0.0
# XXX: should remove outliers
return sum(slopes) / len(slopes)
def getFx(datax, datay, degree):
fp1= sp.polyfit(datax, datay, degree)
#print fp1
#aa = polyval(fp1, 5)
#print aa
fStraight = sp.poly1d(fp1)
return fStraight
def getDiameterQuantilesAlongSinglePath(self,path,G,counter=None):
G=self.filterPathDiameters(path, G,self.__scale)
x=[]
y=[]
length=0
vprop=G.vertex_properties["vp"]
for i in path:
length+=1
if vprop[i]['diameter'] > 0:
x.append(length)
y.append(vprop[i]['diameter'])
coeffs=polyfit(x,y,1)
besty = polyval ( coeffs , x)
self.__io.saveArray(x,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterX')
self.__io.saveArray(y,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterY')
l=len(y)-1
l25=int(l*0.25)
l50=int(l*0.5)
l75=int(l*0.75)
l90=int(l*0.90)
d25=np.average(y[:l25])
d50=np.average(y[l25:l50])
d75=np.average(y[l50:l75])
d90=np.average(y[l90:])
self.__io.saveArray(y,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterHistoTP')
return d25,d50,d75,d90
def regress_col(m, pto, cols, selector, allow_missing = False):
# Discard the constants, we will pick a reference point later
slopes = []
for col in cols:
'''
For each row find an y position
y = row * c0 + c1
'''
rows = []
deps = []
for row in range(m.height()):
fn = m.get_image(col, row)
if fn is None:
if allow_missing:
continue
raise Exception('c%d r%d not in map' % (col, row))
il = pto.get_image_by_fn(fn)
if il is None:
raise Exception('Could not find %s in map' % fn)
rows.append(row)
deps.append(selector(il))
if len(rows) == 0:
if not allow_missing:
raise Exception('No matches')
continue
(c0, c1) = polyfit(rows, deps, 1)
slopes.append(c0)
if len(slopes) == 0:
if not allow_missing:
raise Exception('No matches')
# No dependence
return 0.0
# XXX: should remove outliers
return sum(slopes) / len(slopes)
def get_trendline(xs,ys,params=False):
m,b = scipy.polyfit(xs,ys,1)
ys_trend = (m * xs) + b
if params:
return xs,ys_trend,m,b
else:
return xs,ys_trend
def scatter_fits(sys, scores, values, R, pval, aucname, show=False):
auclabel=get_label(aucname)
if sys=='apo':
format='ko'
label=get_label(sys)
if sys=='bi':
format='ro'
label=get_label(sys)
if sys=='car':
format='bo'
label=get_label(sys)
if pval< 0.0001:
pval=0.0001
pylab.figure()
pylab.plot(scores, values, format)
(ar,br)=polyfit(scores, values, 1)
xr=polyval([ar,br], scores)
pylab.plot(scores,xr,'%s-' % format[0], label='R=%s, pval=%s' %
(round(R,2), round(pval,4)))
if aucname=='types':
pylab.plot(range(0, 15), [0.5]*len(range(0,15)), 'k--', label='Random Disc.')
pylab.ylim(0.3, 0.9)
else:
pylab.ylim(0.5, 1.0)
pylab.xlim(0, 15)
lg=pylab.legend()
lg.draw_frame(False)
pylab.title('%s States' % label)
pylab.xticks(range(0, 15), [' ']*2+ ['inactive']+[' ']*(len(range(0,15))-6)+['active']+[' ']*2)
pylab.xlabel('Pathway Progress')
pylab.ylabel('%s Aucs' % (auclabel))
pylab.savefig('%s_%saucs.png' % (sys, aucname), dpi=300)
if show==True:
pylab.show()
def correct_slope(self,rank=4):
index = np.linspace(0,self.nsamp-1,self.nsamp)
trends = np.zeros((self.nsamp,self.norders))
i = 0
'''
for order in self.orders:
fsubs = np.isfinite(order['flux'])
pars = polyfit(index[fsubs][20:-20],order['flux'][fsubs][20:-20],deg=rank,w=order['uflux'][fsubs][20:-20])
trend = poly1d(pars)
trend_samp = trend(index)
trends[:,i] = trend_samp/np.median(trend_samp)
i += 1
trend_mean = np.median(trends,axis=1)
'''
fluxes = np.zeros((self.nsamp,self.norders))
for i in np.arange(self.norders):
flux = self.orders[i]['flux']
flux = flux/np.median(flux)
fluxes[:,i] = flux
i += 1
trend_mean = np.nanmedian(fluxes,axis=1)
fsubs = np.isfinite(trend_mean)
pars = polyfit(index[fsubs],trend_mean[fsubs],deg=rank)
trend_smooth = poly1d(pars)(index)
for order in self.orders:
order['flux'] /= trend_smooth
def _xcorr_interp(ccc, dt):
"""
Interpolate around the maximum correlation value for sub-sample precision.
:param ccc: Cross-correlation array
:type ccc: numpy.ndarray
:param dt: sample interval
:type dt: float
:return: Position of interpolated maximum in seconds from start of ccc
:rtype: float
"""
if ccc.shape[0] == 1:
cc = ccc[0]
else:
cc = ccc
# Code borrowed from obspy.signal.cross_correlation.xcorr_pick_correction
cc_curvature = np.concatenate((np.zeros(1), np.diff(cc, 2), np.zeros(1)))
cc_t = np.arange(0, len(cc) * dt, dt)
peak_index = cc.argmax()
first_sample = peak_index
# XXX this could be improved..
while first_sample > 0 and cc_curvature[first_sample - 1] <= 0:
first_sample -= 1
last_sample = peak_index
while last_sample < len(cc) - 1 and cc_curvature[last_sample + 1] <= 0:
last_sample += 1
num_samples = last_sample - first_sample + 1
if num_samples < 3:
Logger.warning(
"Fewer than 3 samples selected for fit to cross correlation: "
"{0}, returning maximum in data".format(num_samples))
return np.argmax(cc) * dt, np.amax(cc)
if num_samples < 5:
Logger.debug(
"Fewer than 5 samples selected for fit to cross correlation: "
"{0}".format(num_samples))
coeffs, residual = scipy.polyfit(cc_t[first_sample:last_sample + 1],
cc[first_sample:last_sample + 1],
deg=2,
full=True)[:2]
# check results of fit
if coeffs[0] >= 0:
Logger.info("Fitted parabola opens upwards!")
if residual > 0.1:
Logger.info(
"Residual in quadratic fit to cross correlation maximum larger "
"than 0.1: {0}".format(residual))
# X coordinate of vertex of parabola gives time shift to correct
# differential pick time. Y coordinate gives maximum correlation
# coefficient.
shift = -coeffs[1] / 2.0 / coeffs[0]
coeff = (4 * coeffs[0] * coeffs[2] - coeffs[1]**2) / (4 * coeffs[0])
if coeff < np.amax(ccc) or coeff > 1.0 or not 0 < shift < len(ccc) * dt:
# Sometimes the interpolation returns a worse result.
Logger.warning("Interpolation did not give an accurate result, "
"returning maximum in data")
return np.argmax(ccc) * dt, np.amax(ccc)
return shift, coeff
def get_plot(data, ax, c, l):
ml = data[:, 0]
ex = data[:, 1]
(ar, br) = polyfit(ml, ex, 1)
xr = polyval([ar, br], ml)
ax.scatter(ml, ex, color=c, alpha=0.3, label=l)
#ax.plot(ml,xr)
ax.legend()
def interpolate_flux(freqs, fluxes, freq):
if len(freqs) < 1:
return None
elif len(freqs) == 1:
return fluxes[0]
elif len(freqs) > 1:
(ar, br) = polyfit(freqs, fluxes, 1)
return polyval([ar, br], freq)
def predict(A):
if len(A) > min(3, n):
xtrain = np.linspace(1, len(A), len(A))
ytrain = A
xtest = len(A) + 1
f = poly1d(polyfit(xtrain, ytrain, n))
return f(xtest).round() if all(
f(xtrain).round() == ytrain) else None
def polynomialReg():
sample_rate, samples = wavfile.read('songs/hakuna_matata.wav')
x = np.array(range(0, 100))
y = np.array(samples[5000000:5000100])
plt.subplot(211)
p1 = cp.polyfit(x, y, 1)
p2 = cp.polyfit(x, y, 2)
p3 = cp.polyfit(x, y, 3)
p20 = cp.polyfit(x, y, 20)
#graficamos
plt.plot(x, y, 'r', label="Original")
plt.plot(x, cp.polyval(p1, x), 'b--', label="Grado 1")
plt.plot(x, cp.polyval(p2, x), 'm--', label="Grado 2")
plt.plot(x, cp.polyval(p3, x), 'g--', label="Grado 3")
plt.plot(x, cp.polyval(p20, x), 'k--', label="Grado 20")
plt.legend()
plt.show()
def Polyfit_detrend(x, y): #主程序，相当于C语言的main函数
a, b, c = polyfit(x, y, 2)
y_quad = a * x * x + b * x + c #利用拟合得到的系数，计算x向量对应的y向量
# 拟合结果绘图
# pylab.plot(x, y, 'o') #绘制散点
# pylab.plot(x, y_quad, 'r-') #绘制拟合的曲线
# pylab.show()
return y_quad
def calculate_slow_phi_0s(phi_0s, p_values):
slow_phi_0s = scipy.empty_like(phi_0s)
for i, phi_0 in enumerate(phi_0s):
phi_0_unfolded = unfold(phi_0)
x = arange(len(phi_0_unfolded))
model = polyfit(x, phi_0_unfolded, 3, w=p_values[i])
slow_phi_0s[i] = polyval(model, x)
return slow_phi_0s
