def main(dbname, dbtable, dbuser, dbpass): """Main script execution""" # Query database result = query_database(dbname, dbtable, dbuser, dbpass, 1, 900) # Break result up into separate columns frame_counts, image_scales, widths, heights, proc_times = zip(*result) # Movie area in square pixels and arcseconds area_px = np.array(widths) * np.array(heights) area_as = area_px * np.array(image_scales) ** 2 # Limit outlier influence from server downtime, etc times = np.array(proc_times) # Determine linear fit for number of frames and area (m1, b1) = polyfit(frame_counts, times, 1) (m2, b2) = polyfit(np.sqrt(area_px), times, 1) print "Linear fit for num frames: y = %fx + %f" % (m1, b1) print "Linear fit for movie size: y = %fx + %f" % (m2, b2) # Basic time estimation w1 = 0.8 w2 = 1 - w1 basic_est = lambda x, y: w1 * (m1 * x + b1) + w2 * (m2 * y + b2) evaluate_estimation_fn(basic_est, frame_counts, np.sqrt(area_px), times)
def log_regress(f,xs, tol=0.1): """find root f(x) = 0 using logistic regression, starting with xs""" last_log_xp = None log_xp = None print "initial seeding for log_regress" ys = map(f,xs) log_xs = map(log,xs) plotting = False while last_log_xp is None or abs(exp(log_xp) - exp(last_log_xp)) > tol: print "correlation:",pearsonr(log_xs,ys) #lin = poly1d(polyfit(log_xs,ys,1)) m, b = (polyfit(log_xs,ys,1)) if plotting: lin = poly1d([m,b]) plt.scatter(log_xs,ys) plt.plot(*pl(lin,log_xs)) plt.show() last_log_xp = log_xp or None log_xp = -b/m#secant_interval(lin,min(log_xs),max(log_xs)) log_xs.append(log_xp) yxp = f(exp(log_xp)) ys.append(yxp) print "x:",log_xp and exp(log_xp),"y:",ys[-1],\ "x last:",last_log_xp and exp(last_log_xp),\ "y last:",ys[-2] if len(ys) >= 2 else None #lin = poly1d(polyfit(log_xs,ys,1)) m, b = (polyfit(log_xs,ys,1)) log_xp = -b/m#secant_interval(lin,min(log_xs),max(log_xs)) return exp(log_xp)
def log_regress_spec(f,xs, tol=0.01): """find root f(x) = 0 using logistic regression, starting with xs""" #print "initial seeding for log_regress (unweighted)" ys = map(f,xs) log_xs = map(log,xs) plotting = False honest_guesses = [] while len(honest_guesses) < 2 or abs(honest_guesses[-1] - honest_guesses[-2]) > tol: #print "correlation:",pearsonr(log_xs,ys) #lin = poly1d(polyfit(log_xs,ys,1)) m, b = (polyfit(log_xs,ys,1)) # if plotting: # lin = poly1d([m,b]) # plt.scatter(log_xs,ys) # plt.plot(*pl(lin,log_xs)) # plt.show() honest_guess = -b/m dx = -(honest_guesses[-1] - honest_guess) if honest_guesses else 0 log_xp = honest_guess + 2*dx log_xs.append(log_xp) yxp = f(exp(log_xp)) ys.append(yxp) honest_guesses.append(honest_guess) diff = (abs(exp(honest_guesses[-1]) - exp(honest_guesses[-2])) if len(honest_guesses) > 1 else None) # print "honest_guess:",(honest_guess),"xp:",(log_xp),\ # "y:",yxp, "diff:",diff #lin = poly1d(polyfit(log_xs,ys,1)) m, b = (polyfit(log_xs,ys,1)) log_xp = -b/m#secant_interval(lin,min(log_xs),max(log_xs)) print "final guess: log_xp:",log_xp return exp(log_xp)
def time_fitting(x_fit,y_fit): """Fit a linear relation to the x_fit and y_fit parameters Returns the actual fit and the parameters of the fit, """ import numpy as np x_fit = np.array( x_fit ) y_fit = np.array( y_fit ) ###First fit iteration and remove outliers POLY_FIT_ORDER = 1 slope,intercept = scipy.polyfit(x_fit,y_fit,POLY_FIT_ORDER) fit = scipy.polyval((slope,intercept),x_fit) fit_sigma = fit.std() include_index = np.where(np.abs(fit-y_fit) < 1.5*fit_sigma)[0] if len(include_index) < 4: return None,None,False ###Final Fit x_fit_clipped = x_fit[include_index] y_fit_clipped = y_fit[include_index] parameters = scipy.polyfit(x_fit_clipped,y_fit_clipped,POLY_FIT_ORDER) fit = scipy.polyval(parameters,x_fit) return fit,parameters,True
def trainingAndTesting(self): global train global x, y, xa, xb, ya, yb # separating training from testing data frac = 0.3 split_idx = int(frac * len(xb)) rangeX = range(len(xb)) listX = list(rangeX) logging.info("delta : %i", len(set(rangeX).difference(listX))) shuffled = sp.random.permutation(list(range(len(xb)))) test = sorted(shuffled[:split_idx]) train = sorted(shuffled[split_idx:]) fbt1 = sp.poly1d(sp.polyfit(xb[train], yb[train], 1)) fbt2 = sp.poly1d(sp.polyfit(xb[train], yb[train], 2)) print("fbt2(x)= \n%s" % fbt2) print("fbt2(x)-100,000= \n%s" % (fbt2 - 100000)) print("fbt2(x)= \n%s" % (fbt2)) fbt3 = sp.poly1d(sp.polyfit(xb[train], yb[train], 3)) fbt10 = sp.poly1d(sp.polyfit(xb[train], yb[train], 10)) fbt100 = sp.poly1d(sp.polyfit(xb[train], yb[train], 100)) print("Test errors for only the time after inflection point") for f in [fbt1, fbt2, fbt3, fbt10, fbt100]: print("Error d=%i: %f" % (f.order, self.error(f, xb[test], yb[test])))
def fitKvsPower(self, filename="PowerVGain.dat", zeroed="_0"): if not hasattr(self, "k_avg"): raise RuntimeError, "Must load all data and run calculate() before fitting" gain, self.power = loadtxt(filename, skiprows=1, unpack=True) if zeroed != None: gain_0, power_0 = loadtxt(filename.replace(".", zeroed + ".", 1), skiprows=1, unpack=True) self.power = self.power - power_0 savetxt(filename.replace(".", "_subtract."), self.power, fmt="%.5f") if self.gain.tolist() != gain.tolist(): raise ValueError, "Laser power was not measured over the same gains as calibration measurements:\n\ Laser gain: " + str( gain ) + "\nMeasurement gains: " + str( self.gain ) self.kfitX = polyfit(self.power, self.k_avg[:, 0], 1) self.kfitY = polyfit(self.power, self.k_avg[:, 1], 1) print "Fit parameters for X axis:" print poly1d(self.kfitX, variable="power") print "\nFit parameters for Y axis:" print poly1d(self.kfitY, variable="power")
def main(): data = sp.genfromtxt('./data/web_traffic.tsv', delimiter='\t') x = data[:, 0] y = data[:, 1] x = x[~sp.isnan(y)] y = y[~sp.isnan(y)] fp1 = sp.polyfit(x, y, 1) print('Model parameters for fp1 %s' % fp1) f1 = sp.poly1d(fp1) print('This is the error rate for fp1 %f' % error(f1, x, y)) fp2 = sp.polyfit(x, y, 2) print('Model parameters for fp2 %s' % fp2) f2 = sp.poly1d(fp2) print('This is the error rate for fp2 %f' % error(f2, x, y)) plt.scatter(x, y,color= 'pink') plt.title('My first impression') plt.xlabel('Time') plt.ylabel('#Hits') plt.xticks([w * 7 * 24 for w in range(10)], ['week %i' % w for w in range(10)]) fx = sp.linspace(0, x[-1], 1000) plt.plot(fx, f1(fx), linewidth=3,color='cyan') plt.plot(fx, f2(fx), linewidth=3, linestyle='--',color= 'red') plt.legend(['d = %i' %f1.order, 'd = %i' %f2.order], loc='upper left') plt.autoscale(tight=True) plt.grid() plt.show()
def dsi_adc(q_axis, data, mid_pos): """ TODO Returns ------- ADC4, ADC8 : (I,J,K) volumes Returns TODO """ # Fit with a polynomial the signals S and derive the ADC and the kurtosis # initialization of the output maps I, J, K, N = data.shape ADC4 = np.zeros((I,J,K)) #Ku4 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2])) #P04 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2])) ADC8 = np.zeros(((I,J,K))) #Ku8 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2])) #P08 = np.zeros((b0.shape[0],b0.shape[1],b0.shape[2])) pc = -1 count = 0 n = I * J * K print "Polynomial fitting for each voxel signal over q-axis..." # loop throughout all the voxels of the volume for i in range(b0.shape[0]): for j in range(b0.shape[1]): for k in range(b0.shape[2]): # Percent counter count = count + 1 pcN = int(round( float(100*count)/n )) if pcN > pc and pcN%1 == 0: pc = pcN print str(pc) + " %" # for the current voxel, extract the signal to be fitted S = data[i,j,k,:] # normalization respect to the b0 S = S / S[mid_pos] coeff = sp.polyfit(q_axis,S,8) ADC8[i,j,k] = (-coeff[-3] / (2 * math.pi * math.pi)) # Ku8[i,j,k] = (6 * coeff[-5] / (coeff[-3] * coeff[-3])) - 3 # temp = np.polyval(coeff, q_axis) # P08[i,j,k] = np.sum(temp) coeff = sp.polyfit(q_axis,S,4) ADC4[i,j,k] = (-coeff[-3] / (2 * math.pi * math.pi)) # Ku4[i,j,k] = (6 * coeff[-5] / (coeff[-3] * coeff[-3])) - 3 # temp = np.polyval(coeff, q_axis) # P04[i,j,k] = np.sum(temp) print "[ OK ]" return ADC4, ADC8
def go( mags=(15, 30.1, 0.5), redshifts=(0.01, 12, 0.01), coeff_file="prior_K_zmax7_coeff.dat", outfile="prior_K_extend.dat" ): fp = open(coeff_file) lines = fp.readlines() fp.close() mag_list = np.cast[float](lines[0].split()[2:]) z0 = np.cast[float](lines[1].split()[1:]) gamma = np.cast[float](lines[2].split()[1:]) z_grid = np.arange(redshifts[0], redshifts[1], redshifts[2]) NZ = z_grid.shape[0] mag_grid = np.arange(mags[0], mags[1], mags[2]) NM = mag_grid.shape[0] #### Polynomial extrapolation not reliable # p_z0 = scipy.polyfit(mag_list, z0, order) # z0_grid = np.maximum(scipy.polyval(p_z0, mag_grid), 0.05) # p_gamma = scipy.polyfit(mag_list, gamma, order) # gamma_grid = np.maximum(scipy.polyval(p_gamma, mag_grid), 0.05) #### Interpolations on the defined grid z0_grid = np.interp(mag_grid, mag_list, z0) gamma_grid = np.interp(mag_grid, mag_list, gamma) #### Linear extrapolations of fit coefficients p_z0 = scipy.polyfit(mag_list[:3], z0[:3], 1) z0_grid[mag_grid < mag_list[0]] = np.maximum(scipy.polyval(p_z0, mag_grid[mag_grid < mag_list[0]]), 0.05) p_z0 = scipy.polyfit(mag_list[-3:], z0[-3:], 1) z0_grid[mag_grid > mag_list[-1]] = np.maximum(scipy.polyval(p_z0, mag_grid[mag_grid > mag_list[-1]]), 0.05) p_gamma = scipy.polyfit(mag_list[:3], gamma[:3], 1) gamma_grid[mag_grid < mag_list[0]] = np.maximum(scipy.polyval(p_gamma, mag_grid[mag_grid < mag_list[0]]), 0.05) p_gamma = scipy.polyfit(mag_list[-3:], gamma[-3:], 1) gamma_grid[mag_grid > mag_list[-1]] = np.maximum(scipy.polyval(p_gamma, mag_grid[mag_grid > mag_list[-1]]), 0.05) out_matrix = np.zeros((NZ, NM + 1)) out_matrix[:, 0] = z_grid for i in range(NM): pz = z_grid * np.exp(-(z_grid / z0_grid[i]) ** gamma_grid[i]) pz /= np.trapz(pz, z_grid) plt.plot(z_grid, pz, label=mag_grid[i]) out_matrix[:, i + 1] = pz plt.legend(ncol=3, loc="upper right") header = "# z " for m in mag_grid: header += "%6.1f" % (m) fp = open(outfile, "w") fp.write(header + "\n") np.savetxt(fp, out_matrix, fmt="%6.3e") fp.close()
def generate_report(): """See title""" word_file = open("word_file.txt", 'ab+') bigram_file = open("bigram_file.txt", 'ab+') sorted_words = sorted(words.items(), key = lambda x: x[1]) sorted_bigrams = sorted(bigrams.items(), key = lambda x: x[1]) print(sorted_words, file=word_file) print(sorted_bigrams, file=bigram_file) word_names, word_values = zip(*reversed(sorted_words)) bigram_names, bigram_values = zip(*reversed(sorted_bigrams)) #Create word plot xdata, ydata = range(1,len(word_values)+1), word_values xd, yd = log10(xdata), log10(ydata) polycoef = polyfit(xd,yd,1) yfit = 10** (polycoef[0]*xd+polycoef[1]) plt.figure() plt.loglog(xdata, ydata, 'ro' , xdata, yfit, 'g--') plt.title("Word frequency distribution") plt.ylabel("Word Frequency") plt.xlabel("Words") plt.grid() plt.savefig('Question1_wordplot.png') total_words = sum(word_values) word_percent = [(float(wv)/total_words) for wv in word_values] #word_percent c_values = [(lambda x: x[0]*x[1])(x) for x in zip(xdata, word_percent)] c = sum(c_values)/len(c_values) print("Word c: ", c) #Create bigram plot xdata, ydata = range(1,len(bigram_values)+1), bigram_values xd, yd = log10(xdata), log10(ydata) polycoef = polyfit(xd,yd,1) yfit = 10** (polycoef[0]*xd+polycoef[1]) plt.figure() plt.loglog(xdata, ydata, 'ro' , xdata, yfit, 'g--') plt.title("Bigram frequency distribution") plt.ylabel("Bigram Frequency") plt.xlabel("Bigrams") plt.grid() plt.savefig('Question1_bigramplot.png') total_bigrams = sum(bigram_values) bigram_percent = [(float(bv)/total_bigrams) for bv in bigram_values] #word_percent c_values = [(lambda x: x[0]*x[1])(x) for x in zip(xdata, bigram_percent)] c = sum(c_values)/len(c_values) print("Bigram c: ", c) return
def estimateBeta(priceY,priceX,algo = 'standard'): ''' estimate stock Y vs stock X beta using iterative linear regression. Outliers outside 3 sigma boundary are filtered out Parameters -------- priceX : price series of x (usually market) priceY : price series of y (estimate beta of this price) Returns -------- beta : stockY beta relative to stock X ''' X = pd.DataFrame({'x':priceX,'y':priceY}) if algo=='returns': ret = (X/X.shift(1)-1).dropna().values x = ret[:,0] y = ret[:,1] # filter high values low = np.percentile(x,20) high = np.percentile(x,80) iValid = (x>low) & (x<high) x = x[iValid] y = y[iValid] iteration = 1 nrOutliers = 1 while iteration < 10 and nrOutliers > 0 : (a,b) = polyfit(x,y,1) yf = polyval([a,b],x) #plot(x,y,'x',x,yf,'r-') err = yf-y idxOutlier = abs(err) > 3*np.std(err) nrOutliers =sum(idxOutlier) beta = a #print 'Iteration: %i beta: %.2f outliers: %i' % (iteration,beta, nrOutliers) x = x[~idxOutlier] y = y[~idxOutlier] iteration += 1 elif algo=='log': x = np.log(X['x']) y = np.log(X['y']) (a,b) = polyfit(x,y,1) beta = a elif algo=='standard': ret =np.log(X).diff().dropna() beta = ret['x'].cov(ret['y'])/ret['x'].var() else: raise TypeError("unknown Beta algorithm type, use 'standard', 'log' or 'returns'") return beta
def concat_extrap_ends(x, npts, polyorder=1, lowside=True, highside=True): i=numpy.arange(npts, dtype='float64') if lowside: ans=scipy.polyfit(-1*(i+1.), x[:npts], polyorder) x=numpy.concatenate([scipy.polyval(list(ans), i[::-1]), x]) if highside: ans=scipy.polyfit(-1*(i[::-1]-1.), x[-1*npts:], polyorder) x=numpy.concatenate([x, scipy.polyval(list(ans), i)]) return x
def plotFullModel(dm, sacc): """ """ fig = plt.figure(figsize = (15, 5)) plt.suptitle("Sacc = %s" % sacc) ax1 = plt.subplot(141) lCols = ["blue", "red", "orange", "yellow", "green", "pink"] xVar = "sacc%s_ex" % sacc yVar = "sacc%s_ey" % sacc dm = dm.select("%s != ''" % xVar) dm = dm.select("%s != ''" % yVar) dm = dm.select("%s != -1000" % xVar) dm = dm.select("%s != -1000" % yVar) for a in dm.unique("realAngle"): _dm = dm.select("realAngle == %s" % a) col = lCols.pop() plt.scatter(_dm[xVar], _dm[yVar], color = col, marker = ".", label = a) plt.axvline(constants.xCen, color = gray[3], linestyle = '--') plt.axhline(constants.yCen, color = gray[3], linestyle = '--') ax2 = plt.subplot(142) pm = PivotMatrix(dm, ["stim_type", "realAngle"], ["file"], "xNorm1", colsWithin =True) pm.linePlot(fig = fig) pm.save("PM.csv") pm._print() plt.axhline(0, color = gray[3], linestyle = "--") ax3 = plt.subplot(143) plt.title("object") dmObj= dm.select("stim_type == 'object'") #slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(stimObj["ecc"], stimObj["xNorm1"]) x = dmObj["ecc"] y = dmObj["xNorm%s" % sacc] fit = scipy.polyfit(x,y,1) fit_fn = scipy.poly1d(fit) plt.plot(x,y, 'yo', x, fit_fn(x), '--k') ax4 = plt.subplot(144) plt.title("non-object") dmNo= dm.select("stim_type == 'non-object'") #slope, intercept, r_value, p_value, std_err = scipy.stats.linregress(stimObj["ecc"], stimObj["xNorm1"]) x = dmNo["ecc"] y = dmNo["xNorm%s" % sacc] fit = scipy.polyfit(x,y,1) fit_fn = scipy.poly1d(fit) plt.plot(x,y, 'yo', x, fit_fn(x), '--k') plt.savefig("./plots/Full_model_004C_sacc%s.png" % sacc) plt.show()
def logaod_polyfit(wvl,aod,polynum=False): """ Purpose: Take in an array of aod spectra and calculate the polynomials associated with each log of the spectrum takes in the wvl Input: aod: numpy array of time,wvl wvl: numpy array of wavelengths in nm Output: array of polynomial coefficients for each time point in the log(aod) Keywords: polynum: (optional, defaults to N-2 where N is the number of points in the spectrum) number of polynomials coefficients to return (order of the polynomial), if set to False, uses default N-2 Dependencies: - numpy - scipy Needed Files: - None Modification History: Written: Samuel LeBlanc, NASA Ames Research Center, 2017-11-28 Modified: """ import numpy as np from scipy import polyfit # sanitize inputs shape = aod.shape wvl = np.array(wvl) if not len(wvl.shape)==1: raise ValueError('wvl is not a array with one dimension') if not len(wvl) in shape: raise ValueError('No size of aod is the same as wvl') if not polynum: polynum = len(wvl)-2 if len(shape)>1: ni,n = [(i,j) for (i,j) in enumerate(shape) if not j==len(wvl)][0] cc = np.zeros((polynum,n)) for i in range(n): if ni==0: cc[:,i] = polyfit(np.log(wvl),np.log(aod[i,:]),polynum) else: cc[:,i] = polyfit(np.log(wvl),np.log(aod[:,i]),polynum) else: cc = polyfit(np.log(wvl),np.log(aod),polynum) return cc
def estimateBeta(priceY, priceX, algo="standard"): """ estimate stock Y vs stock X beta using iterative linear regression. Outliers outside 3 sigma boundary are filtered out Parameters -------- priceX : price series of x (usually market) priceY : price series of y (estimate beta of this price) Returns -------- beta : stockY beta relative to stock X """ X = DataFrame({"x": priceX, "y": priceY}) if algo == "returns": ret = (X / X.shift(1) - 1).dropna().values # print len(ret) x = ret[:, 0] y = ret[:, 1] iteration = 1 nrOutliers = 1 while iteration < 10 and nrOutliers > 0: (a, b) = polyfit(x, y, 1) yf = polyval([a, b], x) # plot(x,y,'x',x,yf,'r-') err = yf - y idxOutlier = abs(err) > 3 * np.std(err) nrOutliers = sum(idxOutlier) beta = a # print 'Iteration: %i beta: %.2f outliers: %i' % (iteration,beta, nrOutliers) x = x[~idxOutlier] y = y[~idxOutlier] iteration += 1 elif algo == "log": x = np.log(X["x"]) y = np.log(X["y"]) (a, b) = polyfit(x, y, 1) beta = a elif algo == "standard": ret = np.log(X).diff().dropna() beta = ret["x"].cov(ret["y"]) / ret["x"].var() else: raise TypeError("unknown algorithm type, use 'standard', 'log' or 'returns'") return beta
def intersect2ary(ary1, ary2): """ Takes the two 2-d arrays I{ary1} and I{ary2} and finds their intersection. The arrays are organized [X, Y], where X is a 1-d array for the abscissa and Y is the corresponding 1-d array for the ordinate. """ X1, Y1 = ary1 X2, Y2 = ary2 m, c = polyfit(X1, Y1, 1) n, d = polyfit(X2, Y2, 1) return intersect2(m, c, n, d)
def estimate_rate_func(t, T, N, plot_flag=False, method='central diff'): t_est_pts = scipy.linspace(t.min(), t.max(), N+2) interp_func = scipy.interpolate.interp1d(t,T,'linear') T_est_pts = interp_func(t_est_pts) if plot_flag == True: pylab.figure() pylab.subplot(211) pylab.plot(t_est_pts, T_est_pts,'or') # Estimate slopes slope_pts = scipy.zeros((N,)) T_slope_pts = scipy.zeros((N,)) if method == 'local fit': for i in range(1,(N+1)): mask0 = t > 0.5*(t_est_pts[i-1] + t_est_pts[i]) mask1 = t < 0.5*(t_est_pts[i+1] + t_est_pts[i]) mask = scipy.logical_and(mask0, mask1) t_slope_est = t[mask] T_slope_est = T[mask] local_fit = scipy.polyfit(t_slope_est,T_slope_est,2) dlocal_fit = scipy.polyder(local_fit) slope_pts[i-1] = scipy.polyval(dlocal_fit,t_est_pts[i]) T_slope_pts[i-1] = scipy.polyval(local_fit,t_est_pts[i]) if plot_flag == True: t_slope_fit = scipy.linspace(t_slope_est[0], t_slope_est[-1], 100) T_slope_fit = scipy.polyval(local_fit,t_slope_fit) pylab.plot(t_slope_fit, T_slope_fit,'g') elif method == 'central diff': dt = t_est_pts[1] - t_est_pts[0] slope_pts = (T_est_pts[2:] - T_est_pts[:-2])/(2.0*dt) T_slope_pts = T_est_pts[1:-1] else: raise ValueError, 'unkown method %s'%(method,) # Fit line to slope estimates fit = scipy.polyfit(T_slope_pts, slope_pts,1) if plot_flag == True: T_slope_fit = scipy.linspace(T_slope_pts.min(), T_slope_pts.max(),100) slope_fit = scipy.polyval(fit,T_slope_fit) pylab.subplot(212) pylab.plot(T_slope_fit, slope_fit,'r') pylab.plot(T_slope_pts, slope_pts,'o') pylab.show() rate_slope = fit[0] rate_offset = fit[1] return rate_slope, rate_offset
def pre_edge(energy, mu, group=None, e0=None, step=None, nnorm=3, nvict=0, pre1=None, pre2=-50, norm1=100, norm2=None, _larch=None, **kws): """pre edge, normalization for XAFS """ if _larch is None: raise Warning("cannot remove pre_edge -- larch broken?") if e0 is None or e0 < energy[0] or e0 > energy[-1]: e0 = find_e0(energy, mu, group=group, _larch=_larch) p1 = min(np.where(energy >= e0-10.0)[0]) p2 = max(np.where(energy <= e0+10.0)[0]) ie0 = np.where(energy-e0 == min(abs(energy[p1:p2] - e0)))[0][0] if pre1 is None: pre1 = min(energy) - e0 if norm2 is None: norm2 = max(energy) - e0 p1 = min(np.where(energy >= pre1+e0)[0]) p2 = max(np.where(energy <= pre2+e0)[0]) if p2-p1 < 2: p2 = min(len(energy), p1 + 2) omu = mu*energy**nvict coefs = polyfit(energy[p1:p2], omu[p1:p2], 1) pre_edge = (coefs[0] * energy + coefs[1]) * energy**(-nvict) # normalization p1 = min(np.where(energy >= norm1+e0)[0]) p2 = max(np.where(energy <= norm2+e0)[0]) if p2-p1 < 2: p2 = min(len(energy), p1 + 2) coefs = polyfit(energy[p1:p2], omu[p1:p2], nnorm) post_edge = 0 for n, c in enumerate(reversed(list(coefs))): post_edge += c * energy**(n-nvict) edge_step = post_edge[ie0] - pre_edge[ie0] norm = (mu - pre_edge)/edge_step if _larch.symtable.isgroup(group): setattr(group, 'e0', e0) setattr(group, 'edge_step', edge_step) setattr(group, 'norm', norm) setattr(group, 'pre_edge', pre_edge) setattr(group, 'post_edge', post_edge) else: return edge_step, e0
def main(): data = sp.genfromtxt("web_traffic.tsv", delimiter="\t") plt.xkcd() x = data[:, 0] y = data[:, 1] x = x[~sp.isnan(y)] y = y[~sp.isnan(y)] fp1, _, _, _, _ = sp.polyfit(x, y, 1, full=True) # Here we try 3 degrees of freedom fp2, _, _, _, _ = sp.polyfit(x, y, 3, full=True) f1 = sp.poly1d(fp1) f2 = sp.poly1d(fp2) # We have an obvious inflection point between 3rd and 4th week inflection_in_hours = int(3.5 * 7 * 24) x_before_inflection = x[:inflection_in_hours] x_after_inflection = x[inflection_in_hours:] y_after_inflection = y[inflection_in_hours:] f_after = sp.poly1d(sp.polyfit(x_after_inflection, y_after_inflection, 1)) fx = sp.linspace(0, x[-1], 1000) fx_after = sp.linspace(len(x_before_inflection)+1, x[-1], 1000) plt.scatter(x, y, s=5) plt.title("Web traffic over the last month.") plt.xlabel("Time") plt.ylabel("Hits/hour") plt.xticks([w * 7 * 24 for w in range(10)], ['week {}'.format(w) for w in range(10)]) plt.autoscale(tight=True) plt.plot(fx, f1(fx), linewidth=2) plt.plot(fx, f2(fx), linewidth=2) plt.plot(fx_after, f_after(fx_after), linewidth=3) plt.legend(["d={}".format(f1.order), "d={}".format(f2.order), "d after inflection"], loc="upper left") # plt.grid(True, linestyle="-", color='0.75') plt.show()
def ar1fit(ts): ''' Fits an AR(1) model to the time series data ts. AR(1) is a linear model of the form x_t = beta * x_{t-1} + c + e_{t-1} where beta is the coefficient of term x_{t-1}, c is a constant and x_{t-1} is an i.i.d. noise term. Here we assume that e_{t-1} is normally distributed. Returns the tuple (beta, c, sigma). ''' # Fitting AR(1) entails finding beta, c, and the noise term. # Beta is well approximated by the coefficient of OLS regression # on the lag of the data with itself. Since the noise term is # assumed to be i.i.d. and normal, we must only estimate sigma, # the standard deviation. # Estimate beta - auto regressive parameter x = ts[0:-1] y = ts[1:] p = sp.polyfit(x, y, 1) beta = p[0] # Estimate c c = sp.mean(ts) * (1 - beta) # Estimate the variance from the residuals of the OLS regression. yhat = sp.polyval(p, x) variance = sp.var(y - yhat) sigma = sp.sqrt(variance) return beta, c, sigma
def Gradient(self,Results='',TestSet=False): G = GBR(loss='ls', learning_rate=0.1, n_estimators=100, subsample=1.0, criterion='friedman_mse', min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_depth=3, min_impurity_split=None, init=None, random_state=None, max_features=None, alpha=0.9, verbose=0, max_leaf_nodes=None, warm_start=False, presort='auto') if TestSet==False: GResult = G.fit(self.X,np.ravel(self.y,1)) if Results==True: print(str(GResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(GResult.get_params())) plt.plot(G.fit(self.X,np.ravel(self.y,1)).predict(self.X)) y = np.array(self.y[self.DVCols]) plt.plot(y,'ro') plt.show() else: x_train = self.X[:len(self.X)//2] y_train = np.ravel(self.y,1)[:len(self.y)//2] x_test = self.X[len(self.X)//2:] y_test = np.ravel(self.y,1)[len(self.y)//2:] GResult = G.fit(x_train,y_train) if Results==True: print(str(GResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(GResult.get_params())) GRPredict = GResult.predict(x_test) plt.plot(GRPredict,polyval(polyfit(GRPredict,y_test.reshape(-1),1),GRPredict),'r-',label='predicted') plt.plot(GRPredict,y_test.reshape(-1),'bo') plt.legend() plt.show()
def Neural(self,Results=True,TestSet=False): NN = MLPR(hidden_layer_sizes=(100, ), activation='relu', solver='adam', alpha=0.0001, batch_size='auto', learning_rate='constant', learning_rate_init=0.001, power_t=0.5, max_iter=200, shuffle=True, random_state=None, tol=0.0001, verbose=False, warm_start=False, momentum=0.9, nesterovs_momentum=True, early_stopping=False, validation_fraction=0.1, beta_1=0.9, beta_2=0.999, epsilon=1e-08) if TestSet==False: NNResult = NN.fit(self.X,np.ravel(self.y,1)) if Results==True: print(str(NNResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(NNResult.get_params())) plt.plot(NN.fit(self.X,np.ravel(self.y,1)).predict(self.X)) y = np.array(self.y[self.DVCols]) plt.plot(y,'ro') plt.show() else: x_train = self.X[:len(self.X)//2] y_train = np.ravel(self.y,1)[:len(self.y)//2] x_test = self.X[len(self.X)//2:] y_test = np.ravel(self.y,1)[len(self.y)//2:] NNResult = NN.fit(x_train,y_train) if Results==True: print(str(NNResult.score(self.X,np.ravel(self.y,1))) + '\n' + str(NNResult.get_params())) NNRPredict = NNResult.predict(x_test) plt.plot(NNRPredict,polyval(polyfit(NNRPredict,y_test.reshape(-1),1),NNRPredict),'r-',label='predicted') plt.plot(NNRPredict,y_test.reshape(-1),'bo') plt.legend() plt.show()
def MNK(x, y, st): n = len(x) s = sum(y) fp, residuals, rank, sv, rcond = scipy.polyfit(x, y, st, full=True) f = scipy.poly1d print('a =', round(fp[0], 4)) print('b =', round(fp[1], 4)) print('c =', round(fp[2], 4)) y1 = [fp[0] * x[i]**2 + fp[1] * x[i] + fp[2] for i in range(n)] s1 = sum([1 / x[i] for i in range(n)]) # сумма 1/x s2 = sum([(1 / x[i])**2 for i in range(n)]) # сумма (1/x)**2 s3 = sum([y[i] / x[i] for i in range(n)]) # сумма y/x a = round((s * s2 - s1 * s3) / (n * s2 - s1**2), 3) # коэфициент а с тремя дробными цифрами b = round((n * s3 - s1 * s) / (n * s2 - s1**2), 3) # коэфициент b с тремя дробными цифрами s4 = [a + b / x[i] for i in range(n)] # список значений гиперболической функции so = round( sum([abs(y[i] - y1[i]) for i in range(0, len(x))]) / (len(x) * sum(y)) * 100, 4) # средняя ошибка аппроксимации plt.plot(x, s4, color='black', linewidth=2) plt.legend(loc='best') plt.xlabel('x') plt.ylabel('y') plt.plot(x, y, color='deeppink', linestyle=' ', marker='o') plt.grid(True) plt.show()
def linear_interpolate(self, x, y): new_x, new_y = [], [] for x_i, y_i in zip(x, y): if y_i > 0.0: new_x.append(x_i) new_y.append(y_i) x, y = new_x, new_y half_x = x[len(x) / 4:-len(x) / 4] half_y = y[len(y) / 4:-len(y) / 4] arrayx = np.ndarray(len(half_x)) for val, i in zip(half_x, range(len(half_x))): arrayx[i] = math.log10(val) all_arrayx = np.ndarray(len(x)) for val, i in zip(x, range(len(x))): all_arrayx[i] = math.log10(val) arrayy = np.ndarray(len(half_y)) for val, i in zip(half_y, range(len(half_y))): arrayy[i] = math.log10(val) (ar, br) = polyfit(arrayx, arrayy, 1) xr = polyval([ar, br], all_arrayx) return map(lambda val: 10.0**val, xr), map(lambda val: 10.0**val, all_arrayx), ar, br
def skew_model(self): ''' a model for skewness of theta_1 | theta_0, y_0 skew model is a(y)*theta_0 + b(y) ''' theta_array = self.mid_theta[self.theta_range] query_y = self.query_y skew_slope_dict, skew_intercept_dict = {}, {} x_agg, y_agg = [], [] for y_class in query_y: y = self.mid_y[y_class] theta_trans_mat = trans_prob_given_y(self.marginal, y_class, self.mapping) moments_mat = np.zeros((theta_trans_mat.shape[1], 4)) for j in self.theta_range: prob = theta_trans_mat[:, j] moments_mat[j, :] = moments_given_pdf(self.mid_theta, prob) xp, yp = theta_array, moments_mat[self.theta_range, 2] nan_filter = ~np.isnan(yp) xp, yp = xp[nan_filter], yp[nan_filter] x_agg.extend(xp) y_agg.extend(yp) skew_slope_dict[y], skew_intercept_dict[y] = sci.polyfit(xp, yp, 1) line = lambda x, a: a * x popt, pcov = curve_fit(line, x_agg, y_agg) self.avg_skew_slope = popt[0] return skew_slope_dict, skew_intercept_dict
def test_LGST_1overSqrtN_dependence(self): my_datagen_gateset = self.gateset.depolarize(gate_noise=0.05, spam_noise=0) # !!don't depolarize spam or 1/sqrt(N) dependence saturates!! nSamplesList = np.array([16, 128, 1024, 8192]) diffs = [] for nSamples in nSamplesList: ds = pygsti.construction.generate_fake_data(my_datagen_gateset, self.lgstStrings, nSamples, sampleError='binomial', seed=100) gs_lgst = pygsti.do_lgst(ds, self.specs, self.gateset, svdTruncateTo=4, verbosity=0) gs_lgst_go = pygsti.optimize_gauge( gs_lgst, "target", targetGateset=my_datagen_gateset, spamWeight=1.0, gateWeight=1.0) diffs.append(my_datagen_gateset.frobeniusdist(gs_lgst_go)) diffs = np.array(diffs, 'd') a, b = polyfit(np.log10(nSamplesList), np.log10(diffs), deg=1) #print "\n",nSamplesList; print diffs; print a #DEBUG self.assertLess(a + 0.5, 0.05)
def ivFitSuperconduct(self, vbias, vfb, vbias_offset, show_plot=False, ignore_nans=True): '''Fit the superconducting branch of the IV and return the slope and intercept. ''' fb_diff = np.diff(vfb) nan_indicies = np.where(np.isnan(vfb)) if len(nan_indicies[0]) > 0 and ignore_nans == False: # transition data is at values larger than the NaNs, set ignore_nans to False to turn this on print('not igonring nans') end_sc = int(min(nan_indicies[0]) - 1) else: neg_slope_array = np.where(fb_diff < 0) end_sc = min(neg_slope_array[0] ) - 2 # Go down two point to get away from the edge vbias_sc = vbias[:end_sc] - vbias_offset vfb_sc = vfb[:end_sc] sc_m, sc_b = scipy.polyfit(vbias_sc, vfb_sc, 1) vfb_sc_fit = scipy.polyval([sc_m, sc_b], vbias_sc) sc_fit = [sc_m, sc_b] if show_plot is True: pylab.plot(vbias_sc, vfb_sc_fit, label='SC Branch Fit') pylab.legend() return sc_fit
def mean_var_model(self): ''' a model for the mean of theta_1 | theta_0, y_0 mean model is a(theta)*y + b(theta). For theta in mid_theta[query_theta] we will save a, b in mean_slopes[theta], mean_intercepts[theta] ''' y_array = self.mid_y[self.y_range] query_theta = self.query_theta mean_slope_dict, mean_intercept_dict, var_dict = {}, {}, {} for theta_class in query_theta: theta = self.mid_theta[theta_class] theta_trans_mat = trans_prob_given_theta(self.marginal, theta_class, self.mapping) moments_mat = np.zeros((theta_trans_mat.shape[1], 4)) for j in self.y_range: prob = theta_trans_mat[:, j] moments_mat[j, :] = moments_given_pdf(self.mid_theta, prob) xp, yp = y_array, moments_mat[self.y_range, 0] nan_filter = ~np.isnan(yp) xp, yp = xp[nan_filter], yp[nan_filter] mean_slope_dict[theta], mean_intercept_dict[theta] = sci.polyfit( xp, yp, 1) var_vector = moments_mat[self.y_range, 1] var_vector = var_vector[~np.isnan(var_vector)] var_dict[theta] = np.mean(var_vector) return mean_slope_dict, mean_intercept_dict, var_dict
def line_of_best_fit(degree): plt.plot(dates, prices, 'o') p1 = sp.polyfit(dates, prices, degree) plt.plot(dates, sp.polyval(p1, dates), 'r-') prices_predict = p1[0] * dates + p1[1] # y = mx + b # print("Predicted:\n" , prices_predict, "Actual:\n", prices) plt.show()
def calculate_polyfit(x, y): """ Calculate a linear regression using polyfit instead """ # FIXME(pica) This doesn't work for large ranges (tens of orders of # magnitude). No idea why as the fixRange() function above should make # all values greater than one. The trouble mainly seems to happen when # log10(min(x)) negative. # Check the smallest value in x or y isn't below 2x10-7 otherwise # we hit numerical instabilities. xFactorFix, xFactor = False, 0 yFactorFix, yFactor = False, 0 minX = min(x) minY = min(y) if minX < 2e-7: x, xFactor, xFactorFix = fixRange(x) if minY < 2e-7: y, yFactor, yFactorFix = fixRange(y) (ar, br) = polyfit(x, y, 1) yf = polyval([ar, br], x) xf = x if xFactorFix: xf = xf / 10**xFactor if yFactorFix: yf = yf / 10**yFactor return xf, yf
def test_scipy(): #Sample data creation #number of points n=50 t=linspace(-5,5,n) #parameters a=0.8; b=-4 x=polyval([a,b],t) #add some noise xn=x+randn(n) #Linear regressison -polyfit - polyfit can be used other orders polys (ar,br)=polyfit(t,xn,1) xr=polyval([ar,br],t) #compute the mean square error err=sqrt(sum((xr-xn)**2)/n) print('Linear regression using polyfit') print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, ms error= %.3f' % (a,b,ar,br,err)) #matplotlib ploting title('Linear Regression Example') plot(t,x,'g.--') plot(t,xn,'k.') plot(t,xr,'r.-') legend(['original','plus noise', 'regression']) show() #Linear regression using stats.linregress (a_s,b_s,r,tt,stderr)=stats.linregress(t,xn) print('Linear regression using stats.linregress') print('parameters: a=%.2f b=%.2f \nregression: a=%.2f b=%.2f, std error= %.3f' % (a,b,a_s,b_s,stderr))
def interpolate_rects(self, uid): rects_for_interpolate = self.cars[uid]['rects'].copy() cur_rect = self.cars[uid]['cur_rect'].copy() x, y, w, h = zip(*rects_for_interpolate) fp, residuals, rank, sv, rcond = sp.polyfit(x, y, 3, full=True) f = sp.poly1d(fp) fx = sp.linspace(x[0], x[-1] + 30, 100).astype(int) y_interp = f(fx).astype(int) fp, residuals, rank, sv, rcond = sp.polyfit(w, h, 3, full=True) f = sp.poly1d(fp) fw = sp.linspace(w[0], w[-1] + 30, 100).astype(int) h_interp = f(fw).astype(int) rects = list(zip(fx, y_interp, fw, h_interp)) self.cars[uid]['rects'] = rects cur_rect = rects[-1] self.cars[uid]['cur_rect'] = cur_rect
def linearRegression(histogram): x=[] y=[] for i in range(257): for k in range(257): if histogram.GetBinContent(i,k)!=0: x.append(i) y.append(k) (ar,br)=polyfit(x,y,1) xr=polyval([ar,br],x) err=sqrt(sum((xr-y)**2)/len(y)) if abs(ar)>1.0: (ar,br)=polyfit(y,x,1) xr=polyval([ar,br],y) err=sqrt(sum((xr-x)**2)/len(y)) return err
def linearRegression(list): x = np.linspace(1, len(list), len(list)) # Fit the data to y = mx + c linearModel = polyfit(x, list, 1, full=True) m = linearModel[0][0] res = linearModel[1] return m, res
def test_lagrange_cheby_product(self): import pylab import itertools import scipy test_function = lambda x,y: numpy.cos(CONST_2PI*x*2)*numpy.sin(CONST_2PI*x) #numpy.cos(CONST_2PI*x*4)*(float(y <= 0.5)*2-1)*abs(y-0.2) sample_points=range(10,30) errors=[] n_test_points = 30 for n_sample_points in sample_points: Q = chebyshev_grid(n_sample_points)*0.5+0.5 U = itertools.product(Q,Q) X = [ float(i)/n_test_points for i in range(n_test_points) ] W = itertools.product(X,X) L = lagrange_matrix(Q, X) F = numpy.array( [ [test_function(x,y) for x in Q ] for y in Q ] ) H = numpy.dot( numpy.dot(L, F), numpy.transpose(L) ) G = numpy.array( [ [test_function(x,y) for x in X ] for y in X ] ) errors.append( numpy.log10(numpy_norm(G-H)/numpy_norm(G)+1E-17) ) (slope,offset)=scipy.polyfit(sample_points,errors,1) mean = scipy.mean(errors) #pylab.plot(sample_points,errors) #pylab.show() #print slope,mean self.assertTrue( slope <= -0.4 and mean <= -4 )
def test_LGST_1overSqrtN_dependence(self): my_datagen_gateset = self.model.depolarize(op_noise=0.05, spam_noise=0) # !!don't depolarize spam or 1/sqrt(N) dependence saturates!! nSamplesList = np.array([16, 128, 1024, 8192]) diffs = [] for nSamples in nSamplesList: ds = pygsti.data.simulate_data(my_datagen_gateset, self.lgstStrings, nSamples, sample_error='binomial', seed=100) mdl_lgst = pygsti.run_lgst(ds, self.fiducials, self.fiducials, self.model, svd_truncate_to=4, verbosity=0) mdl_lgst_go = pygsti.gaugeopt_to_target(mdl_lgst, my_datagen_gateset, { 'spam': 1.0, 'gate': 1.0 }, check_jac=True) diffs.append(my_datagen_gateset.frobeniusdist(mdl_lgst_go)) diffs = np.array(diffs, 'd') a, b = polyfit(np.log10(nSamplesList), np.log10(diffs), deg=1) #print "\n",nSamplesList; print diffs; print a #DEBUG self.assertLess(a + 0.5, 0.05)
def h_estimation(p_dict, npoints, alpha=1., Ns=None): """Compute an estimation of the negative entropy of a distribution. The estimation is done using a Bayesian estimator, followed by an extrapolation step to reduce biases in the estimation. Input arguments: p_dict -- a dictionary containing the distribution of states, as created by the function `states2dict` npoints -- total number of points used to estimate the distribution alpha -- paramters of the Dirichlet prior (usually set to 1) """ ns = array([4,2,1], dtype='int64') h_estimate = zeros((len(ns),)) for j, d in enumerate(ns): h_est = zeros((d,)) for i in range(d): p = p_dict[d][i].flatten() # for joint distributions h_est[i] = mean_H_estimate(p + alpha) h_estimate[j] = h_est.mean() # extrapolate if Ns is None: Ns = npoints/ns Ns = Ns.astype('d') h_extr = scipy.polyfit(Ns, Ns*Ns * h_estimate, 2) return h_extr[0]
def predict(A): if len(A) > min(3, n): xtrain = np.linspace(1,len(A),len(A)) ytrain = A xtest = len(A) + 1 f = poly1d(polyfit(xtrain, ytrain, n)) return f(xtest).round() if all(f(xtrain).round() == ytrain) else None
def plot_ci_bootstrap(xs, ys, resid, nboot=500, ax=None): """Return an axes of confidence bands using a bootstrap approach. Notes ----- The bootstrap approach iteratively resampling residuals. It plots `nboot` number of straight lines and outlines the shape of a band. The density of overlapping lines indicates improved confidence. Returns ------- ax : axes - Cluster of lines - Upper and Lower bounds (high and low) (optional) Note: sensitive to outliers References ---------- .. [1] J. Stults. "Visualizing Confidence Intervals", Various Consequences. http://www.variousconsequences.com/2010/02/visualizing-confidence-intervals.html """ if ax is None: ax = plt.gca() bootindex = sp.random.randint for _ in range(nboot): resamp_resid = resid[bootindex(0, len(resid) - 1, len(resid))] # Make coeffs of for polys pc = sp.polyfit(xs, ys + resamp_resid, 1) # Plot bootstrap cluster ax.plot(xs, sp.polyval(pc, xs), "b-", linewidth=2, alpha=3.0 / float(nboot)) return ax
def GetBrineDensityLinearFit(h, T0, gradT, P0, Xs0, gradXs=0, ReturnAll = False, discretization = 100): '''Compute (rho0, xi), the brine density linear fit rho(z) = rho0 + xi*(z-z0), in kg.m-3, for a given temperature T(z) = T0 + gradT+(z-z0) in Celcius, Pressure P(z0) = P0 in Pa and salinity Xs(z) = Xs0 + gradXs*(z-z0) as mass fraction. T0 = T(z0), P0 = P(z0), Xs0 = Xs(z0)''' # variables definitions dz = h/discretization z = arange(0., h*1.00001, dz) rho = array([]) # density # initialization at z = 0 T = T0; P = P0; Xs = Xs0; # incrementation. We move towards the surface. for zz in z: rho = append(rho, GetBrineDensity(T, P, Xs)) T += -gradT*dz Xs += -gradXs*dz P += -rho[-1]*9.81*dz # Linear fit [xi, rho0] = polyfit(z, rho, 1) if ReturnAll == True: return rho0, xi, rho, z else: return rho0, xi
def Asaoka_fit(T0,S0,interval,start_date=0, end_date=0): if end_date ==0: end_date = T0.max() if start_date==0: start_date =T0.min() #now convert T0 to days starting from the origin T0 =(T0-T0[0])/np.timedelta64(1,'D') #user does not specify the end date for fitting, take the maximum start_date = np.datetime64(start_date) end_date = np.datetime64(end_date) n = int(np.floor((end_date-start_date)/np.timedelta64(1,'D')/interval)) T_bar = np.zeros(n) origin = T0[0] for i in np.arange(n): T_bar[i] = origin + i*interval interpolate_1d = interp1d(T0,S0) S1=interpolate_1d(T_bar) #fit the linear curve beta1, beta0 = polyfit(S1[0:-1], S1[1:],1) return beta0, beta1 #we do the interpolation here # fig = plt.figure(figsize=(11.69,8.27),dpi=100) # ax = fig.add_subplot(2,1,1) # ax_Asaoka = fig.add_subplot(2,1,2) # ax_Asaoka.plot(S1[0:-1],S1[1:],'ko',label=r'$s_i$ vs $s_{i-1}$',markersize=3) # ax_Asaoka.set_xlabel(r'$S_{i-1}$') ax_Asaoka.set_ylabel(r'$S_i$')
def get_state_data(state0): state = state0.replace(" ", "-") print(state0) state_data = requests.get( f"https://www.worldometers.info/coronavirus/usa/{state}/") text = bs4.BeautifulSoup(state_data.text, "lxml") data = text.find( "h3", text=f"Daily New Cases in {state0}" ).next_sibling.next_sibling.next_sibling.next_sibling.string dates = json.loads(data[data.find("categories:") + 12:data.find("yAxis:") - 14]) cases = json.loads(data[data.find("data:") + 6:data.find("3-day moving average") - 44]) data_d = text.find( "h3", text=f"Daily New Deaths in {state0}" ).next_sibling.next_sibling.next_sibling.next_sibling.string deaths = json.loads(data_d[data_d.find("data:") + 6:data_d.find("3-day moving average") - 44]) df = pd.DataFrame({"cases": cases, "deaths": deaths}, index=dates) df.index = pd.DatetimeIndex(df.index + ", 2020") tail = df.resample("1W").mean().iloc[-5:-1]["cases"] poly = scipy.polyfit(range(tail.shape[0]), tail.values, 1) return poly
def getDiameterQuantilesAlongSinglePath(self,path,G,counter=None): G=self.filterPathDiameters(path, G,self.__scale) x=[] y=[] length=0 vprop=G.vertex_properties["vp"] for i in path: length+=1 if vprop[i]['diameter'] > 0: x.append(length) y.append(vprop[i]['diameter']) coeffs=polyfit(x,y,1) besty = polyval ( coeffs , x) self.__io.saveArray(x,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterX') self.__io.saveArray(y,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterY') l=len(y)-1 l25=int(l*0.25) l50=int(l*0.5) l75=int(l*0.75) l90=int(l*0.90) d25=np.average(y[:l25]) d50=np.average(y[l25:l50]) d75=np.average(y[l50:l75]) d90=np.average(y[l90:]) self.__io.saveArray(y,self.__io.getHomePath()+'Plots/'+self.__io.getFileName()+'_DiameterHistoTP') return d25,d50,d75,d90
def interpMaskedAreas(x, dtfunc): idx = num.where(dtfunc == -1)[0] # find consecutive points of dtfunc set at -1 diffs = idx - num.roll(idx, 1) setIdx1 = num.where(diffs > 1)[0] setIdx2 = num.where(diffs > 1)[0] - 1 setIdx1 = num.hstack( (num.array([0]), setIdx1) ) setIdx2 = num.hstack( (setIdx2, num.array([-1])) ) # each int in setIdx1 is beginning of a set of points # each int in setIdx2 is end of a set of points for i in range(len(setIdx1)): i1 = setIdx1[i] i2 = setIdx2[i] vals_to_interp = dtfunc[ idx[i1]:idx[i2]+1] # construct interpolation from points on either side of vals # first ydata nVals = len(vals_to_interp) left = num.array( \ [ dtfunc[idx[i1]-1-nVals], dtfunc[idx[i1]-1] ]) right = num.array( \ [ dtfunc[idx[i2]+1], dtfunc[idx[i2]+1+nVals] ]) vals_to_construct_interp = num.hstack( (left, right) ) vtci = vals_to_construct_interp # now xdata leftx = num.array([ x[idx[i1]-1-nVals], x[idx[i1]-1] ]) rightx = num.array([ x[idx[i2]+1], x[idx[i2]+1+nVals] ]) x_vtci = num.hstack( (leftx, rightx) ) # conduct interpolation intpCoeffs = scipy.polyfit(x_vtci, vtci, 3) dtfunc[ idx[i1]:idx[i2]+1 ] = \ scipy.polyval(intpCoeffs, x[ idx[i1]:idx[i2]+1 ]) return dtfunc
def plot_model_order_n(n, x, y): fpn = sp.polyfit(x, y, n) fn = sp.poly1d(fpn) print("Order %d model error: %f" % (fn.order, error(fn, x, y))) fx = sp.linspace(0, x[-1], 1000) if n == 1: linecolor = "red" elif n == 2: linecolor = "black" elif n == 3: linecolor = "green" elif n == 10: linecolor = "violet" elif n == 100: linecolor = "cyan" else: linecolor = "orange" plt.plot(fx, fn(fx), color=linecolor, linewidth=4, label=str(fn.order)) plt.legend(loc="upper left") plt.autoscale(tight=True) plt.ylim(ymin=0) plt.xlim(xmin=0) plt.grid(True)
def main_df(sim_num, rho): d1 = random.Random() d2 = random.Random() d3 = random.Random() pd_limits = [0.0, 0.1] lgds = [ 1.0, 1.0] vol_lgd = [0.2, 0.2] alpha =0.999 extra_corr = correl_function(rho) results = [] asympt_results = [] x_values = [] cdis = [] for i in range(sim_num): pds = np.array([ d1.uniform(*pd_limits), d2.uniform(*pd_limits) ]) weight = d3.uniform(0,1) weights = np.array([weight, 1-weight]) r_intra = np.array([correl_f(pds[0]), correl_f(pds[1])]) exp_loss = sum(weights*lgds*pds) results.append((loss(extra_corr, pds, lgds, weights, r_intra, alpha)-exp_loss)/2) asympt_results.append((sum([w*lgd*loss_var(alpha, pd, corr**2) for (w, lgd, pd, corr) in zip(weights, lgds, pds, r_intra)])-exp_loss)/2) cdis.append(cdi(alpha,weights, lgds, pds, r_intra)) x_values.append(avg(pds) ) #pylab.plot( x_values, results, 'b+') #pylab.plot( x_values, asympt_results, 'go') pylab.ylim(ymin=0.0, ymax=1.0) dfs = [i/j for (i,j) in zip(results, asympt_results)] pylab.plot( cdis, dfs, 'b+') (a,b) = polyfit(cdis, dfs,1) print 'coefficient a = ', a , ' and b = ' , b
def regress_col(m, pto, cols, selector, allow_missing=False): # Discard the constants, we will pick a reference point later slopes = [] for col in cols: ''' For each row find an y position y = row * c0 + c1 ''' rows = [] deps = [] for row in range(m.height()): fn = m.get_image(col, row) if fn is None: if allow_missing: continue raise Exception('c%d r%d not in map' % (col, row)) il = pto.get_image_by_fn(fn) if il is None: raise Exception('Could not find %s in map' % fn) rows.append(row) deps.append(selector(il)) if len(rows) == 0: if not allow_missing: raise Exception('No matches') continue (c0, c1) = polyfit(rows, deps, 1) slopes.append(c0) if len(slopes) == 0: if not allow_missing: raise Exception('No matches') # No dependence return 0.0 # XXX: should remove outliers return sum(slopes) / len(slopes)
def getFx(datax, datay, degree): fp1= sp.polyfit(datax, datay, degree) #print fp1 #aa = polyval(fp1, 5) #print aa fStraight = sp.poly1d(fp1) return fStraight
def regress_col(m, pto, cols, selector, allow_missing = False): # Discard the constants, we will pick a reference point later slopes = [] for col in cols: ''' For each row find an y position y = row * c0 + c1 ''' rows = [] deps = [] for row in range(m.height()): fn = m.get_image(col, row) if fn is None: if allow_missing: continue raise Exception('c%d r%d not in map' % (col, row)) il = pto.get_image_by_fn(fn) if il is None: raise Exception('Could not find %s in map' % fn) rows.append(row) deps.append(selector(il)) if len(rows) == 0: if not allow_missing: raise Exception('No matches') continue (c0, c1) = polyfit(rows, deps, 1) slopes.append(c0) if len(slopes) == 0: if not allow_missing: raise Exception('No matches') # No dependence return 0.0 # XXX: should remove outliers return sum(slopes) / len(slopes)
def get_trendline(xs,ys,params=False): m,b = scipy.polyfit(xs,ys,1) ys_trend = (m * xs) + b if params: return xs,ys_trend,m,b else: return xs,ys_trend
def scatter_fits(sys, scores, values, R, pval, aucname, show=False): auclabel=get_label(aucname) if sys=='apo': format='ko' label=get_label(sys) if sys=='bi': format='ro' label=get_label(sys) if sys=='car': format='bo' label=get_label(sys) if pval< 0.0001: pval=0.0001 pylab.figure() pylab.plot(scores, values, format) (ar,br)=polyfit(scores, values, 1) xr=polyval([ar,br], scores) pylab.plot(scores,xr,'%s-' % format[0], label='R=%s, pval=%s' % (round(R,2), round(pval,4))) if aucname=='types': pylab.plot(range(0, 15), [0.5]*len(range(0,15)), 'k--', label='Random Disc.') pylab.ylim(0.3, 0.9) else: pylab.ylim(0.5, 1.0) pylab.xlim(0, 15) lg=pylab.legend() lg.draw_frame(False) pylab.title('%s States' % label) pylab.xticks(range(0, 15), [' ']*2+ ['inactive']+[' ']*(len(range(0,15))-6)+['active']+[' ']*2) pylab.xlabel('Pathway Progress') pylab.ylabel('%s Aucs' % (auclabel)) pylab.savefig('%s_%saucs.png' % (sys, aucname), dpi=300) if show==True: pylab.show()
def correct_slope(self,rank=4): index = np.linspace(0,self.nsamp-1,self.nsamp) trends = np.zeros((self.nsamp,self.norders)) i = 0 ''' for order in self.orders: fsubs = np.isfinite(order['flux']) pars = polyfit(index[fsubs][20:-20],order['flux'][fsubs][20:-20],deg=rank,w=order['uflux'][fsubs][20:-20]) trend = poly1d(pars) trend_samp = trend(index) trends[:,i] = trend_samp/np.median(trend_samp) i += 1 trend_mean = np.median(trends,axis=1) ''' fluxes = np.zeros((self.nsamp,self.norders)) for i in np.arange(self.norders): flux = self.orders[i]['flux'] flux = flux/np.median(flux) fluxes[:,i] = flux i += 1 trend_mean = np.nanmedian(fluxes,axis=1) fsubs = np.isfinite(trend_mean) pars = polyfit(index[fsubs],trend_mean[fsubs],deg=rank) trend_smooth = poly1d(pars)(index) for order in self.orders: order['flux'] /= trend_smooth
def _xcorr_interp(ccc, dt): """ Interpolate around the maximum correlation value for sub-sample precision. :param ccc: Cross-correlation array :type ccc: numpy.ndarray :param dt: sample interval :type dt: float :return: Position of interpolated maximum in seconds from start of ccc :rtype: float """ if ccc.shape[0] == 1: cc = ccc[0] else: cc = ccc # Code borrowed from obspy.signal.cross_correlation.xcorr_pick_correction cc_curvature = np.concatenate((np.zeros(1), np.diff(cc, 2), np.zeros(1))) cc_t = np.arange(0, len(cc) * dt, dt) peak_index = cc.argmax() first_sample = peak_index # XXX this could be improved.. while first_sample > 0 and cc_curvature[first_sample - 1] <= 0: first_sample -= 1 last_sample = peak_index while last_sample < len(cc) - 1 and cc_curvature[last_sample + 1] <= 0: last_sample += 1 num_samples = last_sample - first_sample + 1 if num_samples < 3: Logger.warning( "Fewer than 3 samples selected for fit to cross correlation: " "{0}, returning maximum in data".format(num_samples)) return np.argmax(cc) * dt, np.amax(cc) if num_samples < 5: Logger.debug( "Fewer than 5 samples selected for fit to cross correlation: " "{0}".format(num_samples)) coeffs, residual = scipy.polyfit(cc_t[first_sample:last_sample + 1], cc[first_sample:last_sample + 1], deg=2, full=True)[:2] # check results of fit if coeffs[0] >= 0: Logger.info("Fitted parabola opens upwards!") if residual > 0.1: Logger.info( "Residual in quadratic fit to cross correlation maximum larger " "than 0.1: {0}".format(residual)) # X coordinate of vertex of parabola gives time shift to correct # differential pick time. Y coordinate gives maximum correlation # coefficient. shift = -coeffs[1] / 2.0 / coeffs[0] coeff = (4 * coeffs[0] * coeffs[2] - coeffs[1]**2) / (4 * coeffs[0]) if coeff < np.amax(ccc) or coeff > 1.0 or not 0 < shift < len(ccc) * dt: # Sometimes the interpolation returns a worse result. Logger.warning("Interpolation did not give an accurate result, " "returning maximum in data") return np.argmax(ccc) * dt, np.amax(ccc) return shift, coeff
def get_plot(data, ax, c, l): ml = data[:, 0] ex = data[:, 1] (ar, br) = polyfit(ml, ex, 1) xr = polyval([ar, br], ml) ax.scatter(ml, ex, color=c, alpha=0.3, label=l) #ax.plot(ml,xr) ax.legend()
def interpolate_flux(freqs, fluxes, freq): if len(freqs) < 1: return None elif len(freqs) == 1: return fluxes[0] elif len(freqs) > 1: (ar, br) = polyfit(freqs, fluxes, 1) return polyval([ar, br], freq)
def predict(A): if len(A) > min(3, n): xtrain = np.linspace(1, len(A), len(A)) ytrain = A xtest = len(A) + 1 f = poly1d(polyfit(xtrain, ytrain, n)) return f(xtest).round() if all( f(xtrain).round() == ytrain) else None
def polynomialReg(): sample_rate, samples = wavfile.read('songs/hakuna_matata.wav') x = np.array(range(0, 100)) y = np.array(samples[5000000:5000100]) plt.subplot(211) p1 = cp.polyfit(x, y, 1) p2 = cp.polyfit(x, y, 2) p3 = cp.polyfit(x, y, 3) p20 = cp.polyfit(x, y, 20) #graficamos plt.plot(x, y, 'r', label="Original") plt.plot(x, cp.polyval(p1, x), 'b--', label="Grado 1") plt.plot(x, cp.polyval(p2, x), 'm--', label="Grado 2") plt.plot(x, cp.polyval(p3, x), 'g--', label="Grado 3") plt.plot(x, cp.polyval(p20, x), 'k--', label="Grado 20") plt.legend() plt.show()
def Polyfit_detrend(x, y): #主程序,相当于C语言的main函数 a, b, c = polyfit(x, y, 2) y_quad = a * x * x + b * x + c #利用拟合得到的系数,计算x向量对应的y向量 # 拟合结果绘图 # pylab.plot(x, y, 'o') #绘制散点 # pylab.plot(x, y_quad, 'r-') #绘制拟合的曲线 # pylab.show() return y_quad
def calculate_slow_phi_0s(phi_0s, p_values): slow_phi_0s = scipy.empty_like(phi_0s) for i, phi_0 in enumerate(phi_0s): phi_0_unfolded = unfold(phi_0) x = arange(len(phi_0_unfolded)) model = polyfit(x, phi_0_unfolded, 3, w=p_values[i]) slow_phi_0s[i] = polyval(model, x) return slow_phi_0s