def precip_accumuation(): RR = np.load("%s/CTSR/%d/R3_methodType/regression_results.npy" % (path, crn)) FS = np.load("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn)) rain_b4 = cf.QRB(yct) rain_af = cf.QRB(yct) bkp_v2 = cf.QRB(1) for line in range(1, RR.shape[0]): print line if FS[line, 2] >=3: osp_b4 = RR[line, 3] lt_b4 = RR[line, 4] #load the precip accumulation table rf_b4 = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" % (path, rn, osp_b4, lt_b4)) rain_b4[line, :] = rf_b4[line, :] osp_af = RR[line, 6] lt_af = RR[line, 7] rf_af = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" % (path, rn, osp_af, lt_af)) rain_af[line, :] = rf_af[line, :] bkp_v2[line, 2] = FS[line, 3] else: rain_af[line, 2:] = np.NAN rain_b4[line, 2:] = np.NAN bkp_v2[line, 2] = np.NAN np.save("%s/CTSR/%d/R3_methodType/rain_b4.npy" % (path, crn), rain_b4) np.save("%s/CTSR/%d/R3_methodType/rain_af.npy" % (path, crn), rain_af) np.save("%s/CTSR/%d/R3_methodType/VPRbrk_BP.npy" % (path, crn), bkp_v2) cf.immkr(bkp_v2, 2)
def results(): res = np.load("./CTSR/5/R2_Chow/Final_ChowResults.npy") count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, crn)) Res_pass = cf.QRB(2) for line in range(1, res.shape[0]): y, x = res[line, :2] if mask[y, x] == 0: Res_pass[line, 2:] = np.NAN elif count[line, 2] == 0: Res_pass[line, 2:] = np.NAN else: if res[line, 3] < 0.05: Res_pass[line, 2] = 1 else: Res_pass[line, 2] = 0 if res[line, 5] < 0.05: Res_pass[line, 3] = 1 else: Res_pass[line, 3] = 0 cf.immkr(Res_pass, 2) cf.immkr(Res_pass, 3) # pdb.set_trace() #Print the % VPR that passed size = sp.stats.itemfreq(Res_pass[1:, 2]) VPRper = size[1, 1]/(size[1, 1]+size[0, 1]) print VPRper*100, "VPR pass rate" #Print the %Res that pass size = sp.stats.itemfreq(Res_pass[1:, 3]) RESper = size[1, 1]/(size[1, 1]+size[0, 1]) print RESper*100, "Residuals pass rate" pass
def neg_grad_timeseries(): """To look at the time series of negative gradients""" i = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rn, rn)) j = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rn, rn)) grad = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rn, rn)) # g, rq, pv for line in range(1, grad.shape[0]): print line y = grad[line, 0] x = grad[line, 1] # if abs(grad[line, 2])>0.001: # grad[line, 2] = np.NAN if y == 256 and x == 55: # >200 and x<75: if grad[line, 2] < 0: if grad[line, 4] < 0.05: if grad[line, 3] > 0.4: print y, x # timeseris ts = np.arange((yct)) + 1982 precip = i[line, 2:] vi = j[line, 2:] fig, ax1 = plt.subplots() ax1.plot(ts, precip, label="precip") ax2 = ax1.twinx() ax2.plot(ts, vi, label="NDVI", color="r") plt.show() # np.save("%s/CTSR/%d/R4_paperfigures/%d__%d_rain.npy" % (path, crn, y, x), precip ) # np.save("%s/CTSR/%d/R4_paperfigures/%d__%d_NDVI.npy" % (path, crn, y, x), vi ) cf.immkr(grad, 2)
def mean_values(): """ builds a singe GRB array that has the mean precip and mean VI values. THere will also be an offset period controled VI and a lt controlled grad """ #load in the data precip = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rn, rn)) VI = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rn, rn)) #load in the lt values pmin = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn)) #"""lt, pval, osp""" cf.immkr(pmin, 2) #lt #load in the grad pgrad = np.load("%s/RESTREND/%d/midputs/r2outs/precip_trend%d.npy" % (path, rn, rn)) #blank array to add means #VI in 2, precip in 3, lt controlled in 4, lt controlled grad in 5 means = cf.QRB(4) #add values to means for line in range(1, VI.shape[0]): y, x = VI[line, 0:2] if mask[y,x] == 0: means[line, 2:]= np.NAN else: means[line, 2] = np.mean(VI[line, 2:]) means[line, 3] = np.mean(precip[line, 2:]) means[line, 4] = np.divide(means[line, 3], pmin[line, 2]) #add a mask for rainfall over a meter if means[line, 3] > 1000: #greater than 1000mm of rainfall means[line, 3] = 1000 #converts both into mm/year then devides means[line, 5] = np.divide(pgrad[line, 2], means[line, 4]*12) * 100 #to get a % #add a mask for gradient over 0.018 if means[line, 5]> 2: means[line, 5] = 2 #titles t0 = str("Mean VI value") t1 = str("Mean RF value (>1000 masked)") t2 = str("Mean RF value LT controlled (mm/month)") t3 = str("Raw precip g vs LT controlled mean as % (>2 masked)") titles = [t0, t1, t2, t3] return means # turn off if i want display #figures immkr(means, 2, cm.rainbow_r, run=7, name=titles[0]) immkr(means, 3, cm.jet, run=8, name=titles[1]) immkr(means, 4, cm.rainbow_r, run=9, name=titles[2]) immkr(means, 5, cm.PiYG, run=10, name=titles[3])
def sensor_mask(): #loade file generated from R_oldscripts/R_sensor_bp_test.py. #This need to be intergrated asap counts = np.load("%s/CTSR/%d/outputs/Sen_mask_count.npy" % (path, rn)) print counts.shape cf.immkr(counts, 2) cf.immkr(counts, 8) pdb.set_trace()
def Quick(): a = cf.QRB(3) for line in range(1, a.shape[0]): y, x = a[line, :2] if mask[y, x] == 0: a[line, 2] = "masked" else: a[line, 2] = "notmasked" cf.immkr(a, 2) pass
def precip_trend(): """Looks at the raw trend in the Precip data""" precip = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rn, rn)) results = cf.QRB(3) # min_p = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn)) # grad = np.zeros([yrg, xrg]) #zero array to pass the residual gradienst into # r2val = np.zeros([yrg, xrg]) #array to pass the r2 values into # pval = np.zeros([yrg, xrg]) #array to pass the r2 values into for row in range(1, precip.shape[0]): y = precip[row, 0] x = precip[row, 1] results[row, 0] = precip[row, 0] results[row, 1] = precip[row, 1] if mask[y, x] == 0: results[row, 2:] = np.NAN elif mask[y, x] == 1: ind = np.arange((yct)) # print ind #the indipendent (rainfall) variable, exog, add constant to this one dep = precip[row, 2:] #the dependent variable, NDVI, endorg, indC = smx.add_constant(ind) #adds the intercept ols = sm.OLS(dep, indC).fit() # print ols.summary() #performs the OLS #the following are constants g = ols.params[1]# * 10 #for scale # print ols.params rq = ols.rsquared pv = ols.pvalues[1] # sys.exit() results[row, 2:] = [g, rq, pv]# print rq, pv # if pv == min_p[row, 3]: # pass # else: # print pv, min_p[row, 3], row # sys.exit() else: print "it went wrong here" sys.exit() #f = cf.immkr(peakNDVI, 2) a = cf.immkr(results, 2) b = cf.immkr(results, 3) c = cf.immkr(results, 4) # plt.show() np.save("%s/RESTREND/%d/midputs/r2outs/precip_trend%d.npy" % (path, rn, rn), results)
def non_parametric_res(): resid = np.load("%s/RESTREND/%d/outputs/olsresiduals%d.npy" % (path, rn, rn)) """the following is to mask out pixels with worring residual values """ for row in range(1, resid.shape[0]): y = resid[row, 0] x = resid[row, 1] if mask[y, x] == 0: pass elif mask[y, x] == 1: test = resid[row, 2:] ts = test.shape[0] un = np.unique(test) if un.shape[0]< yct-2: mask[y, x] = 0 else: pass results = cf.QRB(2) results_ts = cf.QRB(4) ind = np.arange((yct)) for row in range(1, resid.shape[0]): print row y = resid[row, 0] x = resid[row, 1] results[row, 0] = resid[row, 0] results[row, 1] = resid[row, 1] results_ts[row, 0] = resid[row, 0] results_ts[row, 1] = resid[row, 1] if mask[y, x] == 0: results[row, 2:] = np.NAN results_ts[row, 2:] = np.NAN # elif y ==5 and x == 120: # results[row, 2:] = np.NAN # results_ts[row, 2:] = np.NAN elif mask[y, x] == 1: dep = resid[row, 2:] spr= sp.stats.spearmanr(dep, ind) # print y, x # print dep, ind # print y, x, spr # print sp.stats.mstats.theilslopes(dep, ind) ts = sp.stats.mstats.theilslopes(dep, ind) results[row, 2:] = spr results_ts[row, 2:] = ts # sys.exit() # a = cf.immkr(results, 2) # plt.show() # plt.show() np.save("%s/RESTREND/%d/outputs/RES_spearmanr%d.npy" % (path, rn, rn), results) np.save("%s/RESTREND/%d/outputs/RES_theilslopes%d.npy" % (path, rn, rn), results_ts) a = cf.immkr(results_ts, 2) a = cf.immkr(results_ts, 3) a = cf.immkr(results_ts, 4) a = cf.immkr(results_ts, 5)
def p_seperation(): #this needs to put the index """Takes a list of those point that failed the p test and looks to see if seperating them by bfast break points can produce stastically significant results. """ """at first i'm going to see if any pixels have ony a single breakpoint and use that as a test""" p_fails = np.load("%s/RESTREND/%d/midputs/workingfiles/ptest_fail.npy" % (path, rn)) #p fails to be removed in the future count = np.load("%s/CTSR/%d/outputs/Sen_mask_count.npy" % (path, rn)) #this is to create a subset of the data that is easy to look at breaks = np.load("%s/CTSR/%d/outputs/brkpoint_locSEN.npy" % (path, rn)) peaks = np.load("%s/CTSR/%d/outputs/absmax_VI.npy" % (path, rn)) index = cf.QRB(7) #may need a better way to index this n = 0 #just included to give me a count for line in range(1, count.shape[0]): y = count[line, 0] x = count[line, 1] if mask[y, x] == 0: index[line, 2:] = np.NAN elif p_fails[line, 2] == 1: #could be removed to look at everywhere #this may need to be replaced if count[line, 6] == 1: points = peaks[line, 2:] for bp in breaks[line, 2:]: if np.isnan(bp) is True: pass else: bp_num = bp - 1 #to convert from R to python indexing for col in range(0, yct): if bp_num <= points[col]: index[line, 2] = col print line break else: pass # >= is the way i do this, followed by a break n+= 1 else: index[line, 2:] = 0 # May need to vary this later depending how i index in the regression else: index[line, 2:] = 0 # May need to vary this later depending how i index in the regression print n, "here" cf.immkr(p_fails, 2) cf.immkr(index, 2) # pdb.set_trace() np.save("%s/CTSR/%d/outputs/bp_index.npy" % (path, rn), index)
def Year_of_change(): Chowres = np.load("%s/CTSR/%d/R2_Chow/Final_ChowResults.npy" % (path, crn)) bp_loc = np.load("./CTSR/5/R3_methodType/VPRbrk_BP.npy") print bp_loc.shape bp_loc[:, 2] += ys cf.immkr(bp_loc, 2) bins, counts = np.unique(bp_loc[1:, 2], return_counts=True) plt.plot(bins[:24,], counts[:24,]) plt.show() pdb.set_trace() pass
def RES_problemsolver(newest_rn, older_rn): """this is a function that exists to find out why small changes in RESTREND1 deacreased my p values. This may be moved to a problem solver script in the future. Step 1 is to compare the p valuse for runs 4 and 5. Hopefull will be a general purpose problem solver script in the future""" # new_minp = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, newest_rn, newest_rn)) new_minp = np.load("H:/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(newest_rn, newest_rn)) old_minp = np.load("H:/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(older_rn, older_rn)) difference = cf.QRB(1) #old - new. positive values = improvment, negative = reduction (want small p value) # print new_minp.shape, old_minp.shape for line in range(1, new_minp.shape[0]): difference[line, 2] = old_minp[line, 3] - new_minp[line, 3] if new_minp[line, 3] > 0.05: if old_minp[line, 3] <0.05: if old_minp[line, 4] == 3: #answer is the index error correction pass else: sys.exit() print old_minp[line, :] print new_minp[line, :] old_lt = old_minp[line, 2] old_osp = old_minp[line, 4] old_precip = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" % (path, older_rn, old_osp, old_lt)) old_p = old_precip[line, :] for lt in range(0, mlt): for osp in range(0, ofp): pre = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" % (path, newest_rn, osp, lt)) new_pre = pre[line, :] if np.array_equal(old_p, new_pre) is False: #array_equal print lt, osp, "not here" if old_lt == lt+1: if old_osp == osp: print old_p print new_pre else: print lt, osp, "here" print old_p, "good" print new_pre sys.exit() sys.exit() else: pass else: pass cf.immkr(difference, 2)
def bfast_grad(): grad = np.load("%s/CTSR/%d/midputs/r2outs/grad_final.npy" % (path, rn)) # immkr(osp, grad, 2, cm.PiYG, 13) for line in range(1, grad.shape[0]): if abs(grad[line, 2])>3: grad[line, 2] = np.NAN else: pass if grad[line, 2]>1: grad[line, 2] = 1 elif grad[line, 2]<-1: grad[line, 2] = -1 cf.immkr(grad, 2) osp = 0 immkr(osp, grad, 2, cm.PiYG, 13)
def max_r2(): """Function to find the max r2 value of the bf_ofp and lt combinations then build an array that has the r2 value, the osp and column (lt+2) This is a duplicate of the min_simple and included to decide if using r2 values or p values produces better results""" rval_max = cf.QRB(3) #value, osp, col(lt + 2) while True: bf_ofp_rmax = cf.QRB(bf_ofp) # blank array with r2 values bf_ofp_pos = cf.QRB(bf_ofp) # blank array with position for osp in range(bf_ofp): pval = np.load("%s/CTSR/%d/midputs/r2outs/r2val_%d.npy" % (path, rn, osp)) for row in range(1, pval.shape[0]): y = pval[row, 0] x = pval[row, 1] if mask[y, x] == 0: bf_ofp_rmax[row, 2: ] = np.NAN bf_ofp_pos[row, 2: ] = np.NAN else: val = pval[row, 2:] min_pos = np.argmax(val) +2 min_val = np.amax(val) col = osp+2 #plus 2 moves past the y, x colums bf_ofp_rmax[row, col] = min_val bf_ofp_pos[row, col] = min_pos for row in range(1, pval.shape[0]): y = bf_ofp_rmax[row, 0] x = bf_ofp_rmax[row, 1] if mask[y, x] == 0: rval_max[row, 2 ] = np.NAN else: p_bf_ofp = bf_ofp_rmax[row, 2:] lt_bf_ofp = bf_ofp_pos[row, 2:] value = np.amax(p_bf_ofp) osp = np.argmax(p_bf_ofp) col = lt_bf_ofp[osp] rval_max[row, 2:] = value, osp, col break np.save("%s/CTSR/%d/midputs/r2outs/maxr2.npy" % (path, rn,), rval_max) cf.immkr(rval_max, 2) print "maxr2 complete"
def pfail(): """this function will look at the results from the RESTREND run and see which pixels failed the p<0.05 test between VI and precipation""" minp = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn)) p_test = cf.QRB(1) # NAN for ocean, 0 for pass, 1 for fail for line in range(1, minp.shape[0]): y = minp[line, 0] x = minp[line, 1] if mask[y, x] == 0: p_test[line, 2] = np.NAN elif minp[line, 3] > 0.05: #failed the p test p_test[line, 2] = 1 else: p_test[line, 2] = 0 cf.immkr(p_test, 2) np.save("%s/RESTREND/%d/midputs/workingfiles/ptest_fail.npy" % (path, rn), p_test)
def final_sort(): """ Function to add the aditional infomation from the R_R3_VPRbk.R """ #The results from the VPR R analysis VPR_results = np.load("%s/CTSR/%d/R3_methodType/Final_VPRregression.npy" % (path, crn)) # cf.immkr(VPR_results, 3) #The Results from my first sort FS = np.load("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn)) for line in range(1, FS.shape[0]): if VPR_results[line, 3]<= 0.05: FS[line, 2] = 3 elif VPR_results[line, 3]>0.05: FS[line, 2] = 0 cf.immkr(FS, 2) np.save("%s/CTSR/%d/R3_methodType/Final_Sort.npy" % (path, crn), FS)
def grad_viewer(): # rn = 6 results = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rn, rn)) for line in range(1, results.shape[0]): if abs(results[line, 2])>3: results[line, 2] = np.NAN else: pass if results[line, 2] >1: results[line, 2] = 1 elif results[line, 2] <-1: results[line, 2] = -1 cf.immkr(results, 2) osp = 0 immkr(osp, results, 2, cm.PiYG, 13) sys.exit()
def r2_grad_caller(string): """Similar to residual caller but for r2 and grad values. takes a string, either r2val or grad""" result = cf.QRB(1) pvals = np.load("%s/CTSR/%d/midputs/r2outs/minp.npy" % (path, rn,)) for row in range(1, result.shape[0]): y = pvals[row, 0] x = pvals[row, 1] if mask[y, x] == 0: result[row, 2: ] = np.NAN else: print row osp = pvals[row, 3] # lt = pvals[row, 4] - 2 #convert from col to lt var = np.load("%s/CTSR/%d/midputs/r2outs/%s_%d.npy" % (path, rn, string, osp)) result[row, 2] = var[row, pvals[row, 4]] np.save("%s/CTSR/%d/midputs/r2outs/%s_final.npy" % (path, rn, string), result) cf.immkr(result, 2)
def restrend(): resid = np.load("%s/RESTREND/%d/outputs/olsresiduals%d.npy" % (path, rn, rn)) results = cf.QRB(4) for row in range(1, resid.shape[0]): y = resid[row, 0] x = resid[row, 1] results[row, 0] = resid[row, 0] results[row, 1] = resid[row, 1] if mask[y, x] == 0: results[row, 2:] = np.NAN elif mask[y, x] == 1: ind = np.arange((yct)) # print ind #the indipendent (rainfall) variable, exog, add constant to this one dep = resid[row, 2:] #the dependent variable, NDVI, endorg, indC = smx.add_constant(ind) #adds the intercept ols = sm.OLS(dep, indC).fit() #print ols.summary() #performs the OLS #the following are constants g = ols.params[1] m = ols.params[0] rq = ols.rsquared pv = ols.pvalues[1] results[row, 2:] = [g, rq, pv, m]# print rq, pv else: print "it went wrong here" sys.exit() # f = cf.immkr(resid, 2) a = cf.immkr(results, 2) b = cf.immkr(results, 3) c = cf.immkr(results, 4) # plt.show() np.save("%s/RESTREND/%d/outputs/RESTREND%d.npy" % (path, rn, rn), results)
def residual_caller(): """Uses the results of the min_simple to build an array of the residuals""" resid = cf.QRB(yct*12) pvals = np.load("%s/CTSR/%d/midputs/r2outs/minp.npy" % (path, rn,)) for row in range(1, pvals.shape[0]): y = pvals[row, 0] x = pvals[row, 1] if mask[y, x] == 0: resid[row, 2: ] = np.NAN else: print row osp = pvals[row, 3] lt = pvals[row, 4] - 2 #convert from col to lt point_resid = np.load("%s/CTSR/%d/midputs/maxpos/resid_tabs/residuals_%dlt%d.npy" % (path, rn, osp, lt)) resid[row, :] = point_resid[row, :] #i need to build a function that checks the script is accessing the correct residuals np.save("%s/CTSR/%d/midputs/r2outs/residuals.npy" % (path, rn,), resid) cf.immkr(resid, 3)
def min_simple(): """Function to find the min p value of the bf_ofp and lt combinations then build an array that has the p value, the osp and column (lt+2)""" pval_min = cf.QRB(3) #value, osp, col(lt + 2) while True: bf_ofp_pval = cf.QRB(bf_ofp) # blank array with p values bf_ofp_pos = cf.QRB(bf_ofp) # blank array with position for osp in range(bf_ofp): pval = np.load("%s/CTSR/%d/midputs/r2outs/pval_%d.npy" % (path, rn, osp)) for row in range(1, pval.shape[0]): y = pval[row, 0] x = pval[row, 1] if mask[y, x] == 0: bf_ofp_pval[row, 2: ] = np.NAN bf_ofp_pos[row, 2: ] = np.NAN else: val = pval[row, 2:] min_pos = np.argmin(val) +2 min_val = np.amin(val) col = osp+2 #plus 2 moves past the y, x colums bf_ofp_pval[row, col] = min_val bf_ofp_pos[row, col] = min_pos for row in range(1, pval.shape[0]): y = bf_ofp_pval[row, 0] x = bf_ofp_pval[row, 1] if mask[y, x] == 0: pval_min[row, 2 ] = np.NAN else: p_bf_ofp = bf_ofp_pval[row, 2:] lt_bf_ofp = bf_ofp_pos[row, 2:] value = np.amin(p_bf_ofp) osp = np.argmin(p_bf_ofp) col = lt_bf_ofp[osp] pval_min[row, 2:] = value, osp, col break np.save("%s/CTSR/%d/midputs/r2outs/minp.npy" % (path, rn,), pval_min) cf.immkr(pval_min, 2) print "min_simple complete"
def Year_tester(): #to figure out if rabbits made any impact at all #load in the data Sort = np.load("%s/CTSR/%d/R3_methodType/Final_Sort.npy" % (path, crn)) print Sort.shape Sort[1:, 3] += ys for line in range(1,Sort.shape[0]): if Sort[line, 2] == 1: Sort[line, 3] = np.NAN else: pass cf.immkr(Sort, 2) cf.immkr(Sort, 3) bins, counts = np.unique(Sort[1:, 3], return_counts=True) plt.plot(bins[:24,], counts[:24,]) plt.show() pass
def grad_compiler(): """A function to use the min p values to attempt to reconstruct the gradients """ pmin = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn)) grad = cf.QRB(2) for line in range(1, grad.shape[0]): print line y = pmin[line, 0] x = pmin[line, 1] if mask[y, x] == 0: grad[line, 2:] = np.NAN else: col = pmin[line, 2] osp = pmin[line, 4] g = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dslopearray_%d.npy" % (path,rn,osp, rn)) grad[line, 2] = g[line, col] cf.immkr(grad, 2) pdb.set_trace() # grad[1:, ] np.save("%s/RESTREND/%d/midputs/workingfiles/grad_%d.npy" %(path, rn, rn), grad) pass
def grad_comp(): bpc = bp_counter() print bpc # total = [bpc[0], 0, 0] #total number of breapoints with no, partial and complete coverage # totals.append() for bp in range(1, 2): grad = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_grad.npy" % (path, rn, bp)) p_pass = np.load("%s/CTSR/%d/midputs/workingfiles/%d_pass_fail_p.npy" % (path, rn, bp)) count = cf.QRB(bp) for line in range(1, grad.shape[0]): if np.isnan(grad[line, 2]) == True: count[line, 2] = np.NAN elif p_pass[line, 2] ==bp+1: # num = [] for col in range(2, grad.shape[1]): count[line, bp+1]= grad[line, bp+2] - grad[line, bp+1] # if abs(count[line, bp+1]) < 0.001: # print count[line, bp+1] # elif abs(count[line, bp+1]) < 0.005: # print line if grad[line, bp+2] > 0: if grad[line, bp+1] >0: print line # print grad[line, bp+2] - grad[line, bp+1] # nu = np.asarray(num) # n = np.sum(nu) # count[line, 2] = n cf.immkr(count, 2) points = np.asarray(count[1:, 2][~np.isnan(count[1:, 2])]) test = np.histogram(points, bins=np.arange(-0.05, 0.05, 0.005)) # print test # print np.unique(points) # print points plt.figure() plt.hist(points, bins=np.arange(-0.05, 0.05, 0.005)) plt.show()
def pixel_subclass(): """A function to devide the pixels by subclass and see what my results are""" #load the number of breakpoints bp_count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, rnc)) #load the count of the breakpoints # bp_index = np.load("%s/CTSR/%d/midputs/workingfiles/bp_index.npy" % (path, rn)) # bp_loc = np.load("%s/CTSR/%d/outputs/brkpoint_locSEN.npy" % (path, rnc)) #imported to look at the gradients VPR = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rnc, rnc)) # print bp_count.shape # print bp_index.shape res_results = np.load("%s/RESTREND/%d/outputs/RESTREND%d.npy" % (path, rn, rn)) RES_masked = cf.QRB(1) # cf.immkr(res_results, 2) # cf.immkr(res_results, 3) # cf.immkr(res_results, 4) for line in range(1, bp_count.shape[0]): y, x = bp_count[line, 0:2] if mask[y, x] == 0: RES_masked[line, 2] = np.NAN elif pmask[y, x] == 0: RES_masked[line, 2] = np.NAN elif bp_count[line, 2] != 0: RES_masked[line, 2] = np.NAN elif VPR[line, 2]< 0 : RES_masked[line, 2] = np.NAN elif VPR[line, 2] >2: RES_masked[line, 2] = np.NAN elif res_results[line, 4] >0.05: RES_masked[line, 2] = np.NAN else: # print VPR[line, 2] RES_masked[line, 2] = res_results[line, 2] cf.immkr(RES_masked, 2)
def first_sort(): #to find the pixels that need further work #import the relevant data #number of breakpoints count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, crn)) #Values of a standard RESTREND VPR = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rn, rn)) #[g, rq, pv, b] chowres = np.load("%s/CTSR/%d/R2_Chow/Final_ChowResults.npy" % (path, crn)) # cf.immkr(chowres, 3) #blank array for results cat = cf.QRB(2) # data catogeriers for line in range(1, count.shape[0]): y, x = count[line, :2] cat[line, 3] = chowres[line, 6] if mask[y, x] == 0: cat[line, 2] = np.NAN elif VPR[line, 2] < 0: cat[line, 2] = -1 #negative VPR trend elif VPR[line, 4] > 0.05: if chowres[line, 3] < 0.05: ######This means pixels need more work###### cat[line, 2] = 4 #########asap######## else: cat[line, 2] = 0 #no significant breakpoints, unusable pixel elif chowres[line, 5] < 0.05: #has a significant breakpoint, may be split if chowres[line, 3] < 0.05: cat[line, 2] = 3 else: cat[line, 2] = 2 else: cat[line, 2] = 1 cf.immkr(cat, 2) cf.immkr(cat, 3) np.save("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn), cat) # )
def min_p(): minp = cf.QRB(2*ofp) for osp in range(0, ofp): print osp OLS_table = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dltarray_%d.npy" % (path, rn, osp, rn)) # OLS_table = np.load() for row in range(1, OLS_table.shape[0]): # cf.immkr(OLS_table, 3) y = OLS_table[row, 0] x = OLS_table[row, 1] minp[row, 0] = y minp[row, 1] = x if mask[y, x] == 0: minp[row, 2: ] = np.NAN else: pv = OLS_table[row, 3:OLS_table.shape[1]:2] # print pv # sys.exit() pvpos = np.argmin(pv) + 1 minp[row, 2+osp] = pvpos pvval = np.amin(pv) # print pvval minp[row, 2+ofp+osp] = pvval mr2 = cf.QRB(3) for row in range(1, minp.shape[0]): mr2[row, 0] = minp[row, 0] mr2[row, 1] = minp[row, 1] if mask[minp[row, 0], minp[row, 1]] == 0: mr2[row, 2: ] = np.NAN else: p_p = minp[row, 2+ofp:] p_v = np.amin(p_p) offset = np.argmin(p_p) mr2lt = minp[row, 2+offset] #col i think mr2[row, 2] = mr2lt mr2[row, 3] = p_v mr2[row, 4] = offset np.save("%s/RESTREND/%d/midputs/workingfiles/tester_%d.npy" %(path, rn, rn), minp) cf.immkr(mr2, 3) cf.immkr(mr2, 2) cf.immkr(mr2, 4) # sys.exit() # plt.show() np.save("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn), mr2) print mr2.shape, "min_p" return mr2
def bp_regression(): #to recalculate the VPR on either side of the breakpoint FS = np.load("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn)) pv_b4 = cf.QRB(bf_ofp) pv_af = cf.QRB(bf_ofp) lt_b4 = cf.QRB(bf_ofp) lt_af = cf.QRB(bf_ofp) results = cf.QRB(6)# [pval, ops, lt,]before then after bp for osp in range(0, bf_ofp): j = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rn, rn)) pval_b4 = cf.QRB(mlt) pval_af = cf.QRB(mlt) for lt in range(mlt): print osp, lt i = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" % (path, rn, osp, lt)) for line in range(1, i.shape[0]): # print lt, line y = i[line, 0] x = i[line, 1] # print lt, line,# y, x if mask[y,x] == 0: pval_af[line, 2:] = np.NAN pval_b4[line, 2:] = np.NAN elif FS[line, 2] <3: pval_af[line, 2:] = np.NAN pval_b4[line, 2:] = np.NAN else: bkp = FS[line, 3]+2 jb4 = j[line, 2:bkp] jaf = j[line, bkp:] ib4 = i[line, 2:bkp] iaf = i[line, bkp:] # if np.max indC_b4 = smx.add_constant(ib4) indC_af = smx.add_constant(iaf) ols_b4 = sm.OLS(jb4, indC_b4).fit() ols_af = sm.OLS(jaf, indC_af).fit() # print ols_b4.summary() # print ols.params[1] # sys.exit() # g = ols.params[1] #* 1000 # rq = ols.rsquared try: position = lt+2 pval_b4[line, position] = ols_b4.pvalues[1] pval_af[line, position] = ols_af.pvalues[1] except IndexError: print "its indexing incorrectly" pdb.set_trace() for line in range(1, pval_af.shape[0]): if FS[line, 2] >=3: osp_pos = osp+2 vals_b4 = pval_b4[line, 2:] vals_af = pval_af[line, 2:] pv_b4[line, osp_pos] = np.amin(vals_b4) pv_af[line, osp_pos] = np.amin(vals_af) lt_b4[line, osp_pos] = np.argmin(vals_b4) lt_af[line, osp_pos] = np.argmin(vals_af) else: pv_b4[line, 2:] = np.NAN pv_af[line, 2:] = np.NAN lt_b4[line, 2:] = np.NAN lt_af[line, 2:] = np.NAN # pdb.set_trace() # results = cf.QRB(6)# [pval, ops, lt,]before then after bp for line in range(1, pv_b4.shape[0]): if FS[line, 2] >=3: results[line, 2] = np.amin(pv_b4[line, 2:]) results[line, 3] = np.argmin(pv_b4[line, 2:]) results[line, 4] = lt_b4[line, 2+results[line, 3]] results[line, 5] = np.amin(pv_af[line, 2:]) results[line, 6] = np.argmin(pv_af[line, 2:]) results[line, 7] = lt_af[line, 2+results[line, 6]] if results[line, 2] == results[line, 5]: print "some hanky panky is appearing", line else: pass else: results[line, 2:] = np.NAN np.save("%s/CTSR/%d/R3_methodType/regression_results.npy" % (path, crn), results) cf.immkr(results, 2) cf.immkr(results, 3) cf.immkr(results, 4) cf.immkr(results, 5) cf.immkr(results, 6) cf.immkr(results, 7)
def grad_comp(): """funtction to look at grad given the same time period""" osp = 4 for osp in range(0, 4): p_pass = cf.QRB(mlt) p_count = np.zeros((mlt+2)) #a count to the total non zero elements pval = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dpvtest_%d.npy" % (path,rn,osp, rn)) # cf.immkr(pval, 4) for col in range(2, mlt+2): # print col for line in range(1, pval.shape[0]): y = pval[line, 0] x = pval[line, 1] if mask[y, x] == 0: p_pass[line, col] = np.NAN pval[line, col] = np.NAN elif pval[line, col] < 0.05: p_pass[line, col] = 1 p_count[col] += 1 else: p_pass[line, col] = 0 # cf.immkr(p_pass, col) # ptiny p_pass[col, 1:] # print np.sum(p_pass[col, 1:]) # p_count[col] = np.sum(p_pass[col, 1:]) # print p_count coli = np.argmax(p_count) cf.immkr(p_pass, coli) i = 0 if i == 1: colm = [coli, 20, 40, 61] rate = [0.0015, 0.0006, 0.0006, 0.0004] g = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dslopearray_%d.npy" % (path,rn,osp, rn)) for n in range(0, 4): col = colm[n] number = rate[n] print col for line in range(1, g.shape[0]): if g[line, col] > number: g[line, col] = number elif g[line, col] < -number: g[line, col] = -number try: immkr(osp, g, col, cm.PiYG, 1) for line in range(1, g.shape[0]): if p_pass[line, col] == 1: pass else: g[line, col] = np.NAN immkr(osp, g, col, cm.PiYG, 4) immkr(osp, p_pass, col, cm.PiYG, 2) immkr(osp, pval, col, cm.nipy_spectral, 3) # plt.show() except NameError: pdb.set_trace() print "something is wrong" sys.exit() plt.show() print "test" sys.exit()
def cordinates(): """ This is a trial function that will be upgraded later. it will look for a single pixel that meets my criteria [1. pfail, 2. single bp, significant relationship on either side] R_pfail needs to be imporvved to clarify what im passing into this script. Need a good way to subst my pixels """ #load the number or breakpoints bp_count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, rn)) bp_index = np.load("%s/CTSR/%d/midputs/workingfiles/bp_index.npy" % (path, rn)) # print bp_count.shape # print bp_index.shape #load in breakpoint infomation bp = 1 g = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_grad.npy" % (path, rn, bp)) r2 = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_r2.npy" % (path, rn, bp)) p = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_p.npy" % (path, rn, bp)) # cf.immkr(p, 2) # print g.shape # sys.exit() criteria_count = 0 #to speed up calculation return 109857 for line in range(1, bp_count.shape[0]): y, x = bp_count[line, :2] if mask[y, x] == 0: bp_count[line, 2] = np.NAN elif pmask[y,x]==1: #to mask out those spots that passed a p0.05 test bp_count[line, 2] = np.NAN elif bp_count[line, 2] != 1: #to mask out those with too many or too few breakpoints bp_count[line, 2] = np.NAN elif p[line, 2]<0.05 and p[line, 3]<0.05: if p[line, 2]-p[line, 3] > 0.04: print p[line, 2:], line, p[line, 2]-p[line, 3] criteria_count += 1 #to count all those that have passed my filters rnx = 5 i = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rnx, rnx)) precip = i[line, 2:] j = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rnx, rnx)) VI = j[line, 2:] fig, ax1 = plt.subplots() ts = np.asarray(range(ys, yf)) p1, = ax1.plot(ts, precip, label='precip') ax1.set_xlabel('Year') ax1.set_ylabel('Rainfall (mm)') ax2 = ax1.twinx() #plot the VI data p2, = ax2.plot(ts, VI, label='NDVI', color='yellowgreen') #set the NDVI label ax2.set_ylabel("NDVImax") lines = [p1, p2]#, p3, p4] plt.legend(lines, [l.get_label() for l in lines], loc='lower right') plt.title('line %d' % line) plt.show() data = np.load("%s/input_data/%s/%d/VI%d_%d.npy" %(path, vegi, nf, ys, yf)) NDVI = data[line, 2:] size = NDVI.shape[0] # ts = np.asarray(range(ys, yf, 1/12)) ts = np.arange(size) plt.plot(ts, NDVI, label='NDVI', color='yellowgreen') plt.show() #quickly display those that have passed else: bp_count[line, 2] = np.NAN print criteria_count cf.immkr(bp_count, 2)
def onebp_resonly(): regR = np.load("%s/CTSR/%d/R2_Chow/Final_RegrResultsv2.npy" % (path, crn)) Chowres = np.load("%s/CTSR/%d/R2_Chow/Final_ChowResults.npy" % (path, crn)) for line in range(1, regR.shape[0]): y, x = Chowres[line, :2] if Chowres[line, 5] > 0.05: regR[line, 2:] = np.NAN elif Chowres[line, 3] < 0.05: regR[line, 2:] = np.NAN else: pass yearimg = np.zeros((yrg, xrg)) for line in range(1, regR.shape[0]): y, x = Chowres[line, :2] if np.isnan(regR[line, 2]) == True: yearimg[y, x] = np.NAN else: if regR[line, 5] > 0.25: # pre bp p values if regR[line, 9] > 0.25: yearimg[y, x] = 0 elif regR[line, 6] > 0: yearimg[y, x] = 1 elif regR[line, 6] < 0: yearimg[y, x] = -1 else: print "Break 1" sys.exit() elif regR[line, 2] > 0: #pre bp grad > 0 if regR[line, 9] > 0.25: yearimg[y, x] = 3 elif regR[line, 6] > 0: yearimg[y, x] = 4 elif regR[line, 6] < 0: yearimg[y, x] = 2 else: print "Break 2" sys.exit() elif regR[line, 2] < 0: #pre bp grad > 0 if regR[line, 9] > 0.25: yearimg[y, x] = -3 elif regR[line, 6] > 0: yearimg[y, x] = -2 elif regR[line, 6] < 0: yearimg[y, x] = -4 else: print "Break 3" sys.exit() else: print "Break 3" sys.exit() # if Chowres[line, 3] < 0.05: # results[y, x] = 2 # else: # results[y, x] = 1 plt.imshow(yearimg) plt.colorbar() plt.show() cf.immkr(regR, 5) cf.immkr(regR, 9) pass