def precip_accumuation():
RR = np.load("%s/CTSR/%d/R3_methodType/regression_results.npy" % (path, crn))
FS = np.load("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn))
rain_b4 = cf.QRB(yct)
rain_af = cf.QRB(yct)
bkp_v2 = cf.QRB(1)
for line in range(1, RR.shape[0]):
print line
if FS[line, 2] >=3:
osp_b4 = RR[line, 3]
lt_b4 = RR[line, 4]
#load the precip accumulation table
rf_b4 = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" %
(path, rn, osp_b4, lt_b4))
rain_b4[line, :] = rf_b4[line, :]
osp_af = RR[line, 6]
lt_af = RR[line, 7]
rf_af = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" %
(path, rn, osp_af, lt_af))
rain_af[line, :] = rf_af[line, :]
bkp_v2[line, 2] = FS[line, 3]
else:
rain_af[line, 2:] = np.NAN
rain_b4[line, 2:] = np.NAN
bkp_v2[line, 2] = np.NAN
np.save("%s/CTSR/%d/R3_methodType/rain_b4.npy" % (path, crn), rain_b4)
np.save("%s/CTSR/%d/R3_methodType/rain_af.npy" % (path, crn), rain_af)
np.save("%s/CTSR/%d/R3_methodType/VPRbrk_BP.npy" % (path, crn), bkp_v2)
cf.immkr(bkp_v2, 2)
def results():
res = np.load("./CTSR/5/R2_Chow/Final_ChowResults.npy")
count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, crn))
Res_pass = cf.QRB(2)
for line in range(1, res.shape[0]):
y, x = res[line, :2]
if mask[y, x] == 0:
Res_pass[line, 2:] = np.NAN
elif count[line, 2] == 0:
Res_pass[line, 2:] = np.NAN
else:
if res[line, 3] < 0.05:
Res_pass[line, 2] = 1
else:
Res_pass[line, 2] = 0
if res[line, 5] < 0.05:
Res_pass[line, 3] = 1
else:
Res_pass[line, 3] = 0
cf.immkr(Res_pass, 2)
cf.immkr(Res_pass, 3)
# pdb.set_trace()
#Print the % VPR that passed
size = sp.stats.itemfreq(Res_pass[1:, 2])
VPRper = size[1, 1]/(size[1, 1]+size[0, 1])
print VPRper*100, "VPR pass rate"
#Print the %Res that pass
size = sp.stats.itemfreq(Res_pass[1:, 3])
RESper = size[1, 1]/(size[1, 1]+size[0, 1])
print RESper*100, "Residuals pass rate"
pass
def neg_grad_timeseries():
"""To look at the time series of negative gradients"""
i = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rn, rn))
j = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rn, rn))
grad = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rn, rn)) # g, rq, pv
for line in range(1, grad.shape[0]):
print line
y = grad[line, 0]
x = grad[line, 1]
# if abs(grad[line, 2])>0.001:
# grad[line, 2] = np.NAN
if y == 256 and x == 55: # >200 and x<75:
if grad[line, 2] < 0:
if grad[line, 4] < 0.05:
if grad[line, 3] > 0.4:
print y, x
# timeseris
ts = np.arange((yct)) + 1982
precip = i[line, 2:]
vi = j[line, 2:]
fig, ax1 = plt.subplots()
ax1.plot(ts, precip, label="precip")
ax2 = ax1.twinx()
ax2.plot(ts, vi, label="NDVI", color="r")
plt.show()
# np.save("%s/CTSR/%d/R4_paperfigures/%d__%d_rain.npy" % (path, crn, y, x), precip )
# np.save("%s/CTSR/%d/R4_paperfigures/%d__%d_NDVI.npy" % (path, crn, y, x), vi )
cf.immkr(grad, 2)
def mean_values():
"""
builds a singe GRB array that has the mean precip and mean VI values.
THere will also be an offset period controled VI and a lt controlled grad
"""
#load in the data
precip = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rn, rn))
VI = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rn, rn))
#load in the lt values
pmin = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn))
#"""lt, pval, osp"""
cf.immkr(pmin, 2) #lt
#load in the grad
pgrad = np.load("%s/RESTREND/%d/midputs/r2outs/precip_trend%d.npy" % (path, rn, rn))
#blank array to add means
#VI in 2, precip in 3, lt controlled in 4, lt controlled grad in 5
means = cf.QRB(4)
#add values to means
for line in range(1, VI.shape[0]):
y, x = VI[line, 0:2]
if mask[y,x] == 0:
means[line, 2:]= np.NAN
else:
means[line, 2] = np.mean(VI[line, 2:])
means[line, 3] = np.mean(precip[line, 2:])
means[line, 4] = np.divide(means[line, 3], pmin[line, 2])
#add a mask for rainfall over a meter
if means[line, 3] > 1000: #greater than 1000mm of rainfall
means[line, 3] = 1000
#converts both into mm/year then devides
means[line, 5] = np.divide(pgrad[line, 2], means[line, 4]*12) * 100 #to get a %
#add a mask for gradient over 0.018
if means[line, 5]> 2:
means[line, 5] = 2
#titles
t0 = str("Mean VI value")
t1 = str("Mean RF value (>1000 masked)")
t2 = str("Mean RF value LT controlled (mm/month)")
t3 = str("Raw precip g vs LT controlled mean as % (>2 masked)")
titles = [t0, t1, t2, t3]
return means # turn off if i want display
#figures
immkr(means, 2, cm.rainbow_r, run=7, name=titles[0])
immkr(means, 3, cm.jet, run=8, name=titles[1])
immkr(means, 4, cm.rainbow_r, run=9, name=titles[2])
immkr(means, 5, cm.PiYG, run=10, name=titles[3])
def sensor_mask():
#loade file generated from R_oldscripts/R_sensor_bp_test.py.
#This need to be intergrated asap
counts = np.load("%s/CTSR/%d/outputs/Sen_mask_count.npy" % (path, rn))
print counts.shape
cf.immkr(counts, 2)
cf.immkr(counts, 8)
pdb.set_trace()
def Quick(): a = cf.QRB(3) for line in range(1, a.shape[0]): y, x = a[line, :2] if mask[y, x] == 0: a[line, 2] = "masked" else: a[line, 2] = "notmasked" cf.immkr(a, 2) pass
def precip_trend():
"""Looks at the raw trend in the Precip data"""
precip = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rn, rn))
results = cf.QRB(3)
# min_p = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn))
# grad = np.zeros([yrg, xrg]) #zero array to pass the residual gradienst into
# r2val = np.zeros([yrg, xrg]) #array to pass the r2 values into
# pval = np.zeros([yrg, xrg]) #array to pass the r2 values into
for row in range(1, precip.shape[0]):
y = precip[row, 0]
x = precip[row, 1]
results[row, 0] = precip[row, 0]
results[row, 1] = precip[row, 1]
if mask[y, x] == 0:
results[row, 2:] = np.NAN
elif mask[y, x] == 1:
ind = np.arange((yct))
# print ind
#the indipendent (rainfall) variable, exog, add constant to this one
dep = precip[row, 2:]
#the dependent variable, NDVI, endorg,
indC = smx.add_constant(ind)
#adds the intercept
ols = sm.OLS(dep, indC).fit()
# print ols.summary()
#performs the OLS
#the following are constants
g = ols.params[1]# * 10 #for scale
# print ols.params
rq = ols.rsquared
pv = ols.pvalues[1]
# sys.exit()
results[row, 2:] = [g, rq, pv]# print rq, pv
# if pv == min_p[row, 3]:
# pass
# else:
# print pv, min_p[row, 3], row
# sys.exit()
else:
print "it went wrong here"
sys.exit()
#f = cf.immkr(peakNDVI, 2)
a = cf.immkr(results, 2)
b = cf.immkr(results, 3)
c = cf.immkr(results, 4)
# plt.show()
np.save("%s/RESTREND/%d/midputs/r2outs/precip_trend%d.npy" % (path, rn, rn), results)
def non_parametric_res():
resid = np.load("%s/RESTREND/%d/outputs/olsresiduals%d.npy" % (path, rn, rn))
"""the following is to mask out pixels with worring residual values """
for row in range(1, resid.shape[0]):
y = resid[row, 0]
x = resid[row, 1]
if mask[y, x] == 0:
pass
elif mask[y, x] == 1:
test = resid[row, 2:]
ts = test.shape[0]
un = np.unique(test)
if un.shape[0]< yct-2:
mask[y, x] = 0
else:
pass
results = cf.QRB(2)
results_ts = cf.QRB(4)
ind = np.arange((yct))
for row in range(1, resid.shape[0]):
print row
y = resid[row, 0]
x = resid[row, 1]
results[row, 0] = resid[row, 0]
results[row, 1] = resid[row, 1]
results_ts[row, 0] = resid[row, 0]
results_ts[row, 1] = resid[row, 1]
if mask[y, x] == 0:
results[row, 2:] = np.NAN
results_ts[row, 2:] = np.NAN
# elif y ==5 and x == 120:
# results[row, 2:] = np.NAN
# results_ts[row, 2:] = np.NAN
elif mask[y, x] == 1:
dep = resid[row, 2:]
spr= sp.stats.spearmanr(dep, ind)
# print y, x
# print dep, ind
# print y, x, spr
# print sp.stats.mstats.theilslopes(dep, ind)
ts = sp.stats.mstats.theilslopes(dep, ind)
results[row, 2:] = spr
results_ts[row, 2:] = ts
# sys.exit()
# a = cf.immkr(results, 2)
# plt.show()
# plt.show()
np.save("%s/RESTREND/%d/outputs/RES_spearmanr%d.npy" % (path, rn, rn), results)
np.save("%s/RESTREND/%d/outputs/RES_theilslopes%d.npy" % (path, rn, rn), results_ts)
a = cf.immkr(results_ts, 2)
a = cf.immkr(results_ts, 3)
a = cf.immkr(results_ts, 4)
a = cf.immkr(results_ts, 5)
def p_seperation(): #this needs to put the index
"""Takes a list of those point that failed the p test and looks to see if seperating
them by bfast break points can produce stastically significant results. """
"""at first i'm going to see if any pixels have ony a single
breakpoint and use that as a test"""
p_fails = np.load("%s/RESTREND/%d/midputs/workingfiles/ptest_fail.npy" % (path, rn))
#p fails to be removed in the future
count = np.load("%s/CTSR/%d/outputs/Sen_mask_count.npy" % (path, rn))
#this is to create a subset of the data that is easy to look at
breaks = np.load("%s/CTSR/%d/outputs/brkpoint_locSEN.npy" % (path, rn))
peaks = np.load("%s/CTSR/%d/outputs/absmax_VI.npy" % (path, rn))
index = cf.QRB(7) #may need a better way to index this
n = 0 #just included to give me a count
for line in range(1, count.shape[0]):
y = count[line, 0]
x = count[line, 1]
if mask[y, x] == 0:
index[line, 2:] = np.NAN
elif p_fails[line, 2] == 1: #could be removed to look at everywhere
#this may need to be replaced
if count[line, 6] == 1:
points = peaks[line, 2:]
for bp in breaks[line, 2:]:
if np.isnan(bp) is True:
pass
else:
bp_num = bp - 1 #to convert from R to python indexing
for col in range(0, yct):
if bp_num <= points[col]:
index[line, 2] = col
print line
break
else:
pass
# >= is the way i do this, followed by a break
n+= 1
else:
index[line, 2:] = 0
# May need to vary this later depending how i index in the regression
else:
index[line, 2:] = 0
# May need to vary this later depending how i index in the regression
print n, "here"
cf.immkr(p_fails, 2)
cf.immkr(index, 2)
# pdb.set_trace()
np.save("%s/CTSR/%d/outputs/bp_index.npy" % (path, rn), index)
def Year_of_change():
Chowres = np.load("%s/CTSR/%d/R2_Chow/Final_ChowResults.npy" % (path, crn))
bp_loc = np.load("./CTSR/5/R3_methodType/VPRbrk_BP.npy")
print bp_loc.shape
bp_loc[:, 2] += ys
cf.immkr(bp_loc, 2)
bins, counts = np.unique(bp_loc[1:, 2], return_counts=True)
plt.plot(bins[:24,], counts[:24,])
plt.show()
pdb.set_trace()
pass
def RES_problemsolver(newest_rn, older_rn):
"""this is a function that exists to find out why small changes in RESTREND1
deacreased my p values. This may be moved to a problem solver script in the future.
Step 1 is to compare the p valuse for runs 4 and 5. Hopefull will be a general purpose
problem solver script in the future"""
# new_minp = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, newest_rn, newest_rn))
new_minp = np.load("H:/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(newest_rn, newest_rn))
old_minp = np.load("H:/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(older_rn, older_rn))
difference = cf.QRB(1) #old - new. positive values = improvment, negative = reduction (want small p value)
# print new_minp.shape, old_minp.shape
for line in range(1, new_minp.shape[0]):
difference[line, 2] = old_minp[line, 3] - new_minp[line, 3]
if new_minp[line, 3] > 0.05:
if old_minp[line, 3] <0.05:
if old_minp[line, 4] == 3: #answer is the index error correction
pass
else:
sys.exit()
print old_minp[line, :]
print new_minp[line, :]
old_lt = old_minp[line, 2]
old_osp = old_minp[line, 4]
old_precip = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" %
(path, older_rn, old_osp, old_lt))
old_p = old_precip[line, :]
for lt in range(0, mlt):
for osp in range(0, ofp):
pre = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" %
(path, newest_rn, osp, lt))
new_pre = pre[line, :]
if np.array_equal(old_p, new_pre) is False: #array_equal
print lt, osp, "not here"
if old_lt == lt+1:
if old_osp == osp:
print old_p
print new_pre
else:
print lt, osp, "here"
print old_p, "good"
print new_pre
sys.exit()
sys.exit()
else:
pass
else:
pass
cf.immkr(difference, 2)
def bfast_grad():
grad = np.load("%s/CTSR/%d/midputs/r2outs/grad_final.npy" % (path, rn))
# immkr(osp, grad, 2, cm.PiYG, 13)
for line in range(1, grad.shape[0]):
if abs(grad[line, 2])>3:
grad[line, 2] = np.NAN
else:
pass
if grad[line, 2]>1:
grad[line, 2] = 1
elif grad[line, 2]<-1:
grad[line, 2] = -1
cf.immkr(grad, 2)
osp = 0
immkr(osp, grad, 2, cm.PiYG, 13)
def max_r2():
"""Function to find the max r2 value of the bf_ofp and lt
combinations then build an array that has the
r2 value, the osp and column (lt+2)
This is a duplicate of the min_simple and
included to decide if using r2 values or p values
produces better results"""
rval_max = cf.QRB(3) #value, osp, col(lt + 2)
while True:
bf_ofp_rmax = cf.QRB(bf_ofp) # blank array with r2 values
bf_ofp_pos = cf.QRB(bf_ofp) # blank array with position
for osp in range(bf_ofp):
pval = np.load("%s/CTSR/%d/midputs/r2outs/r2val_%d.npy" % (path, rn, osp))
for row in range(1, pval.shape[0]):
y = pval[row, 0]
x = pval[row, 1]
if mask[y, x] == 0:
bf_ofp_rmax[row, 2: ] = np.NAN
bf_ofp_pos[row, 2: ] = np.NAN
else:
val = pval[row, 2:]
min_pos = np.argmax(val) +2
min_val = np.amax(val)
col = osp+2 #plus 2 moves past the y, x colums
bf_ofp_rmax[row, col] = min_val
bf_ofp_pos[row, col] = min_pos
for row in range(1, pval.shape[0]):
y = bf_ofp_rmax[row, 0]
x = bf_ofp_rmax[row, 1]
if mask[y, x] == 0:
rval_max[row, 2 ] = np.NAN
else:
p_bf_ofp = bf_ofp_rmax[row, 2:]
lt_bf_ofp = bf_ofp_pos[row, 2:]
value = np.amax(p_bf_ofp)
osp = np.argmax(p_bf_ofp)
col = lt_bf_ofp[osp]
rval_max[row, 2:] = value, osp, col
break
np.save("%s/CTSR/%d/midputs/r2outs/maxr2.npy" % (path, rn,), rval_max)
cf.immkr(rval_max, 2)
print "maxr2 complete"
def pfail():
"""this function will look at the results from the RESTREND run and see which
pixels failed the p<0.05 test between VI and precipation"""
minp = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn))
p_test = cf.QRB(1) # NAN for ocean, 0 for pass, 1 for fail
for line in range(1, minp.shape[0]):
y = minp[line, 0]
x = minp[line, 1]
if mask[y, x] == 0:
p_test[line, 2] = np.NAN
elif minp[line, 3] > 0.05: #failed the p test
p_test[line, 2] = 1
else:
p_test[line, 2] = 0
cf.immkr(p_test, 2)
np.save("%s/RESTREND/%d/midputs/workingfiles/ptest_fail.npy" % (path, rn), p_test)
def final_sort():
"""
Function to add the aditional infomation from the R_R3_VPRbk.R
"""
#The results from the VPR R analysis
VPR_results = np.load("%s/CTSR/%d/R3_methodType/Final_VPRregression.npy" % (path, crn))
# cf.immkr(VPR_results, 3)
#The Results from my first sort
FS = np.load("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn))
for line in range(1, FS.shape[0]):
if VPR_results[line, 3]<= 0.05:
FS[line, 2] = 3
elif VPR_results[line, 3]>0.05:
FS[line, 2] = 0
cf.immkr(FS, 2)
np.save("%s/CTSR/%d/R3_methodType/Final_Sort.npy" % (path, crn), FS)
def grad_viewer():
# rn = 6
results = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rn, rn))
for line in range(1, results.shape[0]):
if abs(results[line, 2])>3:
results[line, 2] = np.NAN
else:
pass
if results[line, 2] >1:
results[line, 2] = 1
elif results[line, 2] <-1:
results[line, 2] = -1
cf.immkr(results, 2)
osp = 0
immkr(osp, results, 2, cm.PiYG, 13)
sys.exit()
def r2_grad_caller(string):
"""Similar to residual caller but for r2 and grad values.
takes a string, either r2val or grad"""
result = cf.QRB(1)
pvals = np.load("%s/CTSR/%d/midputs/r2outs/minp.npy" % (path, rn,))
for row in range(1, result.shape[0]):
y = pvals[row, 0]
x = pvals[row, 1]
if mask[y, x] == 0:
result[row, 2: ] = np.NAN
else:
print row
osp = pvals[row, 3]
# lt = pvals[row, 4] - 2 #convert from col to lt
var = np.load("%s/CTSR/%d/midputs/r2outs/%s_%d.npy" % (path, rn, string, osp))
result[row, 2] = var[row, pvals[row, 4]]
np.save("%s/CTSR/%d/midputs/r2outs/%s_final.npy" % (path, rn, string), result)
cf.immkr(result, 2)
def restrend():
resid = np.load("%s/RESTREND/%d/outputs/olsresiduals%d.npy" % (path, rn, rn))
results = cf.QRB(4)
for row in range(1, resid.shape[0]):
y = resid[row, 0]
x = resid[row, 1]
results[row, 0] = resid[row, 0]
results[row, 1] = resid[row, 1]
if mask[y, x] == 0:
results[row, 2:] = np.NAN
elif mask[y, x] == 1:
ind = np.arange((yct))
# print ind
#the indipendent (rainfall) variable, exog, add constant to this one
dep = resid[row, 2:]
#the dependent variable, NDVI, endorg,
indC = smx.add_constant(ind)
#adds the intercept
ols = sm.OLS(dep, indC).fit()
#print ols.summary()
#performs the OLS
#the following are constants
g = ols.params[1]
m = ols.params[0]
rq = ols.rsquared
pv = ols.pvalues[1]
results[row, 2:] = [g, rq, pv, m]# print rq, pv
else:
print "it went wrong here"
sys.exit()
# f = cf.immkr(resid, 2)
a = cf.immkr(results, 2)
b = cf.immkr(results, 3)
c = cf.immkr(results, 4)
# plt.show()
np.save("%s/RESTREND/%d/outputs/RESTREND%d.npy" % (path, rn, rn), results)
def residual_caller():
"""Uses the results of the min_simple to build an array of the residuals"""
resid = cf.QRB(yct*12)
pvals = np.load("%s/CTSR/%d/midputs/r2outs/minp.npy" % (path, rn,))
for row in range(1, pvals.shape[0]):
y = pvals[row, 0]
x = pvals[row, 1]
if mask[y, x] == 0:
resid[row, 2: ] = np.NAN
else:
print row
osp = pvals[row, 3]
lt = pvals[row, 4] - 2 #convert from col to lt
point_resid = np.load("%s/CTSR/%d/midputs/maxpos/resid_tabs/residuals_%dlt%d.npy" % (path, rn, osp, lt))
resid[row, :] = point_resid[row, :]
#i need to build a function that checks the script is accessing the correct residuals
np.save("%s/CTSR/%d/midputs/r2outs/residuals.npy" % (path, rn,), resid)
cf.immkr(resid, 3)
def min_simple():
"""Function to find the min p value of the bf_ofp and lt
combinations then build an array that has the
p value, the osp and column (lt+2)"""
pval_min = cf.QRB(3) #value, osp, col(lt + 2)
while True:
bf_ofp_pval = cf.QRB(bf_ofp) # blank array with p values
bf_ofp_pos = cf.QRB(bf_ofp) # blank array with position
for osp in range(bf_ofp):
pval = np.load("%s/CTSR/%d/midputs/r2outs/pval_%d.npy" % (path, rn, osp))
for row in range(1, pval.shape[0]):
y = pval[row, 0]
x = pval[row, 1]
if mask[y, x] == 0:
bf_ofp_pval[row, 2: ] = np.NAN
bf_ofp_pos[row, 2: ] = np.NAN
else:
val = pval[row, 2:]
min_pos = np.argmin(val) +2
min_val = np.amin(val)
col = osp+2 #plus 2 moves past the y, x colums
bf_ofp_pval[row, col] = min_val
bf_ofp_pos[row, col] = min_pos
for row in range(1, pval.shape[0]):
y = bf_ofp_pval[row, 0]
x = bf_ofp_pval[row, 1]
if mask[y, x] == 0:
pval_min[row, 2 ] = np.NAN
else:
p_bf_ofp = bf_ofp_pval[row, 2:]
lt_bf_ofp = bf_ofp_pos[row, 2:]
value = np.amin(p_bf_ofp)
osp = np.argmin(p_bf_ofp)
col = lt_bf_ofp[osp]
pval_min[row, 2:] = value, osp, col
break
np.save("%s/CTSR/%d/midputs/r2outs/minp.npy" % (path, rn,), pval_min)
cf.immkr(pval_min, 2)
print "min_simple complete"
def Year_tester():
#to figure out if rabbits made any impact at all
#load in the data
Sort = np.load("%s/CTSR/%d/R3_methodType/Final_Sort.npy" % (path, crn))
print Sort.shape
Sort[1:, 3] += ys
for line in range(1,Sort.shape[0]):
if Sort[line, 2] == 1:
Sort[line, 3] = np.NAN
else:
pass
cf.immkr(Sort, 2)
cf.immkr(Sort, 3)
bins, counts = np.unique(Sort[1:, 3], return_counts=True)
plt.plot(bins[:24,], counts[:24,])
plt.show()
pass
def grad_compiler():
"""A function to use the min p values to attempt to reconstruct the gradients
"""
pmin = np.load("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn))
grad = cf.QRB(2)
for line in range(1, grad.shape[0]):
print line
y = pmin[line, 0]
x = pmin[line, 1]
if mask[y, x] == 0:
grad[line, 2:] = np.NAN
else:
col = pmin[line, 2]
osp = pmin[line, 4]
g = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dslopearray_%d.npy"
% (path,rn,osp, rn))
grad[line, 2] = g[line, col]
cf.immkr(grad, 2)
pdb.set_trace()
# grad[1:, ]
np.save("%s/RESTREND/%d/midputs/workingfiles/grad_%d.npy" %(path, rn, rn), grad)
pass
def grad_comp():
bpc = bp_counter()
print bpc
# total = [bpc[0], 0, 0] #total number of breapoints with no, partial and complete coverage
# totals.append()
for bp in range(1, 2):
grad = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_grad.npy" % (path, rn, bp))
p_pass = np.load("%s/CTSR/%d/midputs/workingfiles/%d_pass_fail_p.npy" % (path, rn, bp))
count = cf.QRB(bp)
for line in range(1, grad.shape[0]):
if np.isnan(grad[line, 2]) == True:
count[line, 2] = np.NAN
elif p_pass[line, 2] ==bp+1:
# num = []
for col in range(2, grad.shape[1]):
count[line, bp+1]= grad[line, bp+2] - grad[line, bp+1]
# if abs(count[line, bp+1]) < 0.001:
# print count[line, bp+1]
# elif abs(count[line, bp+1]) < 0.005:
# print line
if grad[line, bp+2] > 0:
if grad[line, bp+1] >0:
print line
# print grad[line, bp+2] - grad[line, bp+1]
# nu = np.asarray(num)
# n = np.sum(nu)
# count[line, 2] = n
cf.immkr(count, 2)
points = np.asarray(count[1:, 2][~np.isnan(count[1:, 2])])
test = np.histogram(points, bins=np.arange(-0.05, 0.05, 0.005))
# print test
# print np.unique(points)
# print points
plt.figure()
plt.hist(points, bins=np.arange(-0.05, 0.05, 0.005))
plt.show()
def pixel_subclass():
"""A function to devide the pixels by subclass and see what
my results are"""
#load the number of breakpoints
bp_count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, rnc))
#load the count of the breakpoints
# bp_index = np.load("%s/CTSR/%d/midputs/workingfiles/bp_index.npy" % (path, rn))
# bp_loc = np.load("%s/CTSR/%d/outputs/brkpoint_locSEN.npy" % (path, rnc))
#imported to look at the gradients
VPR = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rnc, rnc))
# print bp_count.shape
# print bp_index.shape
res_results = np.load("%s/RESTREND/%d/outputs/RESTREND%d.npy" % (path, rn, rn))
RES_masked = cf.QRB(1)
# cf.immkr(res_results, 2)
# cf.immkr(res_results, 3)
# cf.immkr(res_results, 4)
for line in range(1, bp_count.shape[0]):
y, x = bp_count[line, 0:2]
if mask[y, x] == 0:
RES_masked[line, 2] = np.NAN
elif pmask[y, x] == 0:
RES_masked[line, 2] = np.NAN
elif bp_count[line, 2] != 0:
RES_masked[line, 2] = np.NAN
elif VPR[line, 2]< 0 :
RES_masked[line, 2] = np.NAN
elif VPR[line, 2] >2:
RES_masked[line, 2] = np.NAN
elif res_results[line, 4] >0.05:
RES_masked[line, 2] = np.NAN
else:
# print VPR[line, 2]
RES_masked[line, 2] = res_results[line, 2]
cf.immkr(RES_masked, 2)
def first_sort():
#to find the pixels that need further work
#import the relevant data
#number of breakpoints
count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, crn))
#Values of a standard RESTREND
VPR = np.load("%s/RESTREND/%d/outputs/olsresults%d.npy" % (path, rn, rn)) #[g, rq, pv, b]
chowres = np.load("%s/CTSR/%d/R2_Chow/Final_ChowResults.npy" % (path, crn))
# cf.immkr(chowres, 3)
#blank array for results
cat = cf.QRB(2) # data catogeriers
for line in range(1, count.shape[0]):
y, x = count[line, :2]
cat[line, 3] = chowres[line, 6]
if mask[y, x] == 0:
cat[line, 2] = np.NAN
elif VPR[line, 2] < 0:
cat[line, 2] = -1 #negative VPR trend
elif VPR[line, 4] > 0.05:
if chowres[line, 3] < 0.05:
######This means pixels need more work######
cat[line, 2] = 4
#########asap########
else:
cat[line, 2] = 0 #no significant breakpoints, unusable pixel
elif chowres[line, 5] < 0.05: #has a significant breakpoint, may be split
if chowres[line, 3] < 0.05:
cat[line, 2] = 3
else:
cat[line, 2] = 2
else:
cat[line, 2] = 1
cf.immkr(cat, 2)
cf.immkr(cat, 3)
np.save("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn), cat) # )
def min_p():
minp = cf.QRB(2*ofp)
for osp in range(0, ofp):
print osp
OLS_table = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dltarray_%d.npy"
% (path, rn, osp, rn))
# OLS_table = np.load()
for row in range(1, OLS_table.shape[0]):
# cf.immkr(OLS_table, 3)
y = OLS_table[row, 0]
x = OLS_table[row, 1]
minp[row, 0] = y
minp[row, 1] = x
if mask[y, x] == 0:
minp[row, 2: ] = np.NAN
else:
pv = OLS_table[row, 3:OLS_table.shape[1]:2]
# print pv
# sys.exit()
pvpos = np.argmin(pv) + 1
minp[row, 2+osp] = pvpos
pvval = np.amin(pv)
# print pvval
minp[row, 2+ofp+osp] = pvval
mr2 = cf.QRB(3)
for row in range(1, minp.shape[0]):
mr2[row, 0] = minp[row, 0]
mr2[row, 1] = minp[row, 1]
if mask[minp[row, 0], minp[row, 1]] == 0:
mr2[row, 2: ] = np.NAN
else:
p_p = minp[row, 2+ofp:]
p_v = np.amin(p_p)
offset = np.argmin(p_p)
mr2lt = minp[row, 2+offset] #col i think
mr2[row, 2] = mr2lt
mr2[row, 3] = p_v
mr2[row, 4] = offset
np.save("%s/RESTREND/%d/midputs/workingfiles/tester_%d.npy" %(path, rn, rn), minp)
cf.immkr(mr2, 3)
cf.immkr(mr2, 2)
cf.immkr(mr2, 4)
# sys.exit()
# plt.show()
np.save("%s/RESTREND/%d/midputs/workingfiles/minp_%d.npy" %(path, rn, rn), mr2)
print mr2.shape, "min_p"
return mr2
def bp_regression():
#to recalculate the VPR on either side of the breakpoint
FS = np.load("%s/CTSR/%d/R3_methodType/firstsort.npy" % (path, crn))
pv_b4 = cf.QRB(bf_ofp)
pv_af = cf.QRB(bf_ofp)
lt_b4 = cf.QRB(bf_ofp)
lt_af = cf.QRB(bf_ofp)
results = cf.QRB(6)# [pval, ops, lt,]before then after bp
for osp in range(0, bf_ofp):
j = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rn, rn))
pval_b4 = cf.QRB(mlt)
pval_af = cf.QRB(mlt)
for lt in range(mlt):
print osp, lt
i = np.load("%s/RESTREND/%d/midputs/maxpos/acu_tabs/precip_%dlt%d.npy" %
(path, rn, osp, lt))
for line in range(1, i.shape[0]):
# print lt, line
y = i[line, 0]
x = i[line, 1]
# print lt, line,# y, x
if mask[y,x] == 0:
pval_af[line, 2:] = np.NAN
pval_b4[line, 2:] = np.NAN
elif FS[line, 2] <3:
pval_af[line, 2:] = np.NAN
pval_b4[line, 2:] = np.NAN
else:
bkp = FS[line, 3]+2
jb4 = j[line, 2:bkp]
jaf = j[line, bkp:]
ib4 = i[line, 2:bkp]
iaf = i[line, bkp:]
# if np.max
indC_b4 = smx.add_constant(ib4)
indC_af = smx.add_constant(iaf)
ols_b4 = sm.OLS(jb4, indC_b4).fit()
ols_af = sm.OLS(jaf, indC_af).fit()
# print ols_b4.summary()
# print ols.params[1]
# sys.exit()
# g = ols.params[1] #* 1000
# rq = ols.rsquared
try:
position = lt+2
pval_b4[line, position] = ols_b4.pvalues[1]
pval_af[line, position] = ols_af.pvalues[1]
except IndexError:
print "its indexing incorrectly"
pdb.set_trace()
for line in range(1, pval_af.shape[0]):
if FS[line, 2] >=3:
osp_pos = osp+2
vals_b4 = pval_b4[line, 2:]
vals_af = pval_af[line, 2:]
pv_b4[line, osp_pos] = np.amin(vals_b4)
pv_af[line, osp_pos] = np.amin(vals_af)
lt_b4[line, osp_pos] = np.argmin(vals_b4)
lt_af[line, osp_pos] = np.argmin(vals_af)
else:
pv_b4[line, 2:] = np.NAN
pv_af[line, 2:] = np.NAN
lt_b4[line, 2:] = np.NAN
lt_af[line, 2:] = np.NAN
# pdb.set_trace()
# results = cf.QRB(6)# [pval, ops, lt,]before then after bp
for line in range(1, pv_b4.shape[0]):
if FS[line, 2] >=3:
results[line, 2] = np.amin(pv_b4[line, 2:])
results[line, 3] = np.argmin(pv_b4[line, 2:])
results[line, 4] = lt_b4[line, 2+results[line, 3]]
results[line, 5] = np.amin(pv_af[line, 2:])
results[line, 6] = np.argmin(pv_af[line, 2:])
results[line, 7] = lt_af[line, 2+results[line, 6]]
if results[line, 2] == results[line, 5]:
print "some hanky panky is appearing", line
else:
pass
else:
results[line, 2:] = np.NAN
np.save("%s/CTSR/%d/R3_methodType/regression_results.npy" % (path, crn), results)
cf.immkr(results, 2)
cf.immkr(results, 3)
cf.immkr(results, 4)
cf.immkr(results, 5)
cf.immkr(results, 6)
cf.immkr(results, 7)
def grad_comp():
"""funtction to look at grad given the same time period"""
osp = 4
for osp in range(0, 4):
p_pass = cf.QRB(mlt)
p_count = np.zeros((mlt+2)) #a count to the total non zero elements
pval = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dpvtest_%d.npy" % (path,rn,osp, rn))
# cf.immkr(pval, 4)
for col in range(2, mlt+2):
# print col
for line in range(1, pval.shape[0]):
y = pval[line, 0]
x = pval[line, 1]
if mask[y, x] == 0:
p_pass[line, col] = np.NAN
pval[line, col] = np.NAN
elif pval[line, col] < 0.05:
p_pass[line, col] = 1
p_count[col] += 1
else:
p_pass[line, col] = 0
# cf.immkr(p_pass, col)
# ptiny p_pass[col, 1:]
# print np.sum(p_pass[col, 1:])
# p_count[col] = np.sum(p_pass[col, 1:])
# print p_count
coli = np.argmax(p_count)
cf.immkr(p_pass, coli)
i = 0
if i == 1:
colm = [coli, 20, 40, 61]
rate = [0.0015, 0.0006, 0.0006, 0.0004]
g = np.load("%s/RESTREND/%d/midputs/workingfiles/OLS_%dslopearray_%d.npy"
% (path,rn,osp, rn))
for n in range(0, 4):
col = colm[n]
number = rate[n]
print col
for line in range(1, g.shape[0]):
if g[line, col] > number:
g[line, col] = number
elif g[line, col] < -number:
g[line, col] = -number
try:
immkr(osp, g, col, cm.PiYG, 1)
for line in range(1, g.shape[0]):
if p_pass[line, col] == 1:
pass
else:
g[line, col] = np.NAN
immkr(osp, g, col, cm.PiYG, 4)
immkr(osp, p_pass, col, cm.PiYG, 2)
immkr(osp, pval, col, cm.nipy_spectral, 3)
# plt.show()
except NameError:
pdb.set_trace()
print "something is wrong"
sys.exit()
plt.show()
print "test"
sys.exit()
def cordinates():
"""
This is a trial function that will be upgraded later. it will look for a single pixel that
meets my criteria [1. pfail, 2. single bp, significant relationship on either side]
R_pfail needs to be imporvved to clarify what im passing into this script. Need a good way to
subst my pixels
"""
#load the number or breakpoints
bp_count = np.load("%s/CTSR/%d/midputs/workingfiles/bp_count.npy" % (path, rn))
bp_index = np.load("%s/CTSR/%d/midputs/workingfiles/bp_index.npy" % (path, rn))
# print bp_count.shape
# print bp_index.shape
#load in breakpoint infomation
bp = 1
g = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_grad.npy" % (path, rn, bp))
r2 = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_r2.npy" % (path, rn, bp))
p = np.load("%s/CTSR/%d/midputs/workingfiles/%d_sort_p.npy" % (path, rn, bp))
# cf.immkr(p, 2)
# print g.shape
# sys.exit()
criteria_count = 0
#to speed up calculation
return 109857
for line in range(1, bp_count.shape[0]):
y, x = bp_count[line, :2]
if mask[y, x] == 0:
bp_count[line, 2] = np.NAN
elif pmask[y,x]==1: #to mask out those spots that passed a p0.05 test
bp_count[line, 2] = np.NAN
elif bp_count[line, 2] != 1: #to mask out those with too many or too few breakpoints
bp_count[line, 2] = np.NAN
elif p[line, 2]<0.05 and p[line, 3]<0.05:
if p[line, 2]-p[line, 3] > 0.04:
print p[line, 2:], line, p[line, 2]-p[line, 3]
criteria_count += 1 #to count all those that have passed my filters
rnx = 5
i = np.load("%s/RESTREND/%d/midputs/maxpos/max_precip_%d.npy" % (path, rnx, rnx))
precip = i[line, 2:]
j = np.load("%s/RESTREND/%d/midputs/maxpos/peakval%d.npy" % (path, rnx, rnx))
VI = j[line, 2:]
fig, ax1 = plt.subplots()
ts = np.asarray(range(ys, yf))
p1, = ax1.plot(ts, precip, label='precip')
ax1.set_xlabel('Year')
ax1.set_ylabel('Rainfall (mm)')
ax2 = ax1.twinx()
#plot the VI data
p2, = ax2.plot(ts, VI, label='NDVI', color='yellowgreen')
#set the NDVI label
ax2.set_ylabel("NDVImax")
lines = [p1, p2]#, p3, p4]
plt.legend(lines, [l.get_label() for l in lines], loc='lower right')
plt.title('line %d' % line)
plt.show()
data = np.load("%s/input_data/%s/%d/VI%d_%d.npy" %(path, vegi, nf, ys, yf))
NDVI = data[line, 2:]
size = NDVI.shape[0]
# ts = np.asarray(range(ys, yf, 1/12))
ts = np.arange(size)
plt.plot(ts, NDVI, label='NDVI', color='yellowgreen')
plt.show()
#quickly display those that have passed
else:
bp_count[line, 2] = np.NAN
print criteria_count
cf.immkr(bp_count, 2)
def onebp_resonly():
regR = np.load("%s/CTSR/%d/R2_Chow/Final_RegrResultsv2.npy" % (path, crn))
Chowres = np.load("%s/CTSR/%d/R2_Chow/Final_ChowResults.npy" % (path, crn))
for line in range(1, regR.shape[0]):
y, x = Chowres[line, :2]
if Chowres[line, 5] > 0.05:
regR[line, 2:] = np.NAN
elif Chowres[line, 3] < 0.05:
regR[line, 2:] = np.NAN
else:
pass
yearimg = np.zeros((yrg, xrg))
for line in range(1, regR.shape[0]):
y, x = Chowres[line, :2]
if np.isnan(regR[line, 2]) == True:
yearimg[y, x] = np.NAN
else:
if regR[line, 5] > 0.25: # pre bp p values
if regR[line, 9] > 0.25:
yearimg[y, x] = 0
elif regR[line, 6] > 0:
yearimg[y, x] = 1
elif regR[line, 6] < 0:
yearimg[y, x] = -1
else:
print "Break 1"
sys.exit()
elif regR[line, 2] > 0: #pre bp grad > 0
if regR[line, 9] > 0.25:
yearimg[y, x] = 3
elif regR[line, 6] > 0:
yearimg[y, x] = 4
elif regR[line, 6] < 0:
yearimg[y, x] = 2
else:
print "Break 2"
sys.exit()
elif regR[line, 2] < 0: #pre bp grad > 0
if regR[line, 9] > 0.25:
yearimg[y, x] = -3
elif regR[line, 6] > 0:
yearimg[y, x] = -2
elif regR[line, 6] < 0:
yearimg[y, x] = -4
else:
print "Break 3"
sys.exit()
else:
print "Break 3"
sys.exit()
# if Chowres[line, 3] < 0.05:
# results[y, x] = 2
# else:
# results[y, x] = 1
plt.imshow(yearimg)
plt.colorbar()
plt.show()
cf.immkr(regR, 5)
cf.immkr(regR, 9)
pass