def evaluate(self, testdata, controldata, datayears,breaks, corrected=True): if len(breaks) < 2: raise AICException(msg="break list is incomplete in AIC.evaluate") try: #print "evaluate", breaks, datayears lo = np.argwhere(datayears==breaks[0])[0][0] hi = np.argwhere(datayears==breaks[-1])[0][0] +1 statslist = [] for b in range(1,len(breaks) -1): mid=np.argwhere(datayears==breaks[b])[0][0] # + 1 # key=(lo, mid) if not (key in self.__segments): if mid - lo <= 3: self.__segments[key]=(10.0e10, mid - lo -1) elif self.__breaksonly: mean=np.mean(controldata[lo:mid]) resid=(controldata[lo:mid]-mean)/(mid-lo)#/mean self.__segments[key]=(np.sum(resid*resid), mid-lo) else: stats = regress.analysed_regress(testdata[lo:mid], controldata[lo:mid]) self.__segments[key]=(stats["SSE"], stats["n"]) statslist.append(self.__segments[key]) lo = mid key=(lo, hi) if not (key in self.__segments): if hi - lo <= 3: self.__segments[key]=(10.0e10, hi - lo-1 ) elif self.__breaksonly: mean=np.mean(controldata[lo:hi]) resid=(controldata[lo:hi]-mean)/(hi-lo-1)#/mean self.__segments[key]=(np.sum(resid*resid), hi-lo) else: stats = regress.analysed_regress(testdata[lo:hi], controldata[lo:hi]) self.__segments[key]=(stats["SSE"], stats["n"]) statslist.append(self.__segments[key]) return self.AIC(statslist, corrected=corrected) except Exception as e: raise AICException(msg="breaklist " +str(breaks)+" gives Exception "+str(e))
def convergentBreaks(testdata, controldata, datayears, model, mode="years", guide="Stability", pr=0.01, screenpr=0.05, trace=False, shallow=False): if not mode in controlmodes: raise CBException("keyword 'mode' should be in "+str(controlmodes)) if guide in ["AIC", "Stability"]: if mode=="flat": ctrl=np.ones(np.shape(controldata)) return convergentBreaks_Inner(testdata, controldata, datayears, ctrl, model, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow) elif mode=="years": return convergentBreaks_Inner(testdata, controldata, datayears, datayears, model, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow) elif mode=="control": return convergentBreaks_Inner(testdata, controldata, datayears, controldata, model, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow) else: rstats = regress.analysed_regress(testdata, controldata) yh, r = regress.residuals(testdata, controldata, rstats) return convergentBreaks_Inner(testdata, controldata, datayears, yh, mode, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow)
def convergentBreaks(testdata, controldata, datayears, model, mode="years", guide="AIC"): if not mode in controlmodes: raise RCTestException("keyword 'mode' should be in "+str(controlmodes)) if guide == "AIC": if mode=="flat": ctrl=np.ones(np.shape(controldata)) return convergentBreaks_Inner(testdata, controldata, datayears, ctrl, model) elif mode=="years": return convergentBreaks_Inner(testdata, controldata, datayears, datayears, model) elif mode=="control": return convergentBreaks_Inner(testdata, controldata, datayears, controldata, model) else: stats = regress.analysed_regress(testdata, controldata) yh, r = regress.residuals(testdata, controldata, stats) return convergentBreaks_Inner(testdata, controldata, datayears, yh, model)
def __init__(self, testdata, controldata, datayears, breaks, corrected=True, breaksonly=False): object.__init__(self) if len(breaks) < 2: raise AICException("break list must include start and end points (len >= 2)") self.__breaksonly=breaksonly self.__segments = {} lo = np.argwhere(datayears==breaks[0])[0][0] hi = np.argwhere(datayears==breaks[-1])[0][0] +1 if self.__breaksonly: mean=np.mean(testdata[lo:hi]) resid=(testdata[lo:hi]-mean) self.__segments[(lo, hi)]=(np.sum(resid*resid), hi-lo)#/mean/(hi-lo-1) else: stats = regress.analysed_regress(testdata[lo:hi], controldata[lo:hi]) self.__segments[(lo, hi)]=(stats["SSE"], stats["n"]) self.__result = self.evaluate( testdata, controldata, datayears, breaks, corrected)
def plot(self, x, y, years, title=0.0, save=True): ys = self.__ys xs = self.__xs print x, y, years #print self.__breakyears.keys() breaks=[years[0]] breaks.extend(np.sort(self.__breakyears.keys())) breaks.extend([years[-1]]) #print breaks fig, ax = plt.subplots() if title != 0.0: ax.set_title(sys.argv[2]+ " "+title) pl = [] #print "LOWESS",lowess(xs,ys) #sys.exit() #print "HERE" pl.append(ax.plot(years, y)) for i in range(len(breaks)-1): try: pt = 0 sYs = np.array(y[breaks[i]-breaks[0]:breaks[i+1]+1-breaks[0]]) sXs = np.array(years[breaks[i]-breaks[0]:breaks[i+1]+1-breaks[0]]) #print breaks[i]-breaks[0], breaks[i+1]+1-breaks[0], sXs, sYs #print "JIM", len(xs), len(ys) pt = 1 stats=regress.analysed_regress(sYs,sXs) pt = 2 yhat, resid=regress.residuals(sYs, sXs, stats) pt = 3 pl.append(ax.plot(sXs, yhat, '-')) except Exception as e: print "Exception at "+str(e),pt if not save: plt.show() else: plt.savefig(sys.argv[2]+ " "+title+'.png') return breaks
if byssummarycounts[k] == maxcount: maxlist.append(k) elif byssummarycounts[k] > maxcount: maxlist = [k] maxcount = byssummarycounts[k] print k, byssummarycounts[k] bysf.write("%s %s\n" % (str(k), str(byssummarycounts[k]))) pl = [] for breakk in maxlist: breaks=[int(b) for b in breakk[1:-1].split(',')] #print "BREAKS",breaks for i in range(1,len(breaks)): try: pt = 0 #print "RANGE",years[0],breaks[i-1],years[0],breaks[i], range(years[0]-breaks[i-1],years[0]-breaks[i]) sYs = np.array(rescaled[breaks[i-1]-years[0]:breaks[i] - years[0]]) sXs = np.array(range(breaks[i-1],breaks[i])) # print "SXY",sYs, sXs #print breaks[i]-breaks[0], breaks[i+1]+1-breaks[0], sXs, sYs #print "JIM", len(xs), len(ys) pt = 1 stats=regress.analysed_regress(sYs,sXs) pt = 2 yhat, resid=regress.residuals(sYs, sXs, stats) pt = 3 pl.append(plt.plot(sXs, yhat, '-')) except Exception as e: print bivariate.ExceptionInfo() print "Exception at ",str(e),pt
def plotbreaks(x, y, breakpoints, title=None, subtitle="", breakyears=None, offset=0): fig = plt.figure() breaklist = [] statslist = [] for i in range(len(breakpoints)): if breakpoints[i] != 0: breaklist.append(i+offset+1) plt.plot(x, y, 'b-') lo = 0 hi = len(y) #now perform and plot regressions for b in breaklist: print b stats = regress.analysed_regress(y[lo:b], x[lo:b]) yh, r = regress.residuals(y[lo:b], x[lo:b], stats) plt.plot(x[lo:b], yh, 'r-') statslist.append(stats) lo = b stats = regress.analysed_regress(y[lo:hi], x[lo:hi]) statslist.append(stats) yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats) plt.plot(x[lo:hi], yh, 'r-') #plot the list of years also (this is probably from the strucchange analysis) if breakyears != None: lo = 0 hi = len(y) breaklist2 = [] statslist2 = [] for yr in breakyears: #x is years breaklist2.append(list(x[:]).index(yr)+scoffset) #now perform and plot regressions for b in breaklist2: print title, "list 2", b stats = regress.analysed_regress(y[lo:b], x[lo:b]) statslist2.append(stats) yh, r = regress.residuals(y[lo:b], x[lo:b], stats) plt.plot(x[lo:b], yh, 'g--') lo = b stats = regress.analysed_regress(y[lo:hi], x[lo:hi]) statslist2.append(stats) yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats) plt.plot(x[lo:hi], yh, 'g--') #plot roger's breaklist as well breaklist3=[] statslist3=[] for b in rogerslist[title]: breaklist3.append(list(x[:]).index(b)) lo = 0 hi = len(y) for b in breaklist3: print title,"list 3",b stats = regress.analysed_regress(y[lo:b], x[lo:b]) statslist3.append(stats) yh, r = regress.residuals(y[lo:b], x[lo:b], stats) plt.plot(x[lo:b], yh, 'r--') lo = b stats = regress.analysed_regress(y[lo:hi], x[lo:hi]) statslist3.append(stats) yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats) plt.plot(x[lo:hi], yh, 'r--') fig.suptitle(RunName+" "+title+"\n"+subtitle) fig.savefig(RunName+" "+title+'.png') plt.show() return statslist, statslist2, statslist3
def plotbreaksSummary(x, y, breaks, title=None, subtitle="", breakyears=None, breakyearsAve=None, offset=0): fig = plt.figure() breaklist = [] statslist = [] for yr in breaks: #x is years breaklist.append(list(x[:]).index(yr)+scoffset+1) #now perform and plot regressions plt.plot(x, y, 'b-') lo = 0 hi = len(y) #now perform and plot regressions if breakyearsAve != None: lo = 0 hi = len(y) breaklist4 = [] statslist4 = [] for yr in breakyearsAve: #x is years breaklist4.append(list(x[:]).index(yr)+scoffset) #now perform and plot regressions for b in breaklist4: print title, "list 2", b stats = regress.analysed_regress(y[lo:b], x[lo:b]) statslist4.append(stats) yh, r = regress.residuals(y[lo:b], x[lo:b], stats) plt.plot(x[lo:b], yh, 'k--') lo = b stats = regress.analysed_regress(y[lo:hi], x[lo:hi]) statslist4.append(stats) yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats) plt.plot(x[lo:hi], yh, 'k--') #plot roger's breaklist as well if breakyears != []: breaklist3=[] statslist3=[] lo = 0 hi = len(y) for yr in breakyears: #x is years breaklist3.append(list(x[:]).index(yr)+scoffset) if breaklist3 != []: for b in breaklist3: print title,"list 3",b stats = regress.analysed_regress(y[lo:b], x[lo:b]) statslist3.append(stats) yh, r = regress.residuals(y[lo:b], x[lo:b], stats) plt.plot(x[lo:b], yh, 'r--') lo = b stats = regress.analysed_regress(y[lo:hi], x[lo:hi]) statslist3.append(stats) yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats) plt.plot(x[lo:hi], yh, 'r--') fig.suptitle(RunName+" "+title+"\n"+subtitle) fig.savefig(RunName+" "+title+'.png') plt.show() return statslist, statslist3