def evaluate(self, testdata, controldata, datayears,breaks, corrected=True):
if len(breaks) < 2:
raise AICException(msg="break list is incomplete in AIC.evaluate")
try:
#print "evaluate", breaks, datayears
lo = np.argwhere(datayears==breaks[0])[0][0]
hi = np.argwhere(datayears==breaks[-1])[0][0] +1
statslist = []
for b in range(1,len(breaks) -1):
mid=np.argwhere(datayears==breaks[b])[0][0] # + 1 #
key=(lo, mid)
if not (key in self.__segments):
if mid - lo <= 3:
self.__segments[key]=(10.0e10, mid - lo -1)
elif self.__breaksonly:
mean=np.mean(controldata[lo:mid])
resid=(controldata[lo:mid]-mean)/(mid-lo)#/mean
self.__segments[key]=(np.sum(resid*resid), mid-lo)
else:
stats = regress.analysed_regress(testdata[lo:mid], controldata[lo:mid])
self.__segments[key]=(stats["SSE"], stats["n"])
statslist.append(self.__segments[key])
lo = mid
key=(lo, hi)
if not (key in self.__segments):
if hi - lo <= 3:
self.__segments[key]=(10.0e10, hi - lo-1 )
elif self.__breaksonly:
mean=np.mean(controldata[lo:hi])
resid=(controldata[lo:hi]-mean)/(hi-lo-1)#/mean
self.__segments[key]=(np.sum(resid*resid), hi-lo)
else:
stats = regress.analysed_regress(testdata[lo:hi], controldata[lo:hi])
self.__segments[key]=(stats["SSE"], stats["n"])
statslist.append(self.__segments[key])
return self.AIC(statslist, corrected=corrected)
except Exception as e:
raise AICException(msg="breaklist " +str(breaks)+" gives Exception "+str(e))
def convergentBreaks(testdata, controldata, datayears, model, mode="years", guide="Stability", pr=0.01, screenpr=0.05, trace=False, shallow=False):
if not mode in controlmodes:
raise CBException("keyword 'mode' should be in "+str(controlmodes))
if guide in ["AIC", "Stability"]:
if mode=="flat":
ctrl=np.ones(np.shape(controldata))
return convergentBreaks_Inner(testdata, controldata, datayears, ctrl, model, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow)
elif mode=="years":
return convergentBreaks_Inner(testdata, controldata, datayears, datayears, model, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow)
elif mode=="control":
return convergentBreaks_Inner(testdata, controldata, datayears, controldata, model, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow)
else:
rstats = regress.analysed_regress(testdata, controldata)
yh, r = regress.residuals(testdata, controldata, rstats)
return convergentBreaks_Inner(testdata, controldata, datayears, yh, mode, pr=pr, screenpr=screenpr, trace=trace, shallow=shallow)
def convergentBreaks(testdata, controldata, datayears, model, mode="years", guide="AIC"):
if not mode in controlmodes:
raise RCTestException("keyword 'mode' should be in "+str(controlmodes))
if guide == "AIC":
if mode=="flat":
ctrl=np.ones(np.shape(controldata))
return convergentBreaks_Inner(testdata, controldata, datayears, ctrl, model)
elif mode=="years":
return convergentBreaks_Inner(testdata, controldata, datayears, datayears, model)
elif mode=="control":
return convergentBreaks_Inner(testdata, controldata, datayears, controldata, model)
else:
stats = regress.analysed_regress(testdata, controldata)
yh, r = regress.residuals(testdata, controldata, stats)
return convergentBreaks_Inner(testdata, controldata, datayears, yh, model)
def __init__(self, testdata, controldata, datayears, breaks, corrected=True, breaksonly=False):
object.__init__(self)
if len(breaks) < 2:
raise AICException("break list must include start and end points (len >= 2)")
self.__breaksonly=breaksonly
self.__segments = {}
lo = np.argwhere(datayears==breaks[0])[0][0]
hi = np.argwhere(datayears==breaks[-1])[0][0] +1
if self.__breaksonly:
mean=np.mean(testdata[lo:hi])
resid=(testdata[lo:hi]-mean)
self.__segments[(lo, hi)]=(np.sum(resid*resid), hi-lo)#/mean/(hi-lo-1)
else:
stats = regress.analysed_regress(testdata[lo:hi], controldata[lo:hi])
self.__segments[(lo, hi)]=(stats["SSE"], stats["n"])
self.__result = self.evaluate( testdata, controldata, datayears, breaks, corrected)
def plot(self, x, y, years, title=0.0, save=True):
ys = self.__ys
xs = self.__xs
print x, y, years
#print self.__breakyears.keys()
breaks=[years[0]]
breaks.extend(np.sort(self.__breakyears.keys()))
breaks.extend([years[-1]])
#print breaks
fig, ax = plt.subplots()
if title != 0.0:
ax.set_title(sys.argv[2]+ " "+title)
pl = []
#print "LOWESS",lowess(xs,ys)
#sys.exit()
#print "HERE"
pl.append(ax.plot(years, y))
for i in range(len(breaks)-1):
try:
pt = 0
sYs = np.array(y[breaks[i]-breaks[0]:breaks[i+1]+1-breaks[0]])
sXs = np.array(years[breaks[i]-breaks[0]:breaks[i+1]+1-breaks[0]])
#print breaks[i]-breaks[0], breaks[i+1]+1-breaks[0], sXs, sYs
#print "JIM", len(xs), len(ys)
pt = 1
stats=regress.analysed_regress(sYs,sXs)
pt = 2
yhat, resid=regress.residuals(sYs, sXs, stats)
pt = 3
pl.append(ax.plot(sXs, yhat, '-'))
except Exception as e:
print "Exception at "+str(e),pt
if not save:
plt.show()
else:
plt.savefig(sys.argv[2]+ " "+title+'.png')
return breaks
if byssummarycounts[k] == maxcount:
maxlist.append(k)
elif byssummarycounts[k] > maxcount:
maxlist = [k]
maxcount = byssummarycounts[k]
print k, byssummarycounts[k]
bysf.write("%s %s\n" % (str(k), str(byssummarycounts[k])))
pl = []
for breakk in maxlist:
breaks=[int(b) for b in breakk[1:-1].split(',')]
#print "BREAKS",breaks
for i in range(1,len(breaks)):
try:
pt = 0
#print "RANGE",years[0],breaks[i-1],years[0],breaks[i], range(years[0]-breaks[i-1],years[0]-breaks[i])
sYs = np.array(rescaled[breaks[i-1]-years[0]:breaks[i] - years[0]])
sXs = np.array(range(breaks[i-1],breaks[i]))
# print "SXY",sYs, sXs
#print breaks[i]-breaks[0], breaks[i+1]+1-breaks[0], sXs, sYs
#print "JIM", len(xs), len(ys)
pt = 1
stats=regress.analysed_regress(sYs,sXs)
pt = 2
yhat, resid=regress.residuals(sYs, sXs, stats)
pt = 3
pl.append(plt.plot(sXs, yhat, '-'))
except Exception as e:
print bivariate.ExceptionInfo()
print "Exception at ",str(e),pt
def plotbreaks(x, y, breakpoints, title=None, subtitle="", breakyears=None, offset=0):
fig = plt.figure()
breaklist = []
statslist = []
for i in range(len(breakpoints)):
if breakpoints[i] != 0:
breaklist.append(i+offset+1)
plt.plot(x, y, 'b-')
lo = 0
hi = len(y)
#now perform and plot regressions
for b in breaklist:
print b
stats = regress.analysed_regress(y[lo:b], x[lo:b])
yh, r = regress.residuals(y[lo:b], x[lo:b], stats)
plt.plot(x[lo:b], yh, 'r-')
statslist.append(stats)
lo = b
stats = regress.analysed_regress(y[lo:hi], x[lo:hi])
statslist.append(stats)
yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats)
plt.plot(x[lo:hi], yh, 'r-')
#plot the list of years also (this is probably from the strucchange analysis)
if breakyears != None:
lo = 0
hi = len(y)
breaklist2 = []
statslist2 = []
for yr in breakyears:
#x is years
breaklist2.append(list(x[:]).index(yr)+scoffset)
#now perform and plot regressions
for b in breaklist2:
print title, "list 2", b
stats = regress.analysed_regress(y[lo:b], x[lo:b])
statslist2.append(stats)
yh, r = regress.residuals(y[lo:b], x[lo:b], stats)
plt.plot(x[lo:b], yh, 'g--')
lo = b
stats = regress.analysed_regress(y[lo:hi], x[lo:hi])
statslist2.append(stats)
yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats)
plt.plot(x[lo:hi], yh, 'g--')
#plot roger's breaklist as well
breaklist3=[]
statslist3=[]
for b in rogerslist[title]:
breaklist3.append(list(x[:]).index(b))
lo = 0
hi = len(y)
for b in breaklist3:
print title,"list 3",b
stats = regress.analysed_regress(y[lo:b], x[lo:b])
statslist3.append(stats)
yh, r = regress.residuals(y[lo:b], x[lo:b], stats)
plt.plot(x[lo:b], yh, 'r--')
lo = b
stats = regress.analysed_regress(y[lo:hi], x[lo:hi])
statslist3.append(stats)
yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats)
plt.plot(x[lo:hi], yh, 'r--')
fig.suptitle(RunName+" "+title+"\n"+subtitle)
fig.savefig(RunName+" "+title+'.png')
plt.show()
return statslist, statslist2, statslist3
def plotbreaksSummary(x, y, breaks, title=None, subtitle="", breakyears=None, breakyearsAve=None, offset=0):
fig = plt.figure()
breaklist = []
statslist = []
for yr in breaks:
#x is years
breaklist.append(list(x[:]).index(yr)+scoffset+1)
#now perform and plot regressions
plt.plot(x, y, 'b-')
lo = 0
hi = len(y)
#now perform and plot regressions
if breakyearsAve != None:
lo = 0
hi = len(y)
breaklist4 = []
statslist4 = []
for yr in breakyearsAve:
#x is years
breaklist4.append(list(x[:]).index(yr)+scoffset)
#now perform and plot regressions
for b in breaklist4:
print title, "list 2", b
stats = regress.analysed_regress(y[lo:b], x[lo:b])
statslist4.append(stats)
yh, r = regress.residuals(y[lo:b], x[lo:b], stats)
plt.plot(x[lo:b], yh, 'k--')
lo = b
stats = regress.analysed_regress(y[lo:hi], x[lo:hi])
statslist4.append(stats)
yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats)
plt.plot(x[lo:hi], yh, 'k--')
#plot roger's breaklist as well
if breakyears != []:
breaklist3=[]
statslist3=[]
lo = 0
hi = len(y)
for yr in breakyears:
#x is years
breaklist3.append(list(x[:]).index(yr)+scoffset)
if breaklist3 != []:
for b in breaklist3:
print title,"list 3",b
stats = regress.analysed_regress(y[lo:b], x[lo:b])
statslist3.append(stats)
yh, r = regress.residuals(y[lo:b], x[lo:b], stats)
plt.plot(x[lo:b], yh, 'r--')
lo = b
stats = regress.analysed_regress(y[lo:hi], x[lo:hi])
statslist3.append(stats)
yh, r = regress.residuals(y[lo:hi], x[lo:hi], stats)
plt.plot(x[lo:hi], yh, 'r--')
fig.suptitle(RunName+" "+title+"\n"+subtitle)
fig.savefig(RunName+" "+title+'.png')
plt.show()
return statslist, statslist3
