def ks_plots(plots, flavour, species):
pvals = []
for plotname in plots:
if plotname in ignore:
continue
plot = plots[plotname]
L, U, N, T, x, plotfile = plot['attrs']
sw, aw = abb_init_kernels(L, U, N, flavour, plotname)
nsw, naw = plot['nsw'], plot['naw']
for j, t in enumerate(T):
if t in sw.years:
if j > 0:
if species == 'sw':
ccdf, dcdf, d, p = ks1(sw.meas['SW'][t], nsw[j], x)
z = np.sort(sw.meas['SW'][t])
elif species == 'aw':
ccdf, dcdf, d, p = ks1(aw.meas['AW'][t], naw[j], x)
z = np.sort(aw.meas['AW'][t])
pvals.append(p)
# # histogram of pvals
# plt.figure()
# plt.hist(pvals, bins=21)
# plt.xlabel('p-value')
# plt.ylabel('frequency')
# plt.savefig('plots/kshist_%s_%s.png' % (flavour, species))
# plt.close()
# sorted scatter plot
plt.figure()
x = range(len(pvals))
plt.plot(x, sorted(pvals), '.k')
plt.plot(x, len(x)*[0.05], '-r')
plt.xlabel('index')
plt.ylabel('p-value')
plt.savefig('plots/ksscatter_%s_%s.png' % (flavour, species))
plt.close()
pvals = np.asarray(pvals)
n = len(pvals)
nsig = np.count_nonzero(pvals > 0.05)
print flavour, species, n, nsig, n - nsig
return pvals
###############################################################################
# main
logging.info("START: %s", time.asctime())
for flavour in flavours:
logging.info("FLAVOUR: %s", flavour)
plots = {}
for plotfile in glob('kernels/abb/*.csv'):
if plotfile == 'kernels/abb/plotsizes.csv':
continue
plotname = plotfile.split('/')[-1].split('.')[0]
sw, aw = abb_init_kernels(L, U, N, flavour, plotname)
logging.info('PLOT: %s %f', plotname, sw.plot_size)
T = range(sw.first_year, sw.last_year+10)
n = np.zeros((len(T)+1, 2, len(sw.x)))
for j, t in enumerate(T):
logging.debug('year: %d', t)
if t in sw.years:
from_data = True
for k in [ sw, aw ]:
k.set_n0(t)
else:
def ks_compare(plots1, plots2, flavour, species):
pvals1 = []
pvals2 = []
for plotname in plots1:
if plotname in ignore:
continue
plot1 = plots1[plotname]
plot2 = plots2[plotname]
L, U, N, T, x, plotfile = plot1['attrs']
sw, aw = abb_init_kernels(L, U, N, flavour, plotname)
nsw1, naw1 = plot1['nsw'], plot1['naw']
nsw2, naw2 = plot2['nsw'], plot2['naw']
for j, t in enumerate(T):
if t in sw.years:
if j > 0:
if species == 'sw':
ccdf, dcdf, d, p = ks1(sw.meas['SW'][t], nsw1[j], x)
# pvals1.append(p)
pvals1.append(d)
ccdf, dcdf, d, p = ks1(sw.meas['SW'][t], nsw2[j], x)
# pvals2.append(p)
pvals2.append(d)
elif species == 'aw':
ccdf, dcdf, d, p = ks1(aw.meas['AW'][t], naw1[j], x)
# pvals1.append(p)
pvals1.append(d)
ccdf, dcdf, d, p = ks1(aw.meas['AW'][t], naw2[j], x)
# pvals2.append(p)
pvals2.append(d)
# pmax = 1.0
pmax = 0.6
# p-val vs p-val (actually d-val vs d-val for now...)
plt.figure()
plt.plot(pvals1, pvals2, 'ok', alpha=0.5)
# plt.axhline(y=0.05, color='black', linestyle='--')
# plt.axvline(x=0.05, color='black', linestyle='--')
plt.plot([0.0, pmax], [0.0, pmax], color='black')
# plt.xlabel('p-value (without competition)')
# plt.ylabel('p-value (with competition)')
plt.xlabel('KS statistic (without competition)')
plt.ylabel('KS statistic (with competition)')
dvals1 = np.asarray(pvals1)
dvals2 = np.asarray(pvals2)
print 'species', species
print 'npoints', len(dvals1)
print 'nbetter', np.count_nonzero(dvals1 > dvals2)
plt.savefig('plots/ksscatter_%s_comp.png' % species)
plt.close()
def comp_ks_plots(plots1, plots2, flavour):
for plotname in plots1:
if plotname != '237':
continue
plot1 = plots1[plotname]
plot2 = plots2[plotname]
L, U, N, T, x, plotfile = plot1['attrs']
dx = float(U - L) / N
nsw1, naw1 = plot1['nsw'], plot1['naw']
nsw2, naw2 = plot2['nsw'], plot2['naw']
sw1, aw1 = abb_init_kernels(L, U, N, flavour, plotname)
sw2, aw2 = abb_init_kernels(L, U, N, flavour, plotname)
for j, t in enumerate(T):
if t in sw1.years and j > 0:
if t != 2000:
continue
fig, ax = plt.subplots(2, 2)
meassw = np.asarray(sw1.meas['SW'][t])
measaw = np.asarray(aw1.meas['AW'][t])
ytop = max(nsw1[j].max(), nsw2[j].max(), naw1[j].max(), naw2[j].max()) * 1.2
# spruce
weights = 1e4 / sw1.plot_size * (dx/4) * np.ones(meassw.shape)
ax[0,0].hist(meassw, bins=N/4, weights=weights,
histtype='stepfilled', color='0.75', edgecolor='0.75')
ax[0,0].plot(sw1.x, nsw1[j], color='r', linestyle='-', linewidth=1)
ax[0,0].plot(sw2.x, nsw2[j], color='b', linestyle=':', linewidth=1)
# ax[0,0].legend(loc='best')
ax[0,0].set_ylabel('stems per hectare')
ax[0,0].set_title('spruce')
ax[0,0].set_ylim(0, ytop)
plt.setp(ax[0,0].get_xticklabels(), visible=False)
# aspen
weights = 1e4 / aw1.plot_size * (dx/4) * np.ones(measaw.shape)
ax[0,1].hist(measaw, bins=N/4, weights=weights,
histtype='stepfilled', color='0.75', edgecolor='0.75')
ax[0,1].plot(aw1.x, naw1[j], color='r', linestyle='-', linewidth=1)
ax[0,1].plot(aw2.x, naw2[j], color='b', linestyle=':', linewidth=1)
ax[0,1].set_ylim(0, ytop)
ax[0,1].set_title('aspen')
plt.setp(ax[0,1].get_xticklabels(), visible=False)
plt.setp(ax[0,1].get_yticklabels(), visible=False)
# spruce ks
ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw1[j], x)
z = np.sort(sw1.meas['SW'][t])
ax[1,0].plot(z, dcdf, color='0.75', linestyle='--', linewidth=2)
ax[1,0].plot(z, ccdf, '-r')
ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw2[j], x)
ax[1,0].plot(z, ccdf, ':b')
ax[1,0].set_xlabel('dbh (mm)')
ax[1,0].set_ylabel('cdf')
ax[1,0].set_xlim(L, U)
ax[1,0].set_ylim(0, 1)
# aspen ks
ccdf, dcdf, d, p = ks1(aw1.meas['AW'][t], naw1[j], x)
z = np.sort(aw1.meas['AW'][t])
ax[1,1].plot(z, dcdf, color='0.75', linestyle='--', linewidth=2, label='data')
ax[1,1].plot(z, ccdf, '-r', label='w/ competition')
ccdf, dcdf, d, p = ks1(aw1.meas['AW'][t], naw2[j], x)
ax[1,1].plot(z, ccdf, ':b', label='w/o competition')
ax[1,1].set_xlabel('dbh (mm)')
ax[1,1].set_ylabel('cdf')
ax[1,1].set_xlim(L, U)
ax[1,1].set_ylim(0, 1)
ax[1,1].legend(loc='lower right', prop={'size': 10})
plt.setp(ax[1,1].get_yticklabels(), visible=False)
fig.savefig('plots/projkscomp_%s_%d.png' % (plotname, t))
fig.savefig('plots/projkscomp_%s_%d.pdf' % (plotname, t))
# plt.show()
plt.close()
def comp_ks_plots(plots1, plots2, flavour):
for plotname in plots1:
if plotname != '237':
continue
plot1 = plots1[plotname]
plot2 = plots2[plotname]
L, U, N, T, x, plotfile = plot1['attrs']
dx = float(U - L) / N
nsw1, naw1 = plot1['nsw'], plot1['naw']
nsw2, naw2 = plot2['nsw'], plot2['naw']
sw1, aw1 = abb_init_kernels(L, U, N, flavour, plotname)
sw2, aw2 = abb_init_kernels(L, U, N, flavour, plotname)
for j, t in enumerate(T):
if t in sw1.years and j > 0:
if t != 2000:
continue
#fig, ax = plt.subplots(ncols=2)
plt.figure(1)
meassw = np.asarray(sw1.meas['SW'][t])
# measaw = np.asarray(aw1.meas['AW'][t])
# ytop = max(nsw1[j].max(), nsw2[j].max(), naw1[j].max(), naw2[j].max()) * 1.2
# spruce
weights = 1e4 / sw1.plot_size * (dx/4) * np.ones(meassw.shape)
plt.hist(meassw, bins=N/4, weights=weights,
histtype='stepfilled', color='0.5', edgecolor='0.5')
plt.plot(sw1.x, nsw1[j], color='r', linestyle='-', linewidth=1)
plt.plot(sw2.x, nsw2[j], color='b', linestyle='-', linewidth=1)
plt.ylabel('stems per hectare')
plt.xlabel('size')
plt.xlim(L, 500)
plt.savefig('plots/projkscompswonly_%s_%d_sph.png' % (plotname, t))
# spruce ks
ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw1[j], x)
z = np.sort(sw1.meas['SW'][t])
plt.figure(2)
plt.plot(z, dcdf, color='0.5', linestyle='-', linewidth=2)
plt.plot(z, ccdf, '-r')
ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw2[j], x)
plt.plot(z, ccdf, '-b')
plt.xlabel('size')
plt.ylabel('cumulative distribution function')
plt.legend(['measurements', 'with comp', 'without comp'], loc='lower right')
plt.xlim(L, 500)
plt.ylim(0, 1)
plt.savefig('plots/projkscompswonly_%s_%d_cdf.png' % (plotname, t))
plt.close()
def obsprd_plot(plots, flavour):
pops = {
'obs': { 'sw': [], 'aw': [] },
'prd': { 'sw': [], 'aw': [] },
}
for plotname in plots:
if plotname in ignore:
continue
plot = plots[plotname]
L, U, N, T, x, plotfile = plot['attrs']
nsw, naw = plot['nsw'], plot['naw']
sw, aw = abb_init_kernels(L, U, N, flavour, plotname)
for obs_year in sw.years[1:]:
pops['obs']['sw'].append(len(sw.meas['SW'][obs_year]) / sw.plot_size * 1e4)
pops['obs']['aw'].append(len(aw.meas['AW'][obs_year]) / sw.plot_size * 1e4)
j = T.index(obs_year)
pops['prd']['sw'].append(sw.population(nsw[j]))
pops['prd']['aw'].append(aw.population(naw[j]))
tmp = np.asarray(pops['obs']['sw'])
if any(tmp > 15000):
print plotname
stats = { 'sw': {}, 'aw': {} }
for species in [ 'sw', 'aw' ]:
from utils.stats import dent_blackie, theil, MSEP
from scipy.stats import ttest_ind, mannwhitneyu, pearsonr
obs = pops['obs'][species]
prd = pops['prd'][species]
F, pval, (a, b) = dent_blackie(obs, prd)
stats[species]['F'] = (F, pval)
stats[species]['theil'] = theil(obs, prd)
stats[species]['t'] = ttest_ind(prd, obs)
stats[species]['mannwhitneyu'] = mannwhitneyu(prd, obs)
stats[species]['pearsonr'] = pearsonr(prd, obs)
mc, sc, rc, msep = MSEP(obs, prd)
stats[species]['msep'] = msep
stats[species]['mc'] = mc
stats[species]['sc'] = sc
stats[species]['rc'] = rc
x = np.asarray(sorted(prd))
yhat = a + b*x
yex = x
plt.figure()
plt.plot(prd, obs, 'ok', label='data', alpha=0.5)
plt.plot(x, yex, '-r', label='y=x')
plt.plot(x, yhat, '-b', label='best fit')
plt.ylabel('observed')
plt.xlabel('predicted')
plt.legend(loc='best')
plt.savefig('plots/obsprd_%s_%s.png' % (flavour, species))
plt.close()
# plt.title({'sw': 'Spruce', 'aw': 'Aspen'}[species])
return stats
###############################################################################
# main
loss = []
init = []
for plotname in plots:
plot = plots[plotname]
L, U, N, T, x, plotfile = plot['attrs']
dx = float(U - L) / N
nsw, naw = plot['nsw'], plot['naw']
sw, aw = abb_init_kernels(L, U, N, 'eviction', plotname)
for i, t in enumerate(sw.years[:-1]):
j0 = T.index(sw.years[i]) + 1
j1 = T.index(sw.years[i+1])
psw0 = sw.population(nsw[j0])
psw1 = sw.population(nsw[j1])
loss.append(abs(psw0 - psw1) / abs(psw0))
j0 = T.index(sw.years[i])
j1 = j0 + 1
with open("out/abb_sw_mort_model.pkl", "r") as f:
plots_model = pickle.load(f)
###############################################################################
# histogram of annual mortality rates (model only)
mrstats = {"model": {}, "const": {}}
mrates = []
for plotname, plot in plots_model.iteritems():
L, U, N, T, x, plotfile = plot["attrs"]
sw, aw = abb_init_kernels(L, U, N, "sw_mort_model", plotname)
nsw, naw = plot["nsw"], plot["naw"]
for j in range(1, nsw.shape[0] - 1):
mr = sw.population(nsw[j + 1]) / sw.population(nsw[j])
if mr <= 1.0:
mrates.append(mr)
mrates = np.asarray(mrates)
mrstats["model"]["mean"] = np.mean(mrates)
mrstats["model"]["var"] = np.var(mrates)
plt.figure()
plt.hist(mrates, bins=501)
plt.xlim([0.9, 1.0])
