def ks_plots(plots, flavour, species): pvals = [] for plotname in plots: if plotname in ignore: continue plot = plots[plotname] L, U, N, T, x, plotfile = plot['attrs'] sw, aw = abb_init_kernels(L, U, N, flavour, plotname) nsw, naw = plot['nsw'], plot['naw'] for j, t in enumerate(T): if t in sw.years: if j > 0: if species == 'sw': ccdf, dcdf, d, p = ks1(sw.meas['SW'][t], nsw[j], x) z = np.sort(sw.meas['SW'][t]) elif species == 'aw': ccdf, dcdf, d, p = ks1(aw.meas['AW'][t], naw[j], x) z = np.sort(aw.meas['AW'][t]) pvals.append(p) # # histogram of pvals # plt.figure() # plt.hist(pvals, bins=21) # plt.xlabel('p-value') # plt.ylabel('frequency') # plt.savefig('plots/kshist_%s_%s.png' % (flavour, species)) # plt.close() # sorted scatter plot plt.figure() x = range(len(pvals)) plt.plot(x, sorted(pvals), '.k') plt.plot(x, len(x)*[0.05], '-r') plt.xlabel('index') plt.ylabel('p-value') plt.savefig('plots/ksscatter_%s_%s.png' % (flavour, species)) plt.close() pvals = np.asarray(pvals) n = len(pvals) nsig = np.count_nonzero(pvals > 0.05) print flavour, species, n, nsig, n - nsig return pvals
############################################################################### # main logging.info("START: %s", time.asctime()) for flavour in flavours: logging.info("FLAVOUR: %s", flavour) plots = {} for plotfile in glob('kernels/abb/*.csv'): if plotfile == 'kernels/abb/plotsizes.csv': continue plotname = plotfile.split('/')[-1].split('.')[0] sw, aw = abb_init_kernels(L, U, N, flavour, plotname) logging.info('PLOT: %s %f', plotname, sw.plot_size) T = range(sw.first_year, sw.last_year+10) n = np.zeros((len(T)+1, 2, len(sw.x))) for j, t in enumerate(T): logging.debug('year: %d', t) if t in sw.years: from_data = True for k in [ sw, aw ]: k.set_n0(t) else:
def ks_compare(plots1, plots2, flavour, species): pvals1 = [] pvals2 = [] for plotname in plots1: if plotname in ignore: continue plot1 = plots1[plotname] plot2 = plots2[plotname] L, U, N, T, x, plotfile = plot1['attrs'] sw, aw = abb_init_kernels(L, U, N, flavour, plotname) nsw1, naw1 = plot1['nsw'], plot1['naw'] nsw2, naw2 = plot2['nsw'], plot2['naw'] for j, t in enumerate(T): if t in sw.years: if j > 0: if species == 'sw': ccdf, dcdf, d, p = ks1(sw.meas['SW'][t], nsw1[j], x) # pvals1.append(p) pvals1.append(d) ccdf, dcdf, d, p = ks1(sw.meas['SW'][t], nsw2[j], x) # pvals2.append(p) pvals2.append(d) elif species == 'aw': ccdf, dcdf, d, p = ks1(aw.meas['AW'][t], naw1[j], x) # pvals1.append(p) pvals1.append(d) ccdf, dcdf, d, p = ks1(aw.meas['AW'][t], naw2[j], x) # pvals2.append(p) pvals2.append(d) # pmax = 1.0 pmax = 0.6 # p-val vs p-val (actually d-val vs d-val for now...) plt.figure() plt.plot(pvals1, pvals2, 'ok', alpha=0.5) # plt.axhline(y=0.05, color='black', linestyle='--') # plt.axvline(x=0.05, color='black', linestyle='--') plt.plot([0.0, pmax], [0.0, pmax], color='black') # plt.xlabel('p-value (without competition)') # plt.ylabel('p-value (with competition)') plt.xlabel('KS statistic (without competition)') plt.ylabel('KS statistic (with competition)') dvals1 = np.asarray(pvals1) dvals2 = np.asarray(pvals2) print 'species', species print 'npoints', len(dvals1) print 'nbetter', np.count_nonzero(dvals1 > dvals2) plt.savefig('plots/ksscatter_%s_comp.png' % species) plt.close()
def comp_ks_plots(plots1, plots2, flavour): for plotname in plots1: if plotname != '237': continue plot1 = plots1[plotname] plot2 = plots2[plotname] L, U, N, T, x, plotfile = plot1['attrs'] dx = float(U - L) / N nsw1, naw1 = plot1['nsw'], plot1['naw'] nsw2, naw2 = plot2['nsw'], plot2['naw'] sw1, aw1 = abb_init_kernels(L, U, N, flavour, plotname) sw2, aw2 = abb_init_kernels(L, U, N, flavour, plotname) for j, t in enumerate(T): if t in sw1.years and j > 0: if t != 2000: continue fig, ax = plt.subplots(2, 2) meassw = np.asarray(sw1.meas['SW'][t]) measaw = np.asarray(aw1.meas['AW'][t]) ytop = max(nsw1[j].max(), nsw2[j].max(), naw1[j].max(), naw2[j].max()) * 1.2 # spruce weights = 1e4 / sw1.plot_size * (dx/4) * np.ones(meassw.shape) ax[0,0].hist(meassw, bins=N/4, weights=weights, histtype='stepfilled', color='0.75', edgecolor='0.75') ax[0,0].plot(sw1.x, nsw1[j], color='r', linestyle='-', linewidth=1) ax[0,0].plot(sw2.x, nsw2[j], color='b', linestyle=':', linewidth=1) # ax[0,0].legend(loc='best') ax[0,0].set_ylabel('stems per hectare') ax[0,0].set_title('spruce') ax[0,0].set_ylim(0, ytop) plt.setp(ax[0,0].get_xticklabels(), visible=False) # aspen weights = 1e4 / aw1.plot_size * (dx/4) * np.ones(measaw.shape) ax[0,1].hist(measaw, bins=N/4, weights=weights, histtype='stepfilled', color='0.75', edgecolor='0.75') ax[0,1].plot(aw1.x, naw1[j], color='r', linestyle='-', linewidth=1) ax[0,1].plot(aw2.x, naw2[j], color='b', linestyle=':', linewidth=1) ax[0,1].set_ylim(0, ytop) ax[0,1].set_title('aspen') plt.setp(ax[0,1].get_xticklabels(), visible=False) plt.setp(ax[0,1].get_yticklabels(), visible=False) # spruce ks ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw1[j], x) z = np.sort(sw1.meas['SW'][t]) ax[1,0].plot(z, dcdf, color='0.75', linestyle='--', linewidth=2) ax[1,0].plot(z, ccdf, '-r') ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw2[j], x) ax[1,0].plot(z, ccdf, ':b') ax[1,0].set_xlabel('dbh (mm)') ax[1,0].set_ylabel('cdf') ax[1,0].set_xlim(L, U) ax[1,0].set_ylim(0, 1) # aspen ks ccdf, dcdf, d, p = ks1(aw1.meas['AW'][t], naw1[j], x) z = np.sort(aw1.meas['AW'][t]) ax[1,1].plot(z, dcdf, color='0.75', linestyle='--', linewidth=2, label='data') ax[1,1].plot(z, ccdf, '-r', label='w/ competition') ccdf, dcdf, d, p = ks1(aw1.meas['AW'][t], naw2[j], x) ax[1,1].plot(z, ccdf, ':b', label='w/o competition') ax[1,1].set_xlabel('dbh (mm)') ax[1,1].set_ylabel('cdf') ax[1,1].set_xlim(L, U) ax[1,1].set_ylim(0, 1) ax[1,1].legend(loc='lower right', prop={'size': 10}) plt.setp(ax[1,1].get_yticklabels(), visible=False) fig.savefig('plots/projkscomp_%s_%d.png' % (plotname, t)) fig.savefig('plots/projkscomp_%s_%d.pdf' % (plotname, t)) # plt.show() plt.close()
def comp_ks_plots(plots1, plots2, flavour): for plotname in plots1: if plotname != '237': continue plot1 = plots1[plotname] plot2 = plots2[plotname] L, U, N, T, x, plotfile = plot1['attrs'] dx = float(U - L) / N nsw1, naw1 = plot1['nsw'], plot1['naw'] nsw2, naw2 = plot2['nsw'], plot2['naw'] sw1, aw1 = abb_init_kernels(L, U, N, flavour, plotname) sw2, aw2 = abb_init_kernels(L, U, N, flavour, plotname) for j, t in enumerate(T): if t in sw1.years and j > 0: if t != 2000: continue #fig, ax = plt.subplots(ncols=2) plt.figure(1) meassw = np.asarray(sw1.meas['SW'][t]) # measaw = np.asarray(aw1.meas['AW'][t]) # ytop = max(nsw1[j].max(), nsw2[j].max(), naw1[j].max(), naw2[j].max()) * 1.2 # spruce weights = 1e4 / sw1.plot_size * (dx/4) * np.ones(meassw.shape) plt.hist(meassw, bins=N/4, weights=weights, histtype='stepfilled', color='0.5', edgecolor='0.5') plt.plot(sw1.x, nsw1[j], color='r', linestyle='-', linewidth=1) plt.plot(sw2.x, nsw2[j], color='b', linestyle='-', linewidth=1) plt.ylabel('stems per hectare') plt.xlabel('size') plt.xlim(L, 500) plt.savefig('plots/projkscompswonly_%s_%d_sph.png' % (plotname, t)) # spruce ks ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw1[j], x) z = np.sort(sw1.meas['SW'][t]) plt.figure(2) plt.plot(z, dcdf, color='0.5', linestyle='-', linewidth=2) plt.plot(z, ccdf, '-r') ccdf, dcdf, d, p = ks1(sw1.meas['SW'][t], nsw2[j], x) plt.plot(z, ccdf, '-b') plt.xlabel('size') plt.ylabel('cumulative distribution function') plt.legend(['measurements', 'with comp', 'without comp'], loc='lower right') plt.xlim(L, 500) plt.ylim(0, 1) plt.savefig('plots/projkscompswonly_%s_%d_cdf.png' % (plotname, t)) plt.close()
def obsprd_plot(plots, flavour): pops = { 'obs': { 'sw': [], 'aw': [] }, 'prd': { 'sw': [], 'aw': [] }, } for plotname in plots: if plotname in ignore: continue plot = plots[plotname] L, U, N, T, x, plotfile = plot['attrs'] nsw, naw = plot['nsw'], plot['naw'] sw, aw = abb_init_kernels(L, U, N, flavour, plotname) for obs_year in sw.years[1:]: pops['obs']['sw'].append(len(sw.meas['SW'][obs_year]) / sw.plot_size * 1e4) pops['obs']['aw'].append(len(aw.meas['AW'][obs_year]) / sw.plot_size * 1e4) j = T.index(obs_year) pops['prd']['sw'].append(sw.population(nsw[j])) pops['prd']['aw'].append(aw.population(naw[j])) tmp = np.asarray(pops['obs']['sw']) if any(tmp > 15000): print plotname stats = { 'sw': {}, 'aw': {} } for species in [ 'sw', 'aw' ]: from utils.stats import dent_blackie, theil, MSEP from scipy.stats import ttest_ind, mannwhitneyu, pearsonr obs = pops['obs'][species] prd = pops['prd'][species] F, pval, (a, b) = dent_blackie(obs, prd) stats[species]['F'] = (F, pval) stats[species]['theil'] = theil(obs, prd) stats[species]['t'] = ttest_ind(prd, obs) stats[species]['mannwhitneyu'] = mannwhitneyu(prd, obs) stats[species]['pearsonr'] = pearsonr(prd, obs) mc, sc, rc, msep = MSEP(obs, prd) stats[species]['msep'] = msep stats[species]['mc'] = mc stats[species]['sc'] = sc stats[species]['rc'] = rc x = np.asarray(sorted(prd)) yhat = a + b*x yex = x plt.figure() plt.plot(prd, obs, 'ok', label='data', alpha=0.5) plt.plot(x, yex, '-r', label='y=x') plt.plot(x, yhat, '-b', label='best fit') plt.ylabel('observed') plt.xlabel('predicted') plt.legend(loc='best') plt.savefig('plots/obsprd_%s_%s.png' % (flavour, species)) plt.close() # plt.title({'sw': 'Spruce', 'aw': 'Aspen'}[species]) return stats
############################################################################### # main loss = [] init = [] for plotname in plots: plot = plots[plotname] L, U, N, T, x, plotfile = plot['attrs'] dx = float(U - L) / N nsw, naw = plot['nsw'], plot['naw'] sw, aw = abb_init_kernels(L, U, N, 'eviction', plotname) for i, t in enumerate(sw.years[:-1]): j0 = T.index(sw.years[i]) + 1 j1 = T.index(sw.years[i+1]) psw0 = sw.population(nsw[j0]) psw1 = sw.population(nsw[j1]) loss.append(abs(psw0 - psw1) / abs(psw0)) j0 = T.index(sw.years[i]) j1 = j0 + 1
with open("out/abb_sw_mort_model.pkl", "r") as f: plots_model = pickle.load(f) ############################################################################### # histogram of annual mortality rates (model only) mrstats = {"model": {}, "const": {}} mrates = [] for plotname, plot in plots_model.iteritems(): L, U, N, T, x, plotfile = plot["attrs"] sw, aw = abb_init_kernels(L, U, N, "sw_mort_model", plotname) nsw, naw = plot["nsw"], plot["naw"] for j in range(1, nsw.shape[0] - 1): mr = sw.population(nsw[j + 1]) / sw.population(nsw[j]) if mr <= 1.0: mrates.append(mr) mrates = np.asarray(mrates) mrstats["model"]["mean"] = np.mean(mrates) mrstats["model"]["var"] = np.var(mrates) plt.figure() plt.hist(mrates, bins=501) plt.xlim([0.9, 1.0])