def read_hpd_rasters(years, regions):
if regions:
with rasterio.open(regions) as regions_ds:
# Adjust read area so raster is the full 1440x720 resolution
regions = regions_ds.read(1,
masked=True,
boundless=True,
window=regions_ds.window(*(-180, -90,
180, 90)))
regions = ma.masked_equal(regions, -99)
regions = ma.masked_equal(regions, regions_ds.nodata)
with Dataset(utils.luh2_static()) as static:
carea = static.variables['carea'][:]
#hpop = np.zeros((len(years), len(np.unique(regions.compressed())) + 1))
hpop = np.zeros((len(years), 196))
for idx, year in enumerate(years):
with rasterio.open(utils.outfn('luh2',
'historical-hpd-%d.tif' % year)) as ds:
hpd = ds.read(1,
masked=True,
boundless=True,
window=ds.window(*(-180, -90, 180, 90)))
hp = carea * hpd
hpop[idx, 0] = hp.sum()
cids, hpop[idx, 1:] = sum_by(regions, hp)
fips = list(map(cid_to_fips, cids))
hpd = pd.DataFrame(hpop, index=years, columns=['Global'] + fips)
hpd = hpd.T
hpd['fips'] = hpd.index
hpd = hpd.melt(id_vars='fips',
value_vars=range(1950, 2011, 10),
var_name='year',
value_name='HPD')
hpd.year = hpd.year.astype(int)
return hpd
def main(version):
fname = '%s/hyde/hyde-%s.nc' % (utils.outdir(), version)
#fname = 'netcdf:%s/hyde/hyde-%s.nc:popc' % (utils.outdir(), version)
uncodes = '%s/luh2/un_codes-full.tif' % utils.outdir()
oname = '%s/luh2/hyde.nc' % utils.outdir()
variables = [('popd', 'f4', 'ppl/km^2', -9999)]
affine, lats, lons, res, cfudge = get_transform(
uncodes, 'netcdf:' + fname + ':popc')
arr = ma.empty((len(lats), len(lons)), fill_value=-9999)
with rasterio.open(utils.luh2_static('carea')) as carea_ds:
carea = carea_ds.read(1, masked=True)
with rasterio.open('netcdf:' + fname + ':popc') as ds:
years = tuple(
map(lambda idx: int(ds.tags(idx)['NETCDF_DIM_time']), ds.indexes))
with Dataset(oname, 'w') as out:
init_nc(out, affine.to_gdal(), lats, lons, years, variables)
print(ds.name)
print(years)
#with click.progressbar(enumerate(years), length=len(years)) as bar:
#for idx, year in bar:
for idx, year in zip(ds.indexes, years):
#pdb.set_trace()
#time.sleep(100)
print(idx, year)
resample(ds, idx, res, rwarp.Resampling.average, arr)
out.variables['popd'][idx - 1, :, :] = arr * cfudge / carea
def means(scenario, years):
"""Calculate and plot land use over time.
Also compute what fraction of NPP is taken by humans.
"""
if scenario != 'all':
utils.luh2_check_year(min(years), scenario)
utils.luh2_check_year(max(years), scenario)
with Dataset(utils.luh2_static()) as static:
carea = static.variables['carea'][:]
land = 1 - static.variables['icwtr'][:]
total = (carea * land).sum()
if scenario != 'all':
scenarios = (scenario, )
else:
scenarios = ('historical', ) + tuple(filter(lambda s: s != 'historical',
utils.luh2_scenarios()))
print(scenarios)
storage = {}
for scene in scenarios:
compute_data(scene, storage, np.array([total]))
joblib.dump(storage, 'overtime.dat', compress=True)
def summarize(what, scenario, years, npp):
df = get_ipbes_regions()
for year in years:
template = '%s-%%s-%04d.tif' % (scenario, year)
vnames = ('Abundance', 'CompSimAb') if what == 'ab' else ('Richness',
'CompSimSR')
fnames = [utils.outfn('luh2', template % vname) for vname in vnames]
fnames.append(utils.luh2_static('carea'))
if npp:
fnames.append(npp)
fnames.append(utils.outfn('luh2', 'ipbes-subs.tif'))
sources = [rasterio.open(src) for src in fnames]
bounds = find_bounds(sources)
data = [read_bounded(src, bounds) for src in sources]
bii = ma.prod(data[0:-1], 0)
bii.mask = np.any(tuple(d.mask for d in data[0:-1]), 0)
intact = ma.prod([np.full(data[0].shape, 1), data[2:-1]], 0)
intact.mask = bii.mask
by_subs = weighted_mean_by(data[-1], bii)
intact_by_subs = weighted_mean_by(data[-1], intact)
df[year] = by_subs.Mean / intact_by_subs.Mean
print(df)
def project_year(model, model_dir, what, scenario, year):
print("projecting %s for %d using %s" % (what, year, scenario))
models = select_models(model, model_dir)
# Read Sam's abundance model (forested and non-forested)
modf = modelr.load(models[0])
intercept_f = modf.intercept
predicts.predictify(modf)
modn = modelr.load(models[1])
intercept_n = modn.intercept
predicts.predictify(modn)
# Open forested/non-forested mask layer
fstnf = rasterio.open(utils.luh2_static('fstnf'))
# Import standard PREDICTS rasters
rastersf = predicts.rasterset('luh2', scenario, year, 'f')
rsf = RasterSet(rastersf, mask=fstnf, maskval=0.0)
rastersn = predicts.rasterset('luh2', scenario, year, 'n')
rsn = RasterSet(rastersn, mask=fstnf, maskval=1.0)
#rsn = RasterSet(rastersn)
if what == 'bii':
vname = 'bii'
rsf[vname] = SimpleExpr(
vname, 'exp(%s) / exp(%f)' % (modf.output, intercept_f))
rsn[vname] = SimpleExpr(
vname, 'exp(%s) / exp(%f)' % (modn.output, intercept_n))
rsf[modf.output] = modf
rsn[modn.output] = modn
else:
vname = modf.output
assert modf.output == modn.output
rsf[vname] = modf
rsn[vname] = modn
if what not in rsf:
print('%s not in rasterset' % what)
print(', '.join(sorted(rsf.keys())))
sys.exit(1)
stime = time.time()
datan, meta = rsn.eval(what, quiet=False)
dataf, _ = rsf.eval(what, quiet=True)
data_vals = dataf.filled(0) + datan.filled(0)
data = data_vals.view(ma.MaskedArray)
data.mask = np.logical_and(dataf.mask, datan.mask)
#data = datan
etime = time.time()
print("executed in %6.2fs" % (etime - stime))
oname = '%s/luh2/%s-%s-%d.tif' % (utils.outdir(), scenario, what, year)
with rasterio.open(oname, 'w', **meta) as dst:
dst.write(data.filled(meta['nodata']), indexes=1)
if None:
fig = plt.figure(figsize=(8, 6))
ax = plt.gca()
show(data, cmap='viridis', ax=ax)
plt.savefig('luh2-%s-%d.png' % (scenario, year))
return
def compute_data(scene, storage, total, regions=None):
with Dataset(utils.luh2_static()) as static:
carea = static.variables['carea'][:]
with Dataset(utils.luh2_states(scene)) as ds:
base_year = (850 if scene == 'historical' else 2015)
years = ds.variables['time'][:] + base_year
crop = np.zeros((len(years), total.shape[0]))
past = np.zeros((len(years), total.shape[0]))
prim = np.zeros((len(years), total.shape[0]))
secd = np.zeros((len(years), total.shape[0]))
urbn = np.zeros((len(years), total.shape[0]))
for year in years:
idx = int(year) - base_year
click.echo('year: %d' % int(year))
cr = (ds.variables['c3ann'][idx, :, :] +
ds.variables['c4ann'][idx, :, :] +
ds.variables['c3per'][idx, :, :] +
ds.variables['c4per'][idx, :, :] +
ds.variables['c3nfx'][idx, :, :])
pa = (ds.variables['range'][idx, :, :] +
ds.variables['pastr'][idx, :, :])
pr = (ds.variables['primf'][idx, :, :] +
ds.variables['primn'][idx, :, :])
se = (ds.variables['secdf'][idx, :, :] +
ds.variables['secdn'][idx, :, :])
ur = ds.variables['urban'][idx, :, :]
crop[idx, 0] = (carea * cr).sum() / total[0] * 100
past[idx, 0] = (carea * pa).sum() / total[0] * 100
prim[idx, 0] = (carea * pr).sum() / total[0] * 100
secd[idx, 0] = (carea * se).sum() / total[0] * 100
urbn[idx, 0] = (carea * ur).sum() / total[0] * 100
if regions is not None:
crop[idx, 1:] = sum_by(regions, carea * cr) / total[1:] * 100
past[idx, 1:] = sum_by(regions, carea * pa) / total[1:] * 100
prim[idx, 1:] = sum_by(regions, carea * pr) / total[1:] * 100
secd[idx, 1:] = sum_by(regions, carea * se) / total[1:] * 100
urbn[idx, 1:] = sum_by(regions, carea * ur) / total[1:] * 100
dim2 = total.shape[0]
yy = np.repeat(years, dim2).reshape(years.shape[0], dim2)
storage[scene] = np.stack((yy, crop, past, prim, secd, urbn))
def sums(scenario, years, regions):
"""Calculate and plot land use over time, broken up by IPBES region.
"""
df = get_ipbes_regions()
if scenario != 'all':
utils.luh2_check_year(min(years), scenario)
utils.luh2_check_year(max(years), scenario)
with Dataset(utils.luh2_static()) as static:
carea = static.variables['carea'][:]
land = 1 - static.variables['icwtr'][:]
if regions:
with rasterio.open(regions) as regions_ds:
# Adjust read area so raster is the full 1440x720 resolution
regions = regions_ds.read(1, masked=True,
window=((-25, 695), (0, 1440)),
boundless=True)
total = np.zeros(len(df) + 1)
total_land = ma.masked_array(carea * land)
total[0] = total_land.sum()
if regions is not None:
total[1:] = sum_by(regions, total_land)
total = np.where(total == 0.0, 1e-5, total)
print(total)
print(total[0])
print(total[1:].sum())
if scenario != 'all':
scenarios = (scenario, )
else:
scenarios = ('historical', ) + tuple(filter(lambda s: s != 'historical',
utils.luh2_scenarios()))
storage = {}
for scene in scenarios:
compute_data(scene, storage, total, regions)
joblib.dump(storage, 'overtime.dat', compress=True)
def tarea(bounds=None, height=None):
with rasterio.open(utils.luh2_static('icwtr')) as ds:
ice = read_bounded(ds, bounds, height)
return (1 - ice) * carea(bounds, height)
def carea(bounds=None, height=None):
with rasterio.open(utils.luh2_static('carea')) as ds:
read_bounded(ds, bounds, height)
def summary(what, scenario, years, npp, vsr):
"""Generate a per-year and per-IPBES region summary of a diversity metric.
Diversity metric supported are
- ab: abundance (Abundance)
- sr: species richness (Richness)
- cs-ab: abundance-based compositional similarity (CompSimAb)
- cs-sr: species richness-based compositional similarity (CompSimSR)
- bii-ab: abundance-based BII (BIIAb)
- bii-sr: species richness-based BII (BIISR)
"""
if npp and vsr:
raise RuntimeError('Please specify --npp or --vsr, not both')
df = get_ipbes_regions()
df_global = pd.DataFrame({
'ID': [-1],
'Name': ['Global']
},
columns=('ID', 'Name'))
vname = 'Abundance' if what == 'ab' \
else 'Richness' if what =='sr' \
else 'CompSimAb' if what == 'cs-ab' \
else 'CompSimSR' if what == 'cs-sr' \
else 'BIIAb' if what == 'bii-ab' \
else 'BIISR'
template = '%s-%s-%%04d.tif' % (scenario, vname)
if scenario == 'historical':
# This is horrible kludge!
years = (900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1710,
1720, 1730, 1740, 1750, 1760, 1770, 1780, 1790, 1800, 1810,
1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910,
1920, 1930, 1940, 1950, 1960, 1970, 1980, 1990, 2000, 2001,
2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
2012, 2013, 2014)
for year in years:
fnames = [utils.outfn('luh2', template % year)]
fnames.append(utils.luh2_static('carea'))
if npp:
fnames.append(npp)
if vsr:
fnames.append(vsr)
fnames.append(utils.outfn('luh2', 'ipbes-subs.tif'))
sources = [rasterio.open(src) for src in fnames]
bounds, width, height = find_bounds(sources)
data = [read_bounded(src, bounds, height) for src in sources]
bii = ma.prod(data[0:-1], 0)
bii.mask = np.any(tuple(d.mask for d in data[0:-1]), 0)
intact = ma.prod([np.full(data[0].shape, 1), data[1]], 0)
if (npp or vsr):
intact *= data[-2]
intact.mask = bii.mask
by_subs = weighted_mean_by(data[-1], bii)
intact_by_subs = weighted_mean_by(data[-1], intact)
if 'Cells' not in df.columns:
df['Cells'] = by_subs.Cells
df[year] = by_subs.Mean / intact_by_subs.Mean
if 'Cells' not in df_global.columns:
df_global['Cells'] = (bii.shape[0] * bii.shape[1] -
ma.count_masked(bii))
df_global[year] = ma.average(bii) / ma.average(intact)
if len(years) < 10:
print(df)
weight = "-npp" if npp else "-vsr" if vsr else ""
df.to_csv('%s-%s%s-subreg-%04d-%04d.csv' %
(scenario, vname, weight, years[0], years[-1]))
df_global.to_csv('%s-%s%s-global-%04d-%04d.csv' %
(scenario, vname, weight, years[0], years[-1]))
