def summarize(what, scenario, years, npp):
df = get_ipbes_regions()
for year in years:
template = '%s-%%s-%04d.tif' % (scenario, year)
vnames = ('Abundance', 'CompSimAb') if what == 'ab' else ('Richness',
'CompSimSR')
fnames = [utils.outfn('luh2', template % vname) for vname in vnames]
fnames.append(utils.luh2_static('carea'))
if npp:
fnames.append(npp)
fnames.append(utils.outfn('luh2', 'ipbes-subs.tif'))
sources = [rasterio.open(src) for src in fnames]
bounds = find_bounds(sources)
data = [read_bounded(src, bounds) for src in sources]
bii = ma.prod(data[0:-1], 0)
bii.mask = np.any(tuple(d.mask for d in data[0:-1]), 0)
intact = ma.prod([np.full(data[0].shape, 1), data[2:-1]], 0)
intact.mask = bii.mask
by_subs = weighted_mean_by(data[-1], bii)
intact_by_subs = weighted_mean_by(data[-1], intact)
df[year] = by_subs.Mean / intact_by_subs.Mean
print(df)
def project(year):
print("projecting %d" % year)
# Open the mask shape file
shp_file = 'c:/data/from-adriana/tropicalforests.shp'
shapes = fiona.open(shp_file)
# Read Adriana's abundance model (mainland)
mod = modelr.load('../tropical-bii/simplifiedAbundanceModel.rds')
predicts.predictify(mod)
# Import standard PREDICTS rasters
rasters = predicts.rasterset('1km', '', year, 'medium')
rasters['primary_lightintense'] = SimpleExpr(
'primary_lightintense', 'primary_light + primary_intense')
rasters['cropland_lightintense'] = SimpleExpr(
'cropland_lightintense', 'cropland_light + cropland_intense')
rasters['pasture_lightintense'] = SimpleExpr(
'pasture_lightintense', 'pasture_light + pasture_intense')
rasters['rd_dist'] = Raster('rd_dist', '/out/1km/roads-adrid.tif')
rasters['hpd'] = Raster(
'hpd', '/vsizip//data/from-adriana/HPD01to12.zip/yr%d/hdr.adf' % year)
rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=True)
what = 'LogAbund'
rs[what] = mod
stime = time.time()
rs.write(what, utils.outfn('1km', 'adrid-%d.tif' % year))
etime = time.time()
print("executed in %6.2fs" % (etime - stime))
def read_hpd_rasters(years, regions):
if regions:
with rasterio.open(regions) as regions_ds:
# Adjust read area so raster is the full 1440x720 resolution
regions = regions_ds.read(1,
masked=True,
boundless=True,
window=regions_ds.window(*(-180, -90,
180, 90)))
regions = ma.masked_equal(regions, -99)
regions = ma.masked_equal(regions, regions_ds.nodata)
with Dataset(utils.luh2_static()) as static:
carea = static.variables['carea'][:]
#hpop = np.zeros((len(years), len(np.unique(regions.compressed())) + 1))
hpop = np.zeros((len(years), 196))
for idx, year in enumerate(years):
with rasterio.open(utils.outfn('luh2',
'historical-hpd-%d.tif' % year)) as ds:
hpd = ds.read(1,
masked=True,
boundless=True,
window=ds.window(*(-180, -90, 180, 90)))
hp = carea * hpd
hpop[idx, 0] = hp.sum()
cids, hpop[idx, 1:] = sum_by(regions, hp)
fips = list(map(cid_to_fips, cids))
hpd = pd.DataFrame(hpop, index=years, columns=['Global'] + fips)
hpd = hpd.T
hpd['fips'] = hpd.index
hpd = hpd.melt(id_vars='fips',
value_vars=range(1950, 2011, 10),
var_name='year',
value_name='HPD')
hpd.year = hpd.year.astype(int)
return hpd
def get_ipbes_regions():
with fiona.open(utils.outfn('vector', 'ipbes_land_shape',
'ipbes_land.shp')) as shapes:
props = tuple(filter(lambda x: x.get('type') == 'Land',
(s['properties'] for s in shapes)))
return pd.DataFrame({'ID': tuple(int(s.get('OBJECTID')) for s in props),
'Name': tuple(s.get('IPBES_sub') for s in props)})
def read_data():
print('Cleaning up WB area')
area = cleanup_wb_area(
utils.data_file('area',
'API_AG.LND.TOTL.K2_DS2_en_csv_v2_10181480.csv'))
print('Cleaning up language distance matrix')
language = cleanup_language()
print('Cleaning up polity v4 data')
p4v = cleanup_p4v(utils.data_file('policy', 'p4v2017.xls'), False)
print('Cleaning up world inequality database data')
wid = cleanup_wid_data()
print('Summarizing human population data')
hpop = read_hpd_rasters(
tuple(range(1800, 2000, 10)) + tuple(range(2000, 2015, 1)),
utils.outfn('luh2', 'un_codes-full.tif'))
return area, language, p4v, wid, hpop
'abundance':
Raster(
'abundance', 'C:/ds/temporal-bii/' + version +
'/bii-ab-low-bound-%d.tif' % year),
'comp_sim':
Raster(
'comp_sim', 'C:/ds/temporal-bii/' + version +
'/bii-cs-low-bound-%d.tif' % year),
'bii_ab':
SimpleExpr('bii_ab', 'abundance * comp_sim')
})
# write out bii raster
bii_rs.write(
'bii_ab',
utils.outfn('temporal-bii/' + version, 'bii-%d.tif' % year))
# calculate bounded bii
# clip the abundance rasters
inname = 'C:/ds/temporal-bii/' + version + '/bii-ab-' + str(
year) + '.tif'
outname = 'C:/ds/temporal-bii/' + version + '/bii-ab-bound-' + str(
year) + '.tif'
ru.clip(inname, outname, 0, abmax)
# clip the cs rasters
inname = 'C:/ds/temporal-bii/' + version + '/bii-cs-' + str(
year) + '.tif'
outname = 'C:/ds/temporal-bii/' + version + '/bii-cs-bound-' + str(
year) + '.tif'
ru.clip(inname, outname, 0, csmax)
###Added +100 to DistRd to deal with zero values
# set up the rasterset, cropping to mainlands
rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True)
# if you're projecting the whole world, use this code instead
# rs = RasterSet(rasters)
# evaluate the model
# model is square root abundance so square it
# Note that the intercept value has been calculated for the baseline
# land use when all other variables are held at 0
# Therefore I calculate separatedly an intercept where DistRd is set to
# the max value, i.e. logDistRd_RS = 1.
intercept = mod.partial({'ISL_MAINLMAINLAND': ISLMAIN, 'logDistRd_rs': 1.0})
print("intercept: %.5f" % intercept)
if args.mainland:
assert math.isclose(intercept, 0.67184, rel_tol=0.001)
else:
## FIXME: Replace RHS with the R calculated value
assert math.isclose(intercept, 0.7270164, rel_tol=0.001)
rs[mod.output] = mod
rs['output'] = SimpleExpr(
'output', '(pow(%s, 2) / pow(%f, 2))' % (mod.output, intercept))
fname = 'ab-%s.tif' % ('mainland' if args.mainland else 'islands')
path = ('katia', 'clip' if args.clip else 'no-clip', fname)
rs.write('output', utils.outfn(*path))
'1km/mainland-from-igor-edited.tif')
mask_ds = rasterio.open(mask_file)
# Richness compositional similarity model with updated PriMin-Urb coefficient
mod = modelr.load('/home/vagrant/katia/models/updated_compsim_rich.rds')
predicts.predictify(mod)
# Import standard PREDICTS rasters
rasters = predicts.rasterset('1km', 'medium', year=2005)
# create an ISL_MAINL raster
# set it to Mainlands this time round (set Mainlands to 1 and Islands to 0)
rasters['ISL_MAINMAINLAND'] = SimpleExpr('ISL_MAINMAINLAND', '2')
rasters['ISL_MAINISLAND'] = SimpleExpr('ISL_MAINISLAND', '0')
rasters['adjGeogDist'] = SimpleExpr('adjGeogDist', '0')
rasters['cubeRtEnvDist'] = SimpleExpr('cubeRtEnvDist', '0')
# set up the rasterset, cropping to mainlands
rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True)
# if you're projecting the whole world, use this code instead
#rs = RasterSet(rasters)
# evaluate the model
# model is logit transformed with an adjustment, so back-transformation
rs[mod.output] = mod
rs['output'] = SimpleExpr(
'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' %
(mod.output, mod.intercept))
rs.write('output', utils.outfn('katia', 'bii-sr-cs-mainlands.tif'))
def luh2_secd(ssp):
return 'netcdf:' + .outfn('luh2', 'secd-%s.nc' % ssp)
# create an ISL_MAINL raster
# set it to Mainlands this time round (set Mainlands to 1 and Islands to 0)
rasters['ISL_MAINMAINLAND'] = SimpleExpr('ISL_MAINMAINLAND', ISLMAIN)
rasters['ISL_MAINISLAND'] = SimpleExpr('ISL_MAINISLAND', 0)
rasters['adjGeogDist'] = SimpleExpr('adjGeogDist', 0)
rasters['cubeRtEnvDist'] = SimpleExpr('cubeRtEnvDist', 0)
# set up the rasterset, cropping to mainlands
rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True)
# if you're projecting the whole world, use this code instead
#rs = RasterSet(rasters)
# evaluate the model
# model is logit transformed with an adjustment, so back-transformation
rs[mod.output] = mod
intercept = mod.partial({'ISL_MAINMAINLAND': ISLMAIN})
print("intercept: %.5f" % intercept)
if args.mainland:
assert math.isclose(intercept, 1.939964, rel_tol=0.001)
else:
assert math.isclose(intercept, 1.439107176, rel_tol=0.001)
rs['output'] = SimpleExpr(
'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' %
(mod.output, intercept))
fname = 'ab-cs-%s.tif' % ('mainland' if args.mainland else 'islands')
rs.write('output', utils.outfn('katia', fname))
if args.mainland:
suffix = 'mainland'
ab_max = 1.655183
sr_max = 1.636021
else:
suffix = 'islands'
ab_max = 1.443549
sr_max = 1.413479
folder = 'clip' if args.clip else 'no-clip'
# pull in all the rasters for computing bii
bii_rs = RasterSet({'abundance': Raster('abundance',
utils.outfn('katia',
folder,
'ab-%s.tif' % suffix)),
'comp_sim': Raster('comp_sim',
utils.outfn('katia',
'ab-cs-%s.tif' % suffix)),
'clip_ab': SimpleExpr('clip_ab',
'clip(abundance, 0, %f)' % ab_max),
'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim'),
'bii_ab2': SimpleExpr('bii_ab2', 'clip_ab * comp_sim'),
})
# write out bii raster
bii_rs.write('bii_ab' if args.clip else 'bii_ab2',
utils.outfn('katia', folder,
'abundance-based-bii-%s.tif' % suffix))
def summary(what, scenario, years, npp, vsr):
"""Generate a per-year and per-IPBES region summary of a diversity metric.
Diversity metric supported are
- ab: abundance (Abundance)
- sr: species richness (Richness)
- cs-ab: abundance-based compositional similarity (CompSimAb)
- cs-sr: species richness-based compositional similarity (CompSimSR)
- bii-ab: abundance-based BII (BIIAb)
- bii-sr: species richness-based BII (BIISR)
"""
if npp and vsr:
raise RuntimeError('Please specify --npp or --vsr, not both')
df = get_ipbes_regions()
df_global = pd.DataFrame({
'ID': [-1],
'Name': ['Global']
},
columns=('ID', 'Name'))
vname = 'Abundance' if what == 'ab' \
else 'Richness' if what =='sr' \
else 'CompSimAb' if what == 'cs-ab' \
else 'CompSimSR' if what == 'cs-sr' \
else 'BIIAb' if what == 'bii-ab' \
else 'BIISR'
template = '%s-%s-%%04d.tif' % (scenario, vname)
if scenario == 'historical':
# This is horrible kludge!
years = (900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1710,
1720, 1730, 1740, 1750, 1760, 1770, 1780, 1790, 1800, 1810,
1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910,
1920, 1930, 1940, 1950, 1960, 1970, 1980, 1990, 2000, 2001,
2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
2012, 2013, 2014)
for year in years:
fnames = [utils.outfn('luh2', template % year)]
fnames.append(utils.luh2_static('carea'))
if npp:
fnames.append(npp)
if vsr:
fnames.append(vsr)
fnames.append(utils.outfn('luh2', 'ipbes-subs.tif'))
sources = [rasterio.open(src) for src in fnames]
bounds, width, height = find_bounds(sources)
data = [read_bounded(src, bounds, height) for src in sources]
bii = ma.prod(data[0:-1], 0)
bii.mask = np.any(tuple(d.mask for d in data[0:-1]), 0)
intact = ma.prod([np.full(data[0].shape, 1), data[1]], 0)
if (npp or vsr):
intact *= data[-2]
intact.mask = bii.mask
by_subs = weighted_mean_by(data[-1], bii)
intact_by_subs = weighted_mean_by(data[-1], intact)
if 'Cells' not in df.columns:
df['Cells'] = by_subs.Cells
df[year] = by_subs.Mean / intact_by_subs.Mean
if 'Cells' not in df_global.columns:
df_global['Cells'] = (bii.shape[0] * bii.shape[1] -
ma.count_masked(bii))
df_global[year] = ma.average(bii) / ma.average(intact)
if len(years) < 10:
print(df)
weight = "-npp" if npp else "-vsr" if vsr else ""
df.to_csv('%s-%s%s-subreg-%04d-%04d.csv' %
(scenario, vname, weight, years[0], years[-1]))
df_global.to_csv('%s-%s%s-global-%04d-%04d.csv' %
(scenario, vname, weight, years[0], years[-1]))
# set up the rasterset, cropping by tropical forests
# set all_touched = False as there are some boundary issues with very small countries on the edge
# like Macao
rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=False)
# evaluate the model
# model is square root abundance so square it
# note that the intercept value has been calculated for the baseline land use when all other variables are held at 0
rs[mod.output] = mod
rs['output'] = SimpleExpr(
'output', '(pow(%s, 2) / pow(%f, 2))' % (mod.output, ref))
rs.write(
'output',
utils.outfn('temporal-bii/' + version, 'bii-ab-%d.tif' % year))
# if you're on the final version (i.e. the last time you're going round)
# write out the pressure data
if version == 'v3':
rs.write(
'primary',
utils.outfn('temporal-bii/primary', 'primary-%d.tif' % year))
rs.write(
'secondary',
utils.outfn('temporal-bii/secondary',
'secondary-%d.tif' % year))
rs.write(
'pasture',
utils.outfn('temporal-bii/pasture', 'pasture-%d.tif' % year))
rs.write(
import rasterio
from rasterio.plot import show, show_hist
from projections.rasterset import RasterSet, Raster
from projections.simpleexpr import SimpleExpr
import projections.predicts as predicts
import projections.r2py.modelr as modelr
import projections.utils as utils
import projections.raster_utils as ru
CLIP = 'no-clip'
# pull in all the rasters for computing bii
bii_rs = RasterSet({
'abundance':
Raster('abundance', utils.outfn('katia', CLIP, 'bii-ab-mainlands.tif')),
'comp_sim':
Raster('comp_sim', utils.outfn('katia', 'bii-ab-cs-mainlands.tif')),
'clip_ab':
SimpleExpr('clip_ab', 'clip(abundance, 0, 1.655183)'),
'bii_ab':
SimpleExpr('bii_ab', 'abundance * comp_sim'),
'bii_ab2':
SimpleExpr('bii_ab2', 'clip_ab * comp_sim'),
})
# write out bii raster
bii_rs.write('bii_ab' if CLIP == 'clip' else 'bii_ab2',
utils.outfn('katia', CLIP, 'abundance-based-bii-mainlands.tif'))
# do the same for species richness
# get the differences
# diff = cuberoot(s1) - cuberoot(s2)
# we want cuberoot(s1+1) = 0
# so the difference is 0 - cuberoot(s2)
# then center and scale (subtract mean and divide by sd)
mean_rd50_diff = float(
df[df['Value'] == 'CRootDens_50km_diff']['mean'])
mean_rd1_diff = float(df[df['Value'] == 'CRootDens_1km_diff']['mean'])
sd_rd50_diff = float(df[df['Value'] == 'CRootDens_50km_diff']['sd'])
sd_rd1_diff = float(df[df['Value'] == 'CRootDens_1km_diff']['sd'])
rasters['CRootDens_50km_diff_cs'] = SimpleExpr(
'CRootDens_50km_diff_cs',
'((0 - clip_rd50) - %f) / %f' % (mean_rd50_diff, sd_rd50_diff))
rasters['CRootDens_1km_diff_cs'] = SimpleExpr(
'CRootDens_1km_diff_cs',
'((0 - clip_rd1) - %f) / %f' % (mean_rd1_diff, sd_rd1_diff))
# set up the rasterset, cropping by tropical forests
rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=False)
# evaluate the model
# model is logit transformed with an adjustment, so back-transformation
rs[mod.output] = mod
rs['output'] = SimpleExpr(
'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' %
(mod.output, ref))
rs.write(
'output',
utils.outfn('temporal-bii/' + version, 'bii-cs-%d.tif' % year))
import sys
import rasterio
from projections.rasterset import RasterSet
import projections.predicts as predicts
import projections.utils as utils
# Import standard PREDICTS rasters
rasters = predicts.rasterset('1km', 'medium', year = 2005)
for suffix in ('islands', 'mainland'):
# Open the BII raster file
mask_file = 'C:/Users/katis2/Desktop/Final_projections/Clip_variables/abundance-based-bii-%s.tif' % suffix
mask_ds = rasterio.open(mask_file)
# set up the rasterset, cropping to mainlands
rs = RasterSet(rasters, mask=mask_ds, maskval=-9999, crop=True)
# Run through each land-use
for lu in ('cropland', 'pasture', 'primary', 'secondary', 'urban'):
# And every use intensity
for ui in ('minimal', 'light', 'intense'):
name = '%s_%s' % (lu, ui)
print(name)
oname = utils.outfn('katia', '%s-%s.tif' % (name, suffix))
if os.path.isfile(oname) or name in ('secondary_intense',
'urnan_light'):
continue
rs.write(name, oname)