def summarize(what, scenario, years, npp): df = get_ipbes_regions() for year in years: template = '%s-%%s-%04d.tif' % (scenario, year) vnames = ('Abundance', 'CompSimAb') if what == 'ab' else ('Richness', 'CompSimSR') fnames = [utils.outfn('luh2', template % vname) for vname in vnames] fnames.append(utils.luh2_static('carea')) if npp: fnames.append(npp) fnames.append(utils.outfn('luh2', 'ipbes-subs.tif')) sources = [rasterio.open(src) for src in fnames] bounds = find_bounds(sources) data = [read_bounded(src, bounds) for src in sources] bii = ma.prod(data[0:-1], 0) bii.mask = np.any(tuple(d.mask for d in data[0:-1]), 0) intact = ma.prod([np.full(data[0].shape, 1), data[2:-1]], 0) intact.mask = bii.mask by_subs = weighted_mean_by(data[-1], bii) intact_by_subs = weighted_mean_by(data[-1], intact) df[year] = by_subs.Mean / intact_by_subs.Mean print(df)
def project(year): print("projecting %d" % year) # Open the mask shape file shp_file = 'c:/data/from-adriana/tropicalforests.shp' shapes = fiona.open(shp_file) # Read Adriana's abundance model (mainland) mod = modelr.load('../tropical-bii/simplifiedAbundanceModel.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', '', year, 'medium') rasters['primary_lightintense'] = SimpleExpr( 'primary_lightintense', 'primary_light + primary_intense') rasters['cropland_lightintense'] = SimpleExpr( 'cropland_lightintense', 'cropland_light + cropland_intense') rasters['pasture_lightintense'] = SimpleExpr( 'pasture_lightintense', 'pasture_light + pasture_intense') rasters['rd_dist'] = Raster('rd_dist', '/out/1km/roads-adrid.tif') rasters['hpd'] = Raster( 'hpd', '/vsizip//data/from-adriana/HPD01to12.zip/yr%d/hdr.adf' % year) rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=True) what = 'LogAbund' rs[what] = mod stime = time.time() rs.write(what, utils.outfn('1km', 'adrid-%d.tif' % year)) etime = time.time() print("executed in %6.2fs" % (etime - stime))
def read_hpd_rasters(years, regions): if regions: with rasterio.open(regions) as regions_ds: # Adjust read area so raster is the full 1440x720 resolution regions = regions_ds.read(1, masked=True, boundless=True, window=regions_ds.window(*(-180, -90, 180, 90))) regions = ma.masked_equal(regions, -99) regions = ma.masked_equal(regions, regions_ds.nodata) with Dataset(utils.luh2_static()) as static: carea = static.variables['carea'][:] #hpop = np.zeros((len(years), len(np.unique(regions.compressed())) + 1)) hpop = np.zeros((len(years), 196)) for idx, year in enumerate(years): with rasterio.open(utils.outfn('luh2', 'historical-hpd-%d.tif' % year)) as ds: hpd = ds.read(1, masked=True, boundless=True, window=ds.window(*(-180, -90, 180, 90))) hp = carea * hpd hpop[idx, 0] = hp.sum() cids, hpop[idx, 1:] = sum_by(regions, hp) fips = list(map(cid_to_fips, cids)) hpd = pd.DataFrame(hpop, index=years, columns=['Global'] + fips) hpd = hpd.T hpd['fips'] = hpd.index hpd = hpd.melt(id_vars='fips', value_vars=range(1950, 2011, 10), var_name='year', value_name='HPD') hpd.year = hpd.year.astype(int) return hpd
def get_ipbes_regions(): with fiona.open(utils.outfn('vector', 'ipbes_land_shape', 'ipbes_land.shp')) as shapes: props = tuple(filter(lambda x: x.get('type') == 'Land', (s['properties'] for s in shapes))) return pd.DataFrame({'ID': tuple(int(s.get('OBJECTID')) for s in props), 'Name': tuple(s.get('IPBES_sub') for s in props)})
def read_data(): print('Cleaning up WB area') area = cleanup_wb_area( utils.data_file('area', 'API_AG.LND.TOTL.K2_DS2_en_csv_v2_10181480.csv')) print('Cleaning up language distance matrix') language = cleanup_language() print('Cleaning up polity v4 data') p4v = cleanup_p4v(utils.data_file('policy', 'p4v2017.xls'), False) print('Cleaning up world inequality database data') wid = cleanup_wid_data() print('Summarizing human population data') hpop = read_hpd_rasters( tuple(range(1800, 2000, 10)) + tuple(range(2000, 2015, 1)), utils.outfn('luh2', 'un_codes-full.tif')) return area, language, p4v, wid, hpop
'abundance': Raster( 'abundance', 'C:/ds/temporal-bii/' + version + '/bii-ab-low-bound-%d.tif' % year), 'comp_sim': Raster( 'comp_sim', 'C:/ds/temporal-bii/' + version + '/bii-cs-low-bound-%d.tif' % year), 'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim') }) # write out bii raster bii_rs.write( 'bii_ab', utils.outfn('temporal-bii/' + version, 'bii-%d.tif' % year)) # calculate bounded bii # clip the abundance rasters inname = 'C:/ds/temporal-bii/' + version + '/bii-ab-' + str( year) + '.tif' outname = 'C:/ds/temporal-bii/' + version + '/bii-ab-bound-' + str( year) + '.tif' ru.clip(inname, outname, 0, abmax) # clip the cs rasters inname = 'C:/ds/temporal-bii/' + version + '/bii-cs-' + str( year) + '.tif' outname = 'C:/ds/temporal-bii/' + version + '/bii-cs-bound-' + str( year) + '.tif' ru.clip(inname, outname, 0, csmax)
###Added +100 to DistRd to deal with zero values # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True) # if you're projecting the whole world, use this code instead # rs = RasterSet(rasters) # evaluate the model # model is square root abundance so square it # Note that the intercept value has been calculated for the baseline # land use when all other variables are held at 0 # Therefore I calculate separatedly an intercept where DistRd is set to # the max value, i.e. logDistRd_RS = 1. intercept = mod.partial({'ISL_MAINLMAINLAND': ISLMAIN, 'logDistRd_rs': 1.0}) print("intercept: %.5f" % intercept) if args.mainland: assert math.isclose(intercept, 0.67184, rel_tol=0.001) else: ## FIXME: Replace RHS with the R calculated value assert math.isclose(intercept, 0.7270164, rel_tol=0.001) rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(pow(%s, 2) / pow(%f, 2))' % (mod.output, intercept)) fname = 'ab-%s.tif' % ('mainland' if args.mainland else 'islands') path = ('katia', 'clip' if args.clip else 'no-clip', fname) rs.write('output', utils.outfn(*path))
'1km/mainland-from-igor-edited.tif') mask_ds = rasterio.open(mask_file) # Richness compositional similarity model with updated PriMin-Urb coefficient mod = modelr.load('/home/vagrant/katia/models/updated_compsim_rich.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', 'medium', year=2005) # create an ISL_MAINL raster # set it to Mainlands this time round (set Mainlands to 1 and Islands to 0) rasters['ISL_MAINMAINLAND'] = SimpleExpr('ISL_MAINMAINLAND', '2') rasters['ISL_MAINISLAND'] = SimpleExpr('ISL_MAINISLAND', '0') rasters['adjGeogDist'] = SimpleExpr('adjGeogDist', '0') rasters['cubeRtEnvDist'] = SimpleExpr('cubeRtEnvDist', '0') # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True) # if you're projecting the whole world, use this code instead #rs = RasterSet(rasters) # evaluate the model # model is logit transformed with an adjustment, so back-transformation rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' % (mod.output, mod.intercept)) rs.write('output', utils.outfn('katia', 'bii-sr-cs-mainlands.tif'))
def luh2_secd(ssp): return 'netcdf:' + .outfn('luh2', 'secd-%s.nc' % ssp)
# create an ISL_MAINL raster # set it to Mainlands this time round (set Mainlands to 1 and Islands to 0) rasters['ISL_MAINMAINLAND'] = SimpleExpr('ISL_MAINMAINLAND', ISLMAIN) rasters['ISL_MAINISLAND'] = SimpleExpr('ISL_MAINISLAND', 0) rasters['adjGeogDist'] = SimpleExpr('adjGeogDist', 0) rasters['cubeRtEnvDist'] = SimpleExpr('cubeRtEnvDist', 0) # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True) # if you're projecting the whole world, use this code instead #rs = RasterSet(rasters) # evaluate the model # model is logit transformed with an adjustment, so back-transformation rs[mod.output] = mod intercept = mod.partial({'ISL_MAINMAINLAND': ISLMAIN}) print("intercept: %.5f" % intercept) if args.mainland: assert math.isclose(intercept, 1.939964, rel_tol=0.001) else: assert math.isclose(intercept, 1.439107176, rel_tol=0.001) rs['output'] = SimpleExpr( 'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' % (mod.output, intercept)) fname = 'ab-cs-%s.tif' % ('mainland' if args.mainland else 'islands') rs.write('output', utils.outfn('katia', fname))
if args.mainland: suffix = 'mainland' ab_max = 1.655183 sr_max = 1.636021 else: suffix = 'islands' ab_max = 1.443549 sr_max = 1.413479 folder = 'clip' if args.clip else 'no-clip' # pull in all the rasters for computing bii bii_rs = RasterSet({'abundance': Raster('abundance', utils.outfn('katia', folder, 'ab-%s.tif' % suffix)), 'comp_sim': Raster('comp_sim', utils.outfn('katia', 'ab-cs-%s.tif' % suffix)), 'clip_ab': SimpleExpr('clip_ab', 'clip(abundance, 0, %f)' % ab_max), 'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim'), 'bii_ab2': SimpleExpr('bii_ab2', 'clip_ab * comp_sim'), }) # write out bii raster bii_rs.write('bii_ab' if args.clip else 'bii_ab2', utils.outfn('katia', folder, 'abundance-based-bii-%s.tif' % suffix))
def summary(what, scenario, years, npp, vsr): """Generate a per-year and per-IPBES region summary of a diversity metric. Diversity metric supported are - ab: abundance (Abundance) - sr: species richness (Richness) - cs-ab: abundance-based compositional similarity (CompSimAb) - cs-sr: species richness-based compositional similarity (CompSimSR) - bii-ab: abundance-based BII (BIIAb) - bii-sr: species richness-based BII (BIISR) """ if npp and vsr: raise RuntimeError('Please specify --npp or --vsr, not both') df = get_ipbes_regions() df_global = pd.DataFrame({ 'ID': [-1], 'Name': ['Global'] }, columns=('ID', 'Name')) vname = 'Abundance' if what == 'ab' \ else 'Richness' if what =='sr' \ else 'CompSimAb' if what == 'cs-ab' \ else 'CompSimSR' if what == 'cs-sr' \ else 'BIIAb' if what == 'bii-ab' \ else 'BIISR' template = '%s-%s-%%04d.tif' % (scenario, vname) if scenario == 'historical': # This is horrible kludge! years = (900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1710, 1720, 1730, 1740, 1750, 1760, 1770, 1780, 1790, 1800, 1810, 1820, 1830, 1840, 1850, 1860, 1870, 1880, 1890, 1900, 1910, 1920, 1930, 1940, 1950, 1960, 1970, 1980, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014) for year in years: fnames = [utils.outfn('luh2', template % year)] fnames.append(utils.luh2_static('carea')) if npp: fnames.append(npp) if vsr: fnames.append(vsr) fnames.append(utils.outfn('luh2', 'ipbes-subs.tif')) sources = [rasterio.open(src) for src in fnames] bounds, width, height = find_bounds(sources) data = [read_bounded(src, bounds, height) for src in sources] bii = ma.prod(data[0:-1], 0) bii.mask = np.any(tuple(d.mask for d in data[0:-1]), 0) intact = ma.prod([np.full(data[0].shape, 1), data[1]], 0) if (npp or vsr): intact *= data[-2] intact.mask = bii.mask by_subs = weighted_mean_by(data[-1], bii) intact_by_subs = weighted_mean_by(data[-1], intact) if 'Cells' not in df.columns: df['Cells'] = by_subs.Cells df[year] = by_subs.Mean / intact_by_subs.Mean if 'Cells' not in df_global.columns: df_global['Cells'] = (bii.shape[0] * bii.shape[1] - ma.count_masked(bii)) df_global[year] = ma.average(bii) / ma.average(intact) if len(years) < 10: print(df) weight = "-npp" if npp else "-vsr" if vsr else "" df.to_csv('%s-%s%s-subreg-%04d-%04d.csv' % (scenario, vname, weight, years[0], years[-1])) df_global.to_csv('%s-%s%s-global-%04d-%04d.csv' % (scenario, vname, weight, years[0], years[-1]))
# set up the rasterset, cropping by tropical forests # set all_touched = False as there are some boundary issues with very small countries on the edge # like Macao rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=False) # evaluate the model # model is square root abundance so square it # note that the intercept value has been calculated for the baseline land use when all other variables are held at 0 rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(pow(%s, 2) / pow(%f, 2))' % (mod.output, ref)) rs.write( 'output', utils.outfn('temporal-bii/' + version, 'bii-ab-%d.tif' % year)) # if you're on the final version (i.e. the last time you're going round) # write out the pressure data if version == 'v3': rs.write( 'primary', utils.outfn('temporal-bii/primary', 'primary-%d.tif' % year)) rs.write( 'secondary', utils.outfn('temporal-bii/secondary', 'secondary-%d.tif' % year)) rs.write( 'pasture', utils.outfn('temporal-bii/pasture', 'pasture-%d.tif' % year)) rs.write(
import rasterio from rasterio.plot import show, show_hist from projections.rasterset import RasterSet, Raster from projections.simpleexpr import SimpleExpr import projections.predicts as predicts import projections.r2py.modelr as modelr import projections.utils as utils import projections.raster_utils as ru CLIP = 'no-clip' # pull in all the rasters for computing bii bii_rs = RasterSet({ 'abundance': Raster('abundance', utils.outfn('katia', CLIP, 'bii-ab-mainlands.tif')), 'comp_sim': Raster('comp_sim', utils.outfn('katia', 'bii-ab-cs-mainlands.tif')), 'clip_ab': SimpleExpr('clip_ab', 'clip(abundance, 0, 1.655183)'), 'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim'), 'bii_ab2': SimpleExpr('bii_ab2', 'clip_ab * comp_sim'), }) # write out bii raster bii_rs.write('bii_ab' if CLIP == 'clip' else 'bii_ab2', utils.outfn('katia', CLIP, 'abundance-based-bii-mainlands.tif')) # do the same for species richness
# get the differences # diff = cuberoot(s1) - cuberoot(s2) # we want cuberoot(s1+1) = 0 # so the difference is 0 - cuberoot(s2) # then center and scale (subtract mean and divide by sd) mean_rd50_diff = float( df[df['Value'] == 'CRootDens_50km_diff']['mean']) mean_rd1_diff = float(df[df['Value'] == 'CRootDens_1km_diff']['mean']) sd_rd50_diff = float(df[df['Value'] == 'CRootDens_50km_diff']['sd']) sd_rd1_diff = float(df[df['Value'] == 'CRootDens_1km_diff']['sd']) rasters['CRootDens_50km_diff_cs'] = SimpleExpr( 'CRootDens_50km_diff_cs', '((0 - clip_rd50) - %f) / %f' % (mean_rd50_diff, sd_rd50_diff)) rasters['CRootDens_1km_diff_cs'] = SimpleExpr( 'CRootDens_1km_diff_cs', '((0 - clip_rd1) - %f) / %f' % (mean_rd1_diff, sd_rd1_diff)) # set up the rasterset, cropping by tropical forests rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=False) # evaluate the model # model is logit transformed with an adjustment, so back-transformation rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' % (mod.output, ref)) rs.write( 'output', utils.outfn('temporal-bii/' + version, 'bii-cs-%d.tif' % year))
import sys import rasterio from projections.rasterset import RasterSet import projections.predicts as predicts import projections.utils as utils # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', 'medium', year = 2005) for suffix in ('islands', 'mainland'): # Open the BII raster file mask_file = 'C:/Users/katis2/Desktop/Final_projections/Clip_variables/abundance-based-bii-%s.tif' % suffix mask_ds = rasterio.open(mask_file) # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=-9999, crop=True) # Run through each land-use for lu in ('cropland', 'pasture', 'primary', 'secondary', 'urban'): # And every use intensity for ui in ('minimal', 'light', 'intense'): name = '%s_%s' % (lu, ui) print(name) oname = utils.outfn('katia', '%s-%s.tif' % (name, suffix)) if os.path.isfile(oname) or name in ('secondary_intense', 'urnan_light'): continue rs.write(name, oname)