def project(year): print("projecting %d" % year) # Open the mask shape file shp_file = 'c:/data/from-adriana/tropicalforests.shp' shapes = fiona.open(shp_file) # Read Adriana's abundance model (mainland) mod = modelr.load('../tropical-bii/simplifiedAbundanceModel.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', '', year, 'medium') rasters['primary_lightintense'] = SimpleExpr( 'primary_lightintense', 'primary_light + primary_intense') rasters['cropland_lightintense'] = SimpleExpr( 'cropland_lightintense', 'cropland_light + cropland_intense') rasters['pasture_lightintense'] = SimpleExpr( 'pasture_lightintense', 'pasture_light + pasture_intense') rasters['rd_dist'] = Raster('rd_dist', '/out/1km/roads-adrid.tif') rasters['hpd'] = Raster( 'hpd', '/vsizip//data/from-adriana/HPD01to12.zip/yr%d/hdr.adf' % year) rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=True) what = 'LogAbund' rs[what] = mod stime = time.time() rs.write(what, utils.outfn('1km', 'adrid-%d.tif' % year)) etime = time.time() print("executed in %6.2fs" % (etime - stime))
def project_year(model, model_dir, what, scenario, year): print("projecting %s for %d using %s" % (what, year, scenario)) models = select_models(model, model_dir) # Read Sam's abundance model (forested and non-forested) mod = modelr.load(models[0]) predicts.predictify(mod) # Import standard PREDICTS rasters by_age = 'young_secondary' in mod.syms print('by_age: %s' % str(by_age)) rasters = predicts.rasterset('luh5', scenario, year, by_age) rs = RasterSet(rasters) if what == 'bii': vname = 'bii' rs[vname] = SimpleExpr(vname, '%s / %f' % (mod.output, intercept)) else: vname = mod.output rs[vname] = mod if what not in rs: print('%s not in rasterset' % what) print(', '.join(sorted(rs.keys()))) sys.exit(1) stime = time.time() data, meta = rs.eval(what, quiet=False) etime = time.time() print("executed in %6.2fs" % (etime - stime)) oname = os.path.join(os.environ['OUTDIR'], 'luh5/%s-%s-%d.tif' % (scenario, what, year)) #hpd, _ = rs.eval('hpd', quiet=False) #hpd_max, meta2 = rs.eval('hpd_max', quiet=False) with rasterio.open(oname, 'w', **meta) as dst: #bb = dst.bounds #ul = map(int, ~meta2['affine'] * (bb[0], bb[3])) #lr = map(int, ~meta2['affine'] * (bb[2], bb[1])) #cap = ma.where(hpd > hpd_max[ul[1]:lr[1], ul[0]:lr[0]], True, False) #show(hpd, norm=colors.PowerNorm(gamma=0.2)) #show(cap * 1) #data.mask = np.logical_or(data.mask, cap) dst.write(data.filled(meta['nodata']), indexes=1) if None: fig = plt.figure(figsize=(8, 6)) ax = plt.gca() show(data, cmap='viridis', ax=ax) plt.savefig('luh2-%s-%d.png' % (scenario, year)) return
def project_year(model, model_dir, what, scenario, year): print("projecting %s for %d using %s" % (what, year, scenario)) models = select_models(model, model_dir) # Read Sam's abundance model (forested and non-forested) modf = modelr.load(models[0]) intercept_f = modf.intercept predicts.predictify(modf) modn = modelr.load(models[1]) intercept_n = modn.intercept predicts.predictify(modn) # Open forested/non-forested mask layer fstnf = rasterio.open(utils.luh2_static('fstnf')) # Import standard PREDICTS rasters rastersf = predicts.rasterset('luh2', scenario, year, 'f') rsf = RasterSet(rastersf, mask=fstnf, maskval=0.0) rastersn = predicts.rasterset('luh2', scenario, year, 'n') rsn = RasterSet(rastersn, mask=fstnf, maskval=1.0) #rsn = RasterSet(rastersn) if what == 'bii': vname = 'bii' rsf[vname] = SimpleExpr( vname, 'exp(%s) / exp(%f)' % (modf.output, intercept_f)) rsn[vname] = SimpleExpr( vname, 'exp(%s) / exp(%f)' % (modn.output, intercept_n)) rsf[modf.output] = modf rsn[modn.output] = modn else: vname = modf.output assert modf.output == modn.output rsf[vname] = modf rsn[vname] = modn if what not in rsf: print('%s not in rasterset' % what) print(', '.join(sorted(rsf.keys()))) sys.exit(1) stime = time.time() datan, meta = rsn.eval(what, quiet=False) dataf, _ = rsf.eval(what, quiet=True) data_vals = dataf.filled(0) + datan.filled(0) data = data_vals.view(ma.MaskedArray) data.mask = np.logical_and(dataf.mask, datan.mask) #data = datan etime = time.time() print("executed in %6.2fs" % (etime - stime)) oname = '%s/luh2/%s-%s-%d.tif' % (utils.outdir(), scenario, what, year) with rasterio.open(oname, 'w', **meta) as dst: dst.write(data.filled(meta['nodata']), indexes=1) if None: fig = plt.figure(figsize=(8, 6)) ax = plt.gca() show(data, cmap='viridis', ax=ax) plt.savefig('luh2-%s-%d.png' % (scenario, year)) return
def project_year(model, model_dir, scenario, year): """Run a projection for a single year. Can be called in parallel when projecting a range of years. """ print("projecting %s for %d using %s" % (model, year, scenario)) # Import standard PREDICTS rasters rasters = predicts.rasterset('luh2', scenario, year) rs = RasterSet(rasters) what, model = select_model(model, model_dir) # Read Sam's models if model: mod = modelr.load(model) predicts.predictify(mod) rs[mod.output] = mod if what in ('CompSimAb', 'CompSimSR', 'Abundance', 'Richness'): if what in ('CompSimAb', 'CompSimSR'): expr = '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' else: expr = '(exp(%s) / exp(%f))' rs[what] = SimpleExpr(what, expr % (mod.output, mod.intercept)) if what not in rs: print('%s not in rasterset' % what) print(', '.join(sorted(rs.keys()))) sys.exit(1) stime = time.time() data, meta = rs.eval(what, quiet=True) etime = time.time() print("executed in %6.2fs" % (etime - stime)) oname = '%s/luh2/%s-%s-%04d.tif' % (utils.outdir(), scenario, what, year) with rasterio.open(oname, 'w', **meta) as dst: dst.write(data.filled(meta['nodata']), indexes=1) if None: fig = plt.figure(figsize=(8, 6)) show(data, cmap='viridis', ax=plt.gca()) fig.savefig('luh2-%s-%d.png' % (scenario, year)) return
def luh2_rasterset(scenario, year): rset = luh2_types('historical', 1999) rset_add(rset, 'perennial', 'c3per + c4per') rset_add(rset, 'annual', 'c3ann + c4ann') rset_add(rset, 'nitrogen', 'c3nfx') rset_add(rset, 'rangelands', 'range') rset_add(rset, 'secondaryf', 'secdyf + secdif + secdmf') rset_add(rset, 'secondaryn', 'secdyn + secdin + secdmn') rset_add(rset, 'secondary', 'secdyf + secdif + secdmf + secdyn + secdin + secdmn') return RasterSet(rset)
'1km/mainland-from-igor-edited.tif') mask_ds = rasterio.open(mask_file) # Richness compositional similarity model with updated PriMin-Urb coefficient mod = modelr.load('/home/vagrant/katia/models/updated_compsim_rich.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', 'medium', year=2005) # create an ISL_MAINL raster # set it to Mainlands this time round (set Mainlands to 1 and Islands to 0) rasters['ISL_MAINMAINLAND'] = SimpleExpr('ISL_MAINMAINLAND', '2') rasters['ISL_MAINISLAND'] = SimpleExpr('ISL_MAINISLAND', '0') rasters['adjGeogDist'] = SimpleExpr('adjGeogDist', '0') rasters['cubeRtEnvDist'] = SimpleExpr('cubeRtEnvDist', '0') # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True) # if you're projecting the whole world, use this code instead #rs = RasterSet(rasters) # evaluate the model # model is logit transformed with an adjustment, so back-transformation rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' % (mod.output, mod.intercept)) rs.write('output', utils.outfn('katia', 'bii-sr-cs-mainlands.tif'))
from projections.rasterset import RasterSet, Raster import projections.predicts as predicts import projections.r2py.modelr as modelr # Open the mask shape file shp_file = os.path.join(os.environ['DATA_ROOT'], 'from-adriana/tropicalforests.shp') shapes = fiona.open(shp_file) # Read Adriana's abundance model (mainland) mod = modelr.load(os.path.join(os.environ['MODEL_DIR'], 'ab-model.rds')) predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('luh5', 'historical', 1990, True) rs = RasterSet(rasters, shapes=shapes, all_touched=True) what = mod.output rs[mod.output] = mod stime = time.time() data1, meta_data1 = rs.eval(what) etime = time.time() print("executed in %6.2fs" % (etime - stime)) show(data1) ## ## Compare with good raster ## out = rasterio.open('adrid-good.tif') good = out.read(1, masked=True) diff = np.fabs(data1 - good)
#!/usr/bin/env python from projections.atlas import atlas from projections.rasterset import RasterSet, Raster import projections.predicts as predicts import projections.rds as rds # Read Katia's abundance model (mainland) mod = rds.read('../models/ab-mainland.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('rcp', 'aim', 2020, 'medium') rs = RasterSet(rasters) rs[mod.output()] = mod data = rs.eval(mod.output()) # Display the raster atlas(data, mod.output(), 'viridis')
# get the differences # diff = cuberoot(s1) - cuberoot(s2) # we want cuberoot(s1+1) = 0 # so the difference is 0 - cuberoot(s2) # then center and scale (subtract mean and divide by sd) mean_rd50_diff = float( df[df['Value'] == 'CRootDens_50km_diff']['mean']) mean_rd1_diff = float(df[df['Value'] == 'CRootDens_1km_diff']['mean']) sd_rd50_diff = float(df[df['Value'] == 'CRootDens_50km_diff']['sd']) sd_rd1_diff = float(df[df['Value'] == 'CRootDens_1km_diff']['sd']) rasters['CRootDens_50km_diff_cs'] = SimpleExpr( 'CRootDens_50km_diff_cs', '((0 - clip_rd50) - %f) / %f' % (mean_rd50_diff, sd_rd50_diff)) rasters['CRootDens_1km_diff_cs'] = SimpleExpr( 'CRootDens_1km_diff_cs', '((0 - clip_rd1) - %f) / %f' % (mean_rd1_diff, sd_rd1_diff)) # set up the rasterset, cropping by tropical forests rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=False) # evaluate the model # model is logit transformed with an adjustment, so back-transformation rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' % (mod.output, ref)) rs.write( 'output', utils.outfn('temporal-bii/' + version, 'bii-cs-%d.tif' % year))
# clip the cs rasters so they can't go below 0 but no upper limit inname = 'C:/ds/temporal-bii/' + version + '/bii-cs-' + str( year) + '.tif' outname = 'C:/ds/temporal-bii/' + version + '/bii-cs-low-bound-' + str( year) + '.tif' ru.clip(inname, outname, 0, None) # calculate bii (with only a lower bound) # pull in all the rasters for computing bii bii_rs = RasterSet({ 'abundance': Raster( 'abundance', 'C:/ds/temporal-bii/' + version + '/bii-ab-low-bound-%d.tif' % year), 'comp_sim': Raster( 'comp_sim', 'C:/ds/temporal-bii/' + version + '/bii-cs-low-bound-%d.tif' % year), 'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim') }) # write out bii raster bii_rs.write( 'bii_ab', utils.outfn('temporal-bii/' + version, 'bii-%d.tif' % year)) # calculate bounded bii # clip the abundance rasters inname = 'C:/ds/temporal-bii/' + version + '/bii-ab-' + str( year) + '.tif'
mod = modelr.load('/home/vagrant/katia/models/updated_compsim_abund.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', 'medium', year=2005) # create an ISL_MAINL raster # set it to Mainlands this time round (set Mainlands to 1 and Islands to 0) rasters['ISL_MAINMAINLAND'] = SimpleExpr('ISL_MAINMAINLAND', ISLMAIN) rasters['ISL_MAINISLAND'] = SimpleExpr('ISL_MAINISLAND', 0) rasters['adjGeogDist'] = SimpleExpr('adjGeogDist', 0) rasters['cubeRtEnvDist'] = SimpleExpr('cubeRtEnvDist', 0) # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True) # if you're projecting the whole world, use this code instead #rs = RasterSet(rasters) # evaluate the model # model is logit transformed with an adjustment, so back-transformation rs[mod.output] = mod intercept = mod.partial({'ISL_MAINMAINLAND': ISLMAIN}) print("intercept: %.5f" % intercept) if args.mainland: assert math.isclose(intercept, 1.939964, rel_tol=0.001) else: assert math.isclose(intercept, 1.439107176, rel_tol=0.001) rs['output'] = SimpleExpr( 'output', '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)' %
ab_max = 1.655183 sr_max = 1.636021 else: suffix = 'islands' ab_max = 1.443549 sr_max = 1.413479 folder = 'clip' if args.clip else 'no-clip' # pull in all the rasters for computing bii bii_rs = RasterSet({'abundance': Raster('abundance', utils.outfn('katia', folder, 'ab-%s.tif' % suffix)), 'comp_sim': Raster('comp_sim', utils.outfn('katia', 'ab-cs-%s.tif' % suffix)), 'clip_ab': SimpleExpr('clip_ab', 'clip(abundance, 0, %f)' % ab_max), 'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim'), 'bii_ab2': SimpleExpr('bii_ab2', 'clip_ab * comp_sim'), }) # write out bii raster bii_rs.write('bii_ab' if args.clip else 'bii_ab2', utils.outfn('katia', folder, 'abundance-based-bii-%s.tif' % suffix)) # do the same for species richness # pull in all the rasters for computing bii bii_rs = RasterSet({'sp_rich': Raster('sp_rich', utils.outfn('katia',
'pow(RdDens_50km, 1/3.)') # extract the maximum value max_rd = float(df[df['Value'] == 'CRootRdDens_50km']['max']) # clip to the maximum value rasters['clip_rd'] = SimpleExpr( 'clip_rd', 'clip(Croot_RdDens_50km, 0, %f)' % max_rd) # scale and center mean_rd = float(df[df['Value'] == 'CRootRdDens_50km']['mean']) sd_rd = float(df[df['Value'] == 'CRootRdDens_50km']['sd']) rasters['CRootRdDens_50km_cs'] = SimpleExpr( 'CRootRdDens_50km_cs', '(clip_rd - %f) / %f' % (mean_rd, sd_rd)) # set up the rasterset, cropping by tropical forests # set all_touched = False as there are some boundary issues with very small countries on the edge # like Macao rs = RasterSet(rasters, shapes=shapes, crop=True, all_touched=False) # evaluate the model # model is square root abundance so square it # note that the intercept value has been calculated for the baseline land use when all other variables are held at 0 rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', '(pow(%s, 2) / pow(%f, 2))' % (mod.output, ref)) rs.write( 'output', utils.outfn('temporal-bii/' + version, 'bii-ab-%d.tif' % year)) # if you're on the final version (i.e. the last time you're going round) # write out the pressure data if version == 'v3':
from projections.rasterset import RasterSet, Raster from projections.simpleexpr import SimpleExpr import projections.predicts as predicts import projections.r2py.modelr as modelr import projections.utils as utils import projections.raster_utils as ru CLIP = 'no-clip' # pull in all the rasters for computing bii bii_rs = RasterSet({ 'abundance': Raster('abundance', utils.outfn('katia', CLIP, 'bii-ab-mainlands.tif')), 'comp_sim': Raster('comp_sim', utils.outfn('katia', 'bii-ab-cs-mainlands.tif')), 'clip_ab': SimpleExpr('clip_ab', 'clip(abundance, 0, 1.655183)'), 'bii_ab': SimpleExpr('bii_ab', 'abundance * comp_sim'), 'bii_ab2': SimpleExpr('bii_ab2', 'clip_ab * comp_sim'), }) # write out bii raster bii_rs.write('bii_ab' if CLIP == 'clip' else 'bii_ab2', utils.outfn('katia', CLIP, 'abundance-based-bii-mainlands.tif')) # do the same for species richness # pull in all the rasters for computing bii bii_rs = RasterSet({ 'sp_rich': Raster('sp_rich', utils.outfn('katia', CLIP, 'bii-sr-mainlands.tif')),
'DistRd', os.path.join(utils.data_root(), '1km/rddistwgs.tif')) ###Use new raster ## If args.clip is true, limit the predictor variable values to the max seen ## when fitting the model if args.clip: rasters['clipDistRd'] = SimpleExpr( 'clipDistRd', 'clip(DistRd, %f, %f)' % (RD_DIST_MIN, RD_DIST_MAX)) else: rasters['clipDistRd'] = SimpleExpr('clipDistRd', 'DistRd') rasters['logDistRd_rs'] = SimpleExpr( 'logDistRd_rs', 'scale(log(clipDistRd + 100),' '0.0, 1.0, -1.120966, 12.18216)') ###Added +100 to DistRd to deal with zero values # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=0, crop=True) # if you're projecting the whole world, use this code instead # rs = RasterSet(rasters) # evaluate the model # model is square root abundance so square it # Note that the intercept value has been calculated for the baseline # land use when all other variables are held at 0 # Therefore I calculate separatedly an intercept where DistRd is set to # the max value, i.e. logDistRd_RS = 1. intercept = mod.partial({'ISL_MAINLMAINLAND': ISLMAIN, 'logDistRd_rs': 1.0}) print("intercept: %.5f" % intercept) if args.mainland: assert math.isclose(intercept, 0.67184, rel_tol=0.001)
#!/usr/bin/env python import fiona import time from rasterio.plot import show, show_hist from projections.rasterset import RasterSet, Raster import projections.predicts as predicts import projections.modelr as modelr # Open the mask shape file shp_file = '../../data/from-adriana/tropicalforests.shp' shapes = fiona.open(shp_file) # Read Adriana's abundance model (mainland) mod = modelr.load('../models/ab-corrected.rds') predicts.predictify(mod) # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', 'foo', 2005, 'historical') rs = RasterSet(rasters, shapes=shapes, all_touched=True) rs[mod.output] = mod what = mod.output stime = time.time() data = rs.write(what, 'hires.tif') etime = time.time() print("executed in %6.2fs" % (etime - stime))
import fiona import numpy as np import time import matplotlib.pyplot as plt from rasterio.plot import show, show_hist from projections.atlas import atlas from projections.rasterset import RasterSet, Raster from projections.simpleexpr import SimpleExpr import projections.predicts as predicts import projections.rds as rds # Import standard PREDICTS rasters rasters = predicts.rasterset('rcp', 'aim', 2020, 'medium') rs = RasterSet(rasters) rs['logHPD_rs2'] = SimpleExpr('logHPD_rs', 'scale(log(hpd + 1), 0.0, 1.0)') data1 = rs.eval('logHPD_rs') data2 = rs.eval('logHPD_rs2') data3 = np.where(np.isclose(data1, data2, equal_nan=True), 1, 0) diff = np.fabs(data1 - data2) print("max diff %f" % diff.max()) print("max in hpd_ref: %f" % rs['hpd_ref'].data.values.max()) print("max in hpd: %f" % rs['hpd'].data.dropna().values.max()) fig, ((ax1, ax2, ax3), (hx1, hx2, hx3)) = plt.subplots(2, 3, figsize=(21, 7)) show(data1, ax=ax1, cmap='Greens', title='Global max/min') show(data2, ax=ax2, cmap='Greens', title='Computed max/min') show(diff, ax=ax3, cmap='viridis', title='Difference', vmin=0, vmax=1.0)
) rasters['secondary_vegetation_intense'] = SimpleExpr( 'secondary_vegetation_intense', 'young_secondary_intense + intermediate_secondary_intense + mature_secondary_intense' ) rasters['primary_vegetation_intense'] = SimpleExpr( 'primary_vegetation_intense', 'primary_intense') rasters['primary_vegetation_light'] = SimpleExpr( 'primary_vegetation_light', 'primary_light') rasters['plantation_pri'] = SimpleExpr('plantation_pri', 'perennial + timber') rasters['plantation_pri_intense'] = SimpleExpr( 'plantation_pri_intense', 'perennial_intense + timber_intense') rasters['plantation_pri_light'] = SimpleExpr( 'plantation_pri_light', 'perennial_light + timber_light') rasters['plantation_pri_minimal'] = SimpleExpr( 'plantation_pri_minimal', 'perennial_minimal + timber_minimal') rs = RasterSet(rasters) # back transform (they were transformed using log(x+1) rs[mod.output] = mod rs['output'] = SimpleExpr( 'output', 'clip((exp(%s) - 1) / (exp(%f) - 1), 0, 1e20)' % (mod.output, mod.intercept)) rs.write( 'output', 'D:/victoria_projections/projections/' + what + '-' + scenario + '-' + '%d.tif' % year)
import sys import rasterio from projections.rasterset import RasterSet import projections.predicts as predicts import projections.utils as utils # Import standard PREDICTS rasters rasters = predicts.rasterset('1km', 'medium', year = 2005) for suffix in ('islands', 'mainland'): # Open the BII raster file mask_file = 'C:/Users/katis2/Desktop/Final_projections/Clip_variables/abundance-based-bii-%s.tif' % suffix mask_ds = rasterio.open(mask_file) # set up the rasterset, cropping to mainlands rs = RasterSet(rasters, mask=mask_ds, maskval=-9999, crop=True) # Run through each land-use for lu in ('cropland', 'pasture', 'primary', 'secondary', 'urban'): # And every use intensity for ui in ('minimal', 'light', 'intense'): name = '%s_%s' % (lu, ui) print(name) oname = utils.outfn('katia', '%s-%s.tif' % (name, suffix)) if os.path.isfile(oname) or name in ('secondary_intense', 'urnan_light'): continue rs.write(name, oname)