def tile_names(tile_id, sensit_type):
# Names of the loss, gain, and model extent tiles
if sensit_type == 'legal_Amazon_loss':
loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed)
else:
loss = '{0}.tif'.format(tile_id)
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
model_extent = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_model_extent)
return loss, gain, model_extent
def tile_names(tile_id, sensit_type):
# Names of the input files
loss = '{0}_{1}.tif'.format(tile_id,
cn.pattern_Brazil_annual_loss_processed)
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
extent = '{0}_{1}.tif'.format(
tile_id, cn.pattern_Brazil_forest_extent_2000_processed)
biomass = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_WHRC_biomass_2000_non_mang_non_planted)
return loss, gain, extent, biomass
def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict,
output_pattern_list, no_upload):
# Converts the forest age category decision tree output values to the three age categories--
# 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years
# These are five digits so they can easily be added to the four digits of the continent-ecozone code to make unique codes
# for each continent-ecozone-age combination.
# The key in the dictionary is the forest age category decision tree endpoints.
age_dict = {0: 0, 1: 10000, 2: 20000, 3: 30000}
uu.print_log("Processing:", tile_id)
# Start time
start = datetime.datetime.now()
# Names of the forest age category and continent-ecozone tiles
age_cat = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_age_cat_IPCC)
cont_eco = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_cont_eco_processed)
# Names of the output natural forest gain rate tiles (above and belowground)
AGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[0])
BGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[1])
AGB_IPCC_default_gain_stdev = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[2])
uu.print_log(
" Creating IPCC default biomass gain rates and standard deviation for {}"
.format(tile_id))
# Opens the input tiles if they exist. kips tile if either input doesn't exist.
try:
age_cat_src = rasterio.open(age_cat)
uu.print_log(" Age category tile found for {}".format(tile_id))
except:
return uu.print_log(
" No age category tile found for {}. Skipping tile.".format(
tile_id))
try:
cont_eco_src = rasterio.open(cont_eco)
uu.print_log(" Continent-ecozone tile found for {}".format(tile_id))
except:
return uu.print_log(
" No continent-ecozone tile found for {}. Skipping tile.".format(
tile_id))
# Grabs metadata about the continent ecozone tile, like its location/projection/cellsize
kwargs = cont_eco_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = cont_eco_src.block_windows(1)
# Updates kwargs for the output dataset.
# Need to update data type to float 32 so that it can handle fractional gain rates
kwargs.update(driver='GTiff',
count=1,
compress='lzw',
nodata=0,
dtype='float32')
# The output files, aboveground and belowground biomass gain rates
dst_above = rasterio.open(AGB_IPCC_default_gain_rate, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_above, sensit_type)
dst_above.update_tags(
units='megagrams aboveground biomass (AGB or dry matter)/ha/yr')
dst_above.update_tags(
source='IPCC Guidelines 2019 refinement, forest section, Table 4.9')
dst_above.update_tags(
extent=
'Full model extent, even though these rates will not be used over the full model extent'
)
dst_below = rasterio.open(BGB_IPCC_default_gain_rate, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_below, sensit_type)
dst_below.update_tags(
units='megagrams belowground biomass (AGB or dry matter)/ha/yr')
dst_below.update_tags(
source='IPCC Guidelines 2019 refinement, forest section, Table 4.9')
dst_below.update_tags(
extent=
'Full model extent, even though these rates will not be used over the full model extent'
)
dst_stdev_above = rasterio.open(AGB_IPCC_default_gain_stdev, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_stdev_above, sensit_type)
dst_stdev_above.update_tags(
units=
'standard deviation, in terms of megagrams aboveground biomass (AGB or dry matter)/ha/yr'
)
dst_stdev_above.update_tags(
source='IPCC Guidelines 2019 refinement, forest section, Table 4.9')
dst_stdev_above.update_tags(
extent=
'Full model extent, even though these standard deviations will not be used over the full model extent'
)
# Iterates across the windows (1 pixel strips) of the input tiles
for idx, window in windows:
# Creates a processing window for each input raster
try:
cont_eco_window = cont_eco_src.read(1, window=window)
except:
cont_eco_window = np.zeros((window.height, window.width),
dtype='uint8')
try:
age_cat_window = age_cat_src.read(1, window=window)
except:
age_cat_window = np.zeros((window.height, window.width),
dtype='uint8')
# Recodes the input forest age category array with 10 different decision tree end values into the 3 actual age categories
age_recode = np.vectorize(age_dict.get)(age_cat_window)
# Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes
cont_eco_age = cont_eco_window + age_recode
## Aboveground removal factors
# Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates
gain_rate_AGB = cont_eco_age.astype('float32')
# Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to
# get annual gain rates (metric tons aboveground biomass/yr) for each pixel
for key, value in gain_table_dict.items():
gain_rate_AGB[gain_rate_AGB == key] = value
# Writes the output window to the output file
dst_above.write_band(1, gain_rate_AGB, window=window)
## Belowground removal factors
# Calculates belowground annual removal rates
gain_rate_BGB = gain_rate_AGB * cn.below_to_above_non_mang
# Writes the output window to the output file
dst_below.write_band(1, gain_rate_BGB, window=window)
## Aboveground removal factor standard deviation
# Converts the continent-ecozone array to float so that the values can be replaced with fractional standard deviations
gain_stdev_AGB = cont_eco_age.astype('float32')
# Applies the dictionary of continent-ecozone-age gain rate standard deviations to the continent-ecozone-age array to
# get annual gain rate standard deviations (metric tons aboveground biomass/yr) for each pixel
for key, value in stdev_table_dict.items():
gain_stdev_AGB[gain_stdev_AGB == key] = value
# Writes the output window to the output file
dst_stdev_above.write_band(1, gain_stdev_AGB, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_pattern_list[0], no_upload)
def legal_Amazon_forest_age_category(tile_id, sensit_type, output_pattern):
# Start time
start = datetime.datetime.now()
loss = '{0}_{1}.tif'.format(tile_id,
cn.pattern_Brazil_annual_loss_processed)
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
extent = '{0}_{1}.tif'.format(
tile_id, cn.pattern_Brazil_forest_extent_2000_processed)
biomass = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_WHRC_biomass_2000_non_mang_non_planted)
plantations = uu.sensit_tile_rename(
sensit_type, tile_id, cn.pattern_planted_forest_type_unmasked)
mangroves = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_mangrove_biomass_2000)
# Opens biomass tile
with rasterio.open(loss) as loss_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = loss_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = loss_src.block_windows(1)
# Opens tiles
gain_src = rasterio.open(gain)
extent_src = rasterio.open(extent)
biomass_src = rasterio.open(biomass)
# Checks whether there are mangrove or planted forest tiles. If so, they are opened.
try:
plantations_src = rasterio.open(plantations)
uu.print_log(
" Planted forest tile found for {}".format(tile_id))
except:
uu.print_log(" No planted forest tile for {}".format(tile_id))
try:
mangroves_src = rasterio.open(mangroves)
uu.print_log(" Mangrove tile found for {}".format(tile_id))
except:
uu.print_log(" No mangrove tile for {}".format(tile_id))
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
# Opens the output tile, giving it the arguments of the input tiles
dst = rasterio.open('{0}_{1}.tif'.format(tile_id, output_pattern), 'w',
**kwargs)
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
# Creates windows for each input raster
loss_window = loss_src.read(1, window=window)
gain_window = gain_src.read(1, window=window)
extent_window = extent_src.read(1, window=window)
biomass_window = biomass_src.read(1, window=window)
# Create a 0s array for the output
dst_data = np.zeros((window.height, window.width), dtype='uint8')
# No change pixels (no loss or gain)
dst_data[np.where((biomass_window > 0) & (extent_window == 1)
& (loss_window == 0))] = 3 # primary forest
# Loss-only pixels
dst_data[np.where((biomass_window > 0) & (extent_window == 1)
& (loss_window > 0))] = 6 # primary forest
# Loss-and-gain pixels
dst_data[np.where((extent_window == 1) & (gain_window == 1) &
(loss_window > 0))] = 8 # young secondary forest
if os.path.exists(mangroves):
mangroves_window = mangroves_src.read(1, window=window)
dst_data = np.ma.masked_where(
mangroves_window != 0, dst_data).filled(0).astype('uint8')
if os.path.exists(plantations):
plantations_window = plantations_src.read(1, window=window)
dst_data = np.ma.masked_where(
plantations_window != 0,
dst_data).filled(0).astype('uint8')
# Writes the output window to the output
dst.write_band(1, dst_data, window=window)
uu.end_of_fx_summary(start, tile_id, output_pattern)
def net_calc(tile_id, pattern, sensit_type):
uu.print_log("Calculating net flux for", tile_id)
# Start time
start = datetime.datetime.now()
# Names of the gain and emissions tiles
removals_in = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cumul_gain_AGCO2_BGCO2_all_types)
emissions_in = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil)
# Output net emissions file
net_flux = '{0}_{1}.tif'.format(tile_id, pattern)
try:
removals_src = rasterio.open(removals_in)
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = removals_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = removals_src.block_windows(1)
uu.print_log(" Gross removals tile {} found".format(removals_in))
except:
uu.print_log(" No gross removals tile {} found".format(removals_in))
try:
emissions_src = rasterio.open(emissions_in)
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = emissions_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = emissions_src.block_windows(1)
uu.print_log(" Gross emissions tile {} found".format(emissions_in))
except:
uu.print_log(" No gross emissions tile {} found".format(emissions_in))
# Skips the tile if there is neither a gross emissions nor a gross removals tile.
# This should only occur for biomass_swap sensitivity analysis, which gets its net flux tile list from
# the JPL tile list (some tiles of which have neither emissions nor removals), rather than the union of
# emissions and removals tiles.
try:
kwargs.update(
driver='GTiff',
count=1,
compress='lzw',
nodata=0,
dtype='float32'
)
except:
uu.print_log("No gross emissions or gross removals for {}. Skipping tile.".format(tile_id))
return
# Opens the output tile, giving it the arguments of the input tiles
net_flux_dst = rasterio.open(net_flux, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(net_flux_dst, sensit_type)
net_flux_dst.update_tags(
units='Mg CO2e/ha over model duration (2001-20{})'.format(cn.loss_years))
net_flux_dst.update_tags(
source='Gross emissions - gross removals')
net_flux_dst.update_tags(
extent='Model extent')
net_flux_dst.update_tags(
scale='Negative values are net sinks. Positive values are net sources.')
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
# Creates windows for each input tile
try:
removals_window = removals_src.read(1, window=window).astype('float32')
except:
removals_window = np.zeros((window.height, window.width)).astype('float32')
try:
emissions_window = emissions_src.read(1, window=window).astype('float32')
except:
emissions_window = np.zeros((window.height, window.width)).astype('float32')
# Subtracts gain that from loss
dst_data = emissions_window - removals_window
net_flux_dst.write_band(1, dst_data, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, pattern)
def annual_gain_rate(tile_id, sensit_type, output_pattern_list,
gain_above_dict, gain_below_dict, stdev_dict):
uu.print_log("Processing:", tile_id)
# Start time
start = datetime.datetime.now()
# This is only needed for testing, when a list of tiles might include ones without mangroves.
# When the full model is being run, only tiles with mangroves are included.
mangrove_biomass_tile_list = uu.tile_list_s3(cn.mangrove_biomass_2000_dir)
if tile_id not in mangrove_biomass_tile_list:
uu.print_log(
"{} does not contain mangroves. Skipping tile.".format(tile_id))
return
# Name of the input files
mangrove_biomass = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_mangrove_biomass_2000)
cont_eco = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_cont_eco_processed)
# Names of the output aboveground and belowground mangrove gain rate tiles
AGB_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0])
BGB_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[1])
AGB_gain_stdev = '{0}_{1}.tif'.format(tile_id, output_pattern_list[2])
uu.print_log(
" Reading input files and creating aboveground and belowground biomass gain rates for {}"
.format(tile_id))
cont_eco_src = rasterio.open(cont_eco)
mangrove_AGB_src = rasterio.open(mangrove_biomass)
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = cont_eco_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = cont_eco_src.block_windows(1)
# Updates kwargs for the output dataset.
# Need to update data type to float 32 so that it can handle fractional gain rates
kwargs.update(driver='GTiff',
count=1,
compress='lzw',
nodata=0,
dtype='float32')
dst_above = rasterio.open(AGB_gain_rate, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_above, sensit_type)
dst_above.update_tags(
units='megagrams aboveground biomass (AGB or dry matter)/ha/yr')
dst_above.update_tags(
source='IPCC Guidelines, 2013 Coastal Wetlands Supplement, Table 4.4')
dst_above.update_tags(
extent=
'Simard et al. 2018, based on Giri et al. 2011 (Global Ecol. Biogeogr.) mangrove extent'
)
dst_below = rasterio.open(BGB_gain_rate, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_below, sensit_type)
dst_below.update_tags(
units='megagrams belowground biomass (BGB or dry matter)/ha/yr')
dst_below.update_tags(
source='IPCC Guidelines, 2013 Coastal Wetlands Supplement, Table 4.4')
dst_below.update_tags(
extent=
'Simard et al. 2018, based on Giri et al. 2011 (Global Ecol. Biogeogr.) mangrove extent'
)
dst_stdev_above = rasterio.open(AGB_gain_stdev, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_stdev_above, sensit_type)
dst_stdev_above.update_tags(
units=
'standard deviation, in terms of megagrams aboveground biomass (AGB or dry matter)/ha/yr'
)
dst_stdev_above.update_tags(
source='IPCC Guidelines, 2013 Coastal Wetlands Supplement, Table 4.4')
dst_stdev_above.update_tags(
extent=
'Simard et al. 2018, based on Giri et al. 2011 (Global Ecol. Biogeogr.) mangrove extent'
)
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
# Creates windows for each input raster
cont_eco = cont_eco_src.read(1, window=window)
mangrove_AGB = mangrove_AGB_src.read(1, window=window)
# Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates.
# Creates two copies: one for aboveground gain and one for belowground gain.
# Creating only one copy of the cont_eco raster made it so that belowground gain rates weren't being
# written correctly for some reason.
cont_eco_above = cont_eco.astype('float32')
cont_eco_below = cont_eco.astype('float32')
cont_eco_stdev = cont_eco.astype('float32')
# Reclassifies mangrove biomass to 1 or 0 to make a mask of mangrove pixels.
# Ultimately, only these pixels (ones with mangrove biomass) will get values.
mangrove_AGB[mangrove_AGB > 0] = 1
# Applies the dictionary of continent-ecozone aboveground gain rates to the continent-ecozone array to
# get annual aboveground gain rates (metric tons aboveground biomass/yr) for each pixel
for key, value in gain_above_dict.items():
cont_eco_above[cont_eco_above == key] = value
# Masks out pixels without mangroves, leaving gain rates in only pixels with mangroves
dst_above_data = cont_eco_above * mangrove_AGB
# Writes the output window to the output
dst_above.write_band(1, dst_above_data, window=window)
# Same as above but for belowground gain rates
for key, value in gain_below_dict.items():
cont_eco_below[cont_eco_below == key] = value
dst_below_data = cont_eco_below * mangrove_AGB
dst_below.write_band(1, dst_below_data, window=window)
# Applies the dictionary of continent-ecozone aboveground gain rate standard deviations to the continent-ecozone array to
# get annual aboveground gain rate standard deviations (metric tons aboveground biomass/yr) for each pixel
for key, value in stdev_dict.items():
cont_eco_stdev[cont_eco_stdev == key] = value
# Masks out pixels without mangroves, leaving gain rates in only pixels with mangroves
dst_stdev = cont_eco_stdev * mangrove_AGB
# Writes the output window to the output
dst_stdev_above.write_band(1, dst_stdev, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_pattern_list[0])
def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type, no_upload):
uu.print_log("Assigning forest age categories:", tile_id)
# Start time
start = datetime.datetime.now()
# Gets the bounding coordinates of each tile. Needed to determine if the tile is in the tropics (within 30 deg of the equator)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Default value is that the tile is not in the tropics
tropics = 0
# Criteria for assigning a tile to the tropics
if (ymax > -30) & (ymax <= 30) :
tropics = 1
uu.print_log(" Tile {} in tropics:".format(tile_id), tropics)
# Names of the input tiles
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
model_extent = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_model_extent)
ifl_primary = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_ifl_primary)
cont_eco = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cont_eco_processed)
# Biomass tile name depends on the sensitivity analysis
if sensit_type == 'biomass_swap':
biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_JPL_unmasked_processed)
uu.print_log("Using JPL biomass tile for {} sensitivity analysis".format(sensit_type))
else:
biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_WHRC_biomass_2000_unmasked)
uu.print_log("Using WHRC biomass tile for {} sensitivity analysis".format(sensit_type))
if sensit_type == 'legal_Amazon_loss':
loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed)
uu.print_log("Using PRODES loss tile {0} for {1} sensitivity analysis".format(tile_id, sensit_type))
elif sensit_type == 'Mekong_loss':
loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Mekong_loss_processed)
else:
loss = '{0}_{1}.tif'.format(cn.pattern_loss, tile_id)
uu.print_log("Using Hansen loss tile {0} for {1} model run".format(tile_id, sensit_type))
uu.print_log(" Assigning age categories")
# Opens biomass tile
with rasterio.open(model_extent) as model_extent_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = model_extent_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = model_extent_src.block_windows(1)
# Opens the input tiles if they exist
try:
cont_eco_src = rasterio.open(cont_eco)
uu.print_log(" Continent-ecozone tile found for {}".format(tile_id))
except:
uu.print_log(" No continent-ecozone tile found for {}".format(tile_id))
try:
gain_src = rasterio.open(gain)
uu.print_log(" Gain tile found for {}".format(tile_id))
except:
uu.print_log(" No gain tile found for {}".format(tile_id))
try:
biomass_src = rasterio.open(biomass)
uu.print_log(" Biomass tile found for {}".format(tile_id))
except:
uu.print_log(" No biomass tile found for {}".format(tile_id))
try:
loss_src = rasterio.open(loss)
uu.print_log(" Loss tile found for {}".format(tile_id))
except:
uu.print_log(" No loss tile found for {}".format(tile_id))
try:
ifl_primary_src = rasterio.open(ifl_primary)
uu.print_log(" IFL-primary forest tile found for {}".format(tile_id))
except:
uu.print_log(" No IFL-primary forest tile found for {}".format(tile_id))
# Updates kwargs for the output dataset
kwargs.update(
driver='GTiff',
count=1,
compress='lzw',
nodata=0
)
# Opens the output tile, giving it the arguments of the input tiles
dst = rasterio.open('{0}_{1}.tif'.format(tile_id, pattern), 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst, sensit_type)
dst.update_tags(
key='1: young (<20 year) secondary forest; 2: old (>20 year) secondary forest; 3: primary forest or IFL')
dst.update_tags(
source='Decision tree that uses Hansen gain and loss, IFL/primary forest extent, and aboveground biomass to assign an age category')
dst.update_tags(
extent='Full model extent, even though these age categories will not be used over the full model extent. They apply to just the rates from IPCC defaults.')
uu.print_log(" Assigning IPCC age categories for", tile_id)
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
# Creates windows for each input raster. Only model_extent_src is guaranteed to exist
model_extent_window = model_extent_src.read(1, window=window)
try:
loss_window = loss_src.read(1, window=window)
except:
loss_window = np.zeros((window.height, window.width), dtype='uint8')
try:
gain_window = gain_src.read(1, window=window)
except:
gain_window = np.zeros((window.height, window.width), dtype='uint8')
try:
cont_eco_window = cont_eco_src.read(1, window=window)
except:
cont_eco_window = np.zeros((window.height, window.width), dtype='uint8')
try:
biomass_window = biomass_src.read(1, window=window)
except:
biomass_window = np.zeros((window.height, window.width), dtype='float32')
try:
ifl_primary_window = ifl_primary_src.read(1, window=window)
except:
ifl_primary_window = np.zeros((window.height, window.width), dtype='uint8')
# Creates a numpy array that has the <=20 year secondary forest growth rate x 20
# based on the continent-ecozone code of each pixel (the dictionary).
# This is used to assign pixels to the correct age category.
gain_20_years = np.vectorize(gain_table_dict.get)(cont_eco_window)*20
# Create a 0s array for the output
dst_data = np.zeros((window.height, window.width), dtype='uint8')
# Logic tree for assigning age categories begins here
# Code 1 = young (<20 years) secondary forest, code 2 = old (>20 year) secondary forest, code 3 = primary forest
# model_extent_window ensures that there is both biomass and tree cover in 2000 OR mangroves OR tree cover gain
# WITHOUT pre-2000 plantations
# For every model version except legal_Amazon_loss sensitivity analysis, which has its own rules about age assignment
if sensit_type != 'legal_Amazon_loss':
# No change pixels- no loss or gain
if tropics == 0:
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0))] = 2
if tropics == 1:
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0) & (ifl_primary_window != 1))] = 2
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0) & (ifl_primary_window == 1))] = 3
# Loss-only pixels
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window != 1) & (biomass_window <= gain_20_years))] = 1
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window != 1) & (biomass_window > gain_20_years))] = 2
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window ==1))] = 3
# Gain-only pixels
# If there is gain, the pixel doesn't need biomass or canopy cover. It just needs to be outside of plantations and mangroves.
# The role of model_extent_window here is to exclude the pre-2000 plantations.
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window == 0))] = 1
# Pixels with loss and gain
# If there is gain with loss, the pixel doesn't need biomass or canopy cover. It just needs to be outside of plantations and mangroves.
# The role of model_extent_window here is to exclude the pre-2000 plantations.
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > (cn.gain_years)))] = 1
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > 0) & (loss_window <= (cn.gain_years/2)))] = 1
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > (cn.gain_years/2)) & (loss_window <= cn.gain_years))] = 1
# For legal_Amazon_loss sensitivity analysis
else:
# Non-loss pixels (could have gain or not. Assuming that if within PRODES extent in 2000, there can't be
# gain, so it's a faulty detection. Thus, gain-only pixels are ignored and become part of no change.)
dst_data[np.where((model_extent_window == 1) & (loss_window == 0))] = 3 # primary forest
# Loss-only pixels
dst_data[np.where((model_extent_window == 1) & (loss_window > 0) & (gain_window == 0))] = 3 # primary forest
# Loss-and-gain pixels
dst_data[np.where((model_extent_window == 1) & (loss_window > 0) & (gain_window == 1))] = 2 # young secondary forest
# Writes the output window to the output
dst.write_band(1, dst_data, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def annual_gain_rate_AGC_BGC_all_forest_types(tile_id, output_pattern_list,
sensit_type, no_upload):
uu.print_log("Mapping removal rate source and AGB and BGB removal rates:",
tile_id)
# Start time
start = datetime.datetime.now()
# Names of the input tiles
# Removal factors
model_extent = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_model_extent)
mangrove_AGB = '{0}_{1}.tif'.format(tile_id,
cn.pattern_annual_gain_AGB_mangrove)
mangrove_BGB = '{0}_{1}.tif'.format(tile_id,
cn.pattern_annual_gain_BGB_mangrove)
europe_AGC_BGC = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe)
plantations_AGC_BGC = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)
us_AGC_BGC = uu.sensit_tile_rename(
sensit_type, tile_id, cn.pattern_annual_gain_AGC_BGC_natrl_forest_US)
young_AGC = uu.sensit_tile_rename(
sensit_type, tile_id, cn.pattern_annual_gain_AGC_natrl_forest_young)
age_category = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_age_cat_IPCC)
ipcc_AGB_default = uu.sensit_tile_rename(
sensit_type, tile_id, cn.pattern_annual_gain_AGB_IPCC_defaults)
# Removal factor standard deviations
mangrove_AGB_stdev = '{0}_{1}.tif'.format(
tile_id, cn.pattern_stdev_annual_gain_AGB_mangrove)
europe_AGC_BGC_stdev = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe)
plantations_AGC_BGC_stdev = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked)
us_AGC_BGC_stdev = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US)
young_AGC_stdev = uu.sensit_tile_rename(
sensit_type, tile_id,
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young)
ipcc_AGB_default_stdev = uu.sensit_tile_rename(
sensit_type, tile_id, cn.pattern_stdev_annual_gain_AGB_IPCC_defaults)
# Names of the output tiles
removal_forest_type = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0])
annual_gain_AGC_all_forest_types = '{0}_{1}.tif'.format(
tile_id, output_pattern_list[1])
annual_gain_BGC_all_forest_types = '{0}_{1}.tif'.format(
tile_id, output_pattern_list[2])
annual_gain_AGC_BGC_all_forest_types = '{0}_{1}.tif'.format(
tile_id, output_pattern_list[3]
) # Not used further in the model. Created just for reference.
stdev_annual_gain_AGC_all_forest_types = '{0}_{1}.tif'.format(
tile_id, output_pattern_list[4])
# Opens biomass tile
with rasterio.open(model_extent) as model_extent_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = model_extent_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = model_extent_src.block_windows(1)
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
# Checks whether there are mangrove or planted forest tiles. If so, they are opened.
try:
mangrove_AGB_src = rasterio.open(mangrove_AGB)
mangrove_BGB_src = rasterio.open(mangrove_BGB)
mangrove_AGB_stdev_src = rasterio.open(mangrove_AGB_stdev)
uu.print_log(
" Mangrove tiles (AGB and BGB) for {}".format(tile_id))
except:
uu.print_log(" No mangrove tile for {}".format(tile_id))
try:
europe_AGC_BGC_src = rasterio.open(europe_AGC_BGC)
europe_AGC_BGC_stdev_src = rasterio.open(europe_AGC_BGC_stdev)
uu.print_log(
" Europe removal factor tile for {}".format(tile_id))
except:
uu.print_log(
" No Europe removal factor tile for {}".format(tile_id))
try:
plantations_AGC_BGC_src = rasterio.open(plantations_AGC_BGC)
plantations_AGC_BGC_stdev_src = rasterio.open(
plantations_AGC_BGC_stdev)
uu.print_log(" Planted forest tile for {}".format(tile_id))
except:
uu.print_log(" No planted forest tile for {}".format(tile_id))
try:
us_AGC_BGC_src = rasterio.open(us_AGC_BGC)
us_AGC_BGC_stdev_src = rasterio.open(us_AGC_BGC_stdev)
uu.print_log(" US removal factor tile for {}".format(tile_id))
except:
uu.print_log(
" No US removal factor tile for {}".format(tile_id))
try:
young_AGC_src = rasterio.open(young_AGC)
young_AGC_stdev_src = rasterio.open(young_AGC_stdev)
uu.print_log(
" Young forest removal factor tile for {}".format(tile_id))
except:
uu.print_log(
" No young forest removal factor tile for {}".format(
tile_id))
try:
age_category_src = rasterio.open(age_category)
uu.print_log(" Age category tile for {}".format(tile_id))
except:
uu.print_log(" No age category tile for {}".format(tile_id))
try:
ipcc_AGB_default_src = rasterio.open(ipcc_AGB_default)
ipcc_AGB_default_stdev_src = rasterio.open(ipcc_AGB_default_stdev)
uu.print_log(
" IPCC default removal rate tile for {}".format(tile_id))
except:
uu.print_log(
" No IPCC default removal rate tile for {}".format(tile_id))
# Opens the output tile, giving it the arguments of the input tiles
removal_forest_type_dst = rasterio.open(removal_forest_type, 'w',
**kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(removal_forest_type_dst, sensit_type)
removal_forest_type_dst.update_tags(
key=
'6: mangroves. 5: European-specific rates. 4: planted forests. 3: US-specific rates. 2: young (<20 year) secondary forests. 1: old (>20 year) secondary forests and primary forests. Priority goes to the highest number.'
)
removal_forest_type_dst.update_tags(
source=
'Mangroves: IPCC wetlands supplement. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9'
)
removal_forest_type_dst.update_tags(extent='Full model extent')
# Updates kwargs for the removal rate outputs-- just need to change datatype
kwargs.update(dtype='float32')
annual_gain_AGC_all_forest_types_dst = rasterio.open(
annual_gain_AGC_all_forest_types, 'w', **kwargs)
annual_gain_BGC_all_forest_types_dst = rasterio.open(
annual_gain_BGC_all_forest_types, 'w', **kwargs)
annual_gain_AGC_BGC_all_forest_types_dst = rasterio.open(
annual_gain_AGC_BGC_all_forest_types, 'w', **kwargs)
stdev_annual_gain_AGC_all_forest_types_dst = rasterio.open(
stdev_annual_gain_AGC_all_forest_types, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(annual_gain_AGC_all_forest_types_dst, sensit_type)
annual_gain_AGC_all_forest_types_dst.update_tags(
units='megagrams aboveground carbon/ha/yr')
annual_gain_AGC_all_forest_types_dst.update_tags(
source=
'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9'
)
annual_gain_AGC_all_forest_types_dst.update_tags(
extent='Full model extent')
# Adds metadata tags to the output raster
uu.add_rasterio_tags(annual_gain_BGC_all_forest_types_dst, sensit_type)
annual_gain_BGC_all_forest_types_dst.update_tags(
units='megagrams belowground carbon/ha/yr')
annual_gain_BGC_all_forest_types_dst.update_tags(
source=
'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9'
)
annual_gain_BGC_all_forest_types_dst.update_tags(
extent='Full model extent')
# Adds metadata tags to the output raster
uu.add_rasterio_tags(annual_gain_AGC_BGC_all_forest_types_dst,
sensit_type)
annual_gain_AGC_BGC_all_forest_types_dst.update_tags(
units='megagrams aboveground + belowground carbon/ha/yr')
annual_gain_AGC_BGC_all_forest_types_dst.update_tags(
source=
'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9'
)
annual_gain_AGC_BGC_all_forest_types_dst.update_tags(
extent='Full model extent')
# Adds metadata tags to the output raster
uu.add_rasterio_tags(stdev_annual_gain_AGC_all_forest_types_dst,
sensit_type)
stdev_annual_gain_AGC_all_forest_types_dst.update_tags(
units=
'standard deviation for removal factor, in terms of megagrams aboveground carbon/ha/yr'
)
stdev_annual_gain_AGC_all_forest_types_dst.update_tags(
source=
'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9'
)
stdev_annual_gain_AGC_all_forest_types_dst.update_tags(
extent='Full model extent')
uu.print_log(
" Creating removal model forest type tile, AGC removal factor tile, BGC removal factor tile, and AGC removal factor standard deviation tile for {}"
.format(tile_id))
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
model_extent_window = model_extent_src.read(1, window=window)
# Output rasters' windows
removal_forest_type_window = np.zeros(
(window.height, window.width), dtype='uint8')
annual_gain_AGC_all_forest_types_window = np.zeros(
(window.height, window.width), dtype='float32')
annual_gain_BGC_all_forest_types_window = np.zeros(
(window.height, window.width), dtype='float32')
stdev_annual_gain_AGC_all_forest_types_window = np.zeros(
(window.height, window.width), dtype='float32')
try:
age_category_window = age_category_src.read(1, window=window)
except:
age_category_window = np.zeros((window.height, window.width),
dtype='uint8')
# Lowest priority
try:
ipcc_AGB_default_rate_window = ipcc_AGB_default_src.read(
1, window=window)
ipcc_AGB_default_stdev_window = ipcc_AGB_default_stdev_src.read(
1, window=window)
# In no_primary_gain, the AGB_default_rate_window = 0, so primary forest pixels would not be
# assigned a removal forest type and therefore get exclude from the model later.
# That is incorrect, so using model_extent as the criterion instead allows the primary forest pixels
# that don't have rates under this sensitivity analysis to still be included in the model.
# Unfortunately, model_extent is slightly different from the IPCC rate extent (no IPCC rates where
# there is no ecozone information), but this is a very small difference and not worth worrying about.
if sensit_type == 'no_primary_gain':
removal_forest_type_window = np.where(
model_extent_window != 0, cn.old_natural_rank,
removal_forest_type_window).astype('uint8')
else:
removal_forest_type_window = np.where(
ipcc_AGB_default_rate_window != 0, cn.old_natural_rank,
removal_forest_type_window).astype('uint8')
annual_gain_AGC_all_forest_types_window = np.where(
ipcc_AGB_default_rate_window != 0,
ipcc_AGB_default_rate_window *
cn.biomass_to_c_non_mangrove,
annual_gain_AGC_all_forest_types_window).astype('float32')
annual_gain_BGC_all_forest_types_window = np.where(
ipcc_AGB_default_rate_window != 0,
ipcc_AGB_default_rate_window *
cn.biomass_to_c_non_mangrove * cn.below_to_above_non_mang,
annual_gain_BGC_all_forest_types_window).astype('float32')
stdev_annual_gain_AGC_all_forest_types_window = np.where(
ipcc_AGB_default_stdev_window != 0,
ipcc_AGB_default_stdev_window *
cn.biomass_to_c_non_mangrove,
stdev_annual_gain_AGC_all_forest_types_window).astype(
'float32')
except:
pass
try: # young_AGC_rate_window uses > because of the weird NaN in the tiles. If != is used, the young rate NaN overwrites the IPCC arrays
young_AGC_rate_window = young_AGC_src.read(1, window=window)
young_AGC_stdev_window = young_AGC_stdev_src.read(
1, window=window)
# Using the > with the NaN results in non-fatal "RuntimeWarning: invalid value encountered in greater".
# This isn't actually a problem, so the "with" statement suppresses it, per https://stackoverflow.com/a/58026329/10839927
with np.errstate(invalid='ignore'):
removal_forest_type_window = np.where(
(young_AGC_rate_window > 0) &
(age_category_window == 1), cn.young_natural_rank,
removal_forest_type_window).astype('uint8')
annual_gain_AGC_all_forest_types_window = np.where(
(young_AGC_rate_window > 0) &
(age_category_window == 1), young_AGC_rate_window,
annual_gain_AGC_all_forest_types_window).astype(
'float32')
annual_gain_BGC_all_forest_types_window = np.where(
(young_AGC_rate_window > 0) &
(age_category_window == 1),
young_AGC_rate_window * cn.below_to_above_non_mang,
annual_gain_BGC_all_forest_types_window).astype(
'float32')
stdev_annual_gain_AGC_all_forest_types_window = np.where(
(young_AGC_stdev_window > 0) &
(age_category_window == 1), young_AGC_stdev_window,
stdev_annual_gain_AGC_all_forest_types_window).astype(
'float32')
except:
pass
if sensit_type != 'US_removals':
try:
us_AGC_BGC_rate_window = us_AGC_BGC_src.read(1,
window=window)
us_AGC_BGC_stdev_window = us_AGC_BGC_stdev_src.read(
1, window=window)
removal_forest_type_window = np.where(
us_AGC_BGC_rate_window != 0, cn.US_rank,
removal_forest_type_window).astype('uint8')
annual_gain_AGC_all_forest_types_window = np.where(
us_AGC_BGC_rate_window != 0, us_AGC_BGC_rate_window /
(1 + cn.below_to_above_non_mang),
annual_gain_AGC_all_forest_types_window).astype(
'float32')
annual_gain_BGC_all_forest_types_window = np.where(
us_AGC_BGC_rate_window != 0, (us_AGC_BGC_rate_window) -
(us_AGC_BGC_rate_window /
(1 + cn.below_to_above_non_mang)),
annual_gain_BGC_all_forest_types_window).astype(
'float32')
stdev_annual_gain_AGC_all_forest_types_window = np.where(
us_AGC_BGC_stdev_window != 0, us_AGC_BGC_stdev_window /
(1 + cn.below_to_above_non_mang),
stdev_annual_gain_AGC_all_forest_types_window).astype(
'float32')
except:
pass
try:
plantations_AGC_BGC_rate_window = plantations_AGC_BGC_src.read(
1, window=window)
plantations_AGC_BGC_stdev_window = plantations_AGC_BGC_stdev_src.read(
1, window=window)
removal_forest_type_window = np.where(
plantations_AGC_BGC_rate_window != 0,
cn.planted_forest_rank,
removal_forest_type_window).astype('uint8')
annual_gain_AGC_all_forest_types_window = np.where(
plantations_AGC_BGC_rate_window != 0,
plantations_AGC_BGC_rate_window /
(1 + cn.below_to_above_non_mang),
annual_gain_AGC_all_forest_types_window).astype('float32')
annual_gain_BGC_all_forest_types_window = np.where(
plantations_AGC_BGC_rate_window != 0,
(plantations_AGC_BGC_rate_window) -
(plantations_AGC_BGC_rate_window /
(1 + cn.below_to_above_non_mang)),
annual_gain_BGC_all_forest_types_window).astype('float32')
stdev_annual_gain_AGC_all_forest_types_window = np.where(
plantations_AGC_BGC_stdev_window != 0,
plantations_AGC_BGC_stdev_window /
(1 + cn.below_to_above_non_mang),
stdev_annual_gain_AGC_all_forest_types_window).astype(
'float32')
except:
pass
try:
europe_AGC_BGC_rate_window = europe_AGC_BGC_src.read(
1, window=window)
europe_AGC_BGC_stdev_window = europe_AGC_BGC_stdev_src.read(
1, window=window)
removal_forest_type_window = np.where(
europe_AGC_BGC_rate_window != 0, cn.europe_rank,
removal_forest_type_window).astype('uint8')
annual_gain_AGC_all_forest_types_window = np.where(
europe_AGC_BGC_rate_window != 0,
europe_AGC_BGC_rate_window /
(1 + cn.below_to_above_non_mang),
annual_gain_AGC_all_forest_types_window).astype('float32')
annual_gain_BGC_all_forest_types_window = np.where(
europe_AGC_BGC_rate_window != 0,
(europe_AGC_BGC_rate_window) -
(europe_AGC_BGC_rate_window /
(1 + cn.below_to_above_non_mang)),
annual_gain_BGC_all_forest_types_window).astype('float32')
# NOTE: Nancy Harris thought that the European removal standard deviations were 2x too large,
# per email on 8/30/2020. Thus, simplest fix is to leave original tiles 2x too large and
# correct them only where composited with other stdev sources.
stdev_annual_gain_AGC_all_forest_types_window = np.where(
europe_AGC_BGC_stdev_window != 0,
(europe_AGC_BGC_stdev_window / 2) /
(1 + cn.below_to_above_non_mang),
stdev_annual_gain_AGC_all_forest_types_window).astype(
'float32')
except:
pass
# Highest priority
try:
mangroves_AGB_rate_window = mangrove_AGB_src.read(
1, window=window)
mangroves_BGB_rate_window = mangrove_BGB_src.read(
1, window=window)
mangroves_AGB_stdev_window = mangrove_AGB_stdev_src.read(
1, window=window)
removal_forest_type_window = np.where(
mangroves_AGB_rate_window != 0, cn.mangrove_rank,
removal_forest_type_window).astype('uint8')
annual_gain_AGC_all_forest_types_window = np.where(
mangroves_AGB_rate_window != 0,
mangroves_AGB_rate_window * cn.biomass_to_c_mangrove,
annual_gain_AGC_all_forest_types_window).astype('float32')
annual_gain_BGC_all_forest_types_window = np.where(
mangroves_BGB_rate_window != 0,
mangroves_BGB_rate_window * cn.biomass_to_c_mangrove,
annual_gain_BGC_all_forest_types_window).astype('float32')
stdev_annual_gain_AGC_all_forest_types_window = np.where(
mangroves_AGB_stdev_window != 0,
mangroves_AGB_stdev_window * cn.biomass_to_c_mangrove,
stdev_annual_gain_AGC_all_forest_types_window).astype(
'float32')
except:
pass
# Masks outputs to model output extent
removal_forest_type_window = np.where(model_extent_window == 1,
removal_forest_type_window,
0)
annual_gain_AGC_all_forest_types_window = np.where(
model_extent_window == 1,
annual_gain_AGC_all_forest_types_window, 0)
annual_gain_BGC_all_forest_types_window = np.where(
model_extent_window == 1,
annual_gain_BGC_all_forest_types_window, 0)
annual_gain_AGC_BGC_all_forest_types_window = annual_gain_AGC_all_forest_types_window + annual_gain_BGC_all_forest_types_window
stdev_annual_gain_AGC_all_forest_types_window = np.where(
model_extent_window == 1,
stdev_annual_gain_AGC_all_forest_types_window, 0)
# Writes the outputs window to the output files
removal_forest_type_dst.write_band(1,
removal_forest_type_window,
window=window)
annual_gain_AGC_all_forest_types_dst.write_band(
1, annual_gain_AGC_all_forest_types_window, window=window)
annual_gain_BGC_all_forest_types_dst.write_band(
1, annual_gain_BGC_all_forest_types_window, window=window)
annual_gain_AGC_BGC_all_forest_types_dst.write_band(
1, annual_gain_AGC_BGC_all_forest_types_window, window=window)
stdev_annual_gain_AGC_all_forest_types_dst.write_band(
1,
stdev_annual_gain_AGC_all_forest_types_window,
window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_removal_forest_type,
no_upload)
def gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type):
uu.print_log("Calculating cumulative CO2 removals:", tile_id)
# Start time
start = datetime.datetime.now()
# Names of the input tiles, modified according to sensitivity analysis
gain_rate_AGC = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_annual_gain_AGC_all_types)
gain_rate_BGC = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_annual_gain_BGC_all_types)
gain_year_count = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_gain_year_count)
# Names of the output removal tiles
cumulative_gain_AGCO2 = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[0])
cumulative_gain_BGCO2 = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[1])
cumulative_gain_AGCO2_BGCO2 = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[2])
# Opens the input tiles
gain_rate_AGC_src = rasterio.open(gain_rate_AGC)
gain_rate_BGC_src = rasterio.open(gain_rate_BGC)
gain_year_count_src = rasterio.open(gain_year_count)
# Grabs metadata for an input tile
kwargs = gain_rate_AGC_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = gain_rate_AGC_src.block_windows(1)
# Updates kwargs for the output dataset.
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
# The output files: aboveground gross removals, belowground gross removals, above+belowground gross removals. Adds metadata tags
cumulative_gain_AGCO2_dst = rasterio.open(cumulative_gain_AGCO2, 'w',
**kwargs)
uu.add_rasterio_tags(cumulative_gain_AGCO2_dst, sensit_type)
cumulative_gain_AGCO2_dst.update_tags(
units='megagrams aboveground CO2/ha over entire model period')
cumulative_gain_AGCO2_dst.update_tags(
source='annual removal factors and gain year count')
cumulative_gain_AGCO2_dst.update_tags(extent='Full model extent')
cumulative_gain_BGCO2_dst = rasterio.open(cumulative_gain_BGCO2, 'w',
**kwargs)
uu.add_rasterio_tags(cumulative_gain_BGCO2_dst, sensit_type)
cumulative_gain_BGCO2_dst.update_tags(
units='megagrams belowground CO2/ha over entire model period')
cumulative_gain_BGCO2_dst.update_tags(
source='annual removal factors and gain year count')
cumulative_gain_BGCO2_dst.update_tags(extent='Full model extent')
cumulative_gain_AGCO2_BGCO2_dst = rasterio.open(
cumulative_gain_AGCO2_BGCO2, 'w', **kwargs)
cumulative_gain_AGCO2_BGCO2_dst.update_tags(
units=
'megagrams aboveground+belowground CO2/ha over entire model period')
cumulative_gain_AGCO2_BGCO2_dst.update_tags(
source='annual removal factors and gain year count')
cumulative_gain_AGCO2_BGCO2_dst.update_tags(extent='Full model extent')
# Iterates across the windows (1 pixel strips) of the input tiles
for idx, window in windows:
# Creates a processing window for each input raster
gain_rate_AGC_window = gain_rate_AGC_src.read(1, window=window)
gain_rate_BGC_window = gain_rate_BGC_src.read(1, window=window)
gain_year_count_window = gain_year_count_src.read(1, window=window)
# Converts the annual removal rate into gross removals
cumulative_gain_AGCO2_window = gain_rate_AGC_window * gain_year_count_window * cn.c_to_co2
cumulative_gain_BGCO2_window = gain_rate_BGC_window * gain_year_count_window * cn.c_to_co2
cumulative_gain_AGCO2_BGCO2_window = cumulative_gain_AGCO2_window + cumulative_gain_BGCO2_window
# Writes the output windows to the output files
cumulative_gain_AGCO2_dst.write_band(1,
cumulative_gain_AGCO2_window,
window=window)
cumulative_gain_BGCO2_dst.write_band(1,
cumulative_gain_BGCO2_window,
window=window)
cumulative_gain_AGCO2_BGCO2_dst.write_band(
1, cumulative_gain_AGCO2_BGCO2_window, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_pattern_list[0])
def mp_aggregate_results_to_4_km(sensit_type,
thresh,
tile_id_list,
std_net_flux=None,
run_date=None,
no_upload=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.tile_list_s3(cn.net_flux_dir, sensit_type)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# Files to download for this script
download_dict = {
cn.annual_gain_AGC_all_types_dir:
[cn.pattern_annual_gain_AGC_all_types],
cn.cumul_gain_AGCO2_BGCO2_all_types_dir:
[cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
cn.gross_emis_all_gases_all_drivers_biomass_soil_dir:
[cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
cn.net_flux_dir: [cn.pattern_net_flux]
}
# Checks whether the canopy cover argument is valid
if thresh < 0 or thresh > 99:
uu.exception_log(
no_upload,
'Invalid tcd. Please provide an integer between 0 and 99.')
if uu.check_aws_creds():
# Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles
uu.s3_flexible_download(cn.pixel_area_dir, cn.pattern_pixel_area,
cn.docker_base_dir, sensit_type, tile_id_list)
# Tree cover density, Hansen gain, and mangrove biomass tiles-- necessary for filtering sums to model extent
uu.s3_flexible_download(cn.tcd_dir, cn.pattern_tcd, cn.docker_base_dir,
sensit_type, tile_id_list)
uu.s3_flexible_download(cn.gain_dir, cn.pattern_gain,
cn.docker_base_dir, sensit_type, tile_id_list)
uu.s3_flexible_download(cn.mangrove_biomass_2000_dir,
cn.pattern_mangrove_biomass_2000,
cn.docker_base_dir, sensit_type, tile_id_list)
uu.print_log("Model outputs to process are:", download_dict)
# List of output directories. Modified later for sensitivity analysis.
# Output pattern is determined later.
output_dir_list = [cn.output_aggreg_dir]
# If the model run isn't the standard one, the output directory is changed
if sensit_type != 'std':
uu.print_log(
"Changing output directory and file name pattern based on sensitivity analysis"
)
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Iterates through the types of tiles to be processed
for dir, download_pattern in list(download_dict.items()):
download_pattern_name = download_pattern[0]
# Downloads input files or entire directories, depending on how many tiles are in the tile_id_list, if AWS credentials are found
if uu.check_aws_creds():
uu.s3_flexible_download(dir, download_pattern_name,
cn.docker_base_dir, sensit_type,
tile_id_list)
# Gets an actual tile id to use as a dummy in creating the actual tile pattern
local_tile_list = uu.tile_list_spot_machine(cn.docker_base_dir,
download_pattern_name)
sample_tile_id = uu.get_tile_id(local_tile_list[0])
# Renames the tiles according to the sensitivity analysis before creating dummy tiles.
# The renaming function requires a whole tile name, so this passes a dummy time name that is then stripped a few
# lines later.
tile_id = sample_tile_id # a dummy tile id (but it has to be a real tile id). It is removed later.
output_pattern = uu.sensit_tile_rename(sensit_type, tile_id,
download_pattern_name)
pattern = output_pattern[9:-4]
# For sensitivity analysis runs, only aggregates the tiles if they were created as part of the sensitivity analysis
if (sensit_type != 'std') & (sensit_type not in pattern):
uu.print_log(
"{} not a sensitivity analysis output. Skipping aggregation..."
.format(pattern))
uu.print_log("")
continue
# Lists the tiles of the particular type that is being iterates through.
# Excludes all intermediate files
tile_list = uu.tile_list_spot_machine(".", "{}.tif".format(pattern))
# from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
tile_list = [i for i in tile_list if not ('rewindow' in i)]
tile_list = [i for i in tile_list if not ('0_4deg' in i)]
tile_list = [i for i in tile_list if not ('.ovr' in i)]
# tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif'] # test tiles
uu.print_log("There are {0} tiles to process for pattern {1}".format(
str(len(tile_list)), download_pattern) + "\n")
uu.print_log("Processing:", dir, "; ", pattern)
# Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels,
# which is the resolution of the output tiles. This will allow the 30x30 m pixels in each window to be summed.
# For multiprocessor use. count/2 used about 400 GB of memory on an r4.16xlarge machine, so that was okay.
if cn.count == 96:
if sensit_type == 'biomass_swap':
processes = 12 # 12 processors = XXX GB peak
else:
processes = 16 # 12 processors = 140 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out)
else:
processes = 8
uu.print_log('Rewindow max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(aggregate_results_to_4_km.rewindow, no_upload=no_upload),
tile_list)
# Added these in response to error12: Cannot allocate memory error.
# This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool
# Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory
pool.close()
pool.join()
# # For single processor use
# for tile in tile_list:
#
# aggregate_results_to_4_km.rewindow(til, no_upload)
# Converts the existing (per ha) values to per pixel values (e.g., emissions/ha to emissions/pixel)
# and sums those values in each 400x400 pixel window.
# The sum for each 400x400 pixel window is stored in a 2D array, which is then converted back into a raster at
# 0.1x0.1 degree resolution (approximately 10m in the tropics).
# Each pixel in that raster is the sum of the 30m pixels converted to value/pixel (instead of value/ha).
# The 0.1x0.1 degree tile is output.
# For multiprocessor use. This used about 450 GB of memory with count/2, it's okay on an r4.16xlarge
if cn.count == 96:
if sensit_type == 'biomass_swap':
processes = 10 # 10 processors = XXX GB peak
else:
processes = 12 # 16 processors = 180 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out)
else:
processes = 8
uu.print_log('Conversion to per pixel and aggregate max processors=',
processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(aggregate_results_to_4_km.aggregate,
thresh=thresh,
sensit_type=sensit_type,
no_upload=no_upload), tile_list)
pool.close()
pool.join()
# # For single processor use
# for tile in tile_list:
#
# aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type, no_upload)
# Makes a vrt of all the output 10x10 tiles (10 km resolution)
out_vrt = "{}_0_4deg.vrt".format(pattern)
os.system('gdalbuildvrt -tr 0.04 0.04 {0} *{1}_0_4deg*.tif'.format(
out_vrt, pattern))
# Creates the output name for the 10km map
out_pattern = uu.name_aggregated_output(download_pattern_name, thresh,
sensit_type)
uu.print_log(out_pattern)
# Produces a single raster of all the 10x10 tiles (0.4 degree resolution)
cmd = [
'gdalwarp', '-t_srs', "EPSG:4326", '-overwrite', '-dstnodata', '0',
'-co', 'COMPRESS=LZW', '-tr', '0.04', '0.04', out_vrt,
'{}.tif'.format(out_pattern)
]
uu.log_subprocess_output_full(cmd)
# Adds metadata tags to output rasters
uu.add_universal_metadata_tags('{0}.tif'.format(out_pattern),
sensit_type)
# Units are different for annual removal factor, so metadata has to reflect that
if 'annual_removal_factor' in out_pattern:
cmd = [
'gdal_edit.py', '-mo',
'units=Mg aboveground carbon/yr/pixel, where pixels are 0.04x0.04 degrees',
'-mo',
'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
'-mo', 'extent=Global', '-mo',
'scale=negative values are removals', '-mo',
'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation'
.format(thresh), '{0}.tif'.format(out_pattern)
]
uu.log_subprocess_output_full(cmd)
else:
cmd = [
'gdal_edit.py', '-mo',
'units=Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees',
'-mo',
'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
'-mo', 'extent=Global', '-mo',
'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation'
.format(thresh), '{0}.tif'.format(out_pattern)
]
uu.log_subprocess_output_full(cmd)
# If no_upload flag is not activated, output is uploaded
if not no_upload:
uu.print_log("Tiles processed. Uploading to s3 now...")
uu.upload_final_set(output_dir_list[0], out_pattern)
# Cleans up the folder before starting on the next raster type
vrtList = glob.glob('*vrt')
for vrt in vrtList:
os.remove(vrt)
for tile_name in tile_list:
tile_id = uu.get_tile_id(tile_name)
# os.remove('{0}_{1}.tif'.format(tile_id, pattern))
os.remove('{0}_{1}_rewindow.tif'.format(tile_id, pattern))
os.remove('{0}_{1}_0_4deg.tif'.format(tile_id, pattern))
# Compares the net flux from the standard model and the sensitivity analysis in two ways.
# This does not work for compariing the raw outputs of the biomass_swap and US_removals sensitivity models because their
# extents are different from the standard model's extent (tropics and US tiles vs. global).
# Thus, in order to do this comparison, you need to clip the standard model net flux and US_removals net flux to
# the outline of the US and clip the standard model net flux to the extent of JPL AGB2000.
# Then, manually upload the clipped US_removals and biomass_swap net flux rasters to the spot machine and the
# code below should work.
if sensit_type not in [
'std', 'biomass_swap', 'US_removals', 'legal_Amazon_loss'
]:
if std_net_flux:
uu.print_log(
"Standard aggregated flux results provided. Creating comparison maps."
)
# Copies the standard model aggregation outputs to s3. Only net flux is used, though.
uu.s3_file_download(std_net_flux, cn.docker_base_dir, sensit_type)
# Identifies the standard model net flux map
std_aggreg_flux = os.path.split(std_net_flux)[1]
try:
# Identifies the sensitivity model net flux map
sensit_aggreg_flux = glob.glob(
'net_flux_Mt_CO2e_*{}*'.format(sensit_type))[0]
uu.print_log("Standard model net flux:", std_aggreg_flux)
uu.print_log("Sensitivity model net flux:", sensit_aggreg_flux)
except:
uu.print_log(
'Cannot do comparison. One of the input flux tiles is not valid. Verify that both net flux rasters are on the spot machine.'
)
uu.print_log(
"Creating map of percent difference between standard and {} net flux"
.format(sensit_type))
aggregate_results_to_4_km.percent_diff(std_aggreg_flux,
sensit_aggreg_flux,
sensit_type, no_upload)
uu.print_log(
"Creating map of which pixels change sign and which stay the same between standard and {}"
.format(sensit_type))
aggregate_results_to_4_km.sign_change(std_aggreg_flux,
sensit_aggreg_flux,
sensit_type, no_upload)
# If no_upload flag is not activated, output is uploaded
if not no_upload:
uu.upload_final_set(output_dir_list[0],
cn.pattern_aggreg_sensit_perc_diff)
uu.upload_final_set(output_dir_list[0],
cn.pattern_aggreg_sensit_sign_change)
else:
uu.print_log(
"No standard aggregated flux results provided. Not creating comparison maps."
)
