def tile_names(tile_id, sensit_type): # Names of the loss, gain, and model extent tiles if sensit_type == 'legal_Amazon_loss': loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed) else: loss = '{0}.tif'.format(tile_id) gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id) model_extent = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_model_extent) return loss, gain, model_extent
def tile_names(tile_id, sensit_type): # Names of the input files loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed) gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id) extent = '{0}_{1}.tif'.format( tile_id, cn.pattern_Brazil_forest_extent_2000_processed) biomass = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_WHRC_biomass_2000_non_mang_non_planted) return loss, gain, extent, biomass
def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, output_pattern_list, no_upload): # Converts the forest age category decision tree output values to the three age categories-- # 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years # These are five digits so they can easily be added to the four digits of the continent-ecozone code to make unique codes # for each continent-ecozone-age combination. # The key in the dictionary is the forest age category decision tree endpoints. age_dict = {0: 0, 1: 10000, 2: 20000, 3: 30000} uu.print_log("Processing:", tile_id) # Start time start = datetime.datetime.now() # Names of the forest age category and continent-ecozone tiles age_cat = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_age_cat_IPCC) cont_eco = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cont_eco_processed) # Names of the output natural forest gain rate tiles (above and belowground) AGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0]) BGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[1]) AGB_IPCC_default_gain_stdev = '{0}_{1}.tif'.format(tile_id, output_pattern_list[2]) uu.print_log( " Creating IPCC default biomass gain rates and standard deviation for {}" .format(tile_id)) # Opens the input tiles if they exist. kips tile if either input doesn't exist. try: age_cat_src = rasterio.open(age_cat) uu.print_log(" Age category tile found for {}".format(tile_id)) except: return uu.print_log( " No age category tile found for {}. Skipping tile.".format( tile_id)) try: cont_eco_src = rasterio.open(cont_eco) uu.print_log(" Continent-ecozone tile found for {}".format(tile_id)) except: return uu.print_log( " No continent-ecozone tile found for {}. Skipping tile.".format( tile_id)) # Grabs metadata about the continent ecozone tile, like its location/projection/cellsize kwargs = cont_eco_src.meta # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory windows = cont_eco_src.block_windows(1) # Updates kwargs for the output dataset. # Need to update data type to float 32 so that it can handle fractional gain rates kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0, dtype='float32') # The output files, aboveground and belowground biomass gain rates dst_above = rasterio.open(AGB_IPCC_default_gain_rate, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_above, sensit_type) dst_above.update_tags( units='megagrams aboveground biomass (AGB or dry matter)/ha/yr') dst_above.update_tags( source='IPCC Guidelines 2019 refinement, forest section, Table 4.9') dst_above.update_tags( extent= 'Full model extent, even though these rates will not be used over the full model extent' ) dst_below = rasterio.open(BGB_IPCC_default_gain_rate, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_below, sensit_type) dst_below.update_tags( units='megagrams belowground biomass (AGB or dry matter)/ha/yr') dst_below.update_tags( source='IPCC Guidelines 2019 refinement, forest section, Table 4.9') dst_below.update_tags( extent= 'Full model extent, even though these rates will not be used over the full model extent' ) dst_stdev_above = rasterio.open(AGB_IPCC_default_gain_stdev, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_stdev_above, sensit_type) dst_stdev_above.update_tags( units= 'standard deviation, in terms of megagrams aboveground biomass (AGB or dry matter)/ha/yr' ) dst_stdev_above.update_tags( source='IPCC Guidelines 2019 refinement, forest section, Table 4.9') dst_stdev_above.update_tags( extent= 'Full model extent, even though these standard deviations will not be used over the full model extent' ) # Iterates across the windows (1 pixel strips) of the input tiles for idx, window in windows: # Creates a processing window for each input raster try: cont_eco_window = cont_eco_src.read(1, window=window) except: cont_eco_window = np.zeros((window.height, window.width), dtype='uint8') try: age_cat_window = age_cat_src.read(1, window=window) except: age_cat_window = np.zeros((window.height, window.width), dtype='uint8') # Recodes the input forest age category array with 10 different decision tree end values into the 3 actual age categories age_recode = np.vectorize(age_dict.get)(age_cat_window) # Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes cont_eco_age = cont_eco_window + age_recode ## Aboveground removal factors # Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates gain_rate_AGB = cont_eco_age.astype('float32') # Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to # get annual gain rates (metric tons aboveground biomass/yr) for each pixel for key, value in gain_table_dict.items(): gain_rate_AGB[gain_rate_AGB == key] = value # Writes the output window to the output file dst_above.write_band(1, gain_rate_AGB, window=window) ## Belowground removal factors # Calculates belowground annual removal rates gain_rate_BGB = gain_rate_AGB * cn.below_to_above_non_mang # Writes the output window to the output file dst_below.write_band(1, gain_rate_BGB, window=window) ## Aboveground removal factor standard deviation # Converts the continent-ecozone array to float so that the values can be replaced with fractional standard deviations gain_stdev_AGB = cont_eco_age.astype('float32') # Applies the dictionary of continent-ecozone-age gain rate standard deviations to the continent-ecozone-age array to # get annual gain rate standard deviations (metric tons aboveground biomass/yr) for each pixel for key, value in stdev_table_dict.items(): gain_stdev_AGB[gain_stdev_AGB == key] = value # Writes the output window to the output file dst_stdev_above.write_band(1, gain_stdev_AGB, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_pattern_list[0], no_upload)
def legal_Amazon_forest_age_category(tile_id, sensit_type, output_pattern): # Start time start = datetime.datetime.now() loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed) gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id) extent = '{0}_{1}.tif'.format( tile_id, cn.pattern_Brazil_forest_extent_2000_processed) biomass = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_WHRC_biomass_2000_non_mang_non_planted) plantations = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_planted_forest_type_unmasked) mangroves = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_mangrove_biomass_2000) # Opens biomass tile with rasterio.open(loss) as loss_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = loss_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = loss_src.block_windows(1) # Opens tiles gain_src = rasterio.open(gain) extent_src = rasterio.open(extent) biomass_src = rasterio.open(biomass) # Checks whether there are mangrove or planted forest tiles. If so, they are opened. try: plantations_src = rasterio.open(plantations) uu.print_log( " Planted forest tile found for {}".format(tile_id)) except: uu.print_log(" No planted forest tile for {}".format(tile_id)) try: mangroves_src = rasterio.open(mangroves) uu.print_log(" Mangrove tile found for {}".format(tile_id)) except: uu.print_log(" No mangrove tile for {}".format(tile_id)) # Updates kwargs for the output dataset kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) # Opens the output tile, giving it the arguments of the input tiles dst = rasterio.open('{0}_{1}.tif'.format(tile_id, output_pattern), 'w', **kwargs) # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: # Creates windows for each input raster loss_window = loss_src.read(1, window=window) gain_window = gain_src.read(1, window=window) extent_window = extent_src.read(1, window=window) biomass_window = biomass_src.read(1, window=window) # Create a 0s array for the output dst_data = np.zeros((window.height, window.width), dtype='uint8') # No change pixels (no loss or gain) dst_data[np.where((biomass_window > 0) & (extent_window == 1) & (loss_window == 0))] = 3 # primary forest # Loss-only pixels dst_data[np.where((biomass_window > 0) & (extent_window == 1) & (loss_window > 0))] = 6 # primary forest # Loss-and-gain pixels dst_data[np.where((extent_window == 1) & (gain_window == 1) & (loss_window > 0))] = 8 # young secondary forest if os.path.exists(mangroves): mangroves_window = mangroves_src.read(1, window=window) dst_data = np.ma.masked_where( mangroves_window != 0, dst_data).filled(0).astype('uint8') if os.path.exists(plantations): plantations_window = plantations_src.read(1, window=window) dst_data = np.ma.masked_where( plantations_window != 0, dst_data).filled(0).astype('uint8') # Writes the output window to the output dst.write_band(1, dst_data, window=window) uu.end_of_fx_summary(start, tile_id, output_pattern)
def net_calc(tile_id, pattern, sensit_type): uu.print_log("Calculating net flux for", tile_id) # Start time start = datetime.datetime.now() # Names of the gain and emissions tiles removals_in = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cumul_gain_AGCO2_BGCO2_all_types) emissions_in = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil) # Output net emissions file net_flux = '{0}_{1}.tif'.format(tile_id, pattern) try: removals_src = rasterio.open(removals_in) # Grabs metadata about the tif, like its location/projection/cellsize kwargs = removals_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = removals_src.block_windows(1) uu.print_log(" Gross removals tile {} found".format(removals_in)) except: uu.print_log(" No gross removals tile {} found".format(removals_in)) try: emissions_src = rasterio.open(emissions_in) # Grabs metadata about the tif, like its location/projection/cellsize kwargs = emissions_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = emissions_src.block_windows(1) uu.print_log(" Gross emissions tile {} found".format(emissions_in)) except: uu.print_log(" No gross emissions tile {} found".format(emissions_in)) # Skips the tile if there is neither a gross emissions nor a gross removals tile. # This should only occur for biomass_swap sensitivity analysis, which gets its net flux tile list from # the JPL tile list (some tiles of which have neither emissions nor removals), rather than the union of # emissions and removals tiles. try: kwargs.update( driver='GTiff', count=1, compress='lzw', nodata=0, dtype='float32' ) except: uu.print_log("No gross emissions or gross removals for {}. Skipping tile.".format(tile_id)) return # Opens the output tile, giving it the arguments of the input tiles net_flux_dst = rasterio.open(net_flux, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(net_flux_dst, sensit_type) net_flux_dst.update_tags( units='Mg CO2e/ha over model duration (2001-20{})'.format(cn.loss_years)) net_flux_dst.update_tags( source='Gross emissions - gross removals') net_flux_dst.update_tags( extent='Model extent') net_flux_dst.update_tags( scale='Negative values are net sinks. Positive values are net sources.') # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: # Creates windows for each input tile try: removals_window = removals_src.read(1, window=window).astype('float32') except: removals_window = np.zeros((window.height, window.width)).astype('float32') try: emissions_window = emissions_src.read(1, window=window).astype('float32') except: emissions_window = np.zeros((window.height, window.width)).astype('float32') # Subtracts gain that from loss dst_data = emissions_window - removals_window net_flux_dst.write_band(1, dst_data, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, pattern)
def annual_gain_rate(tile_id, sensit_type, output_pattern_list, gain_above_dict, gain_below_dict, stdev_dict): uu.print_log("Processing:", tile_id) # Start time start = datetime.datetime.now() # This is only needed for testing, when a list of tiles might include ones without mangroves. # When the full model is being run, only tiles with mangroves are included. mangrove_biomass_tile_list = uu.tile_list_s3(cn.mangrove_biomass_2000_dir) if tile_id not in mangrove_biomass_tile_list: uu.print_log( "{} does not contain mangroves. Skipping tile.".format(tile_id)) return # Name of the input files mangrove_biomass = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_mangrove_biomass_2000) cont_eco = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cont_eco_processed) # Names of the output aboveground and belowground mangrove gain rate tiles AGB_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0]) BGB_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[1]) AGB_gain_stdev = '{0}_{1}.tif'.format(tile_id, output_pattern_list[2]) uu.print_log( " Reading input files and creating aboveground and belowground biomass gain rates for {}" .format(tile_id)) cont_eco_src = rasterio.open(cont_eco) mangrove_AGB_src = rasterio.open(mangrove_biomass) # Grabs metadata about the tif, like its location/projection/cellsize kwargs = cont_eco_src.meta # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory windows = cont_eco_src.block_windows(1) # Updates kwargs for the output dataset. # Need to update data type to float 32 so that it can handle fractional gain rates kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0, dtype='float32') dst_above = rasterio.open(AGB_gain_rate, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_above, sensit_type) dst_above.update_tags( units='megagrams aboveground biomass (AGB or dry matter)/ha/yr') dst_above.update_tags( source='IPCC Guidelines, 2013 Coastal Wetlands Supplement, Table 4.4') dst_above.update_tags( extent= 'Simard et al. 2018, based on Giri et al. 2011 (Global Ecol. Biogeogr.) mangrove extent' ) dst_below = rasterio.open(BGB_gain_rate, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_below, sensit_type) dst_below.update_tags( units='megagrams belowground biomass (BGB or dry matter)/ha/yr') dst_below.update_tags( source='IPCC Guidelines, 2013 Coastal Wetlands Supplement, Table 4.4') dst_below.update_tags( extent= 'Simard et al. 2018, based on Giri et al. 2011 (Global Ecol. Biogeogr.) mangrove extent' ) dst_stdev_above = rasterio.open(AGB_gain_stdev, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_stdev_above, sensit_type) dst_stdev_above.update_tags( units= 'standard deviation, in terms of megagrams aboveground biomass (AGB or dry matter)/ha/yr' ) dst_stdev_above.update_tags( source='IPCC Guidelines, 2013 Coastal Wetlands Supplement, Table 4.4') dst_stdev_above.update_tags( extent= 'Simard et al. 2018, based on Giri et al. 2011 (Global Ecol. Biogeogr.) mangrove extent' ) # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: # Creates windows for each input raster cont_eco = cont_eco_src.read(1, window=window) mangrove_AGB = mangrove_AGB_src.read(1, window=window) # Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates. # Creates two copies: one for aboveground gain and one for belowground gain. # Creating only one copy of the cont_eco raster made it so that belowground gain rates weren't being # written correctly for some reason. cont_eco_above = cont_eco.astype('float32') cont_eco_below = cont_eco.astype('float32') cont_eco_stdev = cont_eco.astype('float32') # Reclassifies mangrove biomass to 1 or 0 to make a mask of mangrove pixels. # Ultimately, only these pixels (ones with mangrove biomass) will get values. mangrove_AGB[mangrove_AGB > 0] = 1 # Applies the dictionary of continent-ecozone aboveground gain rates to the continent-ecozone array to # get annual aboveground gain rates (metric tons aboveground biomass/yr) for each pixel for key, value in gain_above_dict.items(): cont_eco_above[cont_eco_above == key] = value # Masks out pixels without mangroves, leaving gain rates in only pixels with mangroves dst_above_data = cont_eco_above * mangrove_AGB # Writes the output window to the output dst_above.write_band(1, dst_above_data, window=window) # Same as above but for belowground gain rates for key, value in gain_below_dict.items(): cont_eco_below[cont_eco_below == key] = value dst_below_data = cont_eco_below * mangrove_AGB dst_below.write_band(1, dst_below_data, window=window) # Applies the dictionary of continent-ecozone aboveground gain rate standard deviations to the continent-ecozone array to # get annual aboveground gain rate standard deviations (metric tons aboveground biomass/yr) for each pixel for key, value in stdev_dict.items(): cont_eco_stdev[cont_eco_stdev == key] = value # Masks out pixels without mangroves, leaving gain rates in only pixels with mangroves dst_stdev = cont_eco_stdev * mangrove_AGB # Writes the output window to the output dst_stdev_above.write_band(1, dst_stdev, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_pattern_list[0])
def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type, no_upload): uu.print_log("Assigning forest age categories:", tile_id) # Start time start = datetime.datetime.now() # Gets the bounding coordinates of each tile. Needed to determine if the tile is in the tropics (within 30 deg of the equator) xmin, ymin, xmax, ymax = uu.coords(tile_id) # Default value is that the tile is not in the tropics tropics = 0 # Criteria for assigning a tile to the tropics if (ymax > -30) & (ymax <= 30) : tropics = 1 uu.print_log(" Tile {} in tropics:".format(tile_id), tropics) # Names of the input tiles gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id) model_extent = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_model_extent) ifl_primary = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_ifl_primary) cont_eco = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cont_eco_processed) # Biomass tile name depends on the sensitivity analysis if sensit_type == 'biomass_swap': biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_JPL_unmasked_processed) uu.print_log("Using JPL biomass tile for {} sensitivity analysis".format(sensit_type)) else: biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_WHRC_biomass_2000_unmasked) uu.print_log("Using WHRC biomass tile for {} sensitivity analysis".format(sensit_type)) if sensit_type == 'legal_Amazon_loss': loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed) uu.print_log("Using PRODES loss tile {0} for {1} sensitivity analysis".format(tile_id, sensit_type)) elif sensit_type == 'Mekong_loss': loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Mekong_loss_processed) else: loss = '{0}_{1}.tif'.format(cn.pattern_loss, tile_id) uu.print_log("Using Hansen loss tile {0} for {1} model run".format(tile_id, sensit_type)) uu.print_log(" Assigning age categories") # Opens biomass tile with rasterio.open(model_extent) as model_extent_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = model_extent_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = model_extent_src.block_windows(1) # Opens the input tiles if they exist try: cont_eco_src = rasterio.open(cont_eco) uu.print_log(" Continent-ecozone tile found for {}".format(tile_id)) except: uu.print_log(" No continent-ecozone tile found for {}".format(tile_id)) try: gain_src = rasterio.open(gain) uu.print_log(" Gain tile found for {}".format(tile_id)) except: uu.print_log(" No gain tile found for {}".format(tile_id)) try: biomass_src = rasterio.open(biomass) uu.print_log(" Biomass tile found for {}".format(tile_id)) except: uu.print_log(" No biomass tile found for {}".format(tile_id)) try: loss_src = rasterio.open(loss) uu.print_log(" Loss tile found for {}".format(tile_id)) except: uu.print_log(" No loss tile found for {}".format(tile_id)) try: ifl_primary_src = rasterio.open(ifl_primary) uu.print_log(" IFL-primary forest tile found for {}".format(tile_id)) except: uu.print_log(" No IFL-primary forest tile found for {}".format(tile_id)) # Updates kwargs for the output dataset kwargs.update( driver='GTiff', count=1, compress='lzw', nodata=0 ) # Opens the output tile, giving it the arguments of the input tiles dst = rasterio.open('{0}_{1}.tif'.format(tile_id, pattern), 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst, sensit_type) dst.update_tags( key='1: young (<20 year) secondary forest; 2: old (>20 year) secondary forest; 3: primary forest or IFL') dst.update_tags( source='Decision tree that uses Hansen gain and loss, IFL/primary forest extent, and aboveground biomass to assign an age category') dst.update_tags( extent='Full model extent, even though these age categories will not be used over the full model extent. They apply to just the rates from IPCC defaults.') uu.print_log(" Assigning IPCC age categories for", tile_id) # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: # Creates windows for each input raster. Only model_extent_src is guaranteed to exist model_extent_window = model_extent_src.read(1, window=window) try: loss_window = loss_src.read(1, window=window) except: loss_window = np.zeros((window.height, window.width), dtype='uint8') try: gain_window = gain_src.read(1, window=window) except: gain_window = np.zeros((window.height, window.width), dtype='uint8') try: cont_eco_window = cont_eco_src.read(1, window=window) except: cont_eco_window = np.zeros((window.height, window.width), dtype='uint8') try: biomass_window = biomass_src.read(1, window=window) except: biomass_window = np.zeros((window.height, window.width), dtype='float32') try: ifl_primary_window = ifl_primary_src.read(1, window=window) except: ifl_primary_window = np.zeros((window.height, window.width), dtype='uint8') # Creates a numpy array that has the <=20 year secondary forest growth rate x 20 # based on the continent-ecozone code of each pixel (the dictionary). # This is used to assign pixels to the correct age category. gain_20_years = np.vectorize(gain_table_dict.get)(cont_eco_window)*20 # Create a 0s array for the output dst_data = np.zeros((window.height, window.width), dtype='uint8') # Logic tree for assigning age categories begins here # Code 1 = young (<20 years) secondary forest, code 2 = old (>20 year) secondary forest, code 3 = primary forest # model_extent_window ensures that there is both biomass and tree cover in 2000 OR mangroves OR tree cover gain # WITHOUT pre-2000 plantations # For every model version except legal_Amazon_loss sensitivity analysis, which has its own rules about age assignment if sensit_type != 'legal_Amazon_loss': # No change pixels- no loss or gain if tropics == 0: dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0))] = 2 if tropics == 1: dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0) & (ifl_primary_window != 1))] = 2 dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0) & (ifl_primary_window == 1))] = 3 # Loss-only pixels dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window != 1) & (biomass_window <= gain_20_years))] = 1 dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window != 1) & (biomass_window > gain_20_years))] = 2 dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window ==1))] = 3 # Gain-only pixels # If there is gain, the pixel doesn't need biomass or canopy cover. It just needs to be outside of plantations and mangroves. # The role of model_extent_window here is to exclude the pre-2000 plantations. dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window == 0))] = 1 # Pixels with loss and gain # If there is gain with loss, the pixel doesn't need biomass or canopy cover. It just needs to be outside of plantations and mangroves. # The role of model_extent_window here is to exclude the pre-2000 plantations. dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > (cn.gain_years)))] = 1 dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > 0) & (loss_window <= (cn.gain_years/2)))] = 1 dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > (cn.gain_years/2)) & (loss_window <= cn.gain_years))] = 1 # For legal_Amazon_loss sensitivity analysis else: # Non-loss pixels (could have gain or not. Assuming that if within PRODES extent in 2000, there can't be # gain, so it's a faulty detection. Thus, gain-only pixels are ignored and become part of no change.) dst_data[np.where((model_extent_window == 1) & (loss_window == 0))] = 3 # primary forest # Loss-only pixels dst_data[np.where((model_extent_window == 1) & (loss_window > 0) & (gain_window == 0))] = 3 # primary forest # Loss-and-gain pixels dst_data[np.where((model_extent_window == 1) & (loss_window > 0) & (gain_window == 1))] = 2 # young secondary forest # Writes the output window to the output dst.write_band(1, dst_data, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def annual_gain_rate_AGC_BGC_all_forest_types(tile_id, output_pattern_list, sensit_type, no_upload): uu.print_log("Mapping removal rate source and AGB and BGB removal rates:", tile_id) # Start time start = datetime.datetime.now() # Names of the input tiles # Removal factors model_extent = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_model_extent) mangrove_AGB = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_AGB_mangrove) mangrove_BGB = '{0}_{1}.tif'.format(tile_id, cn.pattern_annual_gain_BGB_mangrove) europe_AGC_BGC = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe) plantations_AGC_BGC = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked) us_AGC_BGC = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_annual_gain_AGC_BGC_natrl_forest_US) young_AGC = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_annual_gain_AGC_natrl_forest_young) age_category = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_age_cat_IPCC) ipcc_AGB_default = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_annual_gain_AGB_IPCC_defaults) # Removal factor standard deviations mangrove_AGB_stdev = '{0}_{1}.tif'.format( tile_id, cn.pattern_stdev_annual_gain_AGB_mangrove) europe_AGC_BGC_stdev = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe) plantations_AGC_BGC_stdev = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked) us_AGC_BGC_stdev = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US) young_AGC_stdev = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young) ipcc_AGB_default_stdev = uu.sensit_tile_rename( sensit_type, tile_id, cn.pattern_stdev_annual_gain_AGB_IPCC_defaults) # Names of the output tiles removal_forest_type = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0]) annual_gain_AGC_all_forest_types = '{0}_{1}.tif'.format( tile_id, output_pattern_list[1]) annual_gain_BGC_all_forest_types = '{0}_{1}.tif'.format( tile_id, output_pattern_list[2]) annual_gain_AGC_BGC_all_forest_types = '{0}_{1}.tif'.format( tile_id, output_pattern_list[3] ) # Not used further in the model. Created just for reference. stdev_annual_gain_AGC_all_forest_types = '{0}_{1}.tif'.format( tile_id, output_pattern_list[4]) # Opens biomass tile with rasterio.open(model_extent) as model_extent_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = model_extent_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = model_extent_src.block_windows(1) # Updates kwargs for the output dataset kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) # Checks whether there are mangrove or planted forest tiles. If so, they are opened. try: mangrove_AGB_src = rasterio.open(mangrove_AGB) mangrove_BGB_src = rasterio.open(mangrove_BGB) mangrove_AGB_stdev_src = rasterio.open(mangrove_AGB_stdev) uu.print_log( " Mangrove tiles (AGB and BGB) for {}".format(tile_id)) except: uu.print_log(" No mangrove tile for {}".format(tile_id)) try: europe_AGC_BGC_src = rasterio.open(europe_AGC_BGC) europe_AGC_BGC_stdev_src = rasterio.open(europe_AGC_BGC_stdev) uu.print_log( " Europe removal factor tile for {}".format(tile_id)) except: uu.print_log( " No Europe removal factor tile for {}".format(tile_id)) try: plantations_AGC_BGC_src = rasterio.open(plantations_AGC_BGC) plantations_AGC_BGC_stdev_src = rasterio.open( plantations_AGC_BGC_stdev) uu.print_log(" Planted forest tile for {}".format(tile_id)) except: uu.print_log(" No planted forest tile for {}".format(tile_id)) try: us_AGC_BGC_src = rasterio.open(us_AGC_BGC) us_AGC_BGC_stdev_src = rasterio.open(us_AGC_BGC_stdev) uu.print_log(" US removal factor tile for {}".format(tile_id)) except: uu.print_log( " No US removal factor tile for {}".format(tile_id)) try: young_AGC_src = rasterio.open(young_AGC) young_AGC_stdev_src = rasterio.open(young_AGC_stdev) uu.print_log( " Young forest removal factor tile for {}".format(tile_id)) except: uu.print_log( " No young forest removal factor tile for {}".format( tile_id)) try: age_category_src = rasterio.open(age_category) uu.print_log(" Age category tile for {}".format(tile_id)) except: uu.print_log(" No age category tile for {}".format(tile_id)) try: ipcc_AGB_default_src = rasterio.open(ipcc_AGB_default) ipcc_AGB_default_stdev_src = rasterio.open(ipcc_AGB_default_stdev) uu.print_log( " IPCC default removal rate tile for {}".format(tile_id)) except: uu.print_log( " No IPCC default removal rate tile for {}".format(tile_id)) # Opens the output tile, giving it the arguments of the input tiles removal_forest_type_dst = rasterio.open(removal_forest_type, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(removal_forest_type_dst, sensit_type) removal_forest_type_dst.update_tags( key= '6: mangroves. 5: European-specific rates. 4: planted forests. 3: US-specific rates. 2: young (<20 year) secondary forests. 1: old (>20 year) secondary forests and primary forests. Priority goes to the highest number.' ) removal_forest_type_dst.update_tags( source= 'Mangroves: IPCC wetlands supplement. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9' ) removal_forest_type_dst.update_tags(extent='Full model extent') # Updates kwargs for the removal rate outputs-- just need to change datatype kwargs.update(dtype='float32') annual_gain_AGC_all_forest_types_dst = rasterio.open( annual_gain_AGC_all_forest_types, 'w', **kwargs) annual_gain_BGC_all_forest_types_dst = rasterio.open( annual_gain_BGC_all_forest_types, 'w', **kwargs) annual_gain_AGC_BGC_all_forest_types_dst = rasterio.open( annual_gain_AGC_BGC_all_forest_types, 'w', **kwargs) stdev_annual_gain_AGC_all_forest_types_dst = rasterio.open( stdev_annual_gain_AGC_all_forest_types, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(annual_gain_AGC_all_forest_types_dst, sensit_type) annual_gain_AGC_all_forest_types_dst.update_tags( units='megagrams aboveground carbon/ha/yr') annual_gain_AGC_all_forest_types_dst.update_tags( source= 'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9' ) annual_gain_AGC_all_forest_types_dst.update_tags( extent='Full model extent') # Adds metadata tags to the output raster uu.add_rasterio_tags(annual_gain_BGC_all_forest_types_dst, sensit_type) annual_gain_BGC_all_forest_types_dst.update_tags( units='megagrams belowground carbon/ha/yr') annual_gain_BGC_all_forest_types_dst.update_tags( source= 'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9' ) annual_gain_BGC_all_forest_types_dst.update_tags( extent='Full model extent') # Adds metadata tags to the output raster uu.add_rasterio_tags(annual_gain_AGC_BGC_all_forest_types_dst, sensit_type) annual_gain_AGC_BGC_all_forest_types_dst.update_tags( units='megagrams aboveground + belowground carbon/ha/yr') annual_gain_AGC_BGC_all_forest_types_dst.update_tags( source= 'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9' ) annual_gain_AGC_BGC_all_forest_types_dst.update_tags( extent='Full model extent') # Adds metadata tags to the output raster uu.add_rasterio_tags(stdev_annual_gain_AGC_all_forest_types_dst, sensit_type) stdev_annual_gain_AGC_all_forest_types_dst.update_tags( units= 'standard deviation for removal factor, in terms of megagrams aboveground carbon/ha/yr' ) stdev_annual_gain_AGC_all_forest_types_dst.update_tags( source= 'Mangroves: IPCC wetlands supplement Table 4.4. Europe: Liz Goldman. Planted forests: Spatial Database of Planted Forests. USA: US FIA, via Rich Birdsey. Young natural forests: Cook-Patton et al. 2020. Old natural forests: IPCC Forests table 4.9' ) stdev_annual_gain_AGC_all_forest_types_dst.update_tags( extent='Full model extent') uu.print_log( " Creating removal model forest type tile, AGC removal factor tile, BGC removal factor tile, and AGC removal factor standard deviation tile for {}" .format(tile_id)) # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: model_extent_window = model_extent_src.read(1, window=window) # Output rasters' windows removal_forest_type_window = np.zeros( (window.height, window.width), dtype='uint8') annual_gain_AGC_all_forest_types_window = np.zeros( (window.height, window.width), dtype='float32') annual_gain_BGC_all_forest_types_window = np.zeros( (window.height, window.width), dtype='float32') stdev_annual_gain_AGC_all_forest_types_window = np.zeros( (window.height, window.width), dtype='float32') try: age_category_window = age_category_src.read(1, window=window) except: age_category_window = np.zeros((window.height, window.width), dtype='uint8') # Lowest priority try: ipcc_AGB_default_rate_window = ipcc_AGB_default_src.read( 1, window=window) ipcc_AGB_default_stdev_window = ipcc_AGB_default_stdev_src.read( 1, window=window) # In no_primary_gain, the AGB_default_rate_window = 0, so primary forest pixels would not be # assigned a removal forest type and therefore get exclude from the model later. # That is incorrect, so using model_extent as the criterion instead allows the primary forest pixels # that don't have rates under this sensitivity analysis to still be included in the model. # Unfortunately, model_extent is slightly different from the IPCC rate extent (no IPCC rates where # there is no ecozone information), but this is a very small difference and not worth worrying about. if sensit_type == 'no_primary_gain': removal_forest_type_window = np.where( model_extent_window != 0, cn.old_natural_rank, removal_forest_type_window).astype('uint8') else: removal_forest_type_window = np.where( ipcc_AGB_default_rate_window != 0, cn.old_natural_rank, removal_forest_type_window).astype('uint8') annual_gain_AGC_all_forest_types_window = np.where( ipcc_AGB_default_rate_window != 0, ipcc_AGB_default_rate_window * cn.biomass_to_c_non_mangrove, annual_gain_AGC_all_forest_types_window).astype('float32') annual_gain_BGC_all_forest_types_window = np.where( ipcc_AGB_default_rate_window != 0, ipcc_AGB_default_rate_window * cn.biomass_to_c_non_mangrove * cn.below_to_above_non_mang, annual_gain_BGC_all_forest_types_window).astype('float32') stdev_annual_gain_AGC_all_forest_types_window = np.where( ipcc_AGB_default_stdev_window != 0, ipcc_AGB_default_stdev_window * cn.biomass_to_c_non_mangrove, stdev_annual_gain_AGC_all_forest_types_window).astype( 'float32') except: pass try: # young_AGC_rate_window uses > because of the weird NaN in the tiles. If != is used, the young rate NaN overwrites the IPCC arrays young_AGC_rate_window = young_AGC_src.read(1, window=window) young_AGC_stdev_window = young_AGC_stdev_src.read( 1, window=window) # Using the > with the NaN results in non-fatal "RuntimeWarning: invalid value encountered in greater". # This isn't actually a problem, so the "with" statement suppresses it, per https://stackoverflow.com/a/58026329/10839927 with np.errstate(invalid='ignore'): removal_forest_type_window = np.where( (young_AGC_rate_window > 0) & (age_category_window == 1), cn.young_natural_rank, removal_forest_type_window).astype('uint8') annual_gain_AGC_all_forest_types_window = np.where( (young_AGC_rate_window > 0) & (age_category_window == 1), young_AGC_rate_window, annual_gain_AGC_all_forest_types_window).astype( 'float32') annual_gain_BGC_all_forest_types_window = np.where( (young_AGC_rate_window > 0) & (age_category_window == 1), young_AGC_rate_window * cn.below_to_above_non_mang, annual_gain_BGC_all_forest_types_window).astype( 'float32') stdev_annual_gain_AGC_all_forest_types_window = np.where( (young_AGC_stdev_window > 0) & (age_category_window == 1), young_AGC_stdev_window, stdev_annual_gain_AGC_all_forest_types_window).astype( 'float32') except: pass if sensit_type != 'US_removals': try: us_AGC_BGC_rate_window = us_AGC_BGC_src.read(1, window=window) us_AGC_BGC_stdev_window = us_AGC_BGC_stdev_src.read( 1, window=window) removal_forest_type_window = np.where( us_AGC_BGC_rate_window != 0, cn.US_rank, removal_forest_type_window).astype('uint8') annual_gain_AGC_all_forest_types_window = np.where( us_AGC_BGC_rate_window != 0, us_AGC_BGC_rate_window / (1 + cn.below_to_above_non_mang), annual_gain_AGC_all_forest_types_window).astype( 'float32') annual_gain_BGC_all_forest_types_window = np.where( us_AGC_BGC_rate_window != 0, (us_AGC_BGC_rate_window) - (us_AGC_BGC_rate_window / (1 + cn.below_to_above_non_mang)), annual_gain_BGC_all_forest_types_window).astype( 'float32') stdev_annual_gain_AGC_all_forest_types_window = np.where( us_AGC_BGC_stdev_window != 0, us_AGC_BGC_stdev_window / (1 + cn.below_to_above_non_mang), stdev_annual_gain_AGC_all_forest_types_window).astype( 'float32') except: pass try: plantations_AGC_BGC_rate_window = plantations_AGC_BGC_src.read( 1, window=window) plantations_AGC_BGC_stdev_window = plantations_AGC_BGC_stdev_src.read( 1, window=window) removal_forest_type_window = np.where( plantations_AGC_BGC_rate_window != 0, cn.planted_forest_rank, removal_forest_type_window).astype('uint8') annual_gain_AGC_all_forest_types_window = np.where( plantations_AGC_BGC_rate_window != 0, plantations_AGC_BGC_rate_window / (1 + cn.below_to_above_non_mang), annual_gain_AGC_all_forest_types_window).astype('float32') annual_gain_BGC_all_forest_types_window = np.where( plantations_AGC_BGC_rate_window != 0, (plantations_AGC_BGC_rate_window) - (plantations_AGC_BGC_rate_window / (1 + cn.below_to_above_non_mang)), annual_gain_BGC_all_forest_types_window).astype('float32') stdev_annual_gain_AGC_all_forest_types_window = np.where( plantations_AGC_BGC_stdev_window != 0, plantations_AGC_BGC_stdev_window / (1 + cn.below_to_above_non_mang), stdev_annual_gain_AGC_all_forest_types_window).astype( 'float32') except: pass try: europe_AGC_BGC_rate_window = europe_AGC_BGC_src.read( 1, window=window) europe_AGC_BGC_stdev_window = europe_AGC_BGC_stdev_src.read( 1, window=window) removal_forest_type_window = np.where( europe_AGC_BGC_rate_window != 0, cn.europe_rank, removal_forest_type_window).astype('uint8') annual_gain_AGC_all_forest_types_window = np.where( europe_AGC_BGC_rate_window != 0, europe_AGC_BGC_rate_window / (1 + cn.below_to_above_non_mang), annual_gain_AGC_all_forest_types_window).astype('float32') annual_gain_BGC_all_forest_types_window = np.where( europe_AGC_BGC_rate_window != 0, (europe_AGC_BGC_rate_window) - (europe_AGC_BGC_rate_window / (1 + cn.below_to_above_non_mang)), annual_gain_BGC_all_forest_types_window).astype('float32') # NOTE: Nancy Harris thought that the European removal standard deviations were 2x too large, # per email on 8/30/2020. Thus, simplest fix is to leave original tiles 2x too large and # correct them only where composited with other stdev sources. stdev_annual_gain_AGC_all_forest_types_window = np.where( europe_AGC_BGC_stdev_window != 0, (europe_AGC_BGC_stdev_window / 2) / (1 + cn.below_to_above_non_mang), stdev_annual_gain_AGC_all_forest_types_window).astype( 'float32') except: pass # Highest priority try: mangroves_AGB_rate_window = mangrove_AGB_src.read( 1, window=window) mangroves_BGB_rate_window = mangrove_BGB_src.read( 1, window=window) mangroves_AGB_stdev_window = mangrove_AGB_stdev_src.read( 1, window=window) removal_forest_type_window = np.where( mangroves_AGB_rate_window != 0, cn.mangrove_rank, removal_forest_type_window).astype('uint8') annual_gain_AGC_all_forest_types_window = np.where( mangroves_AGB_rate_window != 0, mangroves_AGB_rate_window * cn.biomass_to_c_mangrove, annual_gain_AGC_all_forest_types_window).astype('float32') annual_gain_BGC_all_forest_types_window = np.where( mangroves_BGB_rate_window != 0, mangroves_BGB_rate_window * cn.biomass_to_c_mangrove, annual_gain_BGC_all_forest_types_window).astype('float32') stdev_annual_gain_AGC_all_forest_types_window = np.where( mangroves_AGB_stdev_window != 0, mangroves_AGB_stdev_window * cn.biomass_to_c_mangrove, stdev_annual_gain_AGC_all_forest_types_window).astype( 'float32') except: pass # Masks outputs to model output extent removal_forest_type_window = np.where(model_extent_window == 1, removal_forest_type_window, 0) annual_gain_AGC_all_forest_types_window = np.where( model_extent_window == 1, annual_gain_AGC_all_forest_types_window, 0) annual_gain_BGC_all_forest_types_window = np.where( model_extent_window == 1, annual_gain_BGC_all_forest_types_window, 0) annual_gain_AGC_BGC_all_forest_types_window = annual_gain_AGC_all_forest_types_window + annual_gain_BGC_all_forest_types_window stdev_annual_gain_AGC_all_forest_types_window = np.where( model_extent_window == 1, stdev_annual_gain_AGC_all_forest_types_window, 0) # Writes the outputs window to the output files removal_forest_type_dst.write_band(1, removal_forest_type_window, window=window) annual_gain_AGC_all_forest_types_dst.write_band( 1, annual_gain_AGC_all_forest_types_window, window=window) annual_gain_BGC_all_forest_types_dst.write_band( 1, annual_gain_BGC_all_forest_types_window, window=window) annual_gain_AGC_BGC_all_forest_types_dst.write_band( 1, annual_gain_AGC_BGC_all_forest_types_window, window=window) stdev_annual_gain_AGC_all_forest_types_dst.write_band( 1, stdev_annual_gain_AGC_all_forest_types_window, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_removal_forest_type, no_upload)
def gross_removals_all_forest_types(tile_id, output_pattern_list, sensit_type): uu.print_log("Calculating cumulative CO2 removals:", tile_id) # Start time start = datetime.datetime.now() # Names of the input tiles, modified according to sensitivity analysis gain_rate_AGC = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_annual_gain_AGC_all_types) gain_rate_BGC = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_annual_gain_BGC_all_types) gain_year_count = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_gain_year_count) # Names of the output removal tiles cumulative_gain_AGCO2 = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0]) cumulative_gain_BGCO2 = '{0}_{1}.tif'.format(tile_id, output_pattern_list[1]) cumulative_gain_AGCO2_BGCO2 = '{0}_{1}.tif'.format(tile_id, output_pattern_list[2]) # Opens the input tiles gain_rate_AGC_src = rasterio.open(gain_rate_AGC) gain_rate_BGC_src = rasterio.open(gain_rate_BGC) gain_year_count_src = rasterio.open(gain_year_count) # Grabs metadata for an input tile kwargs = gain_rate_AGC_src.meta # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory windows = gain_rate_AGC_src.block_windows(1) # Updates kwargs for the output dataset. kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) # The output files: aboveground gross removals, belowground gross removals, above+belowground gross removals. Adds metadata tags cumulative_gain_AGCO2_dst = rasterio.open(cumulative_gain_AGCO2, 'w', **kwargs) uu.add_rasterio_tags(cumulative_gain_AGCO2_dst, sensit_type) cumulative_gain_AGCO2_dst.update_tags( units='megagrams aboveground CO2/ha over entire model period') cumulative_gain_AGCO2_dst.update_tags( source='annual removal factors and gain year count') cumulative_gain_AGCO2_dst.update_tags(extent='Full model extent') cumulative_gain_BGCO2_dst = rasterio.open(cumulative_gain_BGCO2, 'w', **kwargs) uu.add_rasterio_tags(cumulative_gain_BGCO2_dst, sensit_type) cumulative_gain_BGCO2_dst.update_tags( units='megagrams belowground CO2/ha over entire model period') cumulative_gain_BGCO2_dst.update_tags( source='annual removal factors and gain year count') cumulative_gain_BGCO2_dst.update_tags(extent='Full model extent') cumulative_gain_AGCO2_BGCO2_dst = rasterio.open( cumulative_gain_AGCO2_BGCO2, 'w', **kwargs) cumulative_gain_AGCO2_BGCO2_dst.update_tags( units= 'megagrams aboveground+belowground CO2/ha over entire model period') cumulative_gain_AGCO2_BGCO2_dst.update_tags( source='annual removal factors and gain year count') cumulative_gain_AGCO2_BGCO2_dst.update_tags(extent='Full model extent') # Iterates across the windows (1 pixel strips) of the input tiles for idx, window in windows: # Creates a processing window for each input raster gain_rate_AGC_window = gain_rate_AGC_src.read(1, window=window) gain_rate_BGC_window = gain_rate_BGC_src.read(1, window=window) gain_year_count_window = gain_year_count_src.read(1, window=window) # Converts the annual removal rate into gross removals cumulative_gain_AGCO2_window = gain_rate_AGC_window * gain_year_count_window * cn.c_to_co2 cumulative_gain_BGCO2_window = gain_rate_BGC_window * gain_year_count_window * cn.c_to_co2 cumulative_gain_AGCO2_BGCO2_window = cumulative_gain_AGCO2_window + cumulative_gain_BGCO2_window # Writes the output windows to the output files cumulative_gain_AGCO2_dst.write_band(1, cumulative_gain_AGCO2_window, window=window) cumulative_gain_BGCO2_dst.write_band(1, cumulative_gain_BGCO2_window, window=window) cumulative_gain_AGCO2_BGCO2_dst.write_band( 1, cumulative_gain_AGCO2_BGCO2_window, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_pattern_list[0])
def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux=None, run_date=None, no_upload=None): os.chdir(cn.docker_base_dir) # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.tile_list_s3(cn.net_flux_dir, sensit_type) uu.print_log(tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # Files to download for this script download_dict = { cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types], cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types], cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil], cn.net_flux_dir: [cn.pattern_net_flux] } # Checks whether the canopy cover argument is valid if thresh < 0 or thresh > 99: uu.exception_log( no_upload, 'Invalid tcd. Please provide an integer between 0 and 99.') if uu.check_aws_creds(): # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles uu.s3_flexible_download(cn.pixel_area_dir, cn.pattern_pixel_area, cn.docker_base_dir, sensit_type, tile_id_list) # Tree cover density, Hansen gain, and mangrove biomass tiles-- necessary for filtering sums to model extent uu.s3_flexible_download(cn.tcd_dir, cn.pattern_tcd, cn.docker_base_dir, sensit_type, tile_id_list) uu.s3_flexible_download(cn.gain_dir, cn.pattern_gain, cn.docker_base_dir, sensit_type, tile_id_list) uu.s3_flexible_download(cn.mangrove_biomass_2000_dir, cn.pattern_mangrove_biomass_2000, cn.docker_base_dir, sensit_type, tile_id_list) uu.print_log("Model outputs to process are:", download_dict) # List of output directories. Modified later for sensitivity analysis. # Output pattern is determined later. output_dir_list = [cn.output_aggreg_dir] # If the model run isn't the standard one, the output directory is changed if sensit_type != 'std': uu.print_log( "Changing output directory and file name pattern based on sensitivity analysis" ) output_dir_list = uu.alter_dirs(sensit_type, output_dir_list) # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # Iterates through the types of tiles to be processed for dir, download_pattern in list(download_dict.items()): download_pattern_name = download_pattern[0] # Downloads input files or entire directories, depending on how many tiles are in the tile_id_list, if AWS credentials are found if uu.check_aws_creds(): uu.s3_flexible_download(dir, download_pattern_name, cn.docker_base_dir, sensit_type, tile_id_list) # Gets an actual tile id to use as a dummy in creating the actual tile pattern local_tile_list = uu.tile_list_spot_machine(cn.docker_base_dir, download_pattern_name) sample_tile_id = uu.get_tile_id(local_tile_list[0]) # Renames the tiles according to the sensitivity analysis before creating dummy tiles. # The renaming function requires a whole tile name, so this passes a dummy time name that is then stripped a few # lines later. tile_id = sample_tile_id # a dummy tile id (but it has to be a real tile id). It is removed later. output_pattern = uu.sensit_tile_rename(sensit_type, tile_id, download_pattern_name) pattern = output_pattern[9:-4] # For sensitivity analysis runs, only aggregates the tiles if they were created as part of the sensitivity analysis if (sensit_type != 'std') & (sensit_type not in pattern): uu.print_log( "{} not a sensitivity analysis output. Skipping aggregation..." .format(pattern)) uu.print_log("") continue # Lists the tiles of the particular type that is being iterates through. # Excludes all intermediate files tile_list = uu.tile_list_spot_machine(".", "{}.tif".format(pattern)) # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring tile_list = [i for i in tile_list if not ('hanson_2013' in i)] tile_list = [i for i in tile_list if not ('rewindow' in i)] tile_list = [i for i in tile_list if not ('0_4deg' in i)] tile_list = [i for i in tile_list if not ('.ovr' in i)] # tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif'] # test tiles uu.print_log("There are {0} tiles to process for pattern {1}".format( str(len(tile_list)), download_pattern) + "\n") uu.print_log("Processing:", dir, "; ", pattern) # Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels, # which is the resolution of the output tiles. This will allow the 30x30 m pixels in each window to be summed. # For multiprocessor use. count/2 used about 400 GB of memory on an r4.16xlarge machine, so that was okay. if cn.count == 96: if sensit_type == 'biomass_swap': processes = 12 # 12 processors = XXX GB peak else: processes = 16 # 12 processors = 140 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out) else: processes = 8 uu.print_log('Rewindow max processors=', processes) pool = multiprocessing.Pool(processes) pool.map( partial(aggregate_results_to_4_km.rewindow, no_upload=no_upload), tile_list) # Added these in response to error12: Cannot allocate memory error. # This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool # Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory pool.close() pool.join() # # For single processor use # for tile in tile_list: # # aggregate_results_to_4_km.rewindow(til, no_upload) # Converts the existing (per ha) values to per pixel values (e.g., emissions/ha to emissions/pixel) # and sums those values in each 400x400 pixel window. # The sum for each 400x400 pixel window is stored in a 2D array, which is then converted back into a raster at # 0.1x0.1 degree resolution (approximately 10m in the tropics). # Each pixel in that raster is the sum of the 30m pixels converted to value/pixel (instead of value/ha). # The 0.1x0.1 degree tile is output. # For multiprocessor use. This used about 450 GB of memory with count/2, it's okay on an r4.16xlarge if cn.count == 96: if sensit_type == 'biomass_swap': processes = 10 # 10 processors = XXX GB peak else: processes = 12 # 16 processors = 180 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out) else: processes = 8 uu.print_log('Conversion to per pixel and aggregate max processors=', processes) pool = multiprocessing.Pool(processes) pool.map( partial(aggregate_results_to_4_km.aggregate, thresh=thresh, sensit_type=sensit_type, no_upload=no_upload), tile_list) pool.close() pool.join() # # For single processor use # for tile in tile_list: # # aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type, no_upload) # Makes a vrt of all the output 10x10 tiles (10 km resolution) out_vrt = "{}_0_4deg.vrt".format(pattern) os.system('gdalbuildvrt -tr 0.04 0.04 {0} *{1}_0_4deg*.tif'.format( out_vrt, pattern)) # Creates the output name for the 10km map out_pattern = uu.name_aggregated_output(download_pattern_name, thresh, sensit_type) uu.print_log(out_pattern) # Produces a single raster of all the 10x10 tiles (0.4 degree resolution) cmd = [ 'gdalwarp', '-t_srs', "EPSG:4326", '-overwrite', '-dstnodata', '0', '-co', 'COMPRESS=LZW', '-tr', '0.04', '0.04', out_vrt, '{}.tif'.format(out_pattern) ] uu.log_subprocess_output_full(cmd) # Adds metadata tags to output rasters uu.add_universal_metadata_tags('{0}.tif'.format(out_pattern), sensit_type) # Units are different for annual removal factor, so metadata has to reflect that if 'annual_removal_factor' in out_pattern: cmd = [ 'gdal_edit.py', '-mo', 'units=Mg aboveground carbon/yr/pixel, where pixels are 0.04x0.04 degrees', '-mo', 'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels', '-mo', 'extent=Global', '-mo', 'scale=negative values are removals', '-mo', 'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation' .format(thresh), '{0}.tif'.format(out_pattern) ] uu.log_subprocess_output_full(cmd) else: cmd = [ 'gdal_edit.py', '-mo', 'units=Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees', '-mo', 'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels', '-mo', 'extent=Global', '-mo', 'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation' .format(thresh), '{0}.tif'.format(out_pattern) ] uu.log_subprocess_output_full(cmd) # If no_upload flag is not activated, output is uploaded if not no_upload: uu.print_log("Tiles processed. Uploading to s3 now...") uu.upload_final_set(output_dir_list[0], out_pattern) # Cleans up the folder before starting on the next raster type vrtList = glob.glob('*vrt') for vrt in vrtList: os.remove(vrt) for tile_name in tile_list: tile_id = uu.get_tile_id(tile_name) # os.remove('{0}_{1}.tif'.format(tile_id, pattern)) os.remove('{0}_{1}_rewindow.tif'.format(tile_id, pattern)) os.remove('{0}_{1}_0_4deg.tif'.format(tile_id, pattern)) # Compares the net flux from the standard model and the sensitivity analysis in two ways. # This does not work for compariing the raw outputs of the biomass_swap and US_removals sensitivity models because their # extents are different from the standard model's extent (tropics and US tiles vs. global). # Thus, in order to do this comparison, you need to clip the standard model net flux and US_removals net flux to # the outline of the US and clip the standard model net flux to the extent of JPL AGB2000. # Then, manually upload the clipped US_removals and biomass_swap net flux rasters to the spot machine and the # code below should work. if sensit_type not in [ 'std', 'biomass_swap', 'US_removals', 'legal_Amazon_loss' ]: if std_net_flux: uu.print_log( "Standard aggregated flux results provided. Creating comparison maps." ) # Copies the standard model aggregation outputs to s3. Only net flux is used, though. uu.s3_file_download(std_net_flux, cn.docker_base_dir, sensit_type) # Identifies the standard model net flux map std_aggreg_flux = os.path.split(std_net_flux)[1] try: # Identifies the sensitivity model net flux map sensit_aggreg_flux = glob.glob( 'net_flux_Mt_CO2e_*{}*'.format(sensit_type))[0] uu.print_log("Standard model net flux:", std_aggreg_flux) uu.print_log("Sensitivity model net flux:", sensit_aggreg_flux) except: uu.print_log( 'Cannot do comparison. One of the input flux tiles is not valid. Verify that both net flux rasters are on the spot machine.' ) uu.print_log( "Creating map of percent difference between standard and {} net flux" .format(sensit_type)) aggregate_results_to_4_km.percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload) uu.print_log( "Creating map of which pixels change sign and which stay the same between standard and {}" .format(sensit_type)) aggregate_results_to_4_km.sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload) # If no_upload flag is not activated, output is uploaded if not no_upload: uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_perc_diff) uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_sign_change) else: uu.print_log( "No standard aggregated flux results provided. Not creating comparison maps." )