def create_mangrove_soil_C(tile_id, no_upload): # Start time start = datetime.datetime.now() # Checks if mangrove biomass exists. If not, it won't create a mangrove soil C tile. if os.path.exists('{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)): uu.print_log("Mangrove aboveground biomass tile found for", tile_id) uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log("Clipping mangrove soil C from mangrove soil vrt for", tile_id) uu.warp_to_Hansen('mangrove_soil_C.vrt', '{0}_mangrove_full_extent.tif'.format(tile_id), xmin, ymin, xmax, ymax, 'Int16') mangrove_soil = '{0}_mangrove_full_extent.tif'.format(tile_id) mangrove_biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000) outname = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id) out = '--outfile={}'.format(outname) calc = '--calc=A*(B>0)' datatype = '--type={}'.format('Int16') uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id) cmd = ['gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite', datatype, '--quiet'] uu.log_subprocess_output_full(cmd) else: uu.print_log("No mangrove aboveground biomass tile for", tile_id) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove', no_upload)
def output_per_pixel(tile_id, input_pattern, output_pattern, sensit_type): uu.print_log("Calculating per pixel values for", tile_id) # Start time start = datetime.datetime.now() # Names of the input biomass and TCD tiles input_model_tile = '{0}_{1}.tif'.format(tile_id, input_pattern) area_tile = 'hanson_2013_area_{}.tif'.format(tile_id) output_model_tile = '{0}_{1}.tif'.format(tile_id, output_pattern) uu.print_log("Converting {} from Mg CO2/ha to Mg CO2/pixel...".format( input_model_tile)) # Equation argument for converting emissions from per hectare to per pixel. # First, multiplies the per hectare emissions by the area of the pixel in m2, then divides by the number of m2 in a hectare. calc = '--calc=A*B/{}'.format(cn.m2_per_ha) out = '--outfile={}'.format(output_model_tile) cmd = [ 'gdal_calc.py', '-A', input_model_tile, '-B', area_tile, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=LZW', '--overwrite', '--quiet' ] uu.log_subprocess_output_full(cmd) uu.print_log( " Per pixel values calculated for {}".format(output_model_tile)) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_pattern)
def create_gain_year_count_gain_only(tile_id): print "Gain pixel-only processing:", tile_id # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, mangrove = tile_names(tile_id) print 'Loss tile is', loss print 'Gain tile is', gain print 'Mangrove biomass tile is', mangrove # Pixels with gain only print "Creating raster of growth years for gain-only pixels" gain_calc = '--calc=(A==0)*(B==1)*(C>0)*({}/2)'.format(cn.gain_years) gain_outfilename = 'growth_years_gain_only_{}.tif'.format(tile_id) gain_outfilearg = '--outfile={}'.format(gain_outfilename) cmd = ['gdal_calc.py', '-A', loss, '-B', gain, '-C', mangrove, gain_calc, gain_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'int16'] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_gain_only')
def legal_Amazon_create_gain_year_count_merge(tile_id, output_pattern): uu.print_log( "Merging loss, gain, no change, and loss/gain pixels into single raster for {}" .format(tile_id)) # start time start = datetime.datetime.now() # The four rasters from above that are to be merged loss_outfilename = '{}_growth_years_loss_only.tif'.format(tile_id) no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id) loss_and_gain_outfilename = '{}_growth_years_loss_and_gain.tif'.format( tile_id) # All four components are merged together to the final output raster age_outfile = '{}_{}.tif'.format(tile_id, output_pattern) cmd = [ 'gdal_merge.py', '-o', age_outfile, loss_outfilename, no_change_outfilename, loss_and_gain_outfilename, '-co', 'COMPRESS=LZW', '-a_nodata', '0', '-ot', 'Byte' ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_pattern)
def create_gain_year_count_loss_and_gain(tile_id): print "Loss and gain pixel processing:", tile_id # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, tcd, biomass = tile_names(tile_id) # Pixels with both loss and gain and not in mangroves or planted forests print "Creating raster of growth years for loss and gain pixels" loss_and_gain_calc = '--calc=((A>0)*(B==1)*(C>0)*((A-1)+({}+1-A)/2))'.format( cn.loss_years) loss_and_gain_outfilename = 'growth_years_loss_and_gain_{}.tif'.format( tile_id) loss_and_gain_outfilearg = '--outfile={}'.format(loss_and_gain_outfilename) cmd = [ 'gdal_calc.py', '-A', loss, '-B', gain, '-C', biomass, loss_and_gain_calc, loss_and_gain_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW' ] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain')
def create_gain_year_count_merge(tile_id): print "Merging:", tile_id # start time start = datetime.datetime.now() # The four rasters from above that are to be merged loss_outfilename = 'growth_years_loss_only_{}.tif'.format(tile_id) gain_outfilename = 'growth_years_gain_only_{}.tif'.format(tile_id) no_change_outfilename = 'growth_years_no_change_{}.tif'.format(tile_id) loss_and_gain_outfilename = 'growth_years_loss_and_gain_{}.tif'.format( tile_id) # All four components are merged together to the final output raster print "Merging loss, gain, no change, and loss/gain pixels into single raster" age_outfile = '{}_{}.tif'.format(tile_id, cn.pattern_gain_year_count_natrl_forest) cmd = [ 'gdal_merge.py', '-o', age_outfile, loss_outfilename, gain_outfilename, no_change_outfilename, loss_and_gain_outfilename, '-co', 'COMPRESS=LZW', '-a_nodata', '0' ] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_gain_year_count_mangrove)
def legal_Amazon_create_gain_year_count_loss_and_gain_standard( tile_id, sensit_type): uu.print_log("Gain year count for loss and gain pixels:", tile_id) # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, extent, biomass = tile_names(tile_id, sensit_type) # Pixels with both loss and gain, and in PRODES forest 2000 loss_and_gain_calc = '--calc=((A>0)*(B==1)*(C==1)*((A-1)+({}+1-A)/2))'.format( cn.loss_years) loss_and_gain_outfilename = '{}_growth_years_loss_and_gain.tif'.format( tile_id) loss_and_gain_outfilearg = '--outfile={}'.format(loss_and_gain_outfilename) cmd = [ 'gdal_calc.py', '-A', loss, '-B', gain, '-C', extent, loss_and_gain_calc, loss_and_gain_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet' ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain')
def create_gain_year_count_no_change_standard(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for pixels with neither loss nor gain:", tile_id) # Names of the loss, gain and tree cover density tiles loss, gain, model_extent = tile_names(tile_id, sensit_type) # start time start = datetime.datetime.now() if os.path.exists(loss): uu.print_log("Loss tile found for {}. Using it in no change pixel gain year count.".format(tile_id)) no_change_calc = '--calc=(A==0)*(B==0)*(C>0)*{}'.format(cn.loss_years) no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id) no_change_outfilearg = '--outfile={}'.format(no_change_outfilename) cmd = ['gdal_calc.py', '-A', loss, '-B', gain, '-C', model_extent, no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet'] uu.log_subprocess_output_full(cmd) else: uu.print_log("No loss tile found for {}. Not using it for no change pixel gain year count.".format(tile_id)) no_change_calc = '--calc=(A==0)*(B>0)*{}'.format(cn.loss_years) no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id) no_change_outfilearg = '--outfile={}'.format(no_change_outfilename) cmd = ['gdal_calc.py', '-A', gain, '-B', model_extent, no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet'] uu.log_subprocess_output_full(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change', no_upload)
def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type): # start time start = datetime.datetime.now() date = datetime.datetime.now() date_formatted = date.strftime("%Y_%m_%d") uu.print_log(sensit_aggreg_flux) uu.print_log(std_aggreg_flux) # This produces errors about dividing by 0. As far as I can tell, those are fine. It's just trying to divide NoData # pixels by NoData pixels, and it doesn't affect the output. # For model v1.2.0, this kept producing incorrect values for the biomass_swap analysis. I don't know why. I ended # up just using raster calculator in ArcMap to create the percent diff raster for biomass_swap. It worked # fine for all the other analyses, though (including legal_Amazon_loss). # Maybe that divide by 0 is throwing off other values now. perc_diff_calc = '--calc=(A-B)/absolute(B)*100' perc_diff_outfilename = '{0}_{1}_{2}.tif'.format( cn.pattern_aggreg_sensit_perc_diff, sensit_type, date_formatted) perc_diff_outfilearg = '--outfile={}'.format(perc_diff_outfilename) # cmd = ['gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux, perc_diff_calc, perc_diff_outfilearg, # '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--quiet'] cmd = [ 'gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux, perc_diff_calc, perc_diff_outfilearg, '--overwrite', '--co', 'COMPRESS=LZW', '--quiet' ] uu.log_subprocess_output_full(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux)
def legal_Amazon_create_gain_year_count_no_change(tile_id, sensit_type): uu.print_log("Gain year count for non-loss pixels:", tile_id) # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, extent, biomass = tile_names(tile_id, sensit_type) # For unclear reasons, gdal_calc doesn't register the 0 (NoData) pixels in the loss tile, so I have to convert it # to a vrt so that the 0 pixels are recognized. loss_vrt = '{}_loss.vrt'.format(tile_id) os.system('gdalbuildvrt -vrtnodata None {0} {1}'.format(loss_vrt, loss)) # Pixels with loss but in areas with PRODES forest 2000 and biomass >0 (same as standard model) no_change_calc = '--calc=(A==0)*(B==1)*(C>0)*{}'.format(cn.loss_years) no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id) no_change_outfilearg = '--outfile={}'.format(no_change_outfilename) cmd = [ 'gdal_calc.py', '-A', loss_vrt, '-B', extent, '-C', biomass, no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet' ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change')
def rasterize_pre_2000_plantations(tile_id): # Start time start = datetime.datetime.now() uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_plant_pre_2000) cmd = [ 'gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=LZW', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-tap', '-ot', 'Byte', '-a_nodata', '0', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '{}.shp'.format(cn.pattern_plant_pre_2000_raw), out_tile ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_plant_pre_2000)
def create_gain_year_count_no_change(tile_id): print "No change pixel processing:", tile_id # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, mangrove = tile_names(tile_id) print 'Loss tile is', loss print 'Gain tile is', gain print 'Mangrove biomass tile is', mangrove # Pixels with neither loss nor gain but in areas with mangroves. # This is the only equation which really differs from the non-mangrove equations; it does not invoke tcd since that is irrelevant for mangroves. print "Creating raster of growth years for no change pixels" no_change_calc = '--calc=(A==0)*(B==0)*(C>0)*{}'.format(cn.loss_years) no_change_outfilename = 'growth_years_no_change_{}.tif'.format(tile_id) no_change_outfilearg = '--outfile={}'.format(no_change_outfilename) cmd = ['gdal_calc.py', '-A', loss, '-B', gain, '-C', mangrove, no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'int16'] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change')
def create_gain_year_count_no_change_legal_Amazon_loss(tile_id, sensit_type, no_upload): uu.print_log( "Gain year count for pixels without loss for legal_Amazon_loss:", tile_id) # Names of the loss, gain and tree cover density tiles loss, gain, model_extent = tile_names(tile_id, sensit_type) # start time start = datetime.datetime.now() # For unclear reasons, gdal_calc doesn't register the 0 (NoData) pixels in the loss tile, so I have to convert it # to a vrt so that the 0 pixels are recognized. # This was the case with PRODES loss in model v.1.1.2. loss_vrt = '{}_loss.vrt'.format(tile_id) os.system('gdalbuildvrt -vrtnodata None {0} {1}'.format(loss_vrt, loss)) no_change_calc = '--calc=(A==0)*(B>0)*{}'.format(cn.loss_years) no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id) no_change_outfilearg = '--outfile={}'.format(no_change_outfilename) cmd = [ 'gdal_calc.py', '-A', loss_vrt, '-B', model_extent, no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet' ] uu.log_subprocess_output_full(cmd) os.remove(loss_vrt) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change', no_upload)
def create_gain_year_count_no_change(tile_id): print "No change pixel processing:", tile_id # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, tcd, biomass = tile_names(tile_id) # Pixels with neither loss nor gain but in areas with tree cover density >0 and not in mangroves or planted forests print "Creating raster of growth years for no change pixels" no_change_calc = '--calc=(A==0)*(B==0)*(C>0)*(D>0)*{}'.format( cn.loss_years) no_change_outfilename = 'growth_years_no_change_{}.tif'.format(tile_id) no_change_outfilearg = '--outfile={}'.format(no_change_outfilename) cmd = [ 'gdal_calc.py', '-A', loss, '-B', gain, '-C', tcd, '-D', biomass, no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW' ] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change')
def create_gain_year_count_loss_only(tile_id, sensit_type, no_upload): uu.print_log("Gain year count for loss only pixels:", tile_id) # start time start = datetime.datetime.now() # Names of the loss, gain and tree cover density tiles loss, gain, model_extent = tile_names(tile_id, sensit_type) if os.path.exists(loss): uu.print_log( "Loss tile found for {}. Using it in loss only pixel gain year count." .format(tile_id)) loss_calc = '--calc=(A>0)*(B==0)*(C>0)*(A-1)' loss_outfilename = '{}_growth_years_loss_only.tif'.format(tile_id) loss_outfilearg = '--outfile={}'.format(loss_outfilename) cmd = [ 'gdal_calc.py', '-A', loss, '-B', gain, '-C', model_extent, loss_calc, loss_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet' ] uu.log_subprocess_output_full(cmd) else: uu.print_log( "No loss tile found for {}. Skipping loss only pixel gain year count." .format(tile_id)) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_only', no_upload)
def create_soil_C_stdev(tile_id, vrt_CI05, vrt_CI95, out_pattern): # Start time start = datetime.datetime.now() # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, out_pattern)
def cumulative_gain_BGC(tile_id): print "Calculating cumulative belowground carbon gain:", tile_id # Start time start = datetime.datetime.now() # Names of the annual gain rate and gain year count tiles gain_rate_BGB = '{0}_{1}.tif'.format( tile_id, cn.pattern_annual_gain_BGB_natrl_forest) gain_year_count = '{0}_{1}.tif'.format( tile_id, cn.pattern_gain_year_count_natrl_forest) # Carbon gain uses non-mangrove non-planted biomass:carbon ratio accum_calc = '--calc=A*B*{}'.format(cn.biomass_to_c_natrl_forest) BGC_accum_outfilename = '{0}_{1}.tif'.format( tile_id, cn.pattern_cumul_gain_BGC_natrl_forest) BGC_accum_outfilearg = '--outfile={}'.format(BGC_accum_outfilename) cmd = [ 'gdal_calc.py', '-A', gain_rate_BGB, '-B', gain_year_count, accum_calc, BGC_accum_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW' ] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_cumul_gain_BGC_natrl_forest)
def create_gain_year_count_loss_and_gain_standard(tile_id, sensit_type): uu.print_log("Loss and gain pixel processing using standard function:", tile_id) # Names of the loss, gain and tree cover density tiles loss, gain, model_extent = tile_names(tile_id, sensit_type) # start time start = datetime.datetime.now() if os.path.exists(loss): uu.print_log( "Loss tile found for {}. Using it in loss and gain pixel gain year count." .format(tile_id)) loss_and_gain_calc = '--calc=((A>0)*(B==1)*(C>0)*((A-1)+floor(({}+1-A)/2)))'.format( cn.loss_years) loss_and_gain_outfilename = '{}_growth_years_loss_and_gain.tif'.format( tile_id) loss_and_gain_outfilearg = '--outfile={}'.format( loss_and_gain_outfilename) cmd = [ 'gdal_calc.py', '-A', loss, '-B', gain, '-C', model_extent, loss_and_gain_calc, loss_and_gain_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet' ] uu.log_subprocess_output_full(cmd) else: uu.print_log( "No loss tile found for {}. Skipping loss and gain pixel gain year count." .format(tile_id)) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain')
def create_combined_soil_C(tile_id, no_upload): # Start time start = datetime.datetime.now() # Input files mangrove_soil = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id) mineral_soil = '{0}_{1}.tif'.format(tile_id, cn.pattern_soil_C_full_extent_2000_non_mang) # Output file combined_soil = '{0}_{1}.tif'.format(tile_id, cn.pattern_soil_C_full_extent_2000) # Checks if mangrove AGB tile exists. If not, mangrove soil C is not combined with mineral soil C. if os.path.exists('{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)): uu.print_log("Mangrove aboveground biomass tile found for", tile_id) mangrove_soil_src = rasterio.open(mangrove_soil) # Grabs metadata for one of the input tiles, like its location/projection/cellsize kwargs = mangrove_soil_src.meta # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory windows = mangrove_soil_src.block_windows(1) mineral_soil_src = rasterio.open(mineral_soil) # Updates kwargs for the output dataset. # Need to update data type to float 32 so that it can handle fractional gain rates kwargs.update( driver='GTiff', count=1, compress='lzw', nodata=0 ) # The output file: soil C with mangrove soil C taking precedence over mineral soil C dst_combined_soil = rasterio.open(combined_soil, 'w', **kwargs) uu.print_log("Replacing mineral soil C pixels with mangrove soil C pixels for", tile_id) # Iterates across the windows (1 pixel strips) of the input tiles for idx, window in windows: mangrove_soil_window = mangrove_soil_src.read(1, window=window) mineral_soil_window = mineral_soil_src.read(1, window=window) combined_soil_window = np.where(mangrove_soil_window>0, mangrove_soil_window, mineral_soil_window) dst_combined_soil.write_band(1, combined_soil_window, window=window) else: uu.print_log("No mangrove aboveground biomass tile for", tile_id) # If there is no mangrove soil C tile, the final output of the mineral soil function needs to receive the # correct final name. os.rename('{0}_{1}.tif'.format(tile_id, cn.pattern_soil_C_full_extent_2000_non_mang), combined_soil) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_soil_C_full_extent_2000, no_upload)
def clip_year_tiles(tile_year_list, no_upload): # Start time start = datetime.datetime.now() tile_id = tile_year_list[0].strip('.tif') year = tile_year_list[1] vrt_name = "global_vrt_{}_wgs84.vrt".format(year) # Gets coordinates of hansen tile uu.print_log("Getting coordinates of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) # Clips vrt to tile extent uu.print_log("Clipping burn year vrt to {0} for {1}".format(tile_id, year)) clipped_raster = "ba_clipped_{0}_{1}.tif".format(year, tile_id) cmd = [ 'gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '-a_nodata', '0' ] cmd += [vrt_name, clipped_raster, '-tr', '.00025', '.00025'] cmd += ['-projwin', str(xmin), str(ymax), str(xmax), str(ymin)] uu.log_subprocess_output_full(cmd) # Calculates year tile values to be equal to year. ex: 17*1 calc = '--calc={}*(A>0)'.format(int(year) - 2000) recoded_output = "ba_{0}_{1}.tif".format(year, tile_id) outfile = '--outfile={}'.format(recoded_output) cmd = [ 'gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0', '--co', 'COMPRESS=LZW', '--quiet' ] uu.log_subprocess_output_full(cmd) # Only copies to s3 if the tile has data. # No tiles for 2000 have data because the burn year is coded as 0, which is NoData. uu.print_log("Checking if {} contains any data...".format(tile_id)) empty = uu.check_for_data(recoded_output) if empty: uu.print_log(" No data found. Not copying {}.".format(tile_id)) else: uu.print_log( " Data found in {}. Copying tile to s3...".format(tile_id)) cmd = [ 'aws', 's3', 'cp', recoded_output, cn.burn_year_warped_to_Hansen_dir ] uu.log_subprocess_output_full(cmd) uu.print_log(" Tile copied to", cn.burn_year_warped_to_Hansen_dir) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year), no_upload)
def calc_emissions(tile_id, emitted_pools, sensit_type, folder, no_upload): uu.print_log("Calculating gross emissions for", tile_id, "using", sensit_type, "model type...") start = datetime.datetime.now() # Runs the correct c++ script given the emitted_pools (biomass+soil or soil_only) and model type selected. # soil_only, no_shiftin_ag, and convert_to_grassland have special gross emissions C++ scripts. # The other sensitivity analyses and the standard model all use the same gross emissions C++ script. if (emitted_pools == 'soil_only') & (sensit_type == 'std'): cmd = [ '{0}/calc_gross_emissions_soil_only.exe'.format( cn.c_emis_compile_dst), tile_id, sensit_type, folder ] elif (emitted_pools == 'biomass_soil') & ( sensit_type in ['convert_to_grassland', 'no_shifting_ag']): cmd = [ '{0}/calc_gross_emissions_{1}.exe'.format(cn.c_emis_compile_dst, sensit_type), tile_id, sensit_type, folder ] # This C++ script has an extra argument that names the input carbon emitted_pools and output emissions correctly elif (emitted_pools == 'biomass_soil') & ( sensit_type not in ['no_shifting_ag', 'convert_to_grassland']): cmd = [ '{0}/calc_gross_emissions_generic.exe'.format( cn.c_emis_compile_dst), tile_id, sensit_type, folder ] else: uu.exception_log(no_upload, 'Pool and/or sensitivity analysis option not valid') uu.log_subprocess_output_full(cmd) # Identifies which pattern to use for counting tile completion pattern = cn.pattern_gross_emis_commod_biomass_soil if (emitted_pools == 'biomass_soil') & (sensit_type == 'std'): pattern = pattern elif (emitted_pools == 'biomass_soil') & (sensit_type != 'std'): pattern = pattern + "_" + sensit_type elif emitted_pools == 'soil_only': pattern = pattern.replace('biomass_soil', 'soil_only') else: uu.exception_log(no_upload, 'Pool option not valid') # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def loss_in_raster(tile_id, raster_type, output_name, lat, mask): uu.print_log("Calculating loss area for tile id {0}...".format(tile_id)) xmin, ymin, xmax, ymax = uu.coords(tile_id) # start time start = datetime.datetime.now() # Name of the loss time loss_tile = '{0}.tif'.format(tile_id) # The raster that loss is being analyzed inside raster_of_interest = '{0}_{1}.tif'.format(tile_id, raster_type) # Output file name outname = '{0}_{1}.tif'.format(tile_id, output_name) # Only processes the tile if it is inside the latitude band (north of the specified latitude) if ymax > lat and os.path.exists(raster_of_interest): uu.print_log("{} inside latitude band and peat tile exists. Processing tile.".format(tile_id)) # If the user has asked to create just a mask of loss as opposed to the actual output values if mask == "True": calc = '--calc=(A>=1)*(A+1)/(A+1)*B' # If the user has asked to output the actual loss values if mask == "False": # Equation argument for converting emissions from per hectare to per pixel. # First, multiplies the per hectare emissions by the area of the pixel in m2, then divides by the number of m2 in a hectare. calc = '--calc=A*B' # Argument for outputting file out = '--outfile={}'.format(outname) uu.print_log("Masking loss in {} by raster of interest...".format(tile_id)) cmd = ['gdal_calc.py', '-A', loss_tile, '-B', raster_of_interest, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=LZW', '--overwrite', '--quiet'] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) uu.print_log("{} masked".format(tile_id)) else: uu.print_log("{} outside of latitude band. Skipped tile.".format(tile_id)) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_name)
def create_mangrove_soil_C(tile_id): # Start time start = datetime.datetime.now() # Checks if mangrove biomass exists. If not, it won't create a mangrove soil C tile. if os.path.exists('{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)): uu.print_log("Mangrove aboveground biomass tile found for", tile_id) uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log("Clipping mangrove soil C from mangrove soil vrt for", tile_id) uu.warp_to_Hansen('mangrove_soil_C.vrt', '{0}_mangrove_full_extent.tif'.format(tile_id), xmin, ymin, xmax, ymax, 'Int16') mangrove_soil = '{0}_mangrove_full_extent.tif'.format(tile_id) mangrove_biomass = '{0}_{1}.tif'.format( tile_id, cn.pattern_mangrove_biomass_2000) outname = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id) out = '--outfile={}'.format(outname) calc = '--calc=A*(B>0)' datatype = '--type={}'.format('Int16') uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id) cmd = [ 'gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite', datatype, '--quiet' ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) else: uu.print_log("No mangrove aboveground biomass tile for", tile_id) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove')
def prep_FIA_regions(tile_id): uu.print_log("Creating Hansen tile for FIA regions") # Start time start = datetime.datetime.now() uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log("Rasterizing FIA region shapefile", tile_id) blocksizex = 1024 blocksizey = 1024 uu.rasterize( '{}.shp'.format(cn.name_FIA_regions_raw[:-4]), "{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed), xmin, ymin, xmax, ymax, blocksizex, blocksizey, '.00025', 'Byte', 'regionCode', '0') uu.print_log("Checking if {} contains any data...".format(tile_id)) no_data = uu.check_for_data("{0}_{1}.tif".format( tile_id, cn.pattern_FIA_regions_processed)) if no_data: uu.print_log(" No data found. Deleting {}.".format(tile_id)) os.remove("{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed)) else: uu.print_log( " Data found in {}. Copying tile to s3...".format(tile_id)) uu.upload_final(cn.FIA_regions_processed_dir, tile_id, cn.pattern_FIA_regions_processed) uu.print_log(" Tile copied to s3") # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_FIA_regions_processed)
def create_combined_ifl_primary(tile_id): # Start time start = datetime.datetime.now() ifl_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl) primary_tile = '{}_primary_2001.tif'.format(tile_id) ifl_primary_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl_primary) uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) # Assigns the correct time (primary forest or ifl) if ymax <= 30 and ymax >= -20: uu.print_log( "{} between 30N and 30S. Using primary forest tile.".format( tile_id)) os.rename(primary_tile, ifl_primary_tile) else: uu.print_log( "{} not between 30N and 30S. Using IFL tile, if it exists.".format( tile_id)) if os.path.exists(ifl_tile): os.rename(ifl_tile, ifl_primary_tile) else: uu.print_log("IFL tile does not exist for {}".format(tile_id)) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_ifl_primary)
def create_gain_year_count_loss_only(tile_id): print "Loss pixel-only processing:", tile_id # Names of the input tiles loss, gain, tcd, biomass = tile_names(tile_id) # start time start = datetime.datetime.now() # Pixels with loss only and not in mangroves or planted forests print "Creating raster of growth years for loss-only pixels" loss_calc = '--calc=(A>0)*(B==0)*(C>0)*(A-1)' loss_outfilename = 'growth_years_loss_only_{}.tif'.format(tile_id) loss_outfilearg = '--outfile={}'.format(loss_outfilename) cmd = [ 'gdal_calc.py', '-A', loss, '-B', gain, '-C', biomass, loss_calc, loss_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW' ] subprocess.check_call(cmd) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_only')
def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict, output_pattern_list, no_upload): # Converts the forest age category decision tree output values to the three age categories-- # 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years # These are five digits so they can easily be added to the four digits of the continent-ecozone code to make unique codes # for each continent-ecozone-age combination. # The key in the dictionary is the forest age category decision tree endpoints. age_dict = {0: 0, 1: 10000, 2: 20000, 3: 30000} uu.print_log("Processing:", tile_id) # Start time start = datetime.datetime.now() # Names of the forest age category and continent-ecozone tiles age_cat = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_age_cat_IPCC) cont_eco = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cont_eco_processed) # Names of the output natural forest gain rate tiles (above and belowground) AGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[0]) BGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id, output_pattern_list[1]) AGB_IPCC_default_gain_stdev = '{0}_{1}.tif'.format(tile_id, output_pattern_list[2]) uu.print_log( " Creating IPCC default biomass gain rates and standard deviation for {}" .format(tile_id)) # Opens the input tiles if they exist. kips tile if either input doesn't exist. try: age_cat_src = rasterio.open(age_cat) uu.print_log(" Age category tile found for {}".format(tile_id)) except: return uu.print_log( " No age category tile found for {}. Skipping tile.".format( tile_id)) try: cont_eco_src = rasterio.open(cont_eco) uu.print_log(" Continent-ecozone tile found for {}".format(tile_id)) except: return uu.print_log( " No continent-ecozone tile found for {}. Skipping tile.".format( tile_id)) # Grabs metadata about the continent ecozone tile, like its location/projection/cellsize kwargs = cont_eco_src.meta # Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory windows = cont_eco_src.block_windows(1) # Updates kwargs for the output dataset. # Need to update data type to float 32 so that it can handle fractional gain rates kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0, dtype='float32') # The output files, aboveground and belowground biomass gain rates dst_above = rasterio.open(AGB_IPCC_default_gain_rate, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_above, sensit_type) dst_above.update_tags( units='megagrams aboveground biomass (AGB or dry matter)/ha/yr') dst_above.update_tags( source='IPCC Guidelines 2019 refinement, forest section, Table 4.9') dst_above.update_tags( extent= 'Full model extent, even though these rates will not be used over the full model extent' ) dst_below = rasterio.open(BGB_IPCC_default_gain_rate, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_below, sensit_type) dst_below.update_tags( units='megagrams belowground biomass (AGB or dry matter)/ha/yr') dst_below.update_tags( source='IPCC Guidelines 2019 refinement, forest section, Table 4.9') dst_below.update_tags( extent= 'Full model extent, even though these rates will not be used over the full model extent' ) dst_stdev_above = rasterio.open(AGB_IPCC_default_gain_stdev, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(dst_stdev_above, sensit_type) dst_stdev_above.update_tags( units= 'standard deviation, in terms of megagrams aboveground biomass (AGB or dry matter)/ha/yr' ) dst_stdev_above.update_tags( source='IPCC Guidelines 2019 refinement, forest section, Table 4.9') dst_stdev_above.update_tags( extent= 'Full model extent, even though these standard deviations will not be used over the full model extent' ) # Iterates across the windows (1 pixel strips) of the input tiles for idx, window in windows: # Creates a processing window for each input raster try: cont_eco_window = cont_eco_src.read(1, window=window) except: cont_eco_window = np.zeros((window.height, window.width), dtype='uint8') try: age_cat_window = age_cat_src.read(1, window=window) except: age_cat_window = np.zeros((window.height, window.width), dtype='uint8') # Recodes the input forest age category array with 10 different decision tree end values into the 3 actual age categories age_recode = np.vectorize(age_dict.get)(age_cat_window) # Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes cont_eco_age = cont_eco_window + age_recode ## Aboveground removal factors # Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates gain_rate_AGB = cont_eco_age.astype('float32') # Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to # get annual gain rates (metric tons aboveground biomass/yr) for each pixel for key, value in gain_table_dict.items(): gain_rate_AGB[gain_rate_AGB == key] = value # Writes the output window to the output file dst_above.write_band(1, gain_rate_AGB, window=window) ## Belowground removal factors # Calculates belowground annual removal rates gain_rate_BGB = gain_rate_AGB * cn.below_to_above_non_mang # Writes the output window to the output file dst_below.write_band(1, gain_rate_BGB, window=window) ## Aboveground removal factor standard deviation # Converts the continent-ecozone array to float so that the values can be replaced with fractional standard deviations gain_stdev_AGB = cont_eco_age.astype('float32') # Applies the dictionary of continent-ecozone-age gain rate standard deviations to the continent-ecozone-age array to # get annual gain rate standard deviations (metric tons aboveground biomass/yr) for each pixel for key, value in stdev_table_dict.items(): gain_stdev_AGB[gain_stdev_AGB == key] = value # Writes the output window to the output file dst_stdev_above.write_band(1, gain_stdev_AGB, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_pattern_list[0], no_upload)
def create_gain_year_count_merge(tile_id, pattern, sensit_type, no_upload): uu.print_log( "Merging loss, gain, no change, and loss/gain pixels into single raster for {}" .format(tile_id)) # start time start = datetime.datetime.now() # The four rasters from above that are to be merged no_change_gain_years = '{}_growth_years_no_change.tif'.format(tile_id) loss_only_gain_years = '{}_growth_years_loss_only.tif'.format(tile_id) gain_only_gain_years = '{}_growth_years_gain_only.tif'.format(tile_id) loss_and_gain_gain_years = '{}_growth_years_loss_and_gain.tif'.format( tile_id) # Names of the output tiles gain_year_count_merged = '{0}_{1}.tif'.format(tile_id, pattern) # Opens no change gain year count tile. This should exist for all tiles. with rasterio.open(no_change_gain_years) as no_change_gain_years_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = no_change_gain_years_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = no_change_gain_years_src.block_windows(1) # Updates kwargs for the output dataset kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) uu.print_log( " No change tile exists for {} by default".format(tile_id)) # Opens the other gain year count tiles. They may not exist for all other tiles. try: loss_only_gain_years_src = rasterio.open(loss_only_gain_years) uu.print_log(" Loss only tile found for {}".format(tile_id)) except: uu.print_log(" No loss only tile found for {}".format(tile_id)) try: gain_only_gain_years_src = rasterio.open(gain_only_gain_years) uu.print_log(" Gain only tile found for {}".format(tile_id)) except: uu.print_log(" No gain only tile found for {}".format(tile_id)) try: loss_and_gain_gain_years_src = rasterio.open( loss_and_gain_gain_years) uu.print_log(" Loss and gain tile found for {}".format(tile_id)) except: uu.print_log( " No loss and gain tile found for {}".format(tile_id)) # Opens the output tile, giving it the arguments of the input tiles gain_year_count_merged_dst = rasterio.open(gain_year_count_merged, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(gain_year_count_merged_dst, sensit_type) gain_year_count_merged_dst.update_tags(units='years') gain_year_count_merged_dst.update_tags(min_possible_value='0') gain_year_count_merged_dst.update_tags( max_possible_value=cn.loss_years) gain_year_count_merged_dst.update_tags( source= 'Gain years are assigned based on the combination of Hansen loss and gain in each pixel. There are four combinations: neither loss nor gain, loss only, gain only, loss and gain.' ) gain_year_count_merged_dst.update_tags(extent='Full model extent') # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: no_change_gain_years_window = no_change_gain_years_src.read( 1, window=window) try: loss_only_gain_years_window = loss_only_gain_years_src.read( 1, window=window) except: loss_only_gain_years_window = np.zeros( (window.height, window.width), dtype='uint8') try: gain_only_gain_years_window = gain_only_gain_years_src.read( 1, window=window) except: gain_only_gain_years_window = np.zeros( (window.height, window.width), dtype='uint8') try: loss_and_gain_gain_years_window = loss_and_gain_gain_years_src.read( 1, window=window) except: loss_and_gain_gain_years_window = np.zeros( (window.height, window.width), dtype='uint8') gain_year_count_merged_window = loss_only_gain_years_window + gain_only_gain_years_window + \ no_change_gain_years_window + loss_and_gain_gain_years_window gain_year_count_merged_dst.write_band( 1, gain_year_count_merged_window, window=window) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def create_supplementary_outputs(tile_id, input_pattern, output_patterns, sensit_type): # start time start = datetime.datetime.now() # Extracts the tile id, tile type, and bounding box for the tile tile_id = uu.get_tile_id(tile_id) # Names of inputs focal_tile = '{0}_{1}.tif'.format(tile_id, input_pattern) pixel_area = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id) tcd = '{0}_{1}.tif'.format(cn.pattern_tcd, tile_id) gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id) mangrove = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000) # Names of outputs. # Requires that output patterns be listed in main script in the correct order for here # (currently, per pixel full extent, per hectare forest extent, per pixel forest extent). per_pixel_full_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[0]) per_hectare_forest_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[1]) per_pixel_forest_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[2]) # Opens input tiles for rasterio in_src = rasterio.open(focal_tile) # Grabs metadata about the tif, like its location/projection/cellsize kwargs = in_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = in_src.block_windows(1) pixel_area_src = rasterio.open(pixel_area) tcd_src = rasterio.open(tcd) gain_src = rasterio.open(gain) try: mangrove_src = rasterio.open(mangrove) uu.print_log(" Mangrove tile found for {}".format(tile_id)) except: uu.print_log(" No mangrove tile found for {}".format(tile_id)) uu.print_log(" Creating outputs for {}...".format(focal_tile)) kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0, dtype='float32') # Opens output tiles, giving them the arguments of the input tiles per_pixel_full_extent_dst = rasterio.open(per_pixel_full_extent, 'w', **kwargs) per_hectare_forest_extent_dst = rasterio.open(per_hectare_forest_extent, 'w', **kwargs) per_pixel_forest_extent_dst = rasterio.open(per_pixel_forest_extent, 'w', **kwargs) # Adds metadata tags to the output rasters uu.add_rasterio_tags(per_pixel_full_extent_dst, sensit_type) per_pixel_full_extent_dst.update_tags( units='Mg CO2e/pixel over model duration (2001-20{})'.format( cn.loss_years)) per_pixel_full_extent_dst.update_tags( source='per hectare full model extent tile') per_pixel_full_extent_dst.update_tags( extent= 'Full model extent: ((TCD2000>0 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations' ) uu.add_rasterio_tags(per_hectare_forest_extent_dst, sensit_type) per_hectare_forest_extent_dst.update_tags( units='Mg CO2e/hectare over model duration (2001-20{})'.format( cn.loss_years)) per_hectare_forest_extent_dst.update_tags( source='per hectare full model extent tile') per_hectare_forest_extent_dst.update_tags( extent= 'Forest extent: ((TCD2000>30 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations' ) uu.add_rasterio_tags(per_pixel_forest_extent_dst, sensit_type) per_pixel_forest_extent_dst.update_tags( units='Mg CO2e/pixel over model duration (2001-20{})'.format( cn.loss_years)) per_pixel_forest_extent_dst.update_tags( source='per hectare forest model extent tile') per_pixel_forest_extent_dst.update_tags( extent= 'Forest extent: ((TCD2000>30 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations' ) if "net_flux" in focal_tile: per_pixel_full_extent_dst.update_tags( scale= 'Negative values are net sinks. Positive values are net sources.') per_hectare_forest_extent_dst.update_tags( scale= 'Negative values are net sinks. Positive values are net sources.') per_pixel_forest_extent_dst.update_tags( scale= 'Negative values are net sinks. Positive values are net sources.') # Iterates across the windows of the input tiles for idx, window in windows: # Creates windows for each input tile in_window = in_src.read(1, window=window) pixel_area_window = pixel_area_src.read(1, window=window) tcd_window = tcd_src.read(1, window=window) gain_window = gain_src.read(1, window=window) try: mangrove_window = mangrove_src.read(1, window=window) except: mangrove_window = np.zeros((window.height, window.width), dtype='uint8') # Output window for per pixel full extent raster dst_window_per_pixel_full_extent = in_window * pixel_area_window / cn.m2_per_ha # Output window for per hectare forest extent raster # QCed this line before publication and then again afterwards in response to question from Lena Schulte-Uebbing at Wageningen Uni. dst_window_per_hectare_forest_extent = np.where( (tcd_window > cn.canopy_threshold) | (gain_window == 1) | (mangrove_window != 0), in_window, 0) # Output window for per pixel forest extent raster dst_window_per_pixel_forest_extent = dst_window_per_hectare_forest_extent * pixel_area_window / cn.m2_per_ha # Writes arrays to output raster per_pixel_full_extent_dst.write_band(1, dst_window_per_pixel_full_extent, window=window) per_hectare_forest_extent_dst.write_band( 1, dst_window_per_hectare_forest_extent, window=window) per_pixel_forest_extent_dst.write_band( 1, dst_window_per_pixel_forest_extent, window=window) uu.print_log(" Output tiles created for {}...".format(tile_id)) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, output_patterns[0])
def hansen_burnyear(tile_id, no_upload): # Start time start = datetime.datetime.now() uu.print_log("Processing", tile_id) # The tiles that are used. out_tile_no_tag is the output before metadata tags are added. out_tile is the output # once metadata tags have been added. out_tile_no_tag = '{0}_{1}_no_tag.tif'.format(tile_id, cn.pattern_burn_year) out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_burn_year) loss = '{0}.tif'.format(tile_id) # Does not continue processing tile if no loss (because there will not be any output) if not os.path.exists(loss): uu.print_log("No loss tile for", tile_id) return else: uu.print_log("Loss tile exists for", tile_id) # Downloads the burned area tiles for each year include = 'ba_*_{}.tif'.format(tile_id) burn_tiles_dir = 'burn_tiles' if not os.path.exists(burn_tiles_dir): os.mkdir(burn_tiles_dir) cmd = [ 'aws', 's3', 'cp', cn.burn_year_warped_to_Hansen_dir, burn_tiles_dir, '--recursive', '--exclude', "*", '--include', include ] uu.log_subprocess_output_full(cmd) # For each year tile, converts to array and stacks them array_list = [] ba_tifs = glob.glob(burn_tiles_dir + '/*{}*'.format(tile_id)) # Skips the tile if it has no burned area data for any year uu.print_log("There are {0} tiles to stack for {1}".format( len(ba_tifs), tile_id)) if len(ba_tifs) == 0: uu.print_log( "Skipping {} because there are no tiles to stack".format(tile_id)) return # NOTE: All of this could pretty easily be done in rasterio. However, Sam's use of GDAL for this still works fine, # so I've left it using GDAL. for ba_tif in ba_tifs: uu.print_log("Creating array with {}".format(ba_tif)) array = utilities.raster_to_array(ba_tif) array_list.append(array) # Stacks arrays from each year uu.print_log("Stacking arrays for", tile_id) stacked_year_array = utilities.stack_arrays(array_list) # Converts Hansen tile to array uu.print_log("Creating loss year array for", tile_id) loss_array = utilities.raster_to_array(loss) # Determines what year to assign burned area lossarray_min1 = np.subtract(loss_array, 1) stack_con = (stacked_year_array >= lossarray_min1) & (stacked_year_array <= loss_array) stack_con2 = stack_con * stacked_year_array lossyear_burn_array = stack_con2.max(0) utilities.array_to_raster_simple(lossyear_burn_array, out_tile_no_tag, loss) # Only copies to s3 if the tile has data uu.print_log("Checking if {} contains any data...".format(tile_id)) empty = uu.check_for_data(out_tile_no_tag) # Checks output for data. There could be burned area but none of it coincides with tree cover loss, # so this is the final check for whether there is any data. if empty: uu.print_log(" No data found. Not copying {}.".format(tile_id)) # Without this, the untagged version is counted and eventually copied to s3 if it has data in it os.remove(out_tile_no_tag) return else: uu.print_log( " Data found in {}. Adding metadata tags...".format(tile_id)) ### Thomas suggested these on 8/19/2020 but they didn't work. The first one wrote the tags but erased all the ### data in the tiles (everything became 0 according to gdalinfo). The second one had some other error. # with rasterio.open(out_tile_no_tag, 'r') as src: # # profile = src.profile # # with rasterio.open(out_tile_no_tag, 'w', **profile) as dst: # # dst.update_tags(units='year (2001, 2002, 2003...)', # source='MODIS collection 6 burned area', # extent='global') # # with rasterio.open(out_tile_no_tag, 'w+') as src: # # dst.update_tags(units='year (2001, 2002, 2003...)', # source='MODIS collection 6 burned area', # extent='global') # All of the below is to add metadata tags to the output burn year masks. # For some reason, just doing what's at https://rasterio.readthedocs.io/en/latest/topics/tags.html # results in the data getting removed. # I found it necessary to copy the desired output and read its windows into a new copy of the file, to which the # metadata tags are added. I'm sure there's an easier way to do this but I couldn't figure out how. # I know it's very convoluted but I really couldn't figure out how to add the tags without erasing the data. copyfile(out_tile_no_tag, out_tile) with rasterio.open(out_tile_no_tag) as out_tile_no_tag_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = out_tile_no_tag_src.meta #### Use profile instead # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = out_tile_no_tag_src.block_windows(1) # Updates kwargs for the output dataset kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) out_tile_tagged = rasterio.open(out_tile, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(out_tile_tagged, 'std') out_tile_tagged.update_tags(units='year (2001, 2002, 2003...)') out_tile_tagged.update_tags( source= 'MODIS collection 6 burned area, https://modis-fire.umd.edu/files/MODIS_C6_BA_User_Guide_1.3.pdf' ) out_tile_tagged.update_tags(extent='global') # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: in_window = out_tile_no_tag_src.read(1, window=window) # Writes the output window to the output out_tile_tagged.write_band(1, in_window, window=window) # Without this, the untagged version is counted and eventually copied to s3 if it has data in it os.remove(out_tile_no_tag) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_burn_year, no_upload)