def create_mangrove_soil_C(tile_id, no_upload):
# Start time
start = datetime.datetime.now()
# Checks if mangrove biomass exists. If not, it won't create a mangrove soil C tile.
if os.path.exists('{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)):
uu.print_log("Mangrove aboveground biomass tile found for", tile_id)
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Clipping mangrove soil C from mangrove soil vrt for", tile_id)
uu.warp_to_Hansen('mangrove_soil_C.vrt', '{0}_mangrove_full_extent.tif'.format(tile_id), xmin, ymin, xmax, ymax, 'Int16')
mangrove_soil = '{0}_mangrove_full_extent.tif'.format(tile_id)
mangrove_biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
outname = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id)
out = '--outfile={}'.format(outname)
calc = '--calc=A*(B>0)'
datatype = '--type={}'.format('Int16')
uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id)
cmd = ['gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass,
calc, out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite', datatype, '--quiet']
uu.log_subprocess_output_full(cmd)
else:
uu.print_log("No mangrove aboveground biomass tile for", tile_id)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove', no_upload)
def output_per_pixel(tile_id, input_pattern, output_pattern, sensit_type):
uu.print_log("Calculating per pixel values for", tile_id)
# Start time
start = datetime.datetime.now()
# Names of the input biomass and TCD tiles
input_model_tile = '{0}_{1}.tif'.format(tile_id, input_pattern)
area_tile = 'hanson_2013_area_{}.tif'.format(tile_id)
output_model_tile = '{0}_{1}.tif'.format(tile_id, output_pattern)
uu.print_log("Converting {} from Mg CO2/ha to Mg CO2/pixel...".format(
input_model_tile))
# Equation argument for converting emissions from per hectare to per pixel.
# First, multiplies the per hectare emissions by the area of the pixel in m2, then divides by the number of m2 in a hectare.
calc = '--calc=A*B/{}'.format(cn.m2_per_ha)
out = '--outfile={}'.format(output_model_tile)
cmd = [
'gdal_calc.py', '-A', input_model_tile, '-B', area_tile, calc, out,
'--NoDataValue=0', '--co', 'COMPRESS=LZW', '--overwrite', '--quiet'
]
uu.log_subprocess_output_full(cmd)
uu.print_log(
" Per pixel values calculated for {}".format(output_model_tile))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_pattern)
def create_gain_year_count_gain_only(tile_id):
print "Gain pixel-only processing:", tile_id
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, mangrove = tile_names(tile_id)
print 'Loss tile is', loss
print 'Gain tile is', gain
print 'Mangrove biomass tile is', mangrove
# Pixels with gain only
print "Creating raster of growth years for gain-only pixels"
gain_calc = '--calc=(A==0)*(B==1)*(C>0)*({}/2)'.format(cn.gain_years)
gain_outfilename = 'growth_years_gain_only_{}.tif'.format(tile_id)
gain_outfilearg = '--outfile={}'.format(gain_outfilename)
cmd = ['gdal_calc.py', '-A', loss, '-B', gain, '-C', mangrove, gain_calc, gain_outfilearg, '--NoDataValue=0',
'--overwrite', '--co', 'COMPRESS=LZW', '--type', 'int16']
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_gain_only')
def legal_Amazon_create_gain_year_count_merge(tile_id, output_pattern):
uu.print_log(
"Merging loss, gain, no change, and loss/gain pixels into single raster for {}"
.format(tile_id))
# start time
start = datetime.datetime.now()
# The four rasters from above that are to be merged
loss_outfilename = '{}_growth_years_loss_only.tif'.format(tile_id)
no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id)
loss_and_gain_outfilename = '{}_growth_years_loss_and_gain.tif'.format(
tile_id)
# All four components are merged together to the final output raster
age_outfile = '{}_{}.tif'.format(tile_id, output_pattern)
cmd = [
'gdal_merge.py', '-o', age_outfile, loss_outfilename,
no_change_outfilename, loss_and_gain_outfilename, '-co',
'COMPRESS=LZW', '-a_nodata', '0', '-ot', 'Byte'
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_pattern)
def create_gain_year_count_loss_and_gain(tile_id):
print "Loss and gain pixel processing:", tile_id
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, tcd, biomass = tile_names(tile_id)
# Pixels with both loss and gain and not in mangroves or planted forests
print "Creating raster of growth years for loss and gain pixels"
loss_and_gain_calc = '--calc=((A>0)*(B==1)*(C>0)*((A-1)+({}+1-A)/2))'.format(
cn.loss_years)
loss_and_gain_outfilename = 'growth_years_loss_and_gain_{}.tif'.format(
tile_id)
loss_and_gain_outfilearg = '--outfile={}'.format(loss_and_gain_outfilename)
cmd = [
'gdal_calc.py', '-A', loss, '-B', gain, '-C', biomass,
loss_and_gain_calc, loss_and_gain_outfilearg, '--NoDataValue=0',
'--overwrite', '--co', 'COMPRESS=LZW'
]
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain')
def create_gain_year_count_merge(tile_id):
print "Merging:", tile_id
# start time
start = datetime.datetime.now()
# The four rasters from above that are to be merged
loss_outfilename = 'growth_years_loss_only_{}.tif'.format(tile_id)
gain_outfilename = 'growth_years_gain_only_{}.tif'.format(tile_id)
no_change_outfilename = 'growth_years_no_change_{}.tif'.format(tile_id)
loss_and_gain_outfilename = 'growth_years_loss_and_gain_{}.tif'.format(
tile_id)
# All four components are merged together to the final output raster
print "Merging loss, gain, no change, and loss/gain pixels into single raster"
age_outfile = '{}_{}.tif'.format(tile_id,
cn.pattern_gain_year_count_natrl_forest)
cmd = [
'gdal_merge.py', '-o', age_outfile, loss_outfilename, gain_outfilename,
no_change_outfilename, loss_and_gain_outfilename, '-co',
'COMPRESS=LZW', '-a_nodata', '0'
]
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_gain_year_count_mangrove)
def legal_Amazon_create_gain_year_count_loss_and_gain_standard(
tile_id, sensit_type):
uu.print_log("Gain year count for loss and gain pixels:", tile_id)
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, extent, biomass = tile_names(tile_id, sensit_type)
# Pixels with both loss and gain, and in PRODES forest 2000
loss_and_gain_calc = '--calc=((A>0)*(B==1)*(C==1)*((A-1)+({}+1-A)/2))'.format(
cn.loss_years)
loss_and_gain_outfilename = '{}_growth_years_loss_and_gain.tif'.format(
tile_id)
loss_and_gain_outfilearg = '--outfile={}'.format(loss_and_gain_outfilename)
cmd = [
'gdal_calc.py', '-A', loss, '-B', gain, '-C', extent,
loss_and_gain_calc, loss_and_gain_outfilearg, '--NoDataValue=0',
'--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet'
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain')
def create_gain_year_count_no_change_standard(tile_id, sensit_type, no_upload):
uu.print_log("Gain year count for pixels with neither loss nor gain:", tile_id)
# Names of the loss, gain and tree cover density tiles
loss, gain, model_extent = tile_names(tile_id, sensit_type)
# start time
start = datetime.datetime.now()
if os.path.exists(loss):
uu.print_log("Loss tile found for {}. Using it in no change pixel gain year count.".format(tile_id))
no_change_calc = '--calc=(A==0)*(B==0)*(C>0)*{}'.format(cn.loss_years)
no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id)
no_change_outfilearg = '--outfile={}'.format(no_change_outfilename)
cmd = ['gdal_calc.py', '-A', loss, '-B', gain, '-C', model_extent, no_change_calc,
no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet']
uu.log_subprocess_output_full(cmd)
else:
uu.print_log("No loss tile found for {}. Not using it for no change pixel gain year count.".format(tile_id))
no_change_calc = '--calc=(A==0)*(B>0)*{}'.format(cn.loss_years)
no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id)
no_change_outfilearg = '--outfile={}'.format(no_change_outfilename)
cmd = ['gdal_calc.py', '-A', gain, '-B', model_extent, no_change_calc,
no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet']
uu.log_subprocess_output_full(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change', no_upload)
def percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type):
# start time
start = datetime.datetime.now()
date = datetime.datetime.now()
date_formatted = date.strftime("%Y_%m_%d")
uu.print_log(sensit_aggreg_flux)
uu.print_log(std_aggreg_flux)
# This produces errors about dividing by 0. As far as I can tell, those are fine. It's just trying to divide NoData
# pixels by NoData pixels, and it doesn't affect the output.
# For model v1.2.0, this kept producing incorrect values for the biomass_swap analysis. I don't know why. I ended
# up just using raster calculator in ArcMap to create the percent diff raster for biomass_swap. It worked
# fine for all the other analyses, though (including legal_Amazon_loss).
# Maybe that divide by 0 is throwing off other values now.
perc_diff_calc = '--calc=(A-B)/absolute(B)*100'
perc_diff_outfilename = '{0}_{1}_{2}.tif'.format(
cn.pattern_aggreg_sensit_perc_diff, sensit_type, date_formatted)
perc_diff_outfilearg = '--outfile={}'.format(perc_diff_outfilename)
# cmd = ['gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux, perc_diff_calc, perc_diff_outfilearg,
# '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--quiet']
cmd = [
'gdal_calc.py', '-A', sensit_aggreg_flux, '-B', std_aggreg_flux,
perc_diff_calc, perc_diff_outfilearg, '--overwrite', '--co',
'COMPRESS=LZW', '--quiet'
]
uu.log_subprocess_output_full(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, 'global', sensit_aggreg_flux)
def legal_Amazon_create_gain_year_count_no_change(tile_id, sensit_type):
uu.print_log("Gain year count for non-loss pixels:", tile_id)
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, extent, biomass = tile_names(tile_id, sensit_type)
# For unclear reasons, gdal_calc doesn't register the 0 (NoData) pixels in the loss tile, so I have to convert it
# to a vrt so that the 0 pixels are recognized.
loss_vrt = '{}_loss.vrt'.format(tile_id)
os.system('gdalbuildvrt -vrtnodata None {0} {1}'.format(loss_vrt, loss))
# Pixels with loss but in areas with PRODES forest 2000 and biomass >0 (same as standard model)
no_change_calc = '--calc=(A==0)*(B==1)*(C>0)*{}'.format(cn.loss_years)
no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id)
no_change_outfilearg = '--outfile={}'.format(no_change_outfilename)
cmd = [
'gdal_calc.py', '-A', loss_vrt, '-B', extent, '-C', biomass,
no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite',
'--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet'
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change')
def rasterize_pre_2000_plantations(tile_id):
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_plant_pre_2000)
cmd = [
'gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=LZW', '-tr',
'{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-tap', '-ot',
'Byte', '-a_nodata', '0', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '{}.shp'.format(cn.pattern_plant_pre_2000_raw), out_tile
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_plant_pre_2000)
def create_gain_year_count_no_change(tile_id):
print "No change pixel processing:", tile_id
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, mangrove = tile_names(tile_id)
print 'Loss tile is', loss
print 'Gain tile is', gain
print 'Mangrove biomass tile is', mangrove
# Pixels with neither loss nor gain but in areas with mangroves.
# This is the only equation which really differs from the non-mangrove equations; it does not invoke tcd since that is irrelevant for mangroves.
print "Creating raster of growth years for no change pixels"
no_change_calc = '--calc=(A==0)*(B==0)*(C>0)*{}'.format(cn.loss_years)
no_change_outfilename = 'growth_years_no_change_{}.tif'.format(tile_id)
no_change_outfilearg = '--outfile={}'.format(no_change_outfilename)
cmd = ['gdal_calc.py', '-A', loss, '-B', gain, '-C', mangrove, no_change_calc, no_change_outfilearg,
'--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type', 'int16']
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change')
def create_gain_year_count_no_change_legal_Amazon_loss(tile_id, sensit_type,
no_upload):
uu.print_log(
"Gain year count for pixels without loss for legal_Amazon_loss:",
tile_id)
# Names of the loss, gain and tree cover density tiles
loss, gain, model_extent = tile_names(tile_id, sensit_type)
# start time
start = datetime.datetime.now()
# For unclear reasons, gdal_calc doesn't register the 0 (NoData) pixels in the loss tile, so I have to convert it
# to a vrt so that the 0 pixels are recognized.
# This was the case with PRODES loss in model v.1.1.2.
loss_vrt = '{}_loss.vrt'.format(tile_id)
os.system('gdalbuildvrt -vrtnodata None {0} {1}'.format(loss_vrt, loss))
no_change_calc = '--calc=(A==0)*(B>0)*{}'.format(cn.loss_years)
no_change_outfilename = '{}_growth_years_no_change.tif'.format(tile_id)
no_change_outfilearg = '--outfile={}'.format(no_change_outfilename)
cmd = [
'gdal_calc.py', '-A', loss_vrt, '-B', model_extent, no_change_calc,
no_change_outfilearg, '--NoDataValue=0', '--overwrite', '--co',
'COMPRESS=LZW', '--type', 'Byte', '--quiet'
]
uu.log_subprocess_output_full(cmd)
os.remove(loss_vrt)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change', no_upload)
def create_gain_year_count_no_change(tile_id):
print "No change pixel processing:", tile_id
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, tcd, biomass = tile_names(tile_id)
# Pixels with neither loss nor gain but in areas with tree cover density >0 and not in mangroves or planted forests
print "Creating raster of growth years for no change pixels"
no_change_calc = '--calc=(A==0)*(B==0)*(C>0)*(D>0)*{}'.format(
cn.loss_years)
no_change_outfilename = 'growth_years_no_change_{}.tif'.format(tile_id)
no_change_outfilearg = '--outfile={}'.format(no_change_outfilename)
cmd = [
'gdal_calc.py', '-A', loss, '-B', gain, '-C', tcd, '-D', biomass,
no_change_calc, no_change_outfilearg, '--NoDataValue=0', '--overwrite',
'--co', 'COMPRESS=LZW'
]
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_no_change')
def create_gain_year_count_loss_only(tile_id, sensit_type, no_upload):
uu.print_log("Gain year count for loss only pixels:", tile_id)
# start time
start = datetime.datetime.now()
# Names of the loss, gain and tree cover density tiles
loss, gain, model_extent = tile_names(tile_id, sensit_type)
if os.path.exists(loss):
uu.print_log(
"Loss tile found for {}. Using it in loss only pixel gain year count."
.format(tile_id))
loss_calc = '--calc=(A>0)*(B==0)*(C>0)*(A-1)'
loss_outfilename = '{}_growth_years_loss_only.tif'.format(tile_id)
loss_outfilearg = '--outfile={}'.format(loss_outfilename)
cmd = [
'gdal_calc.py', '-A', loss, '-B', gain, '-C', model_extent,
loss_calc, loss_outfilearg, '--NoDataValue=0', '--overwrite',
'--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet'
]
uu.log_subprocess_output_full(cmd)
else:
uu.print_log(
"No loss tile found for {}. Skipping loss only pixel gain year count."
.format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_only', no_upload)
def create_soil_C_stdev(tile_id, vrt_CI05, vrt_CI95, out_pattern):
# Start time
start = datetime.datetime.now()
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, out_pattern)
def cumulative_gain_BGC(tile_id):
print "Calculating cumulative belowground carbon gain:", tile_id
# Start time
start = datetime.datetime.now()
# Names of the annual gain rate and gain year count tiles
gain_rate_BGB = '{0}_{1}.tif'.format(
tile_id, cn.pattern_annual_gain_BGB_natrl_forest)
gain_year_count = '{0}_{1}.tif'.format(
tile_id, cn.pattern_gain_year_count_natrl_forest)
# Carbon gain uses non-mangrove non-planted biomass:carbon ratio
accum_calc = '--calc=A*B*{}'.format(cn.biomass_to_c_natrl_forest)
BGC_accum_outfilename = '{0}_{1}.tif'.format(
tile_id, cn.pattern_cumul_gain_BGC_natrl_forest)
BGC_accum_outfilearg = '--outfile={}'.format(BGC_accum_outfilename)
cmd = [
'gdal_calc.py', '-A', gain_rate_BGB, '-B', gain_year_count, accum_calc,
BGC_accum_outfilearg, '--NoDataValue=0', '--overwrite', '--co',
'COMPRESS=LZW'
]
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id,
cn.pattern_cumul_gain_BGC_natrl_forest)
def create_gain_year_count_loss_and_gain_standard(tile_id, sensit_type):
uu.print_log("Loss and gain pixel processing using standard function:",
tile_id)
# Names of the loss, gain and tree cover density tiles
loss, gain, model_extent = tile_names(tile_id, sensit_type)
# start time
start = datetime.datetime.now()
if os.path.exists(loss):
uu.print_log(
"Loss tile found for {}. Using it in loss and gain pixel gain year count."
.format(tile_id))
loss_and_gain_calc = '--calc=((A>0)*(B==1)*(C>0)*((A-1)+floor(({}+1-A)/2)))'.format(
cn.loss_years)
loss_and_gain_outfilename = '{}_growth_years_loss_and_gain.tif'.format(
tile_id)
loss_and_gain_outfilearg = '--outfile={}'.format(
loss_and_gain_outfilename)
cmd = [
'gdal_calc.py', '-A', loss, '-B', gain, '-C', model_extent,
loss_and_gain_calc, loss_and_gain_outfilearg, '--NoDataValue=0',
'--overwrite', '--co', 'COMPRESS=LZW', '--type', 'Byte', '--quiet'
]
uu.log_subprocess_output_full(cmd)
else:
uu.print_log(
"No loss tile found for {}. Skipping loss and gain pixel gain year count."
.format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_and_gain')
def create_combined_soil_C(tile_id, no_upload):
# Start time
start = datetime.datetime.now()
# Input files
mangrove_soil = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id)
mineral_soil = '{0}_{1}.tif'.format(tile_id, cn.pattern_soil_C_full_extent_2000_non_mang)
# Output file
combined_soil = '{0}_{1}.tif'.format(tile_id, cn.pattern_soil_C_full_extent_2000)
# Checks if mangrove AGB tile exists. If not, mangrove soil C is not combined with mineral soil C.
if os.path.exists('{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)):
uu.print_log("Mangrove aboveground biomass tile found for", tile_id)
mangrove_soil_src = rasterio.open(mangrove_soil)
# Grabs metadata for one of the input tiles, like its location/projection/cellsize
kwargs = mangrove_soil_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = mangrove_soil_src.block_windows(1)
mineral_soil_src = rasterio.open(mineral_soil)
# Updates kwargs for the output dataset.
# Need to update data type to float 32 so that it can handle fractional gain rates
kwargs.update(
driver='GTiff',
count=1,
compress='lzw',
nodata=0
)
# The output file: soil C with mangrove soil C taking precedence over mineral soil C
dst_combined_soil = rasterio.open(combined_soil, 'w', **kwargs)
uu.print_log("Replacing mineral soil C pixels with mangrove soil C pixels for", tile_id)
# Iterates across the windows (1 pixel strips) of the input tiles
for idx, window in windows:
mangrove_soil_window = mangrove_soil_src.read(1, window=window)
mineral_soil_window = mineral_soil_src.read(1, window=window)
combined_soil_window = np.where(mangrove_soil_window>0, mangrove_soil_window, mineral_soil_window)
dst_combined_soil.write_band(1, combined_soil_window, window=window)
else:
uu.print_log("No mangrove aboveground biomass tile for", tile_id)
# If there is no mangrove soil C tile, the final output of the mineral soil function needs to receive the
# correct final name.
os.rename('{0}_{1}.tif'.format(tile_id, cn.pattern_soil_C_full_extent_2000_non_mang), combined_soil)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_soil_C_full_extent_2000, no_upload)
def clip_year_tiles(tile_year_list, no_upload):
# Start time
start = datetime.datetime.now()
tile_id = tile_year_list[0].strip('.tif')
year = tile_year_list[1]
vrt_name = "global_vrt_{}_wgs84.vrt".format(year)
# Gets coordinates of hansen tile
uu.print_log("Getting coordinates of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Clips vrt to tile extent
uu.print_log("Clipping burn year vrt to {0} for {1}".format(tile_id, year))
clipped_raster = "ba_clipped_{0}_{1}.tif".format(year, tile_id)
cmd = [
'gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '-a_nodata',
'0'
]
cmd += [vrt_name, clipped_raster, '-tr', '.00025', '.00025']
cmd += ['-projwin', str(xmin), str(ymax), str(xmax), str(ymin)]
uu.log_subprocess_output_full(cmd)
# Calculates year tile values to be equal to year. ex: 17*1
calc = '--calc={}*(A>0)'.format(int(year) - 2000)
recoded_output = "ba_{0}_{1}.tif".format(year, tile_id)
outfile = '--outfile={}'.format(recoded_output)
cmd = [
'gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0',
'--co', 'COMPRESS=LZW', '--quiet'
]
uu.log_subprocess_output_full(cmd)
# Only copies to s3 if the tile has data.
# No tiles for 2000 have data because the burn year is coded as 0, which is NoData.
uu.print_log("Checking if {} contains any data...".format(tile_id))
empty = uu.check_for_data(recoded_output)
if empty:
uu.print_log(" No data found. Not copying {}.".format(tile_id))
else:
uu.print_log(
" Data found in {}. Copying tile to s3...".format(tile_id))
cmd = [
'aws', 's3', 'cp', recoded_output,
cn.burn_year_warped_to_Hansen_dir
]
uu.log_subprocess_output_full(cmd)
uu.print_log(" Tile copied to", cn.burn_year_warped_to_Hansen_dir)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year), no_upload)
def calc_emissions(tile_id, emitted_pools, sensit_type, folder, no_upload):
uu.print_log("Calculating gross emissions for", tile_id, "using",
sensit_type, "model type...")
start = datetime.datetime.now()
# Runs the correct c++ script given the emitted_pools (biomass+soil or soil_only) and model type selected.
# soil_only, no_shiftin_ag, and convert_to_grassland have special gross emissions C++ scripts.
# The other sensitivity analyses and the standard model all use the same gross emissions C++ script.
if (emitted_pools == 'soil_only') & (sensit_type == 'std'):
cmd = [
'{0}/calc_gross_emissions_soil_only.exe'.format(
cn.c_emis_compile_dst), tile_id, sensit_type, folder
]
elif (emitted_pools == 'biomass_soil') & (
sensit_type in ['convert_to_grassland', 'no_shifting_ag']):
cmd = [
'{0}/calc_gross_emissions_{1}.exe'.format(cn.c_emis_compile_dst,
sensit_type), tile_id,
sensit_type, folder
]
# This C++ script has an extra argument that names the input carbon emitted_pools and output emissions correctly
elif (emitted_pools == 'biomass_soil') & (
sensit_type not in ['no_shifting_ag', 'convert_to_grassland']):
cmd = [
'{0}/calc_gross_emissions_generic.exe'.format(
cn.c_emis_compile_dst), tile_id, sensit_type, folder
]
else:
uu.exception_log(no_upload,
'Pool and/or sensitivity analysis option not valid')
uu.log_subprocess_output_full(cmd)
# Identifies which pattern to use for counting tile completion
pattern = cn.pattern_gross_emis_commod_biomass_soil
if (emitted_pools == 'biomass_soil') & (sensit_type == 'std'):
pattern = pattern
elif (emitted_pools == 'biomass_soil') & (sensit_type != 'std'):
pattern = pattern + "_" + sensit_type
elif emitted_pools == 'soil_only':
pattern = pattern.replace('biomass_soil', 'soil_only')
else:
uu.exception_log(no_upload, 'Pool option not valid')
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def loss_in_raster(tile_id, raster_type, output_name, lat, mask):
uu.print_log("Calculating loss area for tile id {0}...".format(tile_id))
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# start time
start = datetime.datetime.now()
# Name of the loss time
loss_tile = '{0}.tif'.format(tile_id)
# The raster that loss is being analyzed inside
raster_of_interest = '{0}_{1}.tif'.format(tile_id, raster_type)
# Output file name
outname = '{0}_{1}.tif'.format(tile_id, output_name)
# Only processes the tile if it is inside the latitude band (north of the specified latitude)
if ymax > lat and os.path.exists(raster_of_interest):
uu.print_log("{} inside latitude band and peat tile exists. Processing tile.".format(tile_id))
# If the user has asked to create just a mask of loss as opposed to the actual output values
if mask == "True":
calc = '--calc=(A>=1)*(A+1)/(A+1)*B'
# If the user has asked to output the actual loss values
if mask == "False":
# Equation argument for converting emissions from per hectare to per pixel.
# First, multiplies the per hectare emissions by the area of the pixel in m2, then divides by the number of m2 in a hectare.
calc = '--calc=A*B'
# Argument for outputting file
out = '--outfile={}'.format(outname)
uu.print_log("Masking loss in {} by raster of interest...".format(tile_id))
cmd = ['gdal_calc.py', '-A', loss_tile, '-B', raster_of_interest, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=LZW',
'--overwrite', '--quiet']
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.print_log("{} masked".format(tile_id))
else:
uu.print_log("{} outside of latitude band. Skipped tile.".format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_name)
def create_mangrove_soil_C(tile_id):
# Start time
start = datetime.datetime.now()
# Checks if mangrove biomass exists. If not, it won't create a mangrove soil C tile.
if os.path.exists('{0}_{1}.tif'.format(tile_id,
cn.pattern_mangrove_biomass_2000)):
uu.print_log("Mangrove aboveground biomass tile found for", tile_id)
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Clipping mangrove soil C from mangrove soil vrt for",
tile_id)
uu.warp_to_Hansen('mangrove_soil_C.vrt',
'{0}_mangrove_full_extent.tif'.format(tile_id), xmin,
ymin, xmax, ymax, 'Int16')
mangrove_soil = '{0}_mangrove_full_extent.tif'.format(tile_id)
mangrove_biomass = '{0}_{1}.tif'.format(
tile_id, cn.pattern_mangrove_biomass_2000)
outname = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id)
out = '--outfile={}'.format(outname)
calc = '--calc=A*(B>0)'
datatype = '--type={}'.format('Int16')
uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id)
cmd = [
'gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass, calc,
out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite',
datatype, '--quiet'
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
else:
uu.print_log("No mangrove aboveground biomass tile for", tile_id)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove')
def prep_FIA_regions(tile_id):
uu.print_log("Creating Hansen tile for FIA regions")
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Rasterizing FIA region shapefile", tile_id)
blocksizex = 1024
blocksizey = 1024
uu.rasterize(
'{}.shp'.format(cn.name_FIA_regions_raw[:-4]),
"{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed), xmin,
ymin, xmax, ymax, blocksizex, blocksizey, '.00025', 'Byte',
'regionCode', '0')
uu.print_log("Checking if {} contains any data...".format(tile_id))
no_data = uu.check_for_data("{0}_{1}.tif".format(
tile_id, cn.pattern_FIA_regions_processed))
if no_data:
uu.print_log(" No data found. Deleting {}.".format(tile_id))
os.remove("{0}_{1}.tif".format(tile_id,
cn.pattern_FIA_regions_processed))
else:
uu.print_log(
" Data found in {}. Copying tile to s3...".format(tile_id))
uu.upload_final(cn.FIA_regions_processed_dir, tile_id,
cn.pattern_FIA_regions_processed)
uu.print_log(" Tile copied to s3")
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_FIA_regions_processed)
def create_combined_ifl_primary(tile_id):
# Start time
start = datetime.datetime.now()
ifl_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl)
primary_tile = '{}_primary_2001.tif'.format(tile_id)
ifl_primary_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl_primary)
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Assigns the correct time (primary forest or ifl)
if ymax <= 30 and ymax >= -20:
uu.print_log(
"{} between 30N and 30S. Using primary forest tile.".format(
tile_id))
os.rename(primary_tile, ifl_primary_tile)
else:
uu.print_log(
"{} not between 30N and 30S. Using IFL tile, if it exists.".format(
tile_id))
if os.path.exists(ifl_tile):
os.rename(ifl_tile, ifl_primary_tile)
else:
uu.print_log("IFL tile does not exist for {}".format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_ifl_primary)
def create_gain_year_count_loss_only(tile_id):
print "Loss pixel-only processing:", tile_id
# Names of the input tiles
loss, gain, tcd, biomass = tile_names(tile_id)
# start time
start = datetime.datetime.now()
# Pixels with loss only and not in mangroves or planted forests
print "Creating raster of growth years for loss-only pixels"
loss_calc = '--calc=(A>0)*(B==0)*(C>0)*(A-1)'
loss_outfilename = 'growth_years_loss_only_{}.tif'.format(tile_id)
loss_outfilearg = '--outfile={}'.format(loss_outfilename)
cmd = [
'gdal_calc.py', '-A', loss, '-B', gain, '-C', biomass, loss_calc,
loss_outfilearg, '--NoDataValue=0', '--overwrite', '--co',
'COMPRESS=LZW'
]
subprocess.check_call(cmd)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'growth_years_loss_only')
def annual_gain_rate(tile_id, sensit_type, gain_table_dict, stdev_table_dict,
output_pattern_list, no_upload):
# Converts the forest age category decision tree output values to the three age categories--
# 10000: primary forest; 20000: secondary forest > 20 years; 30000: secondary forest <= 20 years
# These are five digits so they can easily be added to the four digits of the continent-ecozone code to make unique codes
# for each continent-ecozone-age combination.
# The key in the dictionary is the forest age category decision tree endpoints.
age_dict = {0: 0, 1: 10000, 2: 20000, 3: 30000}
uu.print_log("Processing:", tile_id)
# Start time
start = datetime.datetime.now()
# Names of the forest age category and continent-ecozone tiles
age_cat = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_age_cat_IPCC)
cont_eco = uu.sensit_tile_rename(sensit_type, tile_id,
cn.pattern_cont_eco_processed)
# Names of the output natural forest gain rate tiles (above and belowground)
AGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[0])
BGB_IPCC_default_gain_rate = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[1])
AGB_IPCC_default_gain_stdev = '{0}_{1}.tif'.format(tile_id,
output_pattern_list[2])
uu.print_log(
" Creating IPCC default biomass gain rates and standard deviation for {}"
.format(tile_id))
# Opens the input tiles if they exist. kips tile if either input doesn't exist.
try:
age_cat_src = rasterio.open(age_cat)
uu.print_log(" Age category tile found for {}".format(tile_id))
except:
return uu.print_log(
" No age category tile found for {}. Skipping tile.".format(
tile_id))
try:
cont_eco_src = rasterio.open(cont_eco)
uu.print_log(" Continent-ecozone tile found for {}".format(tile_id))
except:
return uu.print_log(
" No continent-ecozone tile found for {}. Skipping tile.".format(
tile_id))
# Grabs metadata about the continent ecozone tile, like its location/projection/cellsize
kwargs = cont_eco_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = cont_eco_src.block_windows(1)
# Updates kwargs for the output dataset.
# Need to update data type to float 32 so that it can handle fractional gain rates
kwargs.update(driver='GTiff',
count=1,
compress='lzw',
nodata=0,
dtype='float32')
# The output files, aboveground and belowground biomass gain rates
dst_above = rasterio.open(AGB_IPCC_default_gain_rate, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_above, sensit_type)
dst_above.update_tags(
units='megagrams aboveground biomass (AGB or dry matter)/ha/yr')
dst_above.update_tags(
source='IPCC Guidelines 2019 refinement, forest section, Table 4.9')
dst_above.update_tags(
extent=
'Full model extent, even though these rates will not be used over the full model extent'
)
dst_below = rasterio.open(BGB_IPCC_default_gain_rate, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_below, sensit_type)
dst_below.update_tags(
units='megagrams belowground biomass (AGB or dry matter)/ha/yr')
dst_below.update_tags(
source='IPCC Guidelines 2019 refinement, forest section, Table 4.9')
dst_below.update_tags(
extent=
'Full model extent, even though these rates will not be used over the full model extent'
)
dst_stdev_above = rasterio.open(AGB_IPCC_default_gain_stdev, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst_stdev_above, sensit_type)
dst_stdev_above.update_tags(
units=
'standard deviation, in terms of megagrams aboveground biomass (AGB or dry matter)/ha/yr'
)
dst_stdev_above.update_tags(
source='IPCC Guidelines 2019 refinement, forest section, Table 4.9')
dst_stdev_above.update_tags(
extent=
'Full model extent, even though these standard deviations will not be used over the full model extent'
)
# Iterates across the windows (1 pixel strips) of the input tiles
for idx, window in windows:
# Creates a processing window for each input raster
try:
cont_eco_window = cont_eco_src.read(1, window=window)
except:
cont_eco_window = np.zeros((window.height, window.width),
dtype='uint8')
try:
age_cat_window = age_cat_src.read(1, window=window)
except:
age_cat_window = np.zeros((window.height, window.width),
dtype='uint8')
# Recodes the input forest age category array with 10 different decision tree end values into the 3 actual age categories
age_recode = np.vectorize(age_dict.get)(age_cat_window)
# Adds the age category codes to the continent-ecozone codes to create an array of unique continent-ecozone-age codes
cont_eco_age = cont_eco_window + age_recode
## Aboveground removal factors
# Converts the continent-ecozone array to float so that the values can be replaced with fractional gain rates
gain_rate_AGB = cont_eco_age.astype('float32')
# Applies the dictionary of continent-ecozone-age gain rates to the continent-ecozone-age array to
# get annual gain rates (metric tons aboveground biomass/yr) for each pixel
for key, value in gain_table_dict.items():
gain_rate_AGB[gain_rate_AGB == key] = value
# Writes the output window to the output file
dst_above.write_band(1, gain_rate_AGB, window=window)
## Belowground removal factors
# Calculates belowground annual removal rates
gain_rate_BGB = gain_rate_AGB * cn.below_to_above_non_mang
# Writes the output window to the output file
dst_below.write_band(1, gain_rate_BGB, window=window)
## Aboveground removal factor standard deviation
# Converts the continent-ecozone array to float so that the values can be replaced with fractional standard deviations
gain_stdev_AGB = cont_eco_age.astype('float32')
# Applies the dictionary of continent-ecozone-age gain rate standard deviations to the continent-ecozone-age array to
# get annual gain rate standard deviations (metric tons aboveground biomass/yr) for each pixel
for key, value in stdev_table_dict.items():
gain_stdev_AGB[gain_stdev_AGB == key] = value
# Writes the output window to the output file
dst_stdev_above.write_band(1, gain_stdev_AGB, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_pattern_list[0], no_upload)
def create_gain_year_count_merge(tile_id, pattern, sensit_type, no_upload):
uu.print_log(
"Merging loss, gain, no change, and loss/gain pixels into single raster for {}"
.format(tile_id))
# start time
start = datetime.datetime.now()
# The four rasters from above that are to be merged
no_change_gain_years = '{}_growth_years_no_change.tif'.format(tile_id)
loss_only_gain_years = '{}_growth_years_loss_only.tif'.format(tile_id)
gain_only_gain_years = '{}_growth_years_gain_only.tif'.format(tile_id)
loss_and_gain_gain_years = '{}_growth_years_loss_and_gain.tif'.format(
tile_id)
# Names of the output tiles
gain_year_count_merged = '{0}_{1}.tif'.format(tile_id, pattern)
# Opens no change gain year count tile. This should exist for all tiles.
with rasterio.open(no_change_gain_years) as no_change_gain_years_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = no_change_gain_years_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = no_change_gain_years_src.block_windows(1)
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
uu.print_log(
" No change tile exists for {} by default".format(tile_id))
# Opens the other gain year count tiles. They may not exist for all other tiles.
try:
loss_only_gain_years_src = rasterio.open(loss_only_gain_years)
uu.print_log(" Loss only tile found for {}".format(tile_id))
except:
uu.print_log(" No loss only tile found for {}".format(tile_id))
try:
gain_only_gain_years_src = rasterio.open(gain_only_gain_years)
uu.print_log(" Gain only tile found for {}".format(tile_id))
except:
uu.print_log(" No gain only tile found for {}".format(tile_id))
try:
loss_and_gain_gain_years_src = rasterio.open(
loss_and_gain_gain_years)
uu.print_log(" Loss and gain tile found for {}".format(tile_id))
except:
uu.print_log(
" No loss and gain tile found for {}".format(tile_id))
# Opens the output tile, giving it the arguments of the input tiles
gain_year_count_merged_dst = rasterio.open(gain_year_count_merged, 'w',
**kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(gain_year_count_merged_dst, sensit_type)
gain_year_count_merged_dst.update_tags(units='years')
gain_year_count_merged_dst.update_tags(min_possible_value='0')
gain_year_count_merged_dst.update_tags(
max_possible_value=cn.loss_years)
gain_year_count_merged_dst.update_tags(
source=
'Gain years are assigned based on the combination of Hansen loss and gain in each pixel. There are four combinations: neither loss nor gain, loss only, gain only, loss and gain.'
)
gain_year_count_merged_dst.update_tags(extent='Full model extent')
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
no_change_gain_years_window = no_change_gain_years_src.read(
1, window=window)
try:
loss_only_gain_years_window = loss_only_gain_years_src.read(
1, window=window)
except:
loss_only_gain_years_window = np.zeros(
(window.height, window.width), dtype='uint8')
try:
gain_only_gain_years_window = gain_only_gain_years_src.read(
1, window=window)
except:
gain_only_gain_years_window = np.zeros(
(window.height, window.width), dtype='uint8')
try:
loss_and_gain_gain_years_window = loss_and_gain_gain_years_src.read(
1, window=window)
except:
loss_and_gain_gain_years_window = np.zeros(
(window.height, window.width), dtype='uint8')
gain_year_count_merged_window = loss_only_gain_years_window + gain_only_gain_years_window + \
no_change_gain_years_window + loss_and_gain_gain_years_window
gain_year_count_merged_dst.write_band(
1, gain_year_count_merged_window, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def create_supplementary_outputs(tile_id, input_pattern, output_patterns,
sensit_type):
# start time
start = datetime.datetime.now()
# Extracts the tile id, tile type, and bounding box for the tile
tile_id = uu.get_tile_id(tile_id)
# Names of inputs
focal_tile = '{0}_{1}.tif'.format(tile_id, input_pattern)
pixel_area = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)
tcd = '{0}_{1}.tif'.format(cn.pattern_tcd, tile_id)
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
mangrove = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
# Names of outputs.
# Requires that output patterns be listed in main script in the correct order for here
# (currently, per pixel full extent, per hectare forest extent, per pixel forest extent).
per_pixel_full_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[0])
per_hectare_forest_extent = '{0}_{1}.tif'.format(tile_id,
output_patterns[1])
per_pixel_forest_extent = '{0}_{1}.tif'.format(tile_id, output_patterns[2])
# Opens input tiles for rasterio
in_src = rasterio.open(focal_tile)
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = in_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = in_src.block_windows(1)
pixel_area_src = rasterio.open(pixel_area)
tcd_src = rasterio.open(tcd)
gain_src = rasterio.open(gain)
try:
mangrove_src = rasterio.open(mangrove)
uu.print_log(" Mangrove tile found for {}".format(tile_id))
except:
uu.print_log(" No mangrove tile found for {}".format(tile_id))
uu.print_log(" Creating outputs for {}...".format(focal_tile))
kwargs.update(driver='GTiff',
count=1,
compress='lzw',
nodata=0,
dtype='float32')
# Opens output tiles, giving them the arguments of the input tiles
per_pixel_full_extent_dst = rasterio.open(per_pixel_full_extent, 'w',
**kwargs)
per_hectare_forest_extent_dst = rasterio.open(per_hectare_forest_extent,
'w', **kwargs)
per_pixel_forest_extent_dst = rasterio.open(per_pixel_forest_extent, 'w',
**kwargs)
# Adds metadata tags to the output rasters
uu.add_rasterio_tags(per_pixel_full_extent_dst, sensit_type)
per_pixel_full_extent_dst.update_tags(
units='Mg CO2e/pixel over model duration (2001-20{})'.format(
cn.loss_years))
per_pixel_full_extent_dst.update_tags(
source='per hectare full model extent tile')
per_pixel_full_extent_dst.update_tags(
extent=
'Full model extent: ((TCD2000>0 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations'
)
uu.add_rasterio_tags(per_hectare_forest_extent_dst, sensit_type)
per_hectare_forest_extent_dst.update_tags(
units='Mg CO2e/hectare over model duration (2001-20{})'.format(
cn.loss_years))
per_hectare_forest_extent_dst.update_tags(
source='per hectare full model extent tile')
per_hectare_forest_extent_dst.update_tags(
extent=
'Forest extent: ((TCD2000>30 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations'
)
uu.add_rasterio_tags(per_pixel_forest_extent_dst, sensit_type)
per_pixel_forest_extent_dst.update_tags(
units='Mg CO2e/pixel over model duration (2001-20{})'.format(
cn.loss_years))
per_pixel_forest_extent_dst.update_tags(
source='per hectare forest model extent tile')
per_pixel_forest_extent_dst.update_tags(
extent=
'Forest extent: ((TCD2000>30 AND WHRC AGB2000>0) OR Hansen gain=1 OR mangrove AGB2000>0) NOT IN pre-2000 plantations'
)
if "net_flux" in focal_tile:
per_pixel_full_extent_dst.update_tags(
scale=
'Negative values are net sinks. Positive values are net sources.')
per_hectare_forest_extent_dst.update_tags(
scale=
'Negative values are net sinks. Positive values are net sources.')
per_pixel_forest_extent_dst.update_tags(
scale=
'Negative values are net sinks. Positive values are net sources.')
# Iterates across the windows of the input tiles
for idx, window in windows:
# Creates windows for each input tile
in_window = in_src.read(1, window=window)
pixel_area_window = pixel_area_src.read(1, window=window)
tcd_window = tcd_src.read(1, window=window)
gain_window = gain_src.read(1, window=window)
try:
mangrove_window = mangrove_src.read(1, window=window)
except:
mangrove_window = np.zeros((window.height, window.width),
dtype='uint8')
# Output window for per pixel full extent raster
dst_window_per_pixel_full_extent = in_window * pixel_area_window / cn.m2_per_ha
# Output window for per hectare forest extent raster
# QCed this line before publication and then again afterwards in response to question from Lena Schulte-Uebbing at Wageningen Uni.
dst_window_per_hectare_forest_extent = np.where(
(tcd_window > cn.canopy_threshold) | (gain_window == 1) |
(mangrove_window != 0), in_window, 0)
# Output window for per pixel forest extent raster
dst_window_per_pixel_forest_extent = dst_window_per_hectare_forest_extent * pixel_area_window / cn.m2_per_ha
# Writes arrays to output raster
per_pixel_full_extent_dst.write_band(1,
dst_window_per_pixel_full_extent,
window=window)
per_hectare_forest_extent_dst.write_band(
1, dst_window_per_hectare_forest_extent, window=window)
per_pixel_forest_extent_dst.write_band(
1, dst_window_per_pixel_forest_extent, window=window)
uu.print_log(" Output tiles created for {}...".format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_patterns[0])
def hansen_burnyear(tile_id, no_upload):
# Start time
start = datetime.datetime.now()
uu.print_log("Processing", tile_id)
# The tiles that are used. out_tile_no_tag is the output before metadata tags are added. out_tile is the output
# once metadata tags have been added.
out_tile_no_tag = '{0}_{1}_no_tag.tif'.format(tile_id,
cn.pattern_burn_year)
out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_burn_year)
loss = '{0}.tif'.format(tile_id)
# Does not continue processing tile if no loss (because there will not be any output)
if not os.path.exists(loss):
uu.print_log("No loss tile for", tile_id)
return
else:
uu.print_log("Loss tile exists for", tile_id)
# Downloads the burned area tiles for each year
include = 'ba_*_{}.tif'.format(tile_id)
burn_tiles_dir = 'burn_tiles'
if not os.path.exists(burn_tiles_dir):
os.mkdir(burn_tiles_dir)
cmd = [
'aws', 's3', 'cp', cn.burn_year_warped_to_Hansen_dir, burn_tiles_dir,
'--recursive', '--exclude', "*", '--include', include
]
uu.log_subprocess_output_full(cmd)
# For each year tile, converts to array and stacks them
array_list = []
ba_tifs = glob.glob(burn_tiles_dir + '/*{}*'.format(tile_id))
# Skips the tile if it has no burned area data for any year
uu.print_log("There are {0} tiles to stack for {1}".format(
len(ba_tifs), tile_id))
if len(ba_tifs) == 0:
uu.print_log(
"Skipping {} because there are no tiles to stack".format(tile_id))
return
# NOTE: All of this could pretty easily be done in rasterio. However, Sam's use of GDAL for this still works fine,
# so I've left it using GDAL.
for ba_tif in ba_tifs:
uu.print_log("Creating array with {}".format(ba_tif))
array = utilities.raster_to_array(ba_tif)
array_list.append(array)
# Stacks arrays from each year
uu.print_log("Stacking arrays for", tile_id)
stacked_year_array = utilities.stack_arrays(array_list)
# Converts Hansen tile to array
uu.print_log("Creating loss year array for", tile_id)
loss_array = utilities.raster_to_array(loss)
# Determines what year to assign burned area
lossarray_min1 = np.subtract(loss_array, 1)
stack_con = (stacked_year_array >= lossarray_min1) & (stacked_year_array <=
loss_array)
stack_con2 = stack_con * stacked_year_array
lossyear_burn_array = stack_con2.max(0)
utilities.array_to_raster_simple(lossyear_burn_array, out_tile_no_tag,
loss)
# Only copies to s3 if the tile has data
uu.print_log("Checking if {} contains any data...".format(tile_id))
empty = uu.check_for_data(out_tile_no_tag)
# Checks output for data. There could be burned area but none of it coincides with tree cover loss,
# so this is the final check for whether there is any data.
if empty:
uu.print_log(" No data found. Not copying {}.".format(tile_id))
# Without this, the untagged version is counted and eventually copied to s3 if it has data in it
os.remove(out_tile_no_tag)
return
else:
uu.print_log(
" Data found in {}. Adding metadata tags...".format(tile_id))
### Thomas suggested these on 8/19/2020 but they didn't work. The first one wrote the tags but erased all the
### data in the tiles (everything became 0 according to gdalinfo). The second one had some other error.
# with rasterio.open(out_tile_no_tag, 'r') as src:
#
# profile = src.profile
#
# with rasterio.open(out_tile_no_tag, 'w', **profile) as dst:
#
# dst.update_tags(units='year (2001, 2002, 2003...)',
# source='MODIS collection 6 burned area',
# extent='global')
#
# with rasterio.open(out_tile_no_tag, 'w+') as src:
#
# dst.update_tags(units='year (2001, 2002, 2003...)',
# source='MODIS collection 6 burned area',
# extent='global')
# All of the below is to add metadata tags to the output burn year masks.
# For some reason, just doing what's at https://rasterio.readthedocs.io/en/latest/topics/tags.html
# results in the data getting removed.
# I found it necessary to copy the desired output and read its windows into a new copy of the file, to which the
# metadata tags are added. I'm sure there's an easier way to do this but I couldn't figure out how.
# I know it's very convoluted but I really couldn't figure out how to add the tags without erasing the data.
copyfile(out_tile_no_tag, out_tile)
with rasterio.open(out_tile_no_tag) as out_tile_no_tag_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = out_tile_no_tag_src.meta #### Use profile instead
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = out_tile_no_tag_src.block_windows(1)
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
out_tile_tagged = rasterio.open(out_tile, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(out_tile_tagged, 'std')
out_tile_tagged.update_tags(units='year (2001, 2002, 2003...)')
out_tile_tagged.update_tags(
source=
'MODIS collection 6 burned area, https://modis-fire.umd.edu/files/MODIS_C6_BA_User_Guide_1.3.pdf'
)
out_tile_tagged.update_tags(extent='global')
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
in_window = out_tile_no_tag_src.read(1, window=window)
# Writes the output window to the output
out_tile_tagged.write_band(1, in_window, window=window)
# Without this, the untagged version is counted and eventually copied to s3 if it has data in it
os.remove(out_tile_no_tag)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_burn_year, no_upload)
