def create_10x10_plantation_type(tile_id, plant_type_1x1_vrt):
uu.print_log("Getting bounding coordinates for tile", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax)
tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_planted_forest_type_unmasked)
uu.print_log("Rasterizing", tile_10x10)
cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)),
'-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
'-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Byte', plant_type_1x1_vrt, tile_10x10]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.print_log("Checking if {} contains any data...".format(tile_id))
stats = uu.check_for_data(tile_10x10)
if stats[0] > 0:
uu.print_log(" Data found in {}. Copying tile to s3...".format(tile_id))
uu.upload_final(cn.planted_forest_type_unmasked_dir, tile_id, cn.pattern_planted_forest_type_unmasked)
uu.print_log(" Tile converted and copied to s3")
else:
print(" No data found. Not copying {}.".format(tile_id))
def rasterize_pre_2000_plantations(tile_id):
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_plant_pre_2000)
cmd = [
'gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=LZW', '-tr',
'{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-tap', '-ot',
'Byte', '-a_nodata', '0', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '{}.shp'.format(cn.pattern_plant_pre_2000_raw), out_tile
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_plant_pre_2000)
def create_mangrove_soil_C(tile_id, no_upload):
# Start time
start = datetime.datetime.now()
# Checks if mangrove biomass exists. If not, it won't create a mangrove soil C tile.
if os.path.exists('{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)):
uu.print_log("Mangrove aboveground biomass tile found for", tile_id)
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Clipping mangrove soil C from mangrove soil vrt for", tile_id)
uu.warp_to_Hansen('mangrove_soil_C.vrt', '{0}_mangrove_full_extent.tif'.format(tile_id), xmin, ymin, xmax, ymax, 'Int16')
mangrove_soil = '{0}_mangrove_full_extent.tif'.format(tile_id)
mangrove_biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
outname = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id)
out = '--outfile={}'.format(outname)
calc = '--calc=A*(B>0)'
datatype = '--type={}'.format('Int16')
uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id)
cmd = ['gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass,
calc, out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite', datatype, '--quiet']
uu.log_subprocess_output_full(cmd)
else:
uu.print_log("No mangrove aboveground biomass tile for", tile_id)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove', no_upload)
def create_10x10_plantation(tile_id, plant_1x1_vrt):
print "Getting bounding coordinates for tile", tile_id
xmin, ymin, xmax, ymax = uu.coords(tile_id)
print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax
tile_10x10 = '{0}_{1}.tif'.format(
tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)
print "Rasterizing", tile_10x10
cmd = [
'gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)),
'{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-tap', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite',
'-ot', 'Float32', plant_1x1_vrt, tile_10x10
]
subprocess.check_call(cmd)
print "Checking if {} contains any data...".format(tile_id)
stats = uu.check_for_data(tile_10x10)
if stats[0] > 0:
print " Data found in {}. Copying tile to s3...".format(tile_id)
uu.upload_final(cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir,
tile_id,
cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)
print " Tile converted and copied to s3"
else:
print " No data found. Not copying {}.".format(tile_id)
def clip_year_tiles(tile_year_list, no_upload):
# Start time
start = datetime.datetime.now()
tile_id = tile_year_list[0].strip('.tif')
year = tile_year_list[1]
vrt_name = "global_vrt_{}_wgs84.vrt".format(year)
# Gets coordinates of hansen tile
uu.print_log("Getting coordinates of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Clips vrt to tile extent
uu.print_log("Clipping burn year vrt to {0} for {1}".format(tile_id, year))
clipped_raster = "ba_clipped_{0}_{1}.tif".format(year, tile_id)
cmd = [
'gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '-a_nodata',
'0'
]
cmd += [vrt_name, clipped_raster, '-tr', '.00025', '.00025']
cmd += ['-projwin', str(xmin), str(ymax), str(xmax), str(ymin)]
uu.log_subprocess_output_full(cmd)
# Calculates year tile values to be equal to year. ex: 17*1
calc = '--calc={}*(A>0)'.format(int(year) - 2000)
recoded_output = "ba_{0}_{1}.tif".format(year, tile_id)
outfile = '--outfile={}'.format(recoded_output)
cmd = [
'gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0',
'--co', 'COMPRESS=LZW', '--quiet'
]
uu.log_subprocess_output_full(cmd)
# Only copies to s3 if the tile has data.
# No tiles for 2000 have data because the burn year is coded as 0, which is NoData.
uu.print_log("Checking if {} contains any data...".format(tile_id))
empty = uu.check_for_data(recoded_output)
if empty:
uu.print_log(" No data found. Not copying {}.".format(tile_id))
else:
uu.print_log(
" Data found in {}. Copying tile to s3...".format(tile_id))
cmd = [
'aws', 's3', 'cp', recoded_output,
cn.burn_year_warped_to_Hansen_dir
]
uu.log_subprocess_output_full(cmd)
uu.print_log(" Tile copied to", cn.burn_year_warped_to_Hansen_dir)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year), no_upload)
def create_input_files(tile_id):
print "Getting extent of", tile_id
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# # Soil tiles are already processed, so there's no need to include them here.
# # Below is the old code for tile-izing the histosole soil raster.
# # Leaving this in case I ever add in soil processing again.
# print "clip soil"
# extra_param = ['-tr', '.00025', '.00025', '-dstnodata', '0']
# clip_soil_tile = util.clip('hwsd_oc_final.tif', '{}_soil.tif'.format(tile_id), xmin, ymin, xmax, ymax, extra_param)
#
# print "removing no data flag from soil"
# cmd = ['gdal_edit.py', '-unsetnodata', clip_soil_tile]
# subprocess.check_call(cmd)
#
# print "uploading soil tile to s3"
# util.upload(clip_soil_tile, cn.soil_C_processed_dir)
print "Rasterizing ecozone"
rasterized_eco_zone_tile = util.rasterize(
'fao_ecozones_bor_tem_tro.shp',
"{}_fao_ecozones_bor_tem_tro.tif".format(tile_id), xmin, ymin, xmax,
ymax, '.008', 'Byte', 'recode', '0')
print "Resampling eco zone"
resampled_ecozone = util.resample(
rasterized_eco_zone_tile,
"{0}_{1}.tif".format(tile_id, cn.pattern_fao_ecozone_processed))
print "Uploading processed ecozone"
util.upload(resampled_ecozone, cn.fao_ecozone_processed_dir)
print "Clipping srtm"
tile_srtm = util.clip('srtm.vrt', '{}_srtm.tif'.format(tile_id), xmin,
ymin, xmax, ymax)
print "Resampling srtm"
tile_res_srtm = util.resample(
tile_srtm, '{0}_{1}.tif'.format(tile_id, cn.pattern_srtm))
print "Uploading processed srtm"
util.upload(tile_res_srtm, cn.srtm_processed_dir)
print "Clipping precipitation"
clipped_precip_tile = util.clip('add_30s_precip.tif',
'{}_clip_precip.tif'.format(tile_id), xmin,
ymin, xmax, ymax)
print "Resampling precipitation"
resample_precip_tile = util.resample(
clipped_precip_tile, '{0}_{1}.tif'.format(tile_id, cn.pattern_precip))
print "Uploading processed precipitation"
util.upload(resample_precip_tile, cn.precip_processed_dir)
def rasterize_gadm_1x1(tile_id):
uu.print_log("Getting bounding coordinates for tile", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:",
ymax)
# Degrees of tile in x and y dimensions
x_size = abs(int(xmin) - int(xmax))
y_size = abs(int(ymin) - int(ymax))
# Iterates through input 10x10 tile by 1x1 degree
for x in range(x_size):
xmin_1x1 = int(xmin) + x
xmax_1x1 = int(xmin) + x + 1
for y in range(y_size):
ymin_1x1 = int(ymin) + y
ymax_1x1 = int(ymin) + y + 1
uu.print_log(" xmin_1x1:", xmin_1x1, "; xmax_1x1:", xmax_1x1,
"; ymin_1x1", ymin_1x1, "; ymax_1x1:", ymax_1x1)
tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1)
uu.print_log("Rasterizing", tile_1x1)
cmd = [
'gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)),
'{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-te',
str(xmin_1x1),
str(ymin_1x1),
str(xmax_1x1),
str(ymax_1x1), '-burn', '1', '-a_nodata', '0', cn.gadm_iso,
tile_1x1
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Only keeps 1x1 GADM tiles if they actually include a country; many 1x1 tiles created out of 10x10 tiles
# don't actually include a country.
uu.print_log(
"Checking if {} contains any data...".format(tile_1x1))
stats = uu.check_for_data(tile_1x1)
if stats[1] > 0:
uu.print_log(
" Data found in {}. Keeping tile".format(tile_1x1))
else:
uu.print_log(
" No data found in {}. Deleting.".format(tile_1x1))
os.remove(tile_1x1)
def loss_in_raster(tile_id, raster_type, output_name, lat, mask):
uu.print_log("Calculating loss area for tile id {0}...".format(tile_id))
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# start time
start = datetime.datetime.now()
# Name of the loss time
loss_tile = '{0}.tif'.format(tile_id)
# The raster that loss is being analyzed inside
raster_of_interest = '{0}_{1}.tif'.format(tile_id, raster_type)
# Output file name
outname = '{0}_{1}.tif'.format(tile_id, output_name)
# Only processes the tile if it is inside the latitude band (north of the specified latitude)
if ymax > lat and os.path.exists(raster_of_interest):
uu.print_log("{} inside latitude band and peat tile exists. Processing tile.".format(tile_id))
# If the user has asked to create just a mask of loss as opposed to the actual output values
if mask == "True":
calc = '--calc=(A>=1)*(A+1)/(A+1)*B'
# If the user has asked to output the actual loss values
if mask == "False":
# Equation argument for converting emissions from per hectare to per pixel.
# First, multiplies the per hectare emissions by the area of the pixel in m2, then divides by the number of m2 in a hectare.
calc = '--calc=A*B'
# Argument for outputting file
out = '--outfile={}'.format(outname)
uu.print_log("Masking loss in {} by raster of interest...".format(tile_id))
cmd = ['gdal_calc.py', '-A', loss_tile, '-B', raster_of_interest, calc, out, '--NoDataValue=0', '--co', 'COMPRESS=LZW',
'--overwrite', '--quiet']
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.print_log("{} masked".format(tile_id))
else:
uu.print_log("{} outside of latitude band. Skipped tile.".format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, output_name)
def rasterize_gadm_1x1(tile_id):
print "Getting bounding coordinates for tile", tile_id
xmin, ymin, xmax, ymax = uu.coords(tile_id)
print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax
# Degrees of tile in x and y dimensions
x_size = abs(int(xmin) - int(xmax))
y_size = abs(int(ymin) - int(ymax))
# Iterates through input 10x10 tile by 1x1 degree
for x in range(x_size):
xmin_1x1 = int(xmin) + x
xmax_1x1 = int(xmin) + x + 1
for y in range(y_size):
ymin_1x1 = int(ymin) + y
ymax_1x1 = int(ymin) + y + 1
print " xmin_1x1:", xmin_1x1, "; xmax_1x1:", xmax_1x1, "; ymin_1x1", ymin_1x1, "; ymax_1x1:", ymax_1x1
tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1)
print "Rasterizing", tile_1x1
cmd = [
'gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)),
'{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-te',
str(xmin_1x1),
str(ymin_1x1),
str(xmax_1x1),
str(ymax_1x1), '-burn', '1', '-a_nodata', '0', cn.gadm_iso,
tile_1x1
]
subprocess.check_call(cmd)
# Only keeps 1x1 GADM tiles if they actually include a country; many 1x1 tiles created out of 10x10 tiles
# don't actually include a country.
print "Checking if {} contains any data...".format(tile_1x1)
stats = uu.check_for_data(tile_1x1)
if stats[1] > 0:
print " Data found in {}. Keeping tile".format(tile_1x1)
else:
print " No data found in {}. Deleting.".format(tile_1x1)
os.remove(tile_1x1)
def create_mangrove_soil_C(tile_id):
# Start time
start = datetime.datetime.now()
# Checks if mangrove biomass exists. If not, it won't create a mangrove soil C tile.
if os.path.exists('{0}_{1}.tif'.format(tile_id,
cn.pattern_mangrove_biomass_2000)):
uu.print_log("Mangrove aboveground biomass tile found for", tile_id)
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Clipping mangrove soil C from mangrove soil vrt for",
tile_id)
uu.warp_to_Hansen('mangrove_soil_C.vrt',
'{0}_mangrove_full_extent.tif'.format(tile_id), xmin,
ymin, xmax, ymax, 'Int16')
mangrove_soil = '{0}_mangrove_full_extent.tif'.format(tile_id)
mangrove_biomass = '{0}_{1}.tif'.format(
tile_id, cn.pattern_mangrove_biomass_2000)
outname = '{0}_mangrove_masked_to_mangrove.tif'.format(tile_id)
out = '--outfile={}'.format(outname)
calc = '--calc=A*(B>0)'
datatype = '--type={}'.format('Int16')
uu.print_log("Masking mangrove soil to mangrove biomass for", tile_id)
cmd = [
'gdal_calc.py', '-A', mangrove_soil, '-B', mangrove_biomass, calc,
out, '--NoDataValue=0', '--co', 'COMPRESS=DEFLATE', '--overwrite',
datatype, '--quiet'
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
else:
uu.print_log("No mangrove aboveground biomass tile for", tile_id)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, 'mangrove_masked_to_mangrove')
def prep_FIA_regions(tile_id):
uu.print_log("Creating Hansen tile for FIA regions")
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Rasterizing FIA region shapefile", tile_id)
blocksizex = 1024
blocksizey = 1024
uu.rasterize(
'{}.shp'.format(cn.name_FIA_regions_raw[:-4]),
"{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed), xmin,
ymin, xmax, ymax, blocksizex, blocksizey, '.00025', 'Byte',
'regionCode', '0')
uu.print_log("Checking if {} contains any data...".format(tile_id))
no_data = uu.check_for_data("{0}_{1}.tif".format(
tile_id, cn.pattern_FIA_regions_processed))
if no_data:
uu.print_log(" No data found. Deleting {}.".format(tile_id))
os.remove("{0}_{1}.tif".format(tile_id,
cn.pattern_FIA_regions_processed))
else:
uu.print_log(
" Data found in {}. Copying tile to s3...".format(tile_id))
uu.upload_final(cn.FIA_regions_processed_dir, tile_id,
cn.pattern_FIA_regions_processed)
uu.print_log(" Tile copied to s3")
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_FIA_regions_processed)
def create_combined_ifl_primary(tile_id):
# Start time
start = datetime.datetime.now()
ifl_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl)
primary_tile = '{}_primary_2001.tif'.format(tile_id)
ifl_primary_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl_primary)
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Assigns the correct time (primary forest or ifl)
if ymax <= 30 and ymax >= -20:
uu.print_log(
"{} between 30N and 30S. Using primary forest tile.".format(
tile_id))
os.rename(primary_tile, ifl_primary_tile)
else:
uu.print_log(
"{} not between 30N and 30S. Using IFL tile, if it exists.".format(
tile_id))
if os.path.exists(ifl_tile):
os.rename(ifl_tile, ifl_primary_tile)
else:
uu.print_log("IFL tile does not exist for {}".format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_ifl_primary)
def forest_age_category(tile_id, gain_table_dict, pattern, sensit_type, no_upload):
uu.print_log("Assigning forest age categories:", tile_id)
# Start time
start = datetime.datetime.now()
# Gets the bounding coordinates of each tile. Needed to determine if the tile is in the tropics (within 30 deg of the equator)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Default value is that the tile is not in the tropics
tropics = 0
# Criteria for assigning a tile to the tropics
if (ymax > -30) & (ymax <= 30) :
tropics = 1
uu.print_log(" Tile {} in tropics:".format(tile_id), tropics)
# Names of the input tiles
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
model_extent = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_model_extent)
ifl_primary = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_ifl_primary)
cont_eco = uu.sensit_tile_rename(sensit_type, tile_id, cn.pattern_cont_eco_processed)
# Biomass tile name depends on the sensitivity analysis
if sensit_type == 'biomass_swap':
biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_JPL_unmasked_processed)
uu.print_log("Using JPL biomass tile for {} sensitivity analysis".format(sensit_type))
else:
biomass = '{0}_{1}.tif'.format(tile_id, cn.pattern_WHRC_biomass_2000_unmasked)
uu.print_log("Using WHRC biomass tile for {} sensitivity analysis".format(sensit_type))
if sensit_type == 'legal_Amazon_loss':
loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Brazil_annual_loss_processed)
uu.print_log("Using PRODES loss tile {0} for {1} sensitivity analysis".format(tile_id, sensit_type))
elif sensit_type == 'Mekong_loss':
loss = '{0}_{1}.tif'.format(tile_id, cn.pattern_Mekong_loss_processed)
else:
loss = '{0}_{1}.tif'.format(cn.pattern_loss, tile_id)
uu.print_log("Using Hansen loss tile {0} for {1} model run".format(tile_id, sensit_type))
uu.print_log(" Assigning age categories")
# Opens biomass tile
with rasterio.open(model_extent) as model_extent_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = model_extent_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = model_extent_src.block_windows(1)
# Opens the input tiles if they exist
try:
cont_eco_src = rasterio.open(cont_eco)
uu.print_log(" Continent-ecozone tile found for {}".format(tile_id))
except:
uu.print_log(" No continent-ecozone tile found for {}".format(tile_id))
try:
gain_src = rasterio.open(gain)
uu.print_log(" Gain tile found for {}".format(tile_id))
except:
uu.print_log(" No gain tile found for {}".format(tile_id))
try:
biomass_src = rasterio.open(biomass)
uu.print_log(" Biomass tile found for {}".format(tile_id))
except:
uu.print_log(" No biomass tile found for {}".format(tile_id))
try:
loss_src = rasterio.open(loss)
uu.print_log(" Loss tile found for {}".format(tile_id))
except:
uu.print_log(" No loss tile found for {}".format(tile_id))
try:
ifl_primary_src = rasterio.open(ifl_primary)
uu.print_log(" IFL-primary forest tile found for {}".format(tile_id))
except:
uu.print_log(" No IFL-primary forest tile found for {}".format(tile_id))
# Updates kwargs for the output dataset
kwargs.update(
driver='GTiff',
count=1,
compress='lzw',
nodata=0
)
# Opens the output tile, giving it the arguments of the input tiles
dst = rasterio.open('{0}_{1}.tif'.format(tile_id, pattern), 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(dst, sensit_type)
dst.update_tags(
key='1: young (<20 year) secondary forest; 2: old (>20 year) secondary forest; 3: primary forest or IFL')
dst.update_tags(
source='Decision tree that uses Hansen gain and loss, IFL/primary forest extent, and aboveground biomass to assign an age category')
dst.update_tags(
extent='Full model extent, even though these age categories will not be used over the full model extent. They apply to just the rates from IPCC defaults.')
uu.print_log(" Assigning IPCC age categories for", tile_id)
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
# Creates windows for each input raster. Only model_extent_src is guaranteed to exist
model_extent_window = model_extent_src.read(1, window=window)
try:
loss_window = loss_src.read(1, window=window)
except:
loss_window = np.zeros((window.height, window.width), dtype='uint8')
try:
gain_window = gain_src.read(1, window=window)
except:
gain_window = np.zeros((window.height, window.width), dtype='uint8')
try:
cont_eco_window = cont_eco_src.read(1, window=window)
except:
cont_eco_window = np.zeros((window.height, window.width), dtype='uint8')
try:
biomass_window = biomass_src.read(1, window=window)
except:
biomass_window = np.zeros((window.height, window.width), dtype='float32')
try:
ifl_primary_window = ifl_primary_src.read(1, window=window)
except:
ifl_primary_window = np.zeros((window.height, window.width), dtype='uint8')
# Creates a numpy array that has the <=20 year secondary forest growth rate x 20
# based on the continent-ecozone code of each pixel (the dictionary).
# This is used to assign pixels to the correct age category.
gain_20_years = np.vectorize(gain_table_dict.get)(cont_eco_window)*20
# Create a 0s array for the output
dst_data = np.zeros((window.height, window.width), dtype='uint8')
# Logic tree for assigning age categories begins here
# Code 1 = young (<20 years) secondary forest, code 2 = old (>20 year) secondary forest, code 3 = primary forest
# model_extent_window ensures that there is both biomass and tree cover in 2000 OR mangroves OR tree cover gain
# WITHOUT pre-2000 plantations
# For every model version except legal_Amazon_loss sensitivity analysis, which has its own rules about age assignment
if sensit_type != 'legal_Amazon_loss':
# No change pixels- no loss or gain
if tropics == 0:
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0))] = 2
if tropics == 1:
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0) & (ifl_primary_window != 1))] = 2
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window == 0) & (ifl_primary_window == 1))] = 3
# Loss-only pixels
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window != 1) & (biomass_window <= gain_20_years))] = 1
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window != 1) & (biomass_window > gain_20_years))] = 2
dst_data[np.where((model_extent_window > 0) & (gain_window == 0) & (loss_window > 0) & (ifl_primary_window ==1))] = 3
# Gain-only pixels
# If there is gain, the pixel doesn't need biomass or canopy cover. It just needs to be outside of plantations and mangroves.
# The role of model_extent_window here is to exclude the pre-2000 plantations.
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window == 0))] = 1
# Pixels with loss and gain
# If there is gain with loss, the pixel doesn't need biomass or canopy cover. It just needs to be outside of plantations and mangroves.
# The role of model_extent_window here is to exclude the pre-2000 plantations.
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > (cn.gain_years)))] = 1
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > 0) & (loss_window <= (cn.gain_years/2)))] = 1
dst_data[np.where((model_extent_window > 0) & (gain_window == 1) & (loss_window > (cn.gain_years/2)) & (loss_window <= cn.gain_years))] = 1
# For legal_Amazon_loss sensitivity analysis
else:
# Non-loss pixels (could have gain or not. Assuming that if within PRODES extent in 2000, there can't be
# gain, so it's a faulty detection. Thus, gain-only pixels are ignored and become part of no change.)
dst_data[np.where((model_extent_window == 1) & (loss_window == 0))] = 3 # primary forest
# Loss-only pixels
dst_data[np.where((model_extent_window == 1) & (loss_window > 0) & (gain_window == 0))] = 3 # primary forest
# Loss-and-gain pixels
dst_data[np.where((model_extent_window == 1) & (loss_window > 0) & (gain_window == 1))] = 2 # young secondary forest
# Writes the output window to the output
dst.write_band(1, dst_data, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, pattern, no_upload)
def create_continent_ecozone_tiles(tile_id):
uu.print_log("Processing:", tile_id)
# Start time
start = datetime.datetime.now()
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Extent of", tile_id, "-- ymax:", ymax, "; ymin:", ymin,
"; xmax", xmax, "; xmin:", xmin)
uu.print_log(
"Rasterizing ecozone to extent of biomass tile {}".format(tile_id))
cont_eco_raw = "{0}_{1}".format(tile_id, cn.pattern_cont_eco_raw)
# This makes rasters that are made of 1024 x 1024 pixel windows instead of 40000 x 1 pixel windows
# to improve assigning pixels without continent-ecozone codes to a continent-ecozone code.
# This way, pixels without continent-ecozone are assigned a code based on what's in a window nearby, rather
# than a window that spans the entire 10x10 degree tile.
blocksizex = 1024
blocksizey = 1024
uu.rasterize(
'fao_ecozones_fra_2000_continents_assigned_dissolved_FINAL_20180906.shp',
cont_eco_raw, xmin, ymin, xmax, ymax, blocksizex, blocksizey, '.00025',
'Int16', 'gainEcoCon', '0')
# Opens continent-ecozone tile.
# Everything from here down is used to assign pixels without continent ecozone codes to a continent-ecozone in the 1024x1024 windows.
with rasterio.open('{}.tif'.format(cont_eco_raw)) as cont_eco_raw_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = cont_eco_raw_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = cont_eco_raw_src.block_windows(1)
# Updates kwargs for the output dataset.
# Need to update data type to float 32 so that it can handle fractional gain rates
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
# Opens the output tile, giving it the arguments of the input tiles
with rasterio.open(
'{0}_{1}.tif'.format(tile_id, cn.pattern_cont_eco_processed),
'w', **kwargs) as dst:
# Iterates across the windows (1024 x 1024 pixel boxes) of the input tile.
for idx, window in windows:
# Creates windows for each input raster
cont_eco_raw = cont_eco_raw_src.read(1, window=window)
# Turns the 2D array into a 1D array that is n x n long.
# This makes to easier to remove 0s and find the mode of the remaining continent-ecozone codes
cont_eco_raw_flat = cont_eco_raw.flatten()
# Removes all zeros from the array, leaving just pixels with continent-ecozone codes
non_zeros = np.delete(cont_eco_raw_flat,
np.where(cont_eco_raw_flat == 0))
# If there were only pixels without continent-ecozone codes in the array, the mode is assigned 0
if non_zeros.size < 1:
# print " Window is all 0s"
mode = 0
# If there were pixels with continent-ecozone codes, the mode is the most common code among those in the window
else:
mode = stats.mode(non_zeros)[0]
# print " Window is not all 0s. Mode is", mode
cont_eco_processed = cont_eco_raw
# Assigns all pixels without a continent-ecozone code in that window to that most common code
cont_eco_processed[cont_eco_processed == 0] = mode
# Writes the output window to the output.
# Although the windows for the input tiles are 1024 x 1024 pixels,
# the windows for these output files are 40000 x 1 pixels, like all the other tiles in this model,
# so they should work fine with all the other tiles.
dst.write_band(1, cont_eco_processed, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_annual_gain_AGB_mangrove)
def forest_age_category(tile_id, gain_table_dict):
print "Processing:", tile_id
# Gets the bounding coordinates of each tile. Needed to determine if the tile is in the tropics (within 30 deg of the equator)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
print " ymax:", ymax
# Default value is that the tile is not in the tropics
tropics = 0
# Criteria for assigning a tile to the tropics
if (ymax > -30) & (ymax <= 30):
tropics = 1
print " Tile in tropics:", tropics
# start time
start = datetime.datetime.now()
# Names of the input tiles
loss = '{}.tif'.format(tile_id)
gain = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
tcd = '{0}_{1}.tif'.format(cn.pattern_tcd, tile_id)
ifl = '{0}_{1}.tif'.format(tile_id, cn.pattern_ifl)
biomass = '{0}_{1}.tif'.format(
tile_id, cn.pattern_WHRC_biomass_2000_non_mang_non_planted)
cont_eco = '{0}_{1}.tif'.format(tile_id, cn.pattern_cont_eco_processed)
print " Reading input files and evaluating conditions"
# Opens biomass tile
with rasterio.open(loss) as loss_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = loss_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = loss_src.block_windows(1)
# Opens gain tile
with rasterio.open(gain) as gain_src:
# Opens ifl tile
with rasterio.open(ifl) as ifl_src:
# Opens continent-ecozone combinations tile
with rasterio.open(cont_eco) as cont_eco_src:
# Opens biomass 2000 tile
with rasterio.open(biomass) as biomass_src:
# Opens tree cover density tile
with rasterio.open(tcd) as extent_src:
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff',
count=1,
compress='lzw',
nodata=0)
# Opens the output tile, giving it the arguments of the input tiles
with rasterio.open(
'{0}_{1}.tif'.format(
tile_id,
cn.pattern_age_cat_natrl_forest), 'w',
**kwargs) as dst:
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
# Creates windows for each input raster
loss = loss_src.read(1, window=window)
gain = gain_src.read(1, window=window)
tcd = extent_src.read(1, window=window)
ifl = ifl_src.read(1, window=window)
cont_eco = cont_eco_src.read(1,
window=window)
biomass = biomass_src.read(1,
window=window)
# Creates a numpy array that has the <=20 year secondary forest growth rate x 20
# based on the continent-ecozone code of each pixel (the dictionary).
# This is used to assign pixels to the correct age category.
gain_20_years = np.vectorize(
gain_table_dict.get)(cont_eco) * 20
# Create a 0s array for the output
dst_data = np.zeros(
(window.height, window.width),
dtype='uint8')
# Logic tree for assigning age categories begins here
# No change pixels- no loss or gain
if tropics == 0:
dst_data[np.where((biomass > 0)
& (tcd > 0)
& (gain == 0)
& (loss == 0))] = 1
if tropics == 1:
dst_data[np.where((biomass > 0)
& (tcd > 0)
& (gain == 0)
& (loss == 0)
& (ifl != 1))] = 2
dst_data[np.where((biomass > 0)
& (tcd > 0)
& (gain == 0)
& (loss == 0)
& (ifl == 1))] = 3
# Loss-only pixels
dst_data[np.where(
(biomass > 0) & (gain == 0)
& (loss > 0) & (ifl != 1)
& (biomass <= gain_20_years))] = 4
dst_data[np.where(
(biomass > 0) & (gain == 0)
& (loss > 0) & (ifl != 1)
& (biomass > gain_20_years))] = 5
dst_data[np.where((biomass > 0)
& (gain == 0)
& (loss > 0)
& (ifl == 1))] = 6
# Gain-only pixels
dst_data[np.where((biomass > 0)
& (gain == 1)
& (loss == 0))] = 7
# Pixels with loss and gain
dst_data[np.where((biomass > 0)
& (gain == 1)
& (loss >= 13))] = 8
dst_data[np.where((biomass > 0)
& (gain == 1)
& (loss > 0)
& (loss <= 6))] = 9
dst_data[np.where((biomass > 0)
& (gain == 1)
& (loss > 6)
& (loss < 13))] = 10
# Writes the output window to the output
dst.write_band(1, dst_data, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_age_cat_natrl_forest)
def create_peat_mask_tiles(tile_id):
# Start time
start = datetime.datetime.now()
uu.print_log("Getting bounding coordinates for tile", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log(" ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:",
xmin)
out_tile_no_tag = '{0}_{1}_no_tag.tif'.format(tile_id,
cn.pattern_peat_mask)
out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_peat_mask)
# If the tile is outside the band covered by the CIFOR peat raster, SoilGrids250m is used
if ymax > 40 or ymax < -60:
uu.print_log(
"{} is outside CIFOR band. Using SoilGrids250m organic soil mask..."
.format(tile_id))
out_intermediate = '{0}_intermediate.tif'.format(
tile_id, cn.pattern_peat_mask)
# Cuts the SoilGrids250m global raster to the focal tile
uu.warp_to_Hansen('most_likely_soil_class.vrt', out_intermediate, xmin,
ymin, xmax, ymax, 'Byte')
# Removes all non-histosol sub-groups from the SoilGrids raster.
# Ideally, this would be done once on the entire SoilGrids raster in the main function but I didn't think of that.
# Code 14 is the histosol subgroup in SoilGrids250 (https://files.isric.org/soilgrids/latest/data/wrb/MostProbable.qml).
calc = '--calc=(A==14)'
peat_mask_out_filearg = '--outfile={}'.format(out_tile_no_tag)
cmd = [
'gdal_calc.py', '-A', out_intermediate, calc,
peat_mask_out_filearg, '--NoDataValue=0', '--overwrite', '--co',
'COMPRESS=LZW', '--type=Byte', '--quiet'
]
uu.log_subprocess_output_full(cmd)
uu.print_log("{} created.".format(tile_id))
# If the tile is inside the band covered by CIFOR, CIFOR is used (and Jukka in the tiles where it occurs).
# For some reason, the CIFOR raster has a color scheme that makes it symbolized from 0 to 255. This carries
# over to the output file but that seems like a problem with the output symbology, not the values.
# gdalinfo shows that the min and max values are 1, as they should be, and it visualizes correctly in ArcMap.
else:
uu.print_log(
"{} is inside CIFOR band. Using CIFOR/Jukka combination...".format(
tile_id))
# Combines CIFOR and Jukka (if it occurs there)
cmd = [
'gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr',
'{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-tap',
'-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-dstnodata', '0', '-overwrite',
'{}'.format(cn.cifor_peat_file), 'jukka_peat.tif', out_tile_no_tag
]
uu.log_subprocess_output_full(cmd)
uu.print_log("{} created.".format(tile_id))
# All of the below is to add metadata tags to the output peat masks.
# For some reason, just doing what's at https://rasterio.readthedocs.io/en/latest/topics/tags.html
# results in the data getting removed.
# I found it necessary to copy the peat mask and read its windows into a new copy of the file, to which the
# metadata tags are added. I'm sure there's an easier way to do this but I couldn't figure out how.
# I know it's very convoluted but I really couldn't figure out how to add the tags without erasing the data.
# To make it even stranger, adding the tags before the gdal processing seemed to work fine for the non-tropical
# (SoilGrids) tiles but not for the tropical (CIFOR/Jukka) tiles (i.e. data didn't disappear in the non-tropical
# tiles if I added the tags before the GDAL steps but the tropical data did disappear).
copyfile(out_tile_no_tag, out_tile)
uu.print_log("Adding metadata tags to", tile_id)
# Opens the output tile, only so that metadata tags can be added
# Based on https://rasterio.readthedocs.io/en/latest/topics/tags.html
with rasterio.open(out_tile_no_tag) as out_tile_no_tag_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = out_tile_no_tag_src.meta
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = out_tile_no_tag_src.block_windows(1)
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
out_tile_tagged = rasterio.open(out_tile, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(out_tile_tagged, 'std')
out_tile_tagged.update_tags(key='1 = peat. 0 = not peat.')
out_tile_tagged.update_tags(
source=
'Jukka for IDN and MYS; CIFOR for rest of tropics; SoilGrids250 (May 2020) most likely histosol for outside tropics'
)
out_tile_tagged.update_tags(extent='Full extent of input datasets')
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
peat_mask_window = out_tile_no_tag_src.read(1, window=window)
# Writes the output window to the output
out_tile_tagged.write_band(1, peat_mask_window, window=window)
# Otherwise, the untagged version is counted and eventually copied to s3 if it has data in it
os.remove(out_tile_no_tag)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_peat_mask)
def rewindow(tile):
# start time
start = datetime.datetime.now()
uu.print_log(
"Rewindowing {} to 200x200 pixel windows (0.04 degree x 0.04 degree)..."
.format(tile))
# Extracts the tile id, tile type, and bounding box for the tile
tile_id = uu.get_tile_id(tile)
tile_type = uu.get_tile_type(tile)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Raster name for 400x400 pixel tiles (intermediate output)
input_rewindow = '{0}_{1}_rewindow.tif'.format(tile_id, tile_type)
area_tile = '{0}_{1}.tif'.format(cn.pattern_pixel_area, tile_id)
pixel_area_rewindow = '{0}_{1}_rewindow.tif'.format(
cn.pattern_pixel_area, tile_id)
tcd_tile = '{0}_{1}.tif'.format(cn.pattern_tcd, tile_id)
tcd_rewindow = '{0}_{1}_rewindow.tif'.format(cn.pattern_tcd, tile_id)
gain_tile = '{0}_{1}.tif'.format(cn.pattern_gain, tile_id)
gain_rewindow = '{0}_{1}_rewindow.tif'.format(cn.pattern_gain, tile_id)
mangrove_tile = '{0}_{1}.tif'.format(tile_id,
cn.pattern_mangrove_biomass_2000)
mangrove_tile_rewindow = '{0}_{1}_rewindow.tif'.format(
tile_id, cn.pattern_mangrove_biomass_2000)
# Only rewindows the necessary files if they haven't already been processed (just in case
# this was run on the spot machine before)
if not os.path.exists(input_rewindow):
uu.print_log(
"Model output for {} not rewindowed. Rewindowing...".format(
tile_id))
# Converts the tile of interest to the 400x400 pixel windows
cmd = [
'gdalwarp', '-co', 'COMPRESS=LZW', '-overwrite', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-tap', '-tr',
str(cn.Hansen_res),
str(cn.Hansen_res), '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=160',
'-co', 'BLOCKYSIZE=160', tile, input_rewindow
]
uu.log_subprocess_output_full(cmd)
if not os.path.exists(tcd_rewindow):
uu.print_log(
"Canopy cover for {} not rewindowed. Rewindowing...".format(
tile_id))
# Converts the tcd tile to the 400x400 pixel windows
cmd = [
'gdalwarp', '-co', 'COMPRESS=LZW', '-overwrite', '-dstnodata', '0',
'-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-tap', '-tr',
str(cn.Hansen_res),
str(cn.Hansen_res), '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=160',
'-co', 'BLOCKYSIZE=160', tcd_tile, tcd_rewindow
]
uu.log_subprocess_output_full(cmd)
else:
uu.print_log("Canopy cover for {} already rewindowed.".format(tile_id))
if not os.path.exists(pixel_area_rewindow):
uu.print_log(
"Pixel area for {} not rewindowed. Rewindowing...".format(tile_id))
# Converts the pixel area tile to the 400x400 pixel windows
cmd = [
'gdalwarp', '-co', 'COMPRESS=LZW', '-overwrite', '-dstnodata', '0',
'-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-tap', '-tr',
str(cn.Hansen_res),
str(cn.Hansen_res), '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=160',
'-co', 'BLOCKYSIZE=160', area_tile, pixel_area_rewindow
]
uu.log_subprocess_output_full(cmd)
else:
uu.print_log("Pixel area for {} already rewindowed.".format(tile_id))
if not os.path.exists(gain_rewindow):
uu.print_log(
"Hansen gain for {} not rewindowed. Rewindowing...".format(
tile_id))
# Converts the pixel area tile to the 400x400 pixel windows
cmd = [
'gdalwarp', '-co', 'COMPRESS=LZW', '-overwrite', '-dstnodata', '0',
'-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-tap', '-tr',
str(cn.Hansen_res),
str(cn.Hansen_res), '-co', 'TILED=YES', '-co', 'BLOCKXSIZE=160',
'-co', 'BLOCKYSIZE=160', gain_tile, gain_rewindow
]
uu.log_subprocess_output_full(cmd)
else:
uu.print_log("Hansen gain for {} already rewindowed.".format(tile_id))
if os.path.exists(mangrove_tile):
uu.print_log(
"Mangrove for {} not rewindowed. Rewindowing...".format(tile_id))
if not os.path.exists(mangrove_tile_rewindow):
# Converts the pixel area tile to the 400x400 pixel windows
cmd = [
'gdalwarp', '-co', 'COMPRESS=LZW', '-overwrite', '-dstnodata',
'0', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-tap', '-tr',
str(cn.Hansen_res),
str(cn.Hansen_res), '-co', 'TILED=YES', '-co',
'BLOCKXSIZE=160', '-co', 'BLOCKYSIZE=160', mangrove_tile,
mangrove_tile_rewindow
]
uu.log_subprocess_output_full(cmd)
else:
uu.print_log(
"Mangrove tile for {} already rewindowed.".format(tile_id))
else:
uu.print_log("No mangrove tile found for {}".format(tile_id))
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, '{}_rewindow'.format(tile_type))
def aggregate(tile, thresh, sensit_type):
# start time
start = datetime.datetime.now()
# Extracts the tile id, tile type, and bounding box for the tile
tile_id = uu.get_tile_id(tile)
tile_type = uu.get_tile_type(tile)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Name of inputs
focal_tile_rewindow = '{0}_{1}_rewindow.tif'.format(tile_id, tile_type)
pixel_area_rewindow = '{0}_{1}_rewindow.tif'.format(
cn.pattern_pixel_area, tile_id)
tcd_rewindow = '{0}_{1}_rewindow.tif'.format(cn.pattern_tcd, tile_id)
gain_rewindow = '{0}_{1}_rewindow.tif'.format(cn.pattern_gain, tile_id)
mangrove_rewindow = '{0}_{1}_rewindow.tif'.format(
tile_id, cn.pattern_mangrove_biomass_2000)
# Opens input tiles for rasterio
in_src = rasterio.open(focal_tile_rewindow)
pixel_area_src = rasterio.open(pixel_area_rewindow)
tcd_src = rasterio.open(tcd_rewindow)
gain_src = rasterio.open(gain_rewindow)
try:
mangrove_src = rasterio.open(mangrove_rewindow)
uu.print_log(" Mangrove tile found for {}".format(tile_id))
except:
uu.print_log(" No mangrove tile found for {}".format(tile_id))
uu.print_log(" Converting {} to per-pixel values...".format(tile))
# Grabs the windows of the tile (stripes) in order to iterate over the entire tif without running out of memory
windows = in_src.block_windows(1)
#2D array in which the 0.05x0.05 deg aggregated sums will be stored
sum_array = np.zeros([250, 250], 'float32')
out_raster = "{0}_{1}_0_4deg.tif".format(tile_id, tile_type)
# Iterates across the windows (400x400 30m pixels) of the input tile
for idx, window in windows:
# Creates windows for each input tile
in_window = in_src.read(1, window=window)
pixel_area_window = pixel_area_src.read(1, window=window)
tcd_window = tcd_src.read(1, window=window)
gain_window = gain_src.read(1, window=window)
try:
mangrove_window = mangrove_src.read(1, window=window)
except:
mangrove_window = np.zeros((window.height, window.width),
dtype='uint8')
# Applies the tree cover density threshold to the 30x30m pixels
if thresh > 0:
# QCed this line before publication and then again afterwards in response to question from Lena Schulte-Uebbing at Wageningen Uni.
in_window = np.where((tcd_window > thresh) | (gain_window == 1) |
(mangrove_window != 0), in_window, 0)
# Calculates the per-pixel value from the input tile value (/ha to /pixel)
per_pixel_value = in_window * pixel_area_window / cn.m2_per_ha
# Sums the pixels to create a total value for the 0.1x0.1 deg pixel
non_zero_pixel_sum = np.sum(per_pixel_value)
# Stores the resulting value in the array
sum_array[idx[0], idx[1]] = non_zero_pixel_sum
# Converts the annual carbon gain values annual gain in megatonnes and makes negative (because removals are negative)
if cn.pattern_annual_gain_AGC_all_types in tile_type:
sum_array = sum_array / cn.tonnes_to_megatonnes * -1
# Converts the cumulative CO2 gain values to annualized CO2 in megatonnes and makes negative (because removals are negative)
if cn.pattern_cumul_gain_AGCO2_BGCO2_all_types in tile_type:
sum_array = sum_array / cn.loss_years / cn.tonnes_to_megatonnes * -1
# # Converts the cumulative gross emissions CO2 only values to annualized gross emissions CO2e in megatonnes
# if cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil in tile_type:
# sum_array = sum_array / cn.loss_years / cn.tonnes_to_megatonnes
#
# # Converts the cumulative gross emissions non-CO2 values to annualized gross emissions CO2e in megatonnes
# if cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil in tile_type:
# sum_array = sum_array / cn.loss_years / cn.tonnes_to_megatonnes
# Converts the cumulative gross emissions all gases CO2e values to annualized gross emissions CO2e in megatonnes
if cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil in tile_type:
sum_array = sum_array / cn.loss_years / cn.tonnes_to_megatonnes
# Converts the cumulative net flux CO2 values to annualized net flux CO2 in megatonnes
if cn.pattern_net_flux in tile_type:
sum_array = sum_array / cn.loss_years / cn.tonnes_to_megatonnes
uu.print_log(" Creating aggregated tile for {}...".format(tile))
# Converts array to the same output type as the raster that is created below
sum_array = np.float32(sum_array)
# Creates a tile at 0.04x0.04 degree resolution (approximately 10x10 km in the tropics) where the values are
# from the 2D array created by rasterio above
# https://gis.stackexchange.com/questions/279953/numpy-array-to-gtiff-using-rasterio-without-source-raster
with rasterio.open(out_raster,
'w',
driver='GTiff',
compress='lzw',
nodata='0',
dtype='float32',
count=1,
height=250,
width=250,
crs='EPSG:4326',
transform=from_origin(xmin, ymax, 0.04,
0.04)) as aggregated:
aggregated.write(sum_array, 1)
### I don't know why, but update_tags() is adding the tags to the raster but not saving them.
### That is, the tags are printed but not showing up when I do gdalinfo on the raster.
### Instead, I'm using gdal_edit
# print(aggregated)
# aggregated.update_tags(a="1")
# print(aggregated.tags())
# uu.add_rasterio_tags(aggregated, sensit_type)
# print(aggregated.tags())
# if cn.pattern_annual_gain_AGC_all_types in tile_type:
# aggregated.update_tags(units='Mg aboveground carbon/pixel, where pixels are 0.04x0.04 degrees)',
# source='per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
# extent='Global',
# treecover_density_threshold='{0} (only model pixels with canopy cover > {0} are included in aggregation'.format(thresh))
# if cn.pattern_cumul_gain_AGCO2_BGCO2_all_types:
# aggregated.update_tags(units='Mg CO2/yr/pixel, where pixels are 0.04x0.04 degrees)',
# source='per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
# extent='Global',
# treecover_density_threshold='{0} (only model pixels with canopy cover > {0} are included in aggregation'.format(thresh))
# # if cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil in tile_type:
# # aggregated.update_tags(units='Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees)',
# # source='per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
# # extent='Global', gases_included='CO2 only',
# # treecover_density_threshold = '{0} (only model pixels with canopy cover > {0} are included in aggregation'.format(thresh))
# # if cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil in tile_type:
# # aggregated.update_tags(units='Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees)',
# # source='per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
# # extent='Global', gases_included='CH4, N20',
# # treecover_density_threshold='{0} (only model pixels with canopy cover > {0} are included in aggregation'.format(thresh))
# if cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil in tile_type:
# aggregated.update_tags(units='Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees)',
# source='per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
# extent='Global',
# treecover_density_threshold='{0} (only model pixels with canopy cover > {0} are included in aggregation'.format(thresh))
# if cn.pattern_net_flux in tile_type:
# aggregated.update_tags(units='Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees)',
# scale='Negative values are net sinks. Positive values are net sources.',
# source='per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
# extent='Global',
# treecover_density_threshold='{0} (only model pixels with canopy cover > {0} are included in aggregation'.format(thresh))
# print(aggregated.tags())
# aggregated.close()
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, '{}_0_4deg'.format(tile_type))
def create_input_files(tile_id, no_upload):
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
#### NOTE FOR FUTURE REVISIONS: CHANGE TO USE MP_WARP_TO_HANSEN
uu.print_log("Clipping srtm for", tile_id)
uu.warp_to_Hansen('srtm.vrt', '{0}_{1}.tif'.format(tile_id,
cn.pattern_elevation),
xmin, ymin, xmax, ymax, 'Int16')
#### NOTE FOR FUTURE REVISIONS: CHANGE TO USE MP_WARP_TO_HANSEN
uu.print_log("Clipping precipitation for", tile_id)
uu.warp_to_Hansen('add_30s_precip.tif',
'{0}_{1}.tif'.format(tile_id, cn.pattern_precip), xmin,
ymin, xmax, ymax, 'Int32')
uu.print_log(
"Rasterizing ecozone into boreal-temperate-tropical categories for",
tile_id)
blocksizex = 1024
blocksizey = 1024
uu.rasterize(
'fao_ecozones_bor_tem_tro.shp',
"{0}_{1}.tif".format(tile_id,
cn.pattern_bor_tem_trop_intermediate), xmin, ymin,
xmax, ymax, blocksizex, blocksizey, '.00025', 'Int16', 'recode', '0')
# Opens boreal/temperate/tropical ecozone tile.
# Everything from here down is used to assign pixels without boreal-tem-tropical codes to a bor-tem-trop in the 1024x1024 windows.
bor_tem_trop_src = rasterio.open("{0}_{1}.tif".format(
tile_id, cn.pattern_bor_tem_trop_intermediate))
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = bor_tem_trop_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = bor_tem_trop_src.block_windows(1)
# Updates kwargs for the output dataset.
# Need to update data type to float 32 so that it can handle fractional gain rates
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
bor_tem_trop_processed = '{0}_{1}.tif'.format(
tile_id, cn.pattern_bor_tem_trop_processed)
# The output file: aboveground carbon density in the year of tree cover loss for pixels with tree cover loss
dst_bor_tem_trop = rasterio.open(bor_tem_trop_processed, 'w', **kwargs)
# Iterates across the windows (1024 x 1024 pixel boxes) of the input tile.
for idx, window in windows:
# Creates window for input raster
bor_tem_trop_window = bor_tem_trop_src.read(1, window=window)
# Turns the 2D array into a 1D array that is n x n long.
# This makes to easier to remove 0s and find the mode of the remaining bor-tem-tropi codes
bor_tem_trop_flat = bor_tem_trop_window.flatten()
# Removes all zeros from the array, leaving just pixels with bor-tem-trop codes
non_zeros = np.delete(bor_tem_trop_flat,
np.where(bor_tem_trop_flat == 0))
# If there were only pixels without bor-tem-trop codes in the array, the mode is assigned 0
if non_zeros.size < 1:
# print " Window is all 0s"
mode = 0
# If there were pixels with bor-tem-trop codes, the mode is the most common code among those in the window
else:
mode = stats.mode(non_zeros)[0]
# print " Window is not all 0s. Mode is", mode
# Assigns all pixels without a bor-tem-trop code in that window to that most common code
bor_tem_trop_window[bor_tem_trop_window == 0] = mode
# Writes the output window to the output.
# Although the windows for the input tiles are 1024 x 1024 pixels,
# the windows for these output files are 40000 x 1 pixels, like all the other tiles in this model,
# so they should work fine with all the other tiles.
dst_bor_tem_trop.write_band(1, bor_tem_trop_window, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_precip, no_upload)
def create_climate_zone_tiles(tile_id):
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Makes a 10x10 degree chunk of the global climate zone raster conform to Hansen tile properties.
# Rather than the usual 40000x1 windows, this creates 1024x1024 windows for filling in missing values (see below).
# The output of gdalwarp ("climate_zone_intermediate") is not used anywhere else.
uu.print_log("Warping climate zone tile", tile_id)
cmd = [
'gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr',
str(cn.Hansen_res),
str(cn.Hansen_res), '-tap', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-dstnodata', '0', '-ot', 'Byte', '-overwrite', '-co',
'TILED=YES', '-co', 'BLOCKXSIZE=1024', '-co', 'BLOCKYSIZE=1024',
cn.climate_zone_raw, '{0}_{1}.tif'.format(tile_id,
"climate_zone_intermediate")
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Fills in empty pixels in the climate zone raster with whatever value is most common (mode) in its 1024x1024 pixel window.
# That is, any 1024x1024 processing window that has >=1 climate zone pixel in it will have its empty pixels filled in
# with whatever value is most common in that window.
# This extends the climate zone raster out into coastal areas and better covers coasts/islands, meaning that more
# loss pixels will have climate zone pixels available to them during emissions processing.
# Everything from here down is used to assign pixels without climate zone to a climate zone in the 1024x1024 windows.
uu.print_log("Re-tiling climate zone for tile", tile_id)
# Opens climate zone tile
climate_zone_src = rasterio.open("{0}_{1}.tif".format(
tile_id, "climate_zone_intermediate"))
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = climate_zone_src.meta
# Grabs the windows of the tile (stripes) to iterate over the entire tif without running out of memory
windows = climate_zone_src.block_windows(1)
# Updates kwargs for the output dataset.
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
# Output file name
climate_zone_processed = '{0}_{1}.tif'.format(tile_id,
cn.pattern_climate_zone)
# The output file: climate zone with empty pixels filled in
dst_climate_zone = rasterio.open(climate_zone_processed, 'w', **kwargs)
# Iterates across the windows (1024 x 1024 pixel boxes) of the input tile.
for idx, window in windows:
# Creates window for input raster
climate_zone_window = climate_zone_src.read(1, window=window)
# Turns the 2D array into a 1D array that is n x n long.
# This makes to easier to remove 0s and find the mode of the remaining climate zone codes
climate_zone_flat = climate_zone_window.flatten()
# Removes all zeros from the array, leaving just pixels with climate zone codes
non_zeros = np.delete(climate_zone_flat,
np.where(climate_zone_flat == 0))
# If there were only pixels without climate zone codes in the array, the mode is assigned 0
if non_zeros.size < 1:
mode = 0
# If there were pixels with climate zone codes, the mode is the most common code among those in the window
else:
mode = stats.mode(non_zeros)[0]
# Assigns all pixels without a climate zone code in that window to that most common code
climate_zone_window[climate_zone_window == 0] = mode
# Writes the output window to the output.
# Although the windows for the input tiles are 1024 x 1024 pixels,
# the windows for these output files are 40000 x 1 pixels, like all the other tiles in this model,
# so they should work fine with all the other tiles.
dst_climate_zone.write_band(1, climate_zone_window, window=window)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_climate_zone)
