def create_10x10_plantation(tile_id, plant_1x1_vrt):
print "Getting bounding coordinates for tile", tile_id
xmin, ymin, xmax, ymax = uu.coords(tile_id)
print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax
tile_10x10 = '{0}_{1}.tif'.format(
tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)
print "Rasterizing", tile_10x10
cmd = [
'gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)),
'{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-tap', '-te',
str(xmin),
str(ymin),
str(xmax),
str(ymax), '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite',
'-ot', 'Float32', plant_1x1_vrt, tile_10x10
]
subprocess.check_call(cmd)
print "Checking if {} contains any data...".format(tile_id)
stats = uu.check_for_data(tile_10x10)
if stats[0] > 0:
print " Data found in {}. Copying tile to s3...".format(tile_id)
uu.upload_final(cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir,
tile_id,
cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)
print " Tile converted and copied to s3"
else:
print " No data found. Not copying {}.".format(tile_id)
def create_mangrove_tiles(tile_id):
print "Getting bounding coordinates for tile", tile_id
xmin, xmax, ymin, ymax = utilities.coords(tile_id)
print " ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin
print "Creating tile", tile_id
out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000)
cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', '0.00025', '0.00025', '-tap', '-te',
str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-overwrite', utilities.mangrove_vrt, out_tile]
subprocess.check_call(cmd)
print " Tile created"
print "Checking if {} contains any data...".format(tile_id)
stats = uu.check_for_data(out_tile)
if stats[0] > 0:
print " Data found in {}. Copying tile to s3...".format(tile_id)
uu.upload_final(cn.mangrove_biomass_2000_dir, tile_id, cn.pattern_mangrove_biomass_2000)
print " Tile copied to s3"
else:
print " No data found. Not copying {}.".format(tile_id)
def create_10x10_plantation_type(tile_id, plant_type_1x1_vrt):
uu.print_log("Getting bounding coordinates for tile", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax)
tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_planted_forest_type_unmasked)
uu.print_log("Rasterizing", tile_10x10)
cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)),
'-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax),
'-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Byte', plant_type_1x1_vrt, tile_10x10]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.print_log("Checking if {} contains any data...".format(tile_id))
stats = uu.check_for_data(tile_10x10)
if stats[0] > 0:
uu.print_log(" Data found in {}. Copying tile to s3...".format(tile_id))
uu.upload_final(cn.planted_forest_type_unmasked_dir, tile_id, cn.pattern_planted_forest_type_unmasked)
uu.print_log(" Tile converted and copied to s3")
else:
print(" No data found. Not copying {}.".format(tile_id))
def clip_year_tiles(tile_year_list, no_upload):
# Start time
start = datetime.datetime.now()
tile_id = tile_year_list[0].strip('.tif')
year = tile_year_list[1]
vrt_name = "global_vrt_{}_wgs84.vrt".format(year)
# Gets coordinates of hansen tile
uu.print_log("Getting coordinates of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
# Clips vrt to tile extent
uu.print_log("Clipping burn year vrt to {0} for {1}".format(tile_id, year))
clipped_raster = "ba_clipped_{0}_{1}.tif".format(year, tile_id)
cmd = [
'gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '-a_nodata',
'0'
]
cmd += [vrt_name, clipped_raster, '-tr', '.00025', '.00025']
cmd += ['-projwin', str(xmin), str(ymax), str(xmax), str(ymin)]
uu.log_subprocess_output_full(cmd)
# Calculates year tile values to be equal to year. ex: 17*1
calc = '--calc={}*(A>0)'.format(int(year) - 2000)
recoded_output = "ba_{0}_{1}.tif".format(year, tile_id)
outfile = '--outfile={}'.format(recoded_output)
cmd = [
'gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0',
'--co', 'COMPRESS=LZW', '--quiet'
]
uu.log_subprocess_output_full(cmd)
# Only copies to s3 if the tile has data.
# No tiles for 2000 have data because the burn year is coded as 0, which is NoData.
uu.print_log("Checking if {} contains any data...".format(tile_id))
empty = uu.check_for_data(recoded_output)
if empty:
uu.print_log(" No data found. Not copying {}.".format(tile_id))
else:
uu.print_log(
" Data found in {}. Copying tile to s3...".format(tile_id))
cmd = [
'aws', 's3', 'cp', recoded_output,
cn.burn_year_warped_to_Hansen_dir
]
uu.log_subprocess_output_full(cmd)
uu.print_log(" Tile copied to", cn.burn_year_warped_to_Hansen_dir)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year), no_upload)
def rasterize_gadm_1x1(tile_id):
uu.print_log("Getting bounding coordinates for tile", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:",
ymax)
# Degrees of tile in x and y dimensions
x_size = abs(int(xmin) - int(xmax))
y_size = abs(int(ymin) - int(ymax))
# Iterates through input 10x10 tile by 1x1 degree
for x in range(x_size):
xmin_1x1 = int(xmin) + x
xmax_1x1 = int(xmin) + x + 1
for y in range(y_size):
ymin_1x1 = int(ymin) + y
ymax_1x1 = int(ymin) + y + 1
uu.print_log(" xmin_1x1:", xmin_1x1, "; xmax_1x1:", xmax_1x1,
"; ymin_1x1", ymin_1x1, "; ymax_1x1:", ymax_1x1)
tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1)
uu.print_log("Rasterizing", tile_1x1)
cmd = [
'gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)),
'{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-te',
str(xmin_1x1),
str(ymin_1x1),
str(xmax_1x1),
str(ymax_1x1), '-burn', '1', '-a_nodata', '0', cn.gadm_iso,
tile_1x1
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Only keeps 1x1 GADM tiles if they actually include a country; many 1x1 tiles created out of 10x10 tiles
# don't actually include a country.
uu.print_log(
"Checking if {} contains any data...".format(tile_1x1))
stats = uu.check_for_data(tile_1x1)
if stats[1] > 0:
uu.print_log(
" Data found in {}. Keeping tile".format(tile_1x1))
else:
uu.print_log(
" No data found in {}. Deleting.".format(tile_1x1))
os.remove(tile_1x1)
def rasterize_gadm_1x1(tile_id):
print "Getting bounding coordinates for tile", tile_id
xmin, ymin, xmax, ymax = uu.coords(tile_id)
print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax
# Degrees of tile in x and y dimensions
x_size = abs(int(xmin) - int(xmax))
y_size = abs(int(ymin) - int(ymax))
# Iterates through input 10x10 tile by 1x1 degree
for x in range(x_size):
xmin_1x1 = int(xmin) + x
xmax_1x1 = int(xmin) + x + 1
for y in range(y_size):
ymin_1x1 = int(ymin) + y
ymax_1x1 = int(ymin) + y + 1
print " xmin_1x1:", xmin_1x1, "; xmax_1x1:", xmax_1x1, "; ymin_1x1", ymin_1x1, "; ymax_1x1:", ymax_1x1
tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1)
print "Rasterizing", tile_1x1
cmd = [
'gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)),
'{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-te',
str(xmin_1x1),
str(ymin_1x1),
str(xmax_1x1),
str(ymax_1x1), '-burn', '1', '-a_nodata', '0', cn.gadm_iso,
tile_1x1
]
subprocess.check_call(cmd)
# Only keeps 1x1 GADM tiles if they actually include a country; many 1x1 tiles created out of 10x10 tiles
# don't actually include a country.
print "Checking if {} contains any data...".format(tile_1x1)
stats = uu.check_for_data(tile_1x1)
if stats[1] > 0:
print " Data found in {}. Keeping tile".format(tile_1x1)
else:
print " No data found in {}. Deleting.".format(tile_1x1)
os.remove(tile_1x1)
def prep_FIA_regions(tile_id):
uu.print_log("Creating Hansen tile for FIA regions")
# Start time
start = datetime.datetime.now()
uu.print_log("Getting extent of", tile_id)
xmin, ymin, xmax, ymax = uu.coords(tile_id)
uu.print_log("Rasterizing FIA region shapefile", tile_id)
blocksizex = 1024
blocksizey = 1024
uu.rasterize(
'{}.shp'.format(cn.name_FIA_regions_raw[:-4]),
"{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed), xmin,
ymin, xmax, ymax, blocksizex, blocksizey, '.00025', 'Byte',
'regionCode', '0')
uu.print_log("Checking if {} contains any data...".format(tile_id))
no_data = uu.check_for_data("{0}_{1}.tif".format(
tile_id, cn.pattern_FIA_regions_processed))
if no_data:
uu.print_log(" No data found. Deleting {}.".format(tile_id))
os.remove("{0}_{1}.tif".format(tile_id,
cn.pattern_FIA_regions_processed))
else:
uu.print_log(
" Data found in {}. Copying tile to s3...".format(tile_id))
uu.upload_final(cn.FIA_regions_processed_dir, tile_id,
cn.pattern_FIA_regions_processed)
uu.print_log(" Tile copied to s3")
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_FIA_regions_processed)
def hansen_burnyear(tile_id, no_upload):
# Start time
start = datetime.datetime.now()
uu.print_log("Processing", tile_id)
# The tiles that are used. out_tile_no_tag is the output before metadata tags are added. out_tile is the output
# once metadata tags have been added.
out_tile_no_tag = '{0}_{1}_no_tag.tif'.format(tile_id,
cn.pattern_burn_year)
out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_burn_year)
loss = '{0}.tif'.format(tile_id)
# Does not continue processing tile if no loss (because there will not be any output)
if not os.path.exists(loss):
uu.print_log("No loss tile for", tile_id)
return
else:
uu.print_log("Loss tile exists for", tile_id)
# Downloads the burned area tiles for each year
include = 'ba_*_{}.tif'.format(tile_id)
burn_tiles_dir = 'burn_tiles'
if not os.path.exists(burn_tiles_dir):
os.mkdir(burn_tiles_dir)
cmd = [
'aws', 's3', 'cp', cn.burn_year_warped_to_Hansen_dir, burn_tiles_dir,
'--recursive', '--exclude', "*", '--include', include
]
uu.log_subprocess_output_full(cmd)
# For each year tile, converts to array and stacks them
array_list = []
ba_tifs = glob.glob(burn_tiles_dir + '/*{}*'.format(tile_id))
# Skips the tile if it has no burned area data for any year
uu.print_log("There are {0} tiles to stack for {1}".format(
len(ba_tifs), tile_id))
if len(ba_tifs) == 0:
uu.print_log(
"Skipping {} because there are no tiles to stack".format(tile_id))
return
# NOTE: All of this could pretty easily be done in rasterio. However, Sam's use of GDAL for this still works fine,
# so I've left it using GDAL.
for ba_tif in ba_tifs:
uu.print_log("Creating array with {}".format(ba_tif))
array = utilities.raster_to_array(ba_tif)
array_list.append(array)
# Stacks arrays from each year
uu.print_log("Stacking arrays for", tile_id)
stacked_year_array = utilities.stack_arrays(array_list)
# Converts Hansen tile to array
uu.print_log("Creating loss year array for", tile_id)
loss_array = utilities.raster_to_array(loss)
# Determines what year to assign burned area
lossarray_min1 = np.subtract(loss_array, 1)
stack_con = (stacked_year_array >= lossarray_min1) & (stacked_year_array <=
loss_array)
stack_con2 = stack_con * stacked_year_array
lossyear_burn_array = stack_con2.max(0)
utilities.array_to_raster_simple(lossyear_burn_array, out_tile_no_tag,
loss)
# Only copies to s3 if the tile has data
uu.print_log("Checking if {} contains any data...".format(tile_id))
empty = uu.check_for_data(out_tile_no_tag)
# Checks output for data. There could be burned area but none of it coincides with tree cover loss,
# so this is the final check for whether there is any data.
if empty:
uu.print_log(" No data found. Not copying {}.".format(tile_id))
# Without this, the untagged version is counted and eventually copied to s3 if it has data in it
os.remove(out_tile_no_tag)
return
else:
uu.print_log(
" Data found in {}. Adding metadata tags...".format(tile_id))
### Thomas suggested these on 8/19/2020 but they didn't work. The first one wrote the tags but erased all the
### data in the tiles (everything became 0 according to gdalinfo). The second one had some other error.
# with rasterio.open(out_tile_no_tag, 'r') as src:
#
# profile = src.profile
#
# with rasterio.open(out_tile_no_tag, 'w', **profile) as dst:
#
# dst.update_tags(units='year (2001, 2002, 2003...)',
# source='MODIS collection 6 burned area',
# extent='global')
#
# with rasterio.open(out_tile_no_tag, 'w+') as src:
#
# dst.update_tags(units='year (2001, 2002, 2003...)',
# source='MODIS collection 6 burned area',
# extent='global')
# All of the below is to add metadata tags to the output burn year masks.
# For some reason, just doing what's at https://rasterio.readthedocs.io/en/latest/topics/tags.html
# results in the data getting removed.
# I found it necessary to copy the desired output and read its windows into a new copy of the file, to which the
# metadata tags are added. I'm sure there's an easier way to do this but I couldn't figure out how.
# I know it's very convoluted but I really couldn't figure out how to add the tags without erasing the data.
copyfile(out_tile_no_tag, out_tile)
with rasterio.open(out_tile_no_tag) as out_tile_no_tag_src:
# Grabs metadata about the tif, like its location/projection/cellsize
kwargs = out_tile_no_tag_src.meta #### Use profile instead
# Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
windows = out_tile_no_tag_src.block_windows(1)
# Updates kwargs for the output dataset
kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0)
out_tile_tagged = rasterio.open(out_tile, 'w', **kwargs)
# Adds metadata tags to the output raster
uu.add_rasterio_tags(out_tile_tagged, 'std')
out_tile_tagged.update_tags(units='year (2001, 2002, 2003...)')
out_tile_tagged.update_tags(
source=
'MODIS collection 6 burned area, https://modis-fire.umd.edu/files/MODIS_C6_BA_User_Guide_1.3.pdf'
)
out_tile_tagged.update_tags(extent='global')
# Iterates across the windows (1 pixel strips) of the input tile
for idx, window in windows:
in_window = out_tile_no_tag_src.read(1, window=window)
# Writes the output window to the output
out_tile_tagged.write_band(1, in_window, window=window)
# Without this, the untagged version is counted and eventually copied to s3 if it has data in it
os.remove(out_tile_no_tag)
# Prints information about the tile that was just processed
uu.end_of_fx_summary(start, tile_id, cn.pattern_burn_year, no_upload)
