def main ():
no_upload = False
# Create the output log
uu.initiate_log()
os.chdir(cn.docker_base_dir)
# The list of tiles to iterate through
tile_id_list = uu.tile_list_s3(cn.WHRC_biomass_2000_unmasked_dir)
# tile_id_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
# tile_id_list = ['00N_110E'] # test tile
uu.print_log(tile_id_list)
uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# By definition, this script is for the biomass swap analysis (replacing WHRC AGB with Saatchi/JPL AGB)
sensit_type = 'biomass_swap'
# Downloads a pan-tropical raster that has the erroneous integer values in the oceans removed
uu.s3_file_download(cn.JPL_raw_dir, cn.JPL_raw_name, sensit_type)
# Converts the Saatchi AGB vrt to Hansen tiles
source_raster = cn.JPL_raw_name
out_pattern = cn.pattern_JPL_unmasked_processed
dt = 'Float32'
pool = multiprocessing.Pool(cn.count-5) # count-5 peaks at 320GB of memory
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# Checks if each tile has data in it. Only tiles with data are uploaded.
upload_dir = cn.JPL_processed_dir
pattern = cn.pattern_JPL_unmasked_processed
pool = multiprocessing.Pool(cn.count - 5) # count-5 peaks at 410GB of memory
pool.map(partial(uu.check_and_upload, upload_dir=upload_dir, pattern=pattern), tile_id_list)
def mp_mangrove_processing(tile_id_list, run_date=None, no_upload=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.tile_list_s3(cn.pixel_area_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# Downloads zipped raw mangrove files
uu.s3_file_download(
os.path.join(cn.mangrove_biomass_raw_dir,
cn.mangrove_biomass_raw_file), cn.docker_base_dir, 'std')
# Unzips mangrove images into a flat structure (all tifs into main folder using -j argument)
# NOTE: Unzipping some tifs (e.g., Australia, Indonesia) takes a very long time, so don't worry if the script appears to stop on that.
cmd = ['unzip', '-o', '-j', cn.mangrove_biomass_raw_file]
uu.log_subprocess_output_full(cmd)
# Creates vrt for the Saatchi biomass rasters
mangrove_vrt = 'mangrove_biomass.vrt'
os.system('gdalbuildvrt {} *.tif'.format(mangrove_vrt))
# Converts the mangrove AGB vrt into Hansen tiles
source_raster = mangrove_vrt
out_pattern = cn.pattern_mangrove_biomass_2000
dt = 'float32'
processes = int(cn.count / 4)
uu.print_log('Mangrove preprocessing max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt,
no_upload=no_upload), tile_id_list)
# # For single processor use, for testing purposes
# for tile_id in tile_id_list:
#
# mangrove_processing.create_mangrove_tiles(tile_id, source_raster, out_pattern, no_upload)
# Checks if each tile has data in it. Only tiles with data are uploaded.
upload_dir = cn.mangrove_biomass_2000_dir
pattern = cn.pattern_mangrove_biomass_2000
processes = int(cn.count - 5)
uu.print_log('Mangrove check for data max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_upload, upload_dir=upload_dir, pattern=pattern),
tile_id_list)
def mp_continent_ecozone_tiles(tile_id_list, run_date=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(
cn.pattern_WHRC_biomass_2000_non_mang_non_planted,
cn.mangrove_biomass_2000_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# if the continent-ecozone shapefile hasn't already been downloaded, it will be downloaded and unzipped
uu.s3_file_download(cn.cont_eco_s3_zip, cn.docker_base_dir, 'std')
# Unzips ecozone shapefile
cmd = ['unzip', cn.cont_eco_zip]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# List of output directories and output file name patterns
output_dir_list = [cn.cont_eco_raw_dir, cn.cont_eco_dir]
output_pattern_list = [
cn.pattern_cont_eco_raw, cn.pattern_cont_eco_processed
]
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# For multiprocessor use
processes = int(cn.count / 4)
uu.print_log('Continent-ecozone tile creation max processors=', processes)
pool.map(continent_ecozone_tiles.create_continent_ecozone_tiles,
tile_id_list)
# Uploads the continent-ecozone tile to s3 before the codes are expanded to pixels in 1024x1024 windows that don't have codes.
# These are not used for the model. They are for reference and completeness.
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
### mangrove data.
### Output tiles conform to the dimensions, resolution, and other properties of Hansen loss tiles.
import multiprocessing
import mangrove_processing
import sys
import os
import subprocess
import utilities
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu
# Downloads zipped raw mangrove files
uu.s3_file_download(
os.path.join(cn.mangrove_biomass_raw_dir, cn.mangrove_biomass_raw_file),
'.')
# Unzips mangrove images into a flat structure (all tifs into main folder using -j argument)
# NOTE: Unzipping some tifs (e.g., Australia, Indonesia) takes a very long time, so don't worry if the script appears to stop on that.
cmd = ['unzip', '-j', cn.mangrove_biomass_raw_file]
subprocess.check_call(cmd)
# Creates vrt of all raw mangrove tifs
utilities.build_vrt(utilities.mangrove_vrt)
# Iterates through all possible tiles (not just WHRC biomass tiles) to create mangrove biomass tiles that don't have analogous WHRC tiles
total_tile_list = uu.tile_list(cn.pixel_area_dir)
# biomass_tile_list = ['00N_000E', '20S_120W', '00N_120E'] # test tile
print total_tile_list
def mp_prep_other_inputs(tile_id_list, run_date):
os.chdir(cn.docker_base_dir)
sensit_type='std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
set3=cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir
)
uu.print_log(tile_id_list)
uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# List of output directories and output file name patterns
output_dir_list = [cn.climate_zone_processed_dir, cn.plant_pre_2000_processed_dir,
cn.drivers_processed_dir, cn.ifl_primary_processed_dir,
cn.annual_gain_AGC_natrl_forest_young_dir,
cn.stdev_annual_gain_AGC_natrl_forest_young_dir,
cn.annual_gain_AGC_BGC_natrl_forest_Europe_dir,
cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_dir,
cn.FIA_forest_group_processed_dir,
cn.age_cat_natrl_forest_US_dir,
cn.FIA_regions_processed_dir]
output_pattern_list = [cn.pattern_climate_zone, cn.pattern_plant_pre_2000,
cn.pattern_drivers, cn.pattern_ifl_primary,
cn.pattern_annual_gain_AGC_natrl_forest_young,
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young,
cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe,
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe,
cn.pattern_FIA_forest_group_processed,
cn.pattern_age_cat_natrl_forest_US,
cn.pattern_FIA_regions_processed]
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log("Changing output directory and file name pattern based on sensitivity analysis")
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
output_pattern_list = uu.alter_patterns(sensit_type, output_pattern_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Files to process: climate zone, IDN/MYS plantations before 2000, tree cover loss drivers, combine IFL and primary forest
uu.s3_file_download(os.path.join(cn.climate_zone_raw_dir, cn.climate_zone_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.plant_pre_2000_raw_dir, '{}.zip'.format(cn.pattern_plant_pre_2000_raw)), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.drivers_raw_dir, '{}.zip'.format(cn.pattern_drivers_raw)), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.age_cat_natrl_forest_US_raw_dir, cn.name_age_cat_natrl_forest_US_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, sensit_type)
# For some reason, using uu.s3_file_download or otherwise using AWSCLI as a subprocess doesn't work for this raster.
# Thus, using wget instead.
cmd = ['wget', '{}'.format(cn.annual_gain_AGC_natrl_forest_young_raw_URL), '-P', '{}'.format(cn.docker_base_dir)]
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.s3_file_download(cn.stdev_annual_gain_AGC_natrl_forest_young_raw_URL, cn.docker_base_dir, sensit_type)
cmd = ['aws', 's3', 'cp', cn.primary_raw_dir, cn.docker_base_dir, '--recursive']
uu.log_subprocess_output_full(cmd)
uu.s3_flexible_download(cn.ifl_dir, cn.pattern_ifl, cn.docker_base_dir, sensit_type, tile_id_list)
uu.print_log("Unzipping pre-2000 plantations...")
cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_plant_pre_2000_raw)]
uu.log_subprocess_output_full(cmd)
uu.print_log("Unzipping drivers...")
cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_drivers_raw)]
uu.log_subprocess_output_full(cmd)
# Creates tree cover loss driver tiles
source_raster = '{}.tif'.format(cn.pattern_drivers_raw)
out_pattern = cn.pattern_drivers
dt = 'Byte'
if cn.count == 96:
processes = 80 # 45 processors = 70 GB peak; 70 = 90 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating tree cover loss driver tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates young natural forest removal rate tiles
source_raster = cn.name_annual_gain_AGC_natrl_forest_young_raw
out_pattern = cn.pattern_annual_gain_AGC_natrl_forest_young
dt = 'float32'
if cn.count == 96:
processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates young natural forest removal rate standard deviation tiles
source_raster = cn.name_stdev_annual_gain_AGC_natrl_forest_young_raw
out_pattern = cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
dt = 'float32'
if cn.count == 96:
processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates pre-2000 oil palm plantation tiles
if cn.count == 96:
processes = 80 # 45 processors = 100 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating pre-2000 oil palm plantation tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(prep_other_inputs.rasterize_pre_2000_plantations, tile_id_list)
pool.close()
pool.join()
# Creates climate zone tiles
if cn.count == 96:
processes = 80 # 45 processors = 230 GB peak (on second step); 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating climate zone tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(prep_other_inputs.create_climate_zone_tiles, tile_id_list)
pool.close()
pool.join()
# Creates European natural forest removal rate tiles
source_raster = cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw
out_pattern = cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe
dt = 'float32'
if cn.count == 96:
processes = 60 # 32 processors = 60 GB peak; 60 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates European natural forest standard deviation of removal rate tiles
source_raster = cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw
out_pattern = cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe
dt = 'float32'
if cn.count == 96:
processes = 32 # 32 processors = 60 GB peak; 60 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates a vrt of the primary forests with nodata=0 from the continental primary forest rasters
uu.print_log("Creating vrt of humid tropial primary forest...")
primary_vrt = 'primary_2001.vrt'
os.system('gdalbuildvrt -srcnodata 0 {} *2001_primary.tif'.format(primary_vrt))
uu.print_log(" Humid tropical primary forest vrt created")
# Creates primary forest tiles
source_raster = primary_vrt
out_pattern = 'primary_2001'
dt = 'Byte'
if cn.count == 96:
processes = 45 # 45 processors = 650 GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating primary forest tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates a combined IFL/primary forest raster
# Uses very little memory since it's just file renaming
if cn.count == 96:
processes = 60 # 60 processors = 10 GB peak
else:
processes = int(cn.count/2)
uu.print_log("Assigning each tile to ifl2000 or primary forest with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(prep_other_inputs.create_combined_ifl_primary, tile_id_list)
pool.close()
pool.join()
# Creates forest age category tiles for US forests
source_raster = cn.name_age_cat_natrl_forest_US_raw
out_pattern = cn.pattern_age_cat_natrl_forest_US
dt = 'Byte'
if cn.count == 96:
processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating US forest age category tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates forest groups for US forests
source_raster = cn.name_FIA_forest_group_raw
out_pattern = cn.pattern_FIA_forest_group_processed
dt = 'Byte'
if cn.count == 96:
processes = 80 # 32 processors = 25 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating US forest group tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates FIA regions for US forests
source_raster = cn.name_FIA_regions_raw
out_pattern = cn.pattern_FIA_regions_processed
dt = 'Byte'
if cn.count == 96:
processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating US forest region tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
for output_pattern in [cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young]:
# For some reason I can't figure out, the young forest rasters (rate and stdev) have NaN values in some places where 0 (NoData)
# should be. These NaN values show up as values when the check_and_delete_if_empty function runs, making the tiles not
# deleted even if they have no data. However, the light version (which uses gdalinfo rather than rasterio masks) doesn't
# have this problem. So I'm forcing the young forest rates to and stdev to have their emptiness checked by the gdalinfo version.
if output_pattern in [cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young]:
processes = int(cn.count / 2)
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
if cn.count == 96:
processes = 50 # 60 processors = >730 GB peak (for European natural forest forest removal rates); 50 = XXX GB peak
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
elif cn.count <= 2: # For local tests
processes = 1
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
else:
processes = int(cn.count / 2)
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
uu.print_log('\n')
# Uploads output tiles to s3
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu
# tile_list = util.tile_list(cn.natrl_forest_biomass_2000_dir)
tile_list = ['10N_080W', '40N_120E'] # test tiles
print tile_list
print "Downloading raw ecozone and precipitation files"
# Downloads two of the raw input files for creating carbon pools
input_files = [cn.fao_ecozone_raw_dir, cn.precip_raw_dir]
for input in input_files:
uu.s3_file_download('{}'.format(input), '.')
print "Unzipping FAO ecozones"
unzip_zones = ['unzip', '{}'.format(cn.pattern_fao_ecozone_raw), '-d', '.']
subprocess.check_call(unzip_zones)
print "Copying srtm files"
uu.s3_folder_download(cn.srtm_raw_dir, './srtm')
print "Making srtm vrt"
subprocess.check_call('gdalbuildvrt srtm.vrt srtm/*.tif', shell=True)
# # Soil tiles are already processed, so there's no need to include them here.
# # Leaving this in case I ever add in soil processing again.
# print "Copying soil tiles"
# uu.s3_folder_download(cn.soil_C_processed_dir)
def mp_peatland_processing(tile_id_list, run_date=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.tile_list_s3(cn.pixel_area_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# List of output directories and output file name patterns
output_dir_list = [cn.peat_mask_dir]
output_pattern_list = [cn.pattern_peat_mask]
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Download SoilGrids250 most probable soil class rasters.
# There are 459 tiles and it takes about 20 minutes to download them
cmd = [
'wget', '--recursive', '--no-parent', '-nH', '--cut-dirs=7',
'--accept', '*.geotiff', '{}'.format(cn.soilgrids250_peat_url)
]
uu.log_subprocess_output_full(cmd)
uu.print_log("Making SoilGrids250 most likely soil class vrt...")
check_call('gdalbuildvrt most_likely_soil_class.vrt *{}*'.format(
cn.pattern_soilgrids_most_likely_class),
shell=True)
uu.print_log("Done making SoilGrids250 most likely soil class vrt")
# Downloads peat layers
uu.s3_file_download(
os.path.join(cn.peat_unprocessed_dir, cn.cifor_peat_file),
cn.docker_base_dir, sensit_type)
uu.s3_file_download(
os.path.join(cn.peat_unprocessed_dir, cn.jukka_peat_zip),
cn.docker_base_dir, sensit_type)
# Unzips the Jukka peat shapefile (IDN and MYS)
cmd = ['unzip', '-o', '-j', cn.jukka_peat_zip]
uu.log_subprocess_output_full(cmd)
jukka_tif = 'jukka_peat.tif'
# Converts the Jukka peat shapefile to a raster
uu.print_log('Rasterizing jukka peat...')
cmd = [
'gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=LZW', '-tr',
'{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-tap', '-ot',
'Byte', '-a_nodata', '0', cn.jukka_peat_shp, jukka_tif
]
uu.log_subprocess_output_full(cmd)
uu.print_log(' Jukka peat rasterized')
# For multiprocessor use
# count-10 maxes out at about 100 GB on an r5d.16xlarge
processes = cn.count - 5
uu.print_log('Peatland preprocessing max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(peatland_processing.create_peat_mask_tiles, tile_id_list)
pool.close()
pool.join()
# # For single processor use, for testing purposes
# for tile_id in tile_id_list:
#
# peatland_processing.create_peat_mask_tiles(tile_id)
output_pattern = output_pattern_list[0]
processes = 50 # 50 processors = XXX GB peak
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors...".
format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty, output_pattern=output_pattern),
tile_id_list)
pool.close()
pool.join()
uu.print_log("Uploading output files")
uu.upload_final_set(output_dir_list[0], output_pattern_list[0])
def mp_plantation_preparation(gadm_index_shp, planted_index_shp):
os.chdir(cn.docker_base_dir)
# ## Not actually using this but leaving it here in case I want to add this functionality eventually. This
# # was to allow users to run plantations for a select (contiguous) area rather than for the whole planet.
# # List of bounding box coordinates
# bound_list = args.bounding_box
# # Checks if bounding box coordinates are in multiples of 10 (10 degree tiles). If they're not, the script stops.
# for bound in bound_list:
# if bound%10:
# uu.exception_log(bound, 'not a multiple of 10. Please make bounding box coordinates are multiples of 10.')
# Checks the validity of the two arguments. If either one is invalid, the script ends.
if (gadm_index_path not in cn.gadm_plant_1x1_index_dir or planted_index_path not in cn.gadm_plant_1x1_index_dir):
uu.exception_log('Invalid inputs. Please provide None or s3 shapefile locations for both arguments.')
# List of all possible 10x10 Hansen tiles except for those at very extreme latitudes (not just WHRC biomass tiles)
total_tile_list = uu.tile_list_s3(cn.pixel_area_dir)
uu.print_log("Number of possible 10x10 tiles to evaluate:", len(total_tile_list))
# Removes the latitude bands that don't have any planted forests in them according to Liz Goldman.
# i.e., Liz Goldman said by Slack on 1/2/19 that the nothernmost planted forest is 69.5146 and the southernmost is -46.938968.
# This creates a more focused list of 10x10 tiles to iterate through (removes ones that definitely don't have planted forest).
# NOTE: If the planted forest gdb is updated, the list of latitudes to exclude below may need to be changed to not exclude certain latitude bands.
planted_lat_tile_list = [tile for tile in total_tile_list if '90N' not in tile]
planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '80N' not in tile]
planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '50S' not in tile]
planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '60S' not in tile]
planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '70S' not in tile]
planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '80S' not in tile]
# planted_lat_tile_list = ['10N_080W']
uu.print_log(planted_lat_tile_list)
uu.print_log("Number of 10x10 tiles to evaluate after extreme latitudes have been removed:", len(planted_lat_tile_list))
# If a planted forest extent 1x1 tile index shapefile isn't supplied
if 'None' in args.planted_tile_index:
### Entry point 1:
# If no shapefile of 1x1 tiles for countries with planted forests is supplied, 1x1 tiles of country extents will be created.
# This runs the process from the very beginning and will take a few days.
if 'None' in args.gadm_tile_index:
uu.print_log("No GADM 1x1 tile index shapefile provided. Creating 1x1 planted forest country tiles from scratch...")
# Downloads and unzips the GADM shapefile, which will be used to create 1x1 tiles of land areas
uu.s3_file_download(cn.gadm_path, cn.docker_base_dir)
cmd = ['unzip', cn.gadm_zip]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Creates a new GADM shapefile with just the countries that have planted forests in them.
# This limits creation of 1x1 rasters of land area on the countries that have planted forests rather than on all countries.
# NOTE: If the planted forest gdb is updated and has new countries added to it, the planted forest country list
# in constants_and_names.py must be updated, too.
uu.print_log("Creating shapefile of countries with planted forests...")
os.system('''ogr2ogr -sql "SELECT * FROM gadm_3_6_adm2_final WHERE iso IN ({0})" {1} gadm_3_6_adm2_final.shp'''.format(str(cn.plantation_countries)[1:-1], cn.gadm_iso))
# Creates 1x1 degree tiles of countries that have planted forests in them.
# I think this can handle using 50 processors because it's not trying to upload files to s3 and the tiles are small.
# This takes several days to run because it iterates through at least 250 10x10 tiles.
# For multiprocessor use.
processes = 50
uu.print_log('Rasterize GADM 1x1 max processors=', processes)
pool = Pool(processes)
pool.map(plantation_preparation.rasterize_gadm_1x1, planted_lat_tile_list)
pool.close()
pool.join()
# # Creates 1x1 degree tiles of countries that have planted forests in them.
# # For single processor use.
# for tile in planted_lat_tile_list:
#
# plantation_preparation.rasterize_gadm_1x1(tile)
# Creates a shapefile of the boundaries of the 1x1 GADM tiles in countries with planted forests
os.system('''gdaltindex {0}_{1}.shp GADM_*.tif'''.format(cn.pattern_gadm_1x1_index, uu.date_time_today))
cmd = ['aws', 's3', 'cp', cn.docker_base_dir, cn.gadm_plant_1x1_index_dir, '--exclude', '*', '--include', '{}*'.format(cn.pattern_gadm_1x1_index), '--recursive']
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# # Saves the 1x1 country extent tiles to s3
# # Only use if the entire process can't run in one go on the spot machine
# cmd = ['aws', 's3', 'cp', cn.docker_base_dir, 's3://gfw2-data/climate/carbon_model/temp_spotmachine_output/', '--exclude', '*', '--include', 'GADM_*.tif', '--recursive']
# # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
# process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
# with process.stdout:
# uu.log_subprocess_output(process.stdout)
# Delete the aux.xml files
os.system('''rm GADM*.tif.*''')
# List of all 1x1 degree countey extent tiles created
gadm_list_1x1 = uu.tile_list_spot_machine(".", "GADM_")
uu.print_log("List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1)
uu.print_log(len(gadm_list_1x1))
### Entry point 2:
# If a shapefile of the boundaries of 1x1 degree tiles of countries with planted forests is supplied,
# a list of the 1x1 tiles is created from the shapefile.
# This avoids creating the 1x1 country extent tiles all over again because the relevant tile extent are supplied
# in the shapefile.
elif cn.gadm_plant_1x1_index_dir in args.gadm_tile_index:
uu.print_log("Country extent 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest tiles...")
uu.print_log('{}/'.format(gadm_index_path))
# Copies the shapefile of 1x1 tiles of extent of countries with planted forests
cmd = ['aws', 's3', 'cp', '{}/'.format(gadm_index_path), cn.docker_base_dir, '--recursive', '--exclude', '*', '--include', '{}*'.format(gadm_index_shp)]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Gets the attribute table of the country extent 1x1 tile shapefile
gadm = glob.glob('{}*.dbf'.format(cn.pattern_gadm_1x1_index))[0]
# Converts the attribute table to a dataframe
dbf = Dbf5(gadm)
df = dbf.to_dataframe()
# Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list
gadm_list_1x1 = df['location'].tolist()
gadm_list_1x1 = [str(y) for y in gadm_list_1x1]
uu.print_log("List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1)
uu.print_log("There are", len(gadm_list_1x1), "1x1 country extent tiles to iterate through.")
# In case some other arguments are provided
else:
uu.exception_log('Invalid GADM tile index shapefile provided. Please provide a valid shapefile.')
# Creates 1x1 degree tiles of plantation growth wherever there are plantations.
# Because this is iterating through all 1x1 tiles in countries with planted forests, it first checks
# whether each 1x1 tile intersects planted forests before creating a 1x1 planted forest tile for that
# 1x1 country extent tile.
# 55 processors seems to use about 350 GB of memory, which seems fine. But there was some error about "PQconnectdb failed-- sorry, too many clients already".
# So, moved the number of processors down to 48.
# For multiprocessor use
processes = 48
uu.print_log('Create 1x1 plantation from 1x1 gadm max processors=', processes)
pool = Pool(processes)
pool.map(plantation_preparation.create_1x1_plantation_from_1x1_gadm, gadm_list_1x1)
pool.close()
pool.join()
# # Creates 1x1 degree tiles of plantation growth wherever there are plantations
# # For single processor use
# for tile in gadm_list_1x1:
#
# plantation_preparation.create_1x1_plantation(tile)
# Creates a shapefile in which each feature is the extent of a plantation extent tile.
# This index shapefile can be used the next time this process is run if starting with Entry Point 3.
os.system('''gdaltindex {0}_{1}.shp plant_gain_*.tif'''.format(cn.pattern_plant_1x1_index, uu.date_time_today))
cmd = ['aws', 's3', 'cp', cn.docker_base_dir, cn.gadm_plant_1x1_index_dir, '--exclude', '*', '--include', '{}*'.format(cn.pattern_plant_1x1_index), '--recursive']
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
### Entry point 3
# If a shapefile of the extents of 1x1 planted forest tiles is provided.
# This is the part that actually creates the sequestration rate and forest type tiles.
if cn.pattern_plant_1x1_index in args.planted_tile_index:
uu.print_log("Planted forest 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest growth rate and forest type tiles...")
# Copies the shapefile of 1x1 tiles of extent of planted forests
cmd = ['aws', 's3', 'cp', '{}/'.format(planted_index_path), cn.docker_base_dir, '--recursive', '--exclude', '*', '--include',
'{}*'.format(planted_index_shp), '--recursive']
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Gets the attribute table of the planted forest extent 1x1 tile shapefile
gadm = glob.glob('{}*.dbf'.format(cn.pattern_plant_1x1_index))[0]
# Converts the attribute table to a dataframe
dbf = Dbf5(gadm)
df = dbf.to_dataframe()
# Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list
planted_list_1x1 = df['location'].tolist()
planted_list_1x1 = [str(y) for y in planted_list_1x1]
uu.print_log("List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", planted_list_1x1)
uu.print_log("There are", len(planted_list_1x1), "1x1 planted forest extent tiles to iterate through.")
# Creates 1x1 degree tiles of plantation growth and type wherever there are plantations.
# Because this is iterating through only 1x1 tiles that are known to have planted forests (from a previous run
# of this script), it does not need to check whether there are planted forests in this tile. It goes directly
# to intersecting the planted forest table with the 1x1 tile.
# For single processor use
#for tile in planted_list_1x1:
# plantation_preparation.create_1x1_plantation_growth_from_1x1_planted(tile)
# For multiprocessor use
# processes=40 uses about 360 GB of memory. Works on r4.16xlarge with space to spare
# processes=52 uses about 465 GB of memory (quite stably), so this is basically the max.
num_of_processes = 52
pool = Pool(num_of_processes)
pool.map(plantation_preparation.create_1x1_plantation_growth_from_1x1_planted, planted_list_1x1)
pool.close()
pool.join()
# This works with 50 processors on an r4.16xlarge marchine. Uses about 430 GB out of 480 GB.
num_of_processes = 52
pool = Pool(num_of_processes)
processes = 50
uu.print_log('Create 1x1 plantation type max processors=', processes)
pool = Pool(processes)
pool.map(plantation_preparation.create_1x1_plantation_type_from_1x1_planted, planted_list_1x1)
pool.close()
pool.join()
# This rasterizes the plantation removal factor standard deviations
# processes=50 peaks at about 450 GB
num_of_processes = 50
pool = Pool(num_of_processes)
pool.map(plantation_preparation.create_1x1_plantation_stdev_from_1x1_planted, planted_list_1x1)
pool.close()
pool.join()
def mp_create_inputs_for_C_pools(tile_id_list, run_date=None):
os.chdir(cn.docker_base_dir)
sensit_type = 'std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.tile_list_s3(cn.model_extent_dir, sensit_type)
# List of output directories and output file name patterns
output_dir_list = [
cn.bor_tem_trop_processed_dir, cn.elevation_processed_dir,
cn.precip_processed_dir
]
output_pattern_list = [
cn.pattern_bor_tem_trop_processed, cn.pattern_elevation,
cn.pattern_precip
]
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Downloads two of the raw input files for creating carbon emitted_pools
input_files = [cn.fao_ecozone_raw_dir, cn.precip_raw_dir]
for input in input_files:
uu.s3_file_download('{}'.format(input), cn.docker_base_dir,
sensit_type)
uu.print_log(
"Unzipping boreal/temperate/tropical file (from FAO ecozones)")
cmd = [
'unzip', '{}'.format(cn.pattern_fao_ecozone_raw), '-d',
cn.docker_base_dir
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.print_log("Copying elevation (srtm) files")
uu.s3_folder_download(cn.srtm_raw_dir, './srtm', sensit_type)
uu.print_log("Making elevation (srtm) vrt")
check_call(
'gdalbuildvrt srtm.vrt srtm/*.tif', shell=True
) # I don't know how to convert this to output to the pipe, so just leaving as is
# Worked with count/3 on an r4.16xlarge (140 out of 480 GB used). I think it should be fine with count/2 but didn't try it.
processes = int(cn.count / 2)
uu.print_log('Inputs for C emitted_pools max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(create_inputs_for_C_pools.create_input_files, tile_id_list)
# # For single processor use
# for tile_id in tile_id_list:
#
# create_inputs_for_C_pools.create_input_files(tile_id)
uu.print_log("Uploading output files")
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
# For downloading all tiles in the input folders.
input_files = [
cn.natrl_forest_biomass_2000_dir, cn.mangrove_biomass_2000_dir,
cn.fao_ecozone_processed_dir, cn.precip_processed_dir,
cn.soil_C_processed_dir, cn.srtm_processed_dir
]
# for input in input_files:
# uu.s3_folder_download('{}'.format(input), '.')
# For copying individual tiles to spot machine for testing.
for tile in tile_list:
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.natrl_forest_biomass_emitted_dir, tile,
cn.pattern_natrl_forest_biomass_emitted), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.mangrove_biomass_emitted_dir, tile,
cn.pattern_mangrove_biomass_emitted), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.fao_ecozone_processed_dir, tile,
cn.pattern_fao_ecozone_processed), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.precip_processed_dir, tile,
cn.pattern_precip), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.soil_C_processed_dir, tile,
cn.pattern_soil_C), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.srtm_processed_dir, tile, cn.pattern_srtm),
def main():
# Create the output log
uu.initiate_log()
os.chdir(cn.docker_base_dir)
# Files to download for this script.
download_dict = {
cn.gain_dir: [cn.pattern_gain],
cn.annual_gain_AGB_IPCC_defaults_dir:
[cn.pattern_annual_gain_AGB_IPCC_defaults]
}
# List of tiles that could be run. This list is only used to create the FIA region tiles if they don't already exist.
tile_id_list = uu.tile_list_s3(cn.annual_gain_AGB_IPCC_defaults_dir)
# tile_id_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
# tile_id_list = ['50N_130W'] # test tiles
# List of output directories and output file name patterns
output_dir_list = [
cn.US_annual_gain_AGB_natrl_forest_dir,
cn.US_annual_gain_BGB_natrl_forest_dir
]
output_pattern_list = [
cn.pattern_US_annual_gain_AGB_natrl_forest,
cn.pattern_US_annual_gain_BGB_natrl_forest
]
# By definition, this script is for US-specific removals
sensit_type = 'US_removals'
# Counts how many processed FIA region tiles there are on s3 already. 16 tiles cover the continental US.
FIA_regions_tile_count = uu.count_tiles_s3(cn.FIA_regions_processed_dir)
# Only creates FIA region tiles if they don't already exist on s3.
if FIA_regions_tile_count == 16:
uu.print_log("FIA region tiles already created. Copying to s3 now...")
uu.s3_flexible_download(cn.FIA_regions_processed_dir,
cn.pattern_FIA_regions_processed,
cn.docker_base_dir, 'std', 'all')
else:
uu.print_log(
"FIA region tiles do not exist. Creating tiles, then copying to s3 for future use..."
)
uu.s3_file_download(
os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw),
cn.docker_base_dir, 'std')
cmd = ['unzip', '-o', '-j', cn.name_FIA_regions_raw]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Converts the region shapefile to Hansen tiles
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(US_removal_rates.prep_FIA_regions, tile_id_list)
# List of FIA region tiles on the spot machine. Only this list is used for the rest of the script.
US_tile_list = uu.tile_list_spot_machine(
cn.docker_base_dir, '{}.tif'.format(cn.pattern_FIA_regions_processed))
US_tile_id_list = [i[0:8] for i in US_tile_list]
# US_tile_id_list = ['50N_130W'] # For testing
uu.print_log(US_tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(US_tile_id_list))) +
"\n")
# Counts how many processed forest age category tiles there are on s3 already. 16 tiles cover the continental US.
US_age_tile_count = uu.count_tiles_s3(cn.US_forest_age_cat_processed_dir)
# Only creates FIA forest age category tiles if they don't already exist on s3.
if US_age_tile_count == 16:
uu.print_log(
"Forest age category tiles already created. Copying to spot machine now..."
)
uu.s3_flexible_download(cn.US_forest_age_cat_processed_dir,
cn.pattern_US_forest_age_cat_processed, '',
'std', US_tile_id_list)
else:
uu.print_log(
"Southern forest age category tiles do not exist. Creating tiles, then copying to s3 for future use..."
)
uu.s3_file_download(
os.path.join(cn.US_forest_age_cat_raw_dir,
cn.name_US_forest_age_cat_raw), cn.docker_base_dir,
'std')
# Converts the national forest age category raster to Hansen tiles
source_raster = cn.name_US_forest_age_cat_raw
out_pattern = cn.pattern_US_forest_age_cat_processed
dt = 'Int16'
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt), US_tile_id_list)
uu.upload_final_set(cn.US_forest_age_cat_processed_dir,
cn.pattern_US_forest_age_cat_processed)
# Counts how many processed FIA forest group tiles there are on s3 already. 16 tiles cover the continental US.
FIA_forest_group_tile_count = uu.count_tiles_s3(
cn.FIA_forest_group_processed_dir)
# Only creates FIA forest group tiles if they don't already exist on s3.
if FIA_forest_group_tile_count == 16:
uu.print_log(
"FIA forest group tiles already created. Copying to spot machine now..."
)
uu.s3_flexible_download(cn.FIA_forest_group_processed_dir,
cn.pattern_FIA_forest_group_processed, '',
'std', US_tile_id_list)
else:
uu.print_log(
"FIA forest group tiles do not exist. Creating tiles, then copying to s3 for future use..."
)
uu.s3_file_download(
os.path.join(cn.FIA_forest_group_raw_dir,
cn.name_FIA_forest_group_raw), cn.docker_base_dir,
'std')
# Converts the national forest group raster to Hansen forest group tiles
source_raster = cn.name_FIA_forest_group_raw
out_pattern = cn.pattern_FIA_forest_group_processed
dt = 'Byte'
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt), US_tile_id_list)
uu.upload_final_set(cn.FIA_forest_group_processed_dir,
cn.pattern_FIA_forest_group_processed)
# Downloads input files or entire directories, depending on how many tiles are in the tile_id_list
for key, values in download_dict.items():
dir = key
pattern = values[0]
uu.s3_flexible_download(dir, pattern, cn.docker_base_dir, sensit_type,
US_tile_id_list)
# Table with US-specific removal rates
cmd = [
'aws', 's3', 'cp',
os.path.join(cn.gain_spreadsheet_dir, cn.table_US_removal_rate),
cn.docker_base_dir
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Imports the table with the region-group-age AGB removal rates
gain_table = pd.read_excel("{}".format(cn.table_US_removal_rate),
sheet_name="US_rates_for_model")
# Converts gain table from wide to long, so each region-group-age category has its own row
gain_table_group_region_by_age = pd.melt(
gain_table,
id_vars=['FIA_region_code', 'forest_group_code'],
value_vars=['growth_young', 'growth_middle', 'growth_old'])
gain_table_group_region_by_age = gain_table_group_region_by_age.dropna()
# In the forest age category raster, each category has this value
age_dict = {
'growth_young': 1000,
'growth_middle': 2000,
'growth_old': 3000
}
# Creates a unique value for each forest group-region-age category in the table.
# Although these rates are applied to all standard gain model pixels at first, they are not ultimately used for
# pixels that have Hansen gain (see below).
gain_table_group_region_age = gain_table_group_region_by_age.replace(
{"variable": age_dict})
gain_table_group_region_age[
'age_cat'] = gain_table_group_region_age['variable'] * 10
gain_table_group_region_age['group_region_age_combined'] = gain_table_group_region_age['age_cat'] + \
gain_table_group_region_age['forest_group_code']*100 + \
gain_table_group_region_age['FIA_region_code']
# Converts the forest group-region-age codes and corresponding gain rates to a dictionary,
# where the key is the unique group-region-age code and the value is the AGB removal rate.
gain_table_group_region_age_dict = pd.Series(
gain_table_group_region_age.value.values,
index=gain_table_group_region_age.group_region_age_combined).to_dict()
uu.print_log(gain_table_group_region_age_dict)
# Creates a unique value for each forest group-region category using just young forest rates.
# These are assigned to Hansen gain pixels, which automatically get the young forest rate, regardless of the
# forest age category raster.
gain_table_group_region = gain_table_group_region_age.drop(
gain_table_group_region_age[
gain_table_group_region_age.age_cat != 10000].index)
gain_table_group_region['group_region_combined'] = gain_table_group_region['forest_group_code']*100 + \
gain_table_group_region['FIA_region_code']
# Converts the forest group-region codes and corresponding gain rates to a dictionary,
# where the key is the unique group-region code (youngest age category) and the value is the AGB removal rate.
gain_table_group_region_dict = pd.Series(
gain_table_group_region.value.values,
index=gain_table_group_region.group_region_combined).to_dict()
uu.print_log(gain_table_group_region_dict)
# count/2 on a m4.16xlarge maxes out at about 230 GB of memory (processing 16 tiles at once), so it's okay on an m4.16xlarge
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(
partial(
US_removal_rates.US_removal_rate_calc,
gain_table_group_region_age_dict=gain_table_group_region_age_dict,
gain_table_group_region_dict=gain_table_group_region_dict,
output_pattern_list=output_pattern_list,
sensit_type=sensit_type), US_tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in US_tile_id_list:
#
# US_removal_rates.US_removal_rate_calc(tile_id, gain_table_group_region_age_dict, gain_table_group_region_dict,
# output_pattern_list, sensit_type)
# Uploads output tiles to s3
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
# For downloading all tiles in the input folders.
input_files = [
cn.natrl_forest_biomass_2000_dir, cn.mangrove_biomass_2000_dir,
cn.cumul_gain_AGC_mangrove_dir, cn.cumul_gain_AGC_natrl_forest_dir
]
# for input in input_files:
# uu.s3_folder_download('{}'.format(input), '.')
# For copying individual tiles to spot machine for testing.
# The cumulative carbon gain tiles are for adding to the biomass 2000 tiles to get AGC at the time of tree cover loss.
for tile in tile_list:
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.natrl_forest_biomass_2000_dir, tile,
cn.pattern_natrl_forest_biomass_2000), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.mangrove_biomass_raw_dir, tile,
cn.pattern_mangrove_biomass_emitted), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.cumul_gain_AGC_mangrove_dir,
cn.pattern_cumul_gain_AGC_mangrove, tile), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.cumul_gain_AGC_natrl_forest_dir,
cn.pattern_cumul_gain_AGC_natrl_forest, tile),
'.')
print "Creating tiles of emitted biomass (biomass 2000 + biomass accumulation)"
count = multiprocessing.cpu_count()
def mp_prep_other_inputs(tile_id_list, run_date, no_upload=None):
os.chdir(cn.docker_base_dir)
sensit_type = 'std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
### BUG: THIS SHOULD ALSO INCLUDE cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir IN ITS LIST
tile_id_list = uu.create_combined_tile_list(
cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
set3=cn.gain_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
'''
Before processing the driver, it needs to be reprojected from Goode Homolosine to WGS84.
gdal_warp is producing a weird output, so I did it in ArcMap for the 2020 update,
with the output cell size being 0.01 x 0.01 degree and the method being nearest.
arcpy.ProjectRaster_management(in_raster="C:/GIS/Drivers of loss/2020_drivers__tif__from_Forrest_Follett_20210323/FinalClassification_2020_v2__from_Jimmy_MacCarthy_20210323.tif",
out_raster="C:/GIS/Drivers of loss/2020_drivers__tif__from_Forrest_Follett_20210323/Final_Classification_2020__reproj_nearest_0-005_0-005_deg__20210323.tif",
out_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]",
resampling_type="NEAREST", cell_size="0.005 0.005", geographic_transform="",
Registration_Point="",
in_coor_system="PROJCS['WGS_1984_Goode_Homolosine',GEOGCS['GCS_unknown',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Goode_Homolosine'],PARAMETER['False_Easting',0.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',0.0],PARAMETER['Option',1.0],UNIT['Meter',1.0]]",
vertical="NO_VERTICAL")
'''
# List of output directories and output file name patterns
output_dir_list = [
# cn.climate_zone_processed_dir, cn.plant_pre_2000_processed_dir,
cn.drivers_processed_dir
# cn.ifl_primary_processed_dir,
# cn.annual_gain_AGC_natrl_forest_young_dir,
# cn.stdev_annual_gain_AGC_natrl_forest_young_dir,
# cn.annual_gain_AGC_BGC_natrl_forest_Europe_dir,
# cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_dir,
# cn.FIA_forest_group_processed_dir,
# cn.age_cat_natrl_forest_US_dir,
# cn.FIA_regions_processed_dir
]
output_pattern_list = [
# cn.pattern_climate_zone, cn.pattern_plant_pre_2000,
cn.pattern_drivers
# cn.pattern_ifl_primary,
# cn.pattern_annual_gain_AGC_natrl_forest_young,
# cn.pattern_stdev_annual_gain_AGC_natrl_forest_young,
# cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe,
# cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe,
# cn.pattern_FIA_forest_group_processed,
# cn.pattern_age_cat_natrl_forest_US,
# cn.pattern_FIA_regions_processed
]
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log(
"Changing output directory and file name pattern based on sensitivity analysis"
)
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
output_pattern_list = uu.alter_patterns(sensit_type,
output_pattern_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# # Files to process: climate zone, IDN/MYS plantations before 2000, tree cover loss drivers, combine IFL and primary forest
# uu.s3_file_download(os.path.join(cn.climate_zone_raw_dir, cn.climate_zone_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.plant_pre_2000_raw_dir, '{}.zip'.format(cn.pattern_plant_pre_2000_raw)), cn.docker_base_dir, sensit_type)
uu.s3_file_download(
os.path.join(cn.drivers_raw_dir, cn.pattern_drivers_raw),
cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.age_cat_natrl_forest_US_raw_dir, cn.name_age_cat_natrl_forest_US_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, sensit_type)
# # For some reason, using uu.s3_file_download or otherwise using AWSCLI as a subprocess doesn't work for this raster.
# # Thus, using wget instead.
# cmd = ['wget', '{}'.format(cn.annual_gain_AGC_natrl_forest_young_raw_URL), '-P', '{}'.format(cn.docker_base_dir)]
# process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
# with process.stdout:
# uu.log_subprocess_output(process.stdout)
# uu.s3_file_download(cn.stdev_annual_gain_AGC_natrl_forest_young_raw_URL, cn.docker_base_dir, sensit_type)
# cmd = ['aws', 's3', 'cp', cn.primary_raw_dir, cn.docker_base_dir, '--recursive']
# uu.log_subprocess_output_full(cmd)
#
# uu.s3_flexible_download(cn.ifl_dir, cn.pattern_ifl, cn.docker_base_dir, sensit_type, tile_id_list)
#
# uu.print_log("Unzipping pre-2000 plantations...")
# cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_plant_pre_2000_raw)]
# uu.log_subprocess_output_full(cmd)
# Creates tree cover loss driver tiles.
# The raw driver tile should have NoData for unassigned drivers as opposed to 0 for unassigned drivers.
# For the 2020 driver update, I reclassified the 0 values as NoData in ArcMap. I also unprojected the global drivers
# map to WGS84 because running the homolosine projection that Jimmy provided was giving incorrect processed results.
source_raster = cn.pattern_drivers_raw
out_pattern = cn.pattern_drivers
dt = 'Byte'
if cn.count == 96:
processes = 87 # 45 processors = 70 GB peak; 70 = 90 GB peak; 80 = 100 GB peak; 87 = 125 GB peak
else:
processes = int(cn.count / 2)
uu.print_log(
"Creating tree cover loss driver tiles with {} processors...".format(
processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt,
no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
# # Creates young natural forest removal rate tiles
# source_raster = cn.name_annual_gain_AGC_natrl_forest_young_raw
# out_pattern = cn.pattern_annual_gain_AGC_natrl_forest_young
# dt = 'float32'
# if cn.count == 96:
# processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates young natural forest removal rate standard deviation tiles
# source_raster = cn.name_stdev_annual_gain_AGC_natrl_forest_young_raw
# out_pattern = cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
# dt = 'float32'
# if cn.count == 96:
# processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates pre-2000 oil palm plantation tiles
# if cn.count == 96:
# processes = 80 # 45 processors = 100 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating pre-2000 oil palm plantation tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(prep_other_inputs.rasterize_pre_2000_plantations, tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates climate zone tiles
# if cn.count == 96:
# processes = 80 # 45 processors = 230 GB peak (on second step); 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating climate zone tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(prep_other_inputs.create_climate_zone_tiles, tile_id_list)
# pool.close()
# pool.join()
#
# # Creates European natural forest removal rate tiles
# source_raster = cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw
# out_pattern = cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe
# dt = 'float32'
# if cn.count == 96:
# processes = 60 # 32 processors = 60 GB peak; 60 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates European natural forest standard deviation of removal rate tiles
# source_raster = cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw
# out_pattern = cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe
# dt = 'float32'
# if cn.count == 96:
# processes = 32 # 32 processors = 60 GB peak; 60 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates a vrt of the primary forests with nodata=0 from the continental primary forest rasters
# uu.print_log("Creating vrt of humid tropial primary forest...")
# primary_vrt = 'primary_2001.vrt'
# os.system('gdalbuildvrt -srcnodata 0 {} *2001_primary.tif'.format(primary_vrt))
# uu.print_log(" Humid tropical primary forest vrt created")
#
# # Creates primary forest tiles
# source_raster = primary_vrt
# out_pattern = 'primary_2001'
# dt = 'Byte'
# if cn.count == 96:
# processes = 45 # 45 processors = 650 GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating primary forest tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates a combined IFL/primary forest raster
# # Uses very little memory since it's just file renaming
# if cn.count == 96:
# processes = 60 # 60 processors = 10 GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Assigning each tile to ifl2000 or primary forest with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(prep_other_inputs.create_combined_ifl_primary, tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates forest age category tiles for US forests
# source_raster = cn.name_age_cat_natrl_forest_US_raw
# out_pattern = cn.pattern_age_cat_natrl_forest_US
# dt = 'Byte'
# if cn.count == 96:
# processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating US forest age category tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates forest groups for US forests
# source_raster = cn.name_FIA_forest_group_raw
# out_pattern = cn.pattern_FIA_forest_group_processed
# dt = 'Byte'
# if cn.count == 96:
# processes = 80 # 32 processors = 25 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating US forest group tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates FIA regions for US forests
# source_raster = cn.name_FIA_regions_raw
# out_pattern = cn.pattern_FIA_regions_processed
# dt = 'Byte'
# if cn.count == 96:
# processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating US forest region tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
for output_pattern in [
cn.pattern_drivers
# ,cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
]:
# For some reason I can't figure out, the young forest rasters (rate and stdev) have NaN values in some places where 0 (NoData)
# should be. These NaN values show up as values when the check_and_delete_if_empty function runs, making the tiles not
# deleted even if they have no data. However, the light version (which uses gdalinfo rather than rasterio masks) doesn't
# have this problem. So I'm forcing the young forest rates to and stdev to have their emptiness checked by the gdalinfo version.
if output_pattern in [
cn.pattern_annual_gain_AGC_natrl_forest_young,
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
]:
processes = int(cn.count / 2)
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors using light function..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty_light,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
if cn.count == 96:
processes = 50 # 60 processors = >730 GB peak (for European natural forest forest removal rates); 50 = XXX GB peak
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
elif cn.count <= 2: # For local tests
processes = 1
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors using light function..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty_light,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
else:
processes = int(cn.count / 2)
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
uu.print_log('\n')
# Uploads output tiles to s3
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
def mp_create_soil_C(tile_id_list, no_upload=None):
os.chdir(cn.docker_base_dir)
sensit_type = 'std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
set3=cn.gain_dir
)
uu.print_log(tile_id_list)
uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# List of output directories and output file name patterns
output_dir_list = [cn.soil_C_full_extent_2000_non_mang_dir, cn.soil_C_full_extent_2000_dir,
cn.stdev_soil_C_full_extent_2000_dir]
output_pattern_list = [cn.pattern_soil_C_full_extent_2000_non_mang, cn.pattern_soil_C_full_extent_2000,
cn.pattern_stdev_soil_C_full_extent]
### Soil carbon density
uu.print_log("Downloading mangrove soil C rasters")
uu.s3_file_download(os.path.join(cn.mangrove_soil_C_dir, cn.name_mangrove_soil_C), cn.docker_base_dir, sensit_type)
# For downloading all tiles in the input folders.
input_files = [cn.mangrove_biomass_2000_dir]
for input in input_files:
uu.s3_folder_download(input, cn.docker_base_dir, sensit_type)
# Download raw mineral soil C density tiles.
# First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder
# Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it
# There are 12951 tiles and it takes about 3 hours to download them!
cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
'--accept', '*.tif', '{}'.format(cn.mineral_soil_C_url)]
uu.log_subprocess_output_full(cmd)
uu.print_log("Unzipping mangrove soil C rasters...")
cmd = ['unzip', '-j', cn.name_mangrove_soil_C, '-d', cn.docker_base_dir]
uu.log_subprocess_output_full(cmd)
# Mangrove soil receives precedence over mineral soil
uu.print_log("Making mangrove soil C vrt...")
check_call('gdalbuildvrt mangrove_soil_C.vrt *{}*.tif'.format(cn.pattern_mangrove_soil_C_raw), shell=True)
uu.print_log("Done making mangrove soil C vrt")
uu.print_log("Making mangrove soil C tiles...")
if cn.count == 96:
processes = 32 # 32 processors = 570 GB peak
else:
processes = int(cn.count/3)
uu.print_log('Mangrove soil C max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(partial(create_soil_C.create_mangrove_soil_C, no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in tile_id_list:
#
# create_soil_C.create_mangrove_soil_C(tile_id, no_Upload)
uu.print_log('Done making mangrove soil C tiles', '\n')
uu.print_log("Making mineral soil C vrt...")
check_call('gdalbuildvrt mineral_soil_C.vrt *{}*'.format(cn.pattern_mineral_soil_C_raw), shell=True)
uu.print_log("Done making mineral soil C vrt")
# Creates mineral soil C density tiles
source_raster = 'mineral_soil_C.vrt'
out_pattern = cn.pattern_soil_C_full_extent_2000_non_mang
dt = 'Int16'
if cn.count == 96:
processes = 80 # 32 processors = 100 GB peak; 50 = 160 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating mineral soil C density tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt,
no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in tile_id_list:
#
# create_soil_C.create_mineral_soil_C(tile_id)
uu.print_log("Done making non-mangrove soil C tiles", "\n")
output_pattern = cn.pattern_soil_C_full_extent_2000_non_mang
processes = 60 # 50 processors = ~450 GB peak; 60 = XXX GB peak
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
# If no_upload flag is not activated, output is uploaded to s3
if not no_upload:
uu.print_log("Uploading non-mangrove soil C density tiles")
uu.upload_final_set(output_dir_list[0], output_pattern_list[0])
uu.print_log("Making combined (mangrove & non-mangrove) soil C tiles...")
if cn.count == 96:
processes = 45 # 45 processors = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log('Combined soil C max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(partial(create_soil_C.create_combined_soil_C, no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile in tile_list:
#
# create_soil_C.create_combined_soil_C(tile_id, no_upload)
uu.print_log("Done making combined soil C tiles")
# If no_upload flag is not activated, output is uploaded
if not no_upload:
uu.print_log("Uploading combined soil C density tiles")
uu.upload_final_set(output_dir_list[1], output_pattern_list[1])
# Need to delete soil c density rasters because they have the same pattern as the standard deviation rasters
uu.print_log("Deleting raw soil C density rasters")
c_stocks = glob.glob('*{}*'.format(cn.pattern_soil_C_full_extent_2000))
for c_stock in c_stocks:
os.remove(c_stock)
### Soil carbon density uncertainty
# Separate directories for the 5% CI and 95% CI
dir_CI05 = '{0}{1}'.format(cn.docker_base_dir, 'CI05/')
dir_CI95 = '{0}{1}'.format(cn.docker_base_dir, 'CI95/')
vrt_CI05 = 'mineral_soil_C_CI05.vrt'
vrt_CI95 = 'mineral_soil_C_CI95.vrt'
soil_C_stdev_global = 'soil_C_stdev.tif'
# Download raw mineral soil C density 5% CI tiles
# First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder
# Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it
# Like soil C density rasters, there are 12951 tifs and they take about 3 hours to download.
os.mkdir(dir_CI05)
cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
'--directory-prefix={}'.format(dir_CI05),
'--accept', '*.tif', '{}'.format(cn.CI5_mineral_soil_C_url)]
uu.log_subprocess_output_full(cmd)
uu.print_log("Making mineral soil C 5% CI vrt...")
check_call('gdalbuildvrt {0} {1}*{2}*'.format(vrt_CI05, dir_CI05, cn.pattern_uncert_mineral_soil_C_raw), shell=True)
uu.print_log("Done making mineral soil C CI05 vrt")
# Download raw mineral soil C density 5% CI tiles
# Like soil C density rasters, there are 12951 tifs and they take about 3 hours to download.
os.mkdir(dir_CI95)
cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
'--directory-prefix={}'.format(dir_CI95),
'--accept', '*.tif', '{}'.format(cn.CI95_mineral_soil_C_url)]
uu.log_subprocess_output_full(cmd)
uu.print_log("Making mineral soil C 95% CI vrt...")
check_call('gdalbuildvrt {0} {1}*{2}*'.format(vrt_CI95, dir_CI95, cn.pattern_uncert_mineral_soil_C_raw), shell=True)
uu.print_log("Done making mineral soil C CI95 vrt")
uu.print_log("Creating raster of standard deviations in soil C at native SoilGrids250 resolution. This may take a while...")
# global tif with approximation of the soil C stanard deviation (based on the 5% and 95% CIs)
# This takes about 20 minutes. It doesn't show any progress until the last moment, when it quickly counts
# up to 100.
calc = '--calc=(A-B)/3'
out_filearg = '--outfile={}'.format(soil_C_stdev_global)
cmd = ['gdal_calc.py', '-A', vrt_CI95, '-B', vrt_CI05, calc, out_filearg,
'--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type=Float32']
uu.log_subprocess_output_full(cmd)
uu.print_log("{} created.".format(soil_C_stdev_global))
# Creates soil carbon 2000 density standard deviation tiles
out_pattern = cn.pattern_stdev_soil_C_full_extent
dt = 'Float32'
source_raster = soil_C_stdev_global
if cn.count == 96:
processes = 56 # 32 processors = 290 GB peak; 56 = XXX GB peal
else:
processes = 2
uu.print_log("Creating mineral soil C stock stdev tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt,
no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
output_pattern = cn.pattern_stdev_soil_C_full_extent
processes = 50 # 50 processors = 550 GB peak
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
# Checks the gross removals outputs for tiles with no data
for output_pattern in output_pattern_list:
if cn.count <= 2: # For local tests
processes = 1
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(
output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
else:
processes = 55 # 50 processors = XXX GB peak
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
# If no_upload flag is not activated, output is uploaded
if not no_upload:
uu.print_log("Uploading soil C density standard deviation tiles")
uu.upload_final_set(output_dir_list[2], output_pattern_list[2])
def mp_aggregate_results_to_4_km(sensit_type,
thresh,
tile_id_list,
std_net_flux=None,
run_date=None,
no_upload=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.tile_list_s3(cn.net_flux_dir, sensit_type)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# Files to download for this script
download_dict = {
cn.annual_gain_AGC_all_types_dir:
[cn.pattern_annual_gain_AGC_all_types],
cn.cumul_gain_AGCO2_BGCO2_all_types_dir:
[cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
cn.gross_emis_all_gases_all_drivers_biomass_soil_dir:
[cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil],
cn.net_flux_dir: [cn.pattern_net_flux]
}
# Checks whether the canopy cover argument is valid
if thresh < 0 or thresh > 99:
uu.exception_log(
no_upload,
'Invalid tcd. Please provide an integer between 0 and 99.')
if uu.check_aws_creds():
# Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles
uu.s3_flexible_download(cn.pixel_area_dir, cn.pattern_pixel_area,
cn.docker_base_dir, sensit_type, tile_id_list)
# Tree cover density, Hansen gain, and mangrove biomass tiles-- necessary for filtering sums to model extent
uu.s3_flexible_download(cn.tcd_dir, cn.pattern_tcd, cn.docker_base_dir,
sensit_type, tile_id_list)
uu.s3_flexible_download(cn.gain_dir, cn.pattern_gain,
cn.docker_base_dir, sensit_type, tile_id_list)
uu.s3_flexible_download(cn.mangrove_biomass_2000_dir,
cn.pattern_mangrove_biomass_2000,
cn.docker_base_dir, sensit_type, tile_id_list)
uu.print_log("Model outputs to process are:", download_dict)
# List of output directories. Modified later for sensitivity analysis.
# Output pattern is determined later.
output_dir_list = [cn.output_aggreg_dir]
# If the model run isn't the standard one, the output directory is changed
if sensit_type != 'std':
uu.print_log(
"Changing output directory and file name pattern based on sensitivity analysis"
)
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Iterates through the types of tiles to be processed
for dir, download_pattern in list(download_dict.items()):
download_pattern_name = download_pattern[0]
# Downloads input files or entire directories, depending on how many tiles are in the tile_id_list, if AWS credentials are found
if uu.check_aws_creds():
uu.s3_flexible_download(dir, download_pattern_name,
cn.docker_base_dir, sensit_type,
tile_id_list)
# Gets an actual tile id to use as a dummy in creating the actual tile pattern
local_tile_list = uu.tile_list_spot_machine(cn.docker_base_dir,
download_pattern_name)
sample_tile_id = uu.get_tile_id(local_tile_list[0])
# Renames the tiles according to the sensitivity analysis before creating dummy tiles.
# The renaming function requires a whole tile name, so this passes a dummy time name that is then stripped a few
# lines later.
tile_id = sample_tile_id # a dummy tile id (but it has to be a real tile id). It is removed later.
output_pattern = uu.sensit_tile_rename(sensit_type, tile_id,
download_pattern_name)
pattern = output_pattern[9:-4]
# For sensitivity analysis runs, only aggregates the tiles if they were created as part of the sensitivity analysis
if (sensit_type != 'std') & (sensit_type not in pattern):
uu.print_log(
"{} not a sensitivity analysis output. Skipping aggregation..."
.format(pattern))
uu.print_log("")
continue
# Lists the tiles of the particular type that is being iterates through.
# Excludes all intermediate files
tile_list = uu.tile_list_spot_machine(".", "{}.tif".format(pattern))
# from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring
tile_list = [i for i in tile_list if not ('hanson_2013' in i)]
tile_list = [i for i in tile_list if not ('rewindow' in i)]
tile_list = [i for i in tile_list if not ('0_4deg' in i)]
tile_list = [i for i in tile_list if not ('.ovr' in i)]
# tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif'] # test tiles
uu.print_log("There are {0} tiles to process for pattern {1}".format(
str(len(tile_list)), download_pattern) + "\n")
uu.print_log("Processing:", dir, "; ", pattern)
# Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels,
# which is the resolution of the output tiles. This will allow the 30x30 m pixels in each window to be summed.
# For multiprocessor use. count/2 used about 400 GB of memory on an r4.16xlarge machine, so that was okay.
if cn.count == 96:
if sensit_type == 'biomass_swap':
processes = 12 # 12 processors = XXX GB peak
else:
processes = 16 # 12 processors = 140 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out)
else:
processes = 8
uu.print_log('Rewindow max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(aggregate_results_to_4_km.rewindow, no_upload=no_upload),
tile_list)
# Added these in response to error12: Cannot allocate memory error.
# This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool
# Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory
pool.close()
pool.join()
# # For single processor use
# for tile in tile_list:
#
# aggregate_results_to_4_km.rewindow(til, no_upload)
# Converts the existing (per ha) values to per pixel values (e.g., emissions/ha to emissions/pixel)
# and sums those values in each 400x400 pixel window.
# The sum for each 400x400 pixel window is stored in a 2D array, which is then converted back into a raster at
# 0.1x0.1 degree resolution (approximately 10m in the tropics).
# Each pixel in that raster is the sum of the 30m pixels converted to value/pixel (instead of value/ha).
# The 0.1x0.1 degree tile is output.
# For multiprocessor use. This used about 450 GB of memory with count/2, it's okay on an r4.16xlarge
if cn.count == 96:
if sensit_type == 'biomass_swap':
processes = 10 # 10 processors = XXX GB peak
else:
processes = 12 # 16 processors = 180 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out)
else:
processes = 8
uu.print_log('Conversion to per pixel and aggregate max processors=',
processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(aggregate_results_to_4_km.aggregate,
thresh=thresh,
sensit_type=sensit_type,
no_upload=no_upload), tile_list)
pool.close()
pool.join()
# # For single processor use
# for tile in tile_list:
#
# aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type, no_upload)
# Makes a vrt of all the output 10x10 tiles (10 km resolution)
out_vrt = "{}_0_4deg.vrt".format(pattern)
os.system('gdalbuildvrt -tr 0.04 0.04 {0} *{1}_0_4deg*.tif'.format(
out_vrt, pattern))
# Creates the output name for the 10km map
out_pattern = uu.name_aggregated_output(download_pattern_name, thresh,
sensit_type)
uu.print_log(out_pattern)
# Produces a single raster of all the 10x10 tiles (0.4 degree resolution)
cmd = [
'gdalwarp', '-t_srs', "EPSG:4326", '-overwrite', '-dstnodata', '0',
'-co', 'COMPRESS=LZW', '-tr', '0.04', '0.04', out_vrt,
'{}.tif'.format(out_pattern)
]
uu.log_subprocess_output_full(cmd)
# Adds metadata tags to output rasters
uu.add_universal_metadata_tags('{0}.tif'.format(out_pattern),
sensit_type)
# Units are different for annual removal factor, so metadata has to reflect that
if 'annual_removal_factor' in out_pattern:
cmd = [
'gdal_edit.py', '-mo',
'units=Mg aboveground carbon/yr/pixel, where pixels are 0.04x0.04 degrees',
'-mo',
'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
'-mo', 'extent=Global', '-mo',
'scale=negative values are removals', '-mo',
'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation'
.format(thresh), '{0}.tif'.format(out_pattern)
]
uu.log_subprocess_output_full(cmd)
else:
cmd = [
'gdal_edit.py', '-mo',
'units=Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees',
'-mo',
'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels',
'-mo', 'extent=Global', '-mo',
'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation'
.format(thresh), '{0}.tif'.format(out_pattern)
]
uu.log_subprocess_output_full(cmd)
# If no_upload flag is not activated, output is uploaded
if not no_upload:
uu.print_log("Tiles processed. Uploading to s3 now...")
uu.upload_final_set(output_dir_list[0], out_pattern)
# Cleans up the folder before starting on the next raster type
vrtList = glob.glob('*vrt')
for vrt in vrtList:
os.remove(vrt)
for tile_name in tile_list:
tile_id = uu.get_tile_id(tile_name)
# os.remove('{0}_{1}.tif'.format(tile_id, pattern))
os.remove('{0}_{1}_rewindow.tif'.format(tile_id, pattern))
os.remove('{0}_{1}_0_4deg.tif'.format(tile_id, pattern))
# Compares the net flux from the standard model and the sensitivity analysis in two ways.
# This does not work for compariing the raw outputs of the biomass_swap and US_removals sensitivity models because their
# extents are different from the standard model's extent (tropics and US tiles vs. global).
# Thus, in order to do this comparison, you need to clip the standard model net flux and US_removals net flux to
# the outline of the US and clip the standard model net flux to the extent of JPL AGB2000.
# Then, manually upload the clipped US_removals and biomass_swap net flux rasters to the spot machine and the
# code below should work.
if sensit_type not in [
'std', 'biomass_swap', 'US_removals', 'legal_Amazon_loss'
]:
if std_net_flux:
uu.print_log(
"Standard aggregated flux results provided. Creating comparison maps."
)
# Copies the standard model aggregation outputs to s3. Only net flux is used, though.
uu.s3_file_download(std_net_flux, cn.docker_base_dir, sensit_type)
# Identifies the standard model net flux map
std_aggreg_flux = os.path.split(std_net_flux)[1]
try:
# Identifies the sensitivity model net flux map
sensit_aggreg_flux = glob.glob(
'net_flux_Mt_CO2e_*{}*'.format(sensit_type))[0]
uu.print_log("Standard model net flux:", std_aggreg_flux)
uu.print_log("Sensitivity model net flux:", sensit_aggreg_flux)
except:
uu.print_log(
'Cannot do comparison. One of the input flux tiles is not valid. Verify that both net flux rasters are on the spot machine.'
)
uu.print_log(
"Creating map of percent difference between standard and {} net flux"
.format(sensit_type))
aggregate_results_to_4_km.percent_diff(std_aggreg_flux,
sensit_aggreg_flux,
sensit_type, no_upload)
uu.print_log(
"Creating map of which pixels change sign and which stay the same between standard and {}"
.format(sensit_type))
aggregate_results_to_4_km.sign_change(std_aggreg_flux,
sensit_aggreg_flux,
sensit_type, no_upload)
# If no_upload flag is not activated, output is uploaded
if not no_upload:
uu.upload_final_set(output_dir_list[0],
cn.pattern_aggreg_sensit_perc_diff)
uu.upload_final_set(output_dir_list[0],
cn.pattern_aggreg_sensit_sign_change)
else:
uu.print_log(
"No standard aggregated flux results provided. Not creating comparison maps."
)
def main():
parser = argparse.ArgumentParser(
description='Create planted forest carbon gain rate tiles')
parser.add_argument(
'--gadm-tile-index',
'-gi',
required=True,
help=
'Shapefile of 1x1 degree tiles of countries that contain planted forests (i.e. countries with planted forests rasterized to 1x1 deg). If no shapefile, write None.'
)
parser.add_argument(
'--planted-tile-index',
'-pi',
required=True,
help=
'Shapefile of 1x1 degree tiles of that contain planted forests (i.e. planted forest extent rasterized to 1x1 deg). If no shapefile, write None.'
)
args = parser.parse_args()
# Creates the directory and shapefile names for the two possible arguments (index shapefiles)
gadm_index = os.path.split(args.gadm_tile_index)
gadm_index_path = gadm_index[0]
gadm_index_shp = gadm_index[1]
gadm_index_shp = gadm_index_shp[:-4]
planted_index = os.path.split(args.planted_tile_index)
planted_index_path = planted_index[0]
planted_index_shp = planted_index[1]
planted_index_shp = planted_index_shp[:-4]
# Checks the validity of the two arguments. If either one is invalid, the script ends.
if (gadm_index_path not in cn.gadm_plant_1x1_index_dir
or planted_index_path not in cn.gadm_plant_1x1_index_dir):
raise Exception(
'Invalid inputs. Please provide None or s3 shapefile locations for both arguments.'
)
# List of all possible 10x10 Hansen tiles except for those at very extreme latitudes (not just WHRC biomass tiles)
total_tile_list = uu.tile_list(cn.pixel_area_dir)
print "Number of possible 10x10 tiles to evaluate:", len(total_tile_list)
# Removes the latitude bands that don't have any planted forests in them according to Liz Goldman.
# i.e., Liz Goldman said by Slack on 1/2/19 that the nothernmost planted forest is 69.5146 and the southernmost is -46.938968.
# This creates a more focused list of 10x10 tiles to iterate through (removes ones that definitely don't have planted forest).
# NOTE: If the planted forest gdb is updated, the list of latitudes to exclude below may need to be changed to not exclude certain latitude bands.
planted_lat_tile_list = [
tile for tile in total_tile_list if '90N' not in tile
]
planted_lat_tile_list = [
tile for tile in planted_lat_tile_list if '80N' not in tile
]
planted_lat_tile_list = [
tile for tile in planted_lat_tile_list if '50S' not in tile
]
planted_lat_tile_list = [
tile for tile in planted_lat_tile_list if '60S' not in tile
]
planted_lat_tile_list = [
tile for tile in planted_lat_tile_list if '70S' not in tile
]
planted_lat_tile_list = [
tile for tile in planted_lat_tile_list if '80S' not in tile
]
# planted_lat_tile_list = ['10N_080W']
print planted_lat_tile_list
print "Number of 10x10 tiles to evaluate after extreme latitudes have been removed:", len(
planted_lat_tile_list)
# If a planted forest extent 1x1 tile index shapefile isn't supplied
if 'None' in args.planted_tile_index:
### Entry point 1:
# If no shapefile of 1x1 tiles for countries with planted forests is supplied, 1x1 tiles of country extents will be created.
# This runs the process from the very beginning and will take a few days.
if 'None' in args.gadm_tile_index:
print "No GADM 1x1 tile index shapefile provided. Creating 1x1 planted forest country tiles from scratch..."
# Downloads and unzips the GADM shapefile, which will be used to create 1x1 tiles of land areas
uu.s3_file_download(cn.gadm_path, '.')
cmd = ['unzip', cn.gadm_zip]
subprocess.check_call(cmd)
# Creates a new GADM shapefile with just the countries that have planted forests in them.
# This limits creation of 1x1 rasters of land area on the countries that have planted forests rather than on all countries.
# NOTE: If the planted forest gdb is updated and has new countries added to it, the planted forest country list
# in constants_and_names.py must be updated, too.
print "Creating shapefile of countries with planted forests..."
os.system(
'''ogr2ogr -sql "SELECT * FROM gadm_3_6_adm2_final WHERE iso IN ({0})" {1} gadm_3_6_adm2_final.shp'''
.format(str(cn.plantation_countries)[1:-1], cn.gadm_iso))
# Creates 1x1 degree tiles of countries that have planted forests in them.
# I assume this can handle using 50 processors because it's not trying to upload files to s3 and the tiles are small.
# This takes several days to run because it iterates through at least 250 10x10 tiles.
# For multiprocessor use.
num_of_processes = 50
pool = Pool(num_of_processes)
pool.map(plantation_preparation.rasterize_gadm_1x1,
planted_lat_tile_list)
pool.close()
pool.join()
# # Creates 1x1 degree tiles of countries that have planted forests in them.
# # For single processor use.
# for tile in planted_lat_tile_list:
#
# plantation_preparation.rasterize_gadm_1x1(tile)
# Creates a shapefile of the boundaries of the 1x1 GADM tiles in countries with planted forests
os.system('''gdaltindex {0}_{1}.shp GADM_*.tif'''.format(
cn.pattern_gadm_1x1_index, uu.date))
cmd = [
'aws', 's3', 'cp', '.', cn.gadm_plant_1x1_index_dir,
'--exclude', '*', '--include',
'{}*'.format(cn.pattern_gadm_1x1_index), '--recursive'
]
subprocess.check_call(cmd)
# # Saves the 1x1 country extent tiles to s3
# # Only use if the entire process can't run in one go on the spot machine
# cmd = ['aws', 's3', 'cp', '.', 's3://gfw2-data/climate/carbon_model/temp_spotmachine_output/', '--exclude', '*', '--include', 'GADM_*.tif', '--recursive']
# subprocess.check_call(cmd)
# Delete the aux.xml files
os.system('''rm GADM*.tif.*''')
# List of all 1x1 degree countey extent tiles created
gadm_list_1x1 = uu.tile_list_spot_machine(".", "GADM_")
print "List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1
print len(gadm_list_1x1)
### Entry point 2:
# If a shapefile of the boundaries of 1x1 degree tiles of countries with planted forests is supplied,
# a list of the 1x1 tiles is created from the shapefile.
# This avoids creating the 1x1 country extent tiles all over again because the relevant tile extent are supplied
# in the shapefile.
elif cn.gadm_plant_1x1_index_dir in args.gadm_tile_index:
print "Country extent 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest tiles..."
# Copies the shapefile of 1x1 tiles of extent of countries with planted forests
cmd = [
'aws', 's3', 'cp', '{}/'.format(gadm_index_path), '.',
'--recursive', '--exclude', '*', '--include',
'{}*'.format(gadm_index_shp), '--recursive'
]
subprocess.check_call(cmd)
# Gets the attribute table of the country extent 1x1 tile shapefile
gadm = glob.glob('{}*.dbf'.format(cn.pattern_gadm_1x1_index))[0]
# Converts the attribute table to a dataframe
dbf = Dbf5(gadm)
df = dbf.to_dataframe()
# Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list
gadm_list_1x1 = df['location'].tolist()
gadm_list_1x1 = [str(y) for y in gadm_list_1x1]
print "List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1
print "There are", len(
gadm_list_1x1), "1x1 country extent tiles to iterate through."
# In case some other arguments are provided
else:
raise Exception(
'Invalid GADM tile index shapefile provided. Please provide a valid shapefile.'
)
# Creates 1x1 degree tiles of plantation growth wherever there are plantations.
# Because this is iterating through all 1x1 tiles in countries with planted forests, it first checks
# whether each 1x1 tile intersects planted forests before creating a 1x1 planted forest tile for that
# 1x1 country extent tile.
# For multiprocessor use
num_of_processes = 30
pool = Pool(num_of_processes)
pool.map(plantation_preparation.create_1x1_plantation_from_1x1_gadm,
gadm_list_1x1)
pool.close()
pool.join()
# # Creates 1x1 degree tiles of plantation growth wherever there are plantations
# # For single processor use
# for tile in gadm_list_1x1:
#
# plantation_preparation.create_1x1_plantation(tile)
# Creates a shapefile in which each feature is the extent of a plantation extent tile.
# This index shapefile can be used the next time this process is run if starting with Entry Point 3.
os.system('''gdaltindex {0}_{1}.shp plant_*.tif'''.format(
cn.pattern_plant_1x1_index, uu.date))
cmd = [
'aws', 's3', 'cp', '.', cn.gadm_plant_1x1_index_dir, '--exclude',
'*', '--include', '{}*'.format(cn.pattern_plant_1x1_index),
'--recursive'
]
subprocess.check_call(cmd)
### Entry point 3
# If a shapefile of the extents of 1x1 planted forest tiles is provided
if cn.pattern_plant_1x1_index in args.planted_tile_index:
print "Planted forest 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest growth tiles..."
# Copies the shapefile of 1x1 tiles of extent of planted forests
cmd = [
'aws', 's3', 'cp', '{}/'.format(planted_index_path), '.',
'--recursive', '--exclude', '*', '--include',
'{}*'.format(planted_index_shp), '--recursive'
]
subprocess.check_call(cmd)
# Gets the attribute table of the planted forest extent 1x1 tile shapefile
gadm = glob.glob('{}*.dbf'.format(cn.pattern_plant_1x1_index))[0]
# Converts the attribute table to a dataframe
dbf = Dbf5(gadm)
df = dbf.to_dataframe()
# Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list
planted_list_1x1 = df['location'].tolist()
planted_list_1x1 = [str(y) for y in planted_list_1x1]
print "List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", planted_list_1x1
print "There are", len(
planted_list_1x1
), "1x1 planted forest extent tiles to iterate through."
# Creates 1x1 degree tiles of plantation growth wherever there are plantations.
# Because this is iterating through only 1x1 tiles that are known to have planted forests (from a previous run
# of this script), it does not need to check whether there are planted forests in this tile. It goes directly
# to intersecting the planted forest table with the 1x1 tile.
# For multiprocessor use
# This works with 30 processors on an r4.16xlarge.
num_of_processes = 50
pool = Pool(num_of_processes)
pool.map(plantation_preparation.create_1x1_plantation_from_1x1_planted,
planted_list_1x1)
pool.close()
pool.join()
### All entry points meet here: creation of 10x10 degree planted forest tiles from 1x1 degree planted forest tiles
# Name of the vrt of 1x1 planted forest tiles
plant_1x1_vrt = 'plant_1x1.vrt'
# Creates a mosaic of all the 1x1 plantation growth rate tiles
print "Creating vrt of 1x1 plantation growth rate tiles"
os.system('gdalbuildvrt {} plant_*.tif'.format(plant_1x1_vrt))
# Creates 10x10 degree tiles of plantation growth by iterating over the pixel area tiles that are in latitudes with planted forests
# For multiprocessor use
num_of_processes = 20
pool = Pool(num_of_processes)
pool.map(
partial(plantation_preparation.create_10x10_plantation,
plant_1x1_vrt=plant_1x1_vrt), planted_lat_tile_list)
pool.close()
pool.join()