def mp_continent_ecozone_tiles(tile_id_list, run_date=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(
cn.pattern_WHRC_biomass_2000_non_mang_non_planted,
cn.mangrove_biomass_2000_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# if the continent-ecozone shapefile hasn't already been downloaded, it will be downloaded and unzipped
uu.s3_file_download(cn.cont_eco_s3_zip, cn.docker_base_dir, 'std')
# Unzips ecozone shapefile
cmd = ['unzip', cn.cont_eco_zip]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# List of output directories and output file name patterns
output_dir_list = [cn.cont_eco_raw_dir, cn.cont_eco_dir]
output_pattern_list = [
cn.pattern_cont_eco_raw, cn.pattern_cont_eco_processed
]
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# For multiprocessor use
processes = int(cn.count / 4)
uu.print_log('Continent-ecozone tile creation max processors=', processes)
pool.map(continent_ecozone_tiles.create_continent_ecozone_tiles,
tile_id_list)
# Uploads the continent-ecozone tile to s3 before the codes are expanded to pixels in 1024x1024 windows that don't have codes.
# These are not used for the model. They are for reference and completeness.
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
def mp_prep_other_inputs(tile_id_list, run_date):
os.chdir(cn.docker_base_dir)
sensit_type='std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
set3=cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir
)
uu.print_log(tile_id_list)
uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# List of output directories and output file name patterns
output_dir_list = [cn.climate_zone_processed_dir, cn.plant_pre_2000_processed_dir,
cn.drivers_processed_dir, cn.ifl_primary_processed_dir,
cn.annual_gain_AGC_natrl_forest_young_dir,
cn.stdev_annual_gain_AGC_natrl_forest_young_dir,
cn.annual_gain_AGC_BGC_natrl_forest_Europe_dir,
cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_dir,
cn.FIA_forest_group_processed_dir,
cn.age_cat_natrl_forest_US_dir,
cn.FIA_regions_processed_dir]
output_pattern_list = [cn.pattern_climate_zone, cn.pattern_plant_pre_2000,
cn.pattern_drivers, cn.pattern_ifl_primary,
cn.pattern_annual_gain_AGC_natrl_forest_young,
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young,
cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe,
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe,
cn.pattern_FIA_forest_group_processed,
cn.pattern_age_cat_natrl_forest_US,
cn.pattern_FIA_regions_processed]
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log("Changing output directory and file name pattern based on sensitivity analysis")
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
output_pattern_list = uu.alter_patterns(sensit_type, output_pattern_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Files to process: climate zone, IDN/MYS plantations before 2000, tree cover loss drivers, combine IFL and primary forest
uu.s3_file_download(os.path.join(cn.climate_zone_raw_dir, cn.climate_zone_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.plant_pre_2000_raw_dir, '{}.zip'.format(cn.pattern_plant_pre_2000_raw)), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.drivers_raw_dir, '{}.zip'.format(cn.pattern_drivers_raw)), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.age_cat_natrl_forest_US_raw_dir, cn.name_age_cat_natrl_forest_US_raw), cn.docker_base_dir, sensit_type)
uu.s3_file_download(os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, sensit_type)
# For some reason, using uu.s3_file_download or otherwise using AWSCLI as a subprocess doesn't work for this raster.
# Thus, using wget instead.
cmd = ['wget', '{}'.format(cn.annual_gain_AGC_natrl_forest_young_raw_URL), '-P', '{}'.format(cn.docker_base_dir)]
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
uu.s3_file_download(cn.stdev_annual_gain_AGC_natrl_forest_young_raw_URL, cn.docker_base_dir, sensit_type)
cmd = ['aws', 's3', 'cp', cn.primary_raw_dir, cn.docker_base_dir, '--recursive']
uu.log_subprocess_output_full(cmd)
uu.s3_flexible_download(cn.ifl_dir, cn.pattern_ifl, cn.docker_base_dir, sensit_type, tile_id_list)
uu.print_log("Unzipping pre-2000 plantations...")
cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_plant_pre_2000_raw)]
uu.log_subprocess_output_full(cmd)
uu.print_log("Unzipping drivers...")
cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_drivers_raw)]
uu.log_subprocess_output_full(cmd)
# Creates tree cover loss driver tiles
source_raster = '{}.tif'.format(cn.pattern_drivers_raw)
out_pattern = cn.pattern_drivers
dt = 'Byte'
if cn.count == 96:
processes = 80 # 45 processors = 70 GB peak; 70 = 90 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating tree cover loss driver tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates young natural forest removal rate tiles
source_raster = cn.name_annual_gain_AGC_natrl_forest_young_raw
out_pattern = cn.pattern_annual_gain_AGC_natrl_forest_young
dt = 'float32'
if cn.count == 96:
processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates young natural forest removal rate standard deviation tiles
source_raster = cn.name_stdev_annual_gain_AGC_natrl_forest_young_raw
out_pattern = cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
dt = 'float32'
if cn.count == 96:
processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates pre-2000 oil palm plantation tiles
if cn.count == 96:
processes = 80 # 45 processors = 100 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating pre-2000 oil palm plantation tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(prep_other_inputs.rasterize_pre_2000_plantations, tile_id_list)
pool.close()
pool.join()
# Creates climate zone tiles
if cn.count == 96:
processes = 80 # 45 processors = 230 GB peak (on second step); 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating climate zone tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(prep_other_inputs.create_climate_zone_tiles, tile_id_list)
pool.close()
pool.join()
# Creates European natural forest removal rate tiles
source_raster = cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw
out_pattern = cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe
dt = 'float32'
if cn.count == 96:
processes = 60 # 32 processors = 60 GB peak; 60 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates European natural forest standard deviation of removal rate tiles
source_raster = cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw
out_pattern = cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe
dt = 'float32'
if cn.count == 96:
processes = 32 # 32 processors = 60 GB peak; 60 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates a vrt of the primary forests with nodata=0 from the continental primary forest rasters
uu.print_log("Creating vrt of humid tropial primary forest...")
primary_vrt = 'primary_2001.vrt'
os.system('gdalbuildvrt -srcnodata 0 {} *2001_primary.tif'.format(primary_vrt))
uu.print_log(" Humid tropical primary forest vrt created")
# Creates primary forest tiles
source_raster = primary_vrt
out_pattern = 'primary_2001'
dt = 'Byte'
if cn.count == 96:
processes = 45 # 45 processors = 650 GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating primary forest tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates a combined IFL/primary forest raster
# Uses very little memory since it's just file renaming
if cn.count == 96:
processes = 60 # 60 processors = 10 GB peak
else:
processes = int(cn.count/2)
uu.print_log("Assigning each tile to ifl2000 or primary forest with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(prep_other_inputs.create_combined_ifl_primary, tile_id_list)
pool.close()
pool.join()
# Creates forest age category tiles for US forests
source_raster = cn.name_age_cat_natrl_forest_US_raw
out_pattern = cn.pattern_age_cat_natrl_forest_US
dt = 'Byte'
if cn.count == 96:
processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating US forest age category tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates forest groups for US forests
source_raster = cn.name_FIA_forest_group_raw
out_pattern = cn.pattern_FIA_forest_group_processed
dt = 'Byte'
if cn.count == 96:
processes = 80 # 32 processors = 25 GB peak; 80 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating US forest group tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
# Creates FIA regions for US forests
source_raster = cn.name_FIA_regions_raw
out_pattern = cn.pattern_FIA_regions_processed
dt = 'Byte'
if cn.count == 96:
processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
else:
processes = int(cn.count/2)
uu.print_log("Creating US forest region tiles with {} processors...".format(processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
pool.close()
pool.join()
for output_pattern in [cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young]:
# For some reason I can't figure out, the young forest rasters (rate and stdev) have NaN values in some places where 0 (NoData)
# should be. These NaN values show up as values when the check_and_delete_if_empty function runs, making the tiles not
# deleted even if they have no data. However, the light version (which uses gdalinfo rather than rasterio masks) doesn't
# have this problem. So I'm forcing the young forest rates to and stdev to have their emptiness checked by the gdalinfo version.
if output_pattern in [cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young]:
processes = int(cn.count / 2)
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
if cn.count == 96:
processes = 50 # 60 processors = >730 GB peak (for European natural forest forest removal rates); 50 = XXX GB peak
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
elif cn.count <= 2: # For local tests
processes = 1
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
else:
processes = int(cn.count / 2)
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
uu.print_log('\n')
# Uploads output tiles to s3
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
### This script combines the annual gain rate tiles from different forest types (non-mangrove natural forests, mangroves,
### plantations) into combined tiles. It does the same for cumulative gain over the study period.
import multiprocessing
import utilities
import merge_cumulative_annual_gain_all_forest_types
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu
tile_list = uu.create_combined_tile_list(
cn.WHRC_biomass_2000_non_mang_non_planted_dir,
cn.annual_gain_AGB_mangrove_dir,
set3=cn.annual_gain_AGB_planted_forest_non_mangrove_dir)
# tile_list = ['00N_110E'] # test tiles
# tile_list = ['80N_020E', '00N_000E', '00N_020E', '00N_110E'] # test tiles: no mangrove or planted forest, mangrove only, planted forest only, mangrove and planted forest
print tile_list
print "There are {} unique tiles to process".format(str(len(tile_list)))
# For downloading all tiles in the input folders
download_list = [
cn.annual_gain_AGB_natrl_forest_dir, cn.annual_gain_AGB_mangrove_dir,
cn.cumul_gain_AGC_natrl_forest_dir, cn.cumul_gain_AGC_mangrove_dir,
cn.annual_gain_BGB_natrl_forest_dir, cn.annual_gain_BGB_mangrove_dir,
cn.cumul_gain_BGC_natrl_forest_dir, cn.cumul_gain_BGC_mangrove_dir,
cn.annual_gain_AGB_planted_forest_non_mangrove_dir,
cn.annual_gain_BGB_planted_forest_non_mangrove_dir,
cn.cumul_gain_AGC_planted_forest_non_mangrove_dir,
cn.cumul_gain_BGC_planted_forest_non_mangrove_dir
def mp_create_soil_C(tile_id_list):
os.chdir(cn.docker_base_dir)
sensit_type = 'std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(
cn.WHRC_biomass_2000_unmasked_dir, cn.mangrove_biomass_2000_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# List of output directories and output file name patterns
output_dir_list = [
cn.soil_C_full_extent_2000_dir, cn.stdev_soil_C_full_extent_2000_dir
]
output_pattern_list = [
cn.pattern_soil_C_full_extent_2000, cn.pattern_stdev_soil_C_full_extent
]
### Soil carbon density
# uu.print_log("Downloading mangrove soil C rasters")
# uu.s3_file_download(os.path.join(cn.mangrove_soil_C_dir, cn.name_mangrove_soil_C), cn.docker_base_dir, sensit_type)
#
# # For downloading all tiles in the input folders.
# input_files = [cn.mangrove_biomass_2000_dir]
#
# for input in input_files:
# uu.s3_folder_download(input, cn.docker_base_dir, sensit_type)
#
# # Download raw mineral soil C density tiles.
# # First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder
# # Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it
# # There are 12951 tiles and it takes about 3 hours to download them!
# cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
# '--accept', '*.tif', '{}'.format(cn.mineral_soil_C_url)]
# uu.log_subprocess_output_full(cmd)
#
# uu.print_log("Unzipping mangrove soil C rasters...")
# cmd = ['unzip', '-j', cn.name_mangrove_soil_C, '-d', cn.docker_base_dir]
# uu.log_subprocess_output_full(cmd)
#
# # Mangrove soil receives precedence over mineral soil
# uu.print_log("Making mangrove soil C vrt...")
# check_call('gdalbuildvrt mangrove_soil_C.vrt *{}*.tif'.format(cn.pattern_mangrove_soil_C_raw), shell=True)
# uu.print_log("Done making mangrove soil C vrt")
#
# uu.print_log("Making mangrove soil C tiles...")
#
# if cn.count == 96:
# processes = 32 # 32 processors = 570 GB peak
# else:
# processes = int(cn.count/3)
# uu.print_log('Mangrove soil C max processors=', processes)
# pool = multiprocessing.Pool(processes)
# pool.map(create_soil_C.create_mangrove_soil_C, tile_id_list)
# pool.close()
# pool.join()
#
# # # For single processor use
# # for tile_id in tile_id_list:
# #
# # create_soil_C.create_mangrove_soil_C(tile_id)
#
# uu.print_log('Done making mangrove soil C tiles', '\n')
#
# uu.print_log("Making mineral soil C vrt...")
# check_call('gdalbuildvrt mineral_soil_C.vrt *{}*'.format(cn.pattern_mineral_soil_C_raw), shell=True)
# uu.print_log("Done making mineral soil C vrt")
#
# # Creates mineral soil C density tiles
# source_raster = 'mineral_soil_C.vrt'
# out_pattern = 'mineral_soil'
# dt = 'Int16'
# if cn.count == 96:
# processes = 50 # 32 processors = 100 GB peak; 50 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating mineral soil C density tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list)
# pool.close()
# pool.join()
#
# # # For single processor use
# # for tile_id in tile_id_list:
# #
# # create_soil_C.create_mineral_soil_C(tile_id)
#
# uu.print_log("Done making mineral soil C tiles", "\n")
#
#
# uu.print_log("Making combined (mangrove & non-mangrove) soil C tiles...")
#
# if cn.count == 96:
# processes = 45 # 45 processors = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log('Combined soil C max processors=', processes)
# pool = multiprocessing.Pool(processes)
# pool.map(create_soil_C.create_combined_soil_C, tile_id_list)
# pool.close()
# pool.join()
#
# # # For single processor use
# # for tile in tile_list:
# #
# # create_soil_C.create_combined_soil_C(tile_id)
#
# uu.print_log("Done making combined soil C tiles")
#
# uu.print_log("Uploading soil C density tiles")
# uu.upload_final_set(output_dir_list[0], output_pattern_list[0])
#
# # Need to delete soil c density rasters because they have the same pattern as the standard deviation rasters
# uu.print_log("Deleting raw soil C density rasters")
# c_stocks = glob.glob('*{}*'.format(cn.pattern_soil_C_full_extent_2000))
# for c_stock in c_stocks:
# os.remove(c_stock)
### Soil carbon density uncertainty
# Separate directories for the 5% CI and 95% CI
dir_CI05 = '{0}{1}'.format(cn.docker_base_dir, 'CI05/')
dir_CI95 = '{0}{1}'.format(cn.docker_base_dir, 'CI95/')
vrt_CI05 = 'mineral_soil_C_CI05.vrt'
vrt_CI95 = 'mineral_soil_C_CI95.vrt'
soil_C_stdev_global = 'soil_C_stdev.tif'
# # Download raw mineral soil C density 5% CI tiles
# # First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder
# # Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it
# # Like soil C density rasters, there are 12951 tifs and they take about 3 hours to download.
# os.mkdir(dir_CI05)
#
# cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
# '--directory-prefix={}'.format(dir_CI05),
# '--accept', '*.tif', '{}'.format(cn.CI5_mineral_soil_C_url)]
# uu.log_subprocess_output_full(cmd)
#
# uu.print_log("Making mineral soil C 5% CI vrt...")
# check_call('gdalbuildvrt {0} {1}*{2}*'.format(vrt_CI05, dir_CI05, cn.pattern_uncert_mineral_soil_C_raw), shell=True)
# uu.print_log("Done making mineral soil C CI05 vrt")
#
# # Download raw mineral soil C density 5% CI tiles
# # Like soil C density rasters, there are 12951 tifs and they take about 3 hours to download.
# os.mkdir(dir_CI95)
#
# cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
# '--directory-prefix={}'.format(dir_CI95),
# '--accept', '*.tif', '{}'.format(cn.CI95_mineral_soil_C_url)]
# uu.log_subprocess_output_full(cmd)
#
# uu.print_log("Making mineral soil C 95% CI vrt...")
# check_call('gdalbuildvrt {0} {1}*{2}*'.format(vrt_CI95, dir_CI95, cn.pattern_uncert_mineral_soil_C_raw), shell=True)
# uu.print_log("Done making mineral soil C CI95 vrt")
uu.print_log(
"Creating raster of standard deviations in soil C at native SoilGrids250 resolution. This may take a while..."
)
# global tif with approximation of the soil C stanard deviation (based on the 5% and 95% CIs)
# This takes about 20 minutes. It doesn't show any progress until the last moment, when it quickly counts
# up to 100.
calc = '--calc=(A-B)/3'
out_filearg = '--outfile={}'.format(soil_C_stdev_global)
cmd = [
'gdal_calc.py', '-A', vrt_CI95, '-B', vrt_CI05, calc, out_filearg,
'--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW',
'--type=Float32'
]
uu.log_subprocess_output_full(cmd)
uu.print_log("{} created.".format(soil_C_stdev_global))
# Creates soil carbon 2000 density standard deviation tiles
out_pattern = cn.pattern_stdev_soil_C_full_extent
dt = 'Float32'
source_raster = soil_C_stdev_global
if cn.count == 96:
processes = 56 # 32 processors = 290 GB peak; 56 = XXX GB peal
else:
processes = 2
uu.print_log(
"Creating mineral soil C stock stdev tiles with {} processors...".
format(processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt), tile_id_list)
pool.close()
pool.join()
output_pattern = cn.pattern_stdev_soil_C_full_extent
processes = 50 # 50 processors = 550 GB peak
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors...".
format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty, output_pattern=output_pattern),
tile_id_list)
pool.close()
pool.join()
uu.print_log("Uploading soil C density standard deviation tiles")
uu.upload_final_set(output_dir_list[1], output_pattern_list[1])
# if the continent-ecozone shapefile hasn't already been downloaded, it will be downloaded and unzipped
if not os.path.exists(cn.cont_eco_zip):
# Downloads ecozone shapefile
utilities.s3_file_download(
'{}'.format(cn.cont_eco_s3_zip),
'.',
)
# Unzips ecozone shapefile
cmd = ['unzip', cn.cont_eco_zip]
subprocess.check_call(cmd)
biomass_tile_list = uu.create_combined_tile_list(
cn.pattern_WHRC_biomass_2000_non_mang_non_planted,
cn.mangrove_biomass_2000_dir)
# biomass_tile_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
# biomass_tile_list = ['20S_110E'] # test tile
print biomass_tile_list
print "There are {} tiles to process".format(str(len(biomass_tile_list)))
# For multiprocessor use
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=count / 4)
pool.map(continent_ecozone_tiles.create_continent_ecozone_tiles,
biomass_tile_list)
print "Done processing tiles. Now uploading them to s3..."
# Uploads the continent-ecozone tile to s3 before the codes are expanded to pixels in 1024x1024 windows that don't have codes.
def mp_model_extent(sensit_type, tile_id_list, run_date = None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model. Which biomass tiles to use depends on sensitivity analysis
if sensit_type == 'biomass_swap':
tile_id_list = uu.tile_list_s3(cn.JPL_processed_dir, sensit_type)
elif sensit_type == 'legal_Amazon_loss':
tile_id_list = uu.tile_list_s3(cn.Brazil_forest_extent_2000_processed_dir, sensit_type)
else:
tile_id_list = uu.create_combined_tile_list(cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
cn.gain_dir, cn.tcd_dir
)
uu.print_log(tile_id_list)
uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# Files to download for this script.
download_dict = {
cn.mangrove_biomass_2000_dir: [cn.pattern_mangrove_biomass_2000],
cn.gain_dir: [cn.pattern_gain],
cn.plant_pre_2000_processed_dir: [cn.pattern_plant_pre_2000]
}
if sensit_type == 'legal_Amazon_loss':
download_dict[cn.Brazil_forest_extent_2000_processed_dir] = [cn.pattern_Brazil_forest_extent_2000_processed]
else:
download_dict[cn.tcd_dir] = [cn.pattern_tcd]
if sensit_type == 'biomass_swap':
download_dict[cn.JPL_processed_dir] = [cn.pattern_JPL_unmasked_processed]
else:
download_dict[cn.WHRC_biomass_2000_unmasked_dir] = [cn.pattern_WHRC_biomass_2000_unmasked]
# List of output directories and output file name patterns
output_dir_list = [cn.model_extent_dir]
output_pattern_list = [cn.pattern_model_extent]
# Downloads input files or entire directories, depending on how many tiles are in the tile_id_list
for key, values in download_dict.items():
dir = key
pattern = values[0]
uu.s3_flexible_download(dir, pattern, cn.docker_base_dir, sensit_type, tile_id_list)
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log("Changing output directory and file name pattern based on sensitivity analysis")
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
output_pattern_list = uu.alter_patterns(sensit_type, output_pattern_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Creates a single filename pattern to pass to the multiprocessor call
pattern = output_pattern_list[0]
# This configuration of the multiprocessing call is necessary for passing multiple arguments to the main function
# It is based on the example here: http://spencerimp.blogspot.com/2015/12/python-multiprocess-with-multiple.html
if cn.count == 96:
if sensit_type == 'biomass_swap':
processes = 38
else:
processes = 42 # 30 processors = 480 GB peak (sporadic decreases followed by sustained increases);
# 36 = 550 GB peak; 40 = 590 GB peak; 42 = XXX GB peak
else:
processes = 3
uu.print_log('Removal model forest extent processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(partial(model_extent.model_extent, pattern=pattern, sensit_type=sensit_type), tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in tile_id_list:
# model_extent.model_extent(tile_id, pattern, sensit_type)
output_pattern = output_pattern_list[0]
if cn.count <= 2: # For local tests
processes = 1
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
else:
processes = 50 # 50 processors = XXX GB peak
uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
# Uploads output tiles to s3
uu.upload_final_set(output_dir_list[0], output_pattern_list[0])
def mp_create_soil_C(tile_id_list, no_upload=None):
os.chdir(cn.docker_base_dir)
sensit_type = 'std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(
cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
set3=cn.gain_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# List of output directories and output file name patterns
output_dir_list = [
cn.soil_C_full_extent_2000_non_mang_dir,
cn.soil_C_full_extent_2000_dir, cn.stdev_soil_C_full_extent_2000_dir
]
output_pattern_list = [
cn.pattern_soil_C_full_extent_2000_non_mang,
cn.pattern_soil_C_full_extent_2000, cn.pattern_stdev_soil_C_full_extent
]
# ### Soil carbon density
#
# uu.print_log("Downloading mangrove soil C rasters")
# uu.s3_file_download(os.path.join(cn.mangrove_soil_C_dir, cn.name_mangrove_soil_C), cn.docker_base_dir, sensit_type)
#
# # For downloading all tiles in the input folders.
# input_files = [cn.mangrove_biomass_2000_dir]
#
# for input in input_files:
# uu.s3_folder_download(input, cn.docker_base_dir, sensit_type)
#
# # Download raw mineral soil C density tiles.
# # First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder
# # Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it
# # There are 12951 tiles and it takes about 3 hours to download them!
# cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*',
# '--accept', '*.tif', '{}'.format(cn.mineral_soil_C_url)]
# uu.log_subprocess_output_full(cmd)
#
# uu.print_log("Unzipping mangrove soil C rasters...")
# cmd = ['unzip', '-j', cn.name_mangrove_soil_C, '-d', cn.docker_base_dir]
# uu.log_subprocess_output_full(cmd)
#
# # Mangrove soil receives precedence over mineral soil
# uu.print_log("Making mangrove soil C vrt...")
# check_call('gdalbuildvrt mangrove_soil_C.vrt *{}*.tif'.format(cn.pattern_mangrove_soil_C_raw), shell=True)
# uu.print_log("Done making mangrove soil C vrt")
#
# uu.print_log("Making mangrove soil C tiles...")
#
# if cn.count == 96:
# processes = 32 # 32 processors = 570 GB peak
# else:
# processes = int(cn.count/3)
# uu.print_log('Mangrove soil C max processors=', processes)
# pool = multiprocessing.Pool(processes)
# pool.map(partial(create_soil_C.create_mangrove_soil_C, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # # For single processor use
# # for tile_id in tile_id_list:
# #
# # create_soil_C.create_mangrove_soil_C(tile_id, no_Upload)
#
# uu.print_log('Done making mangrove soil C tiles', '\n')
uu.print_log("Making mineral soil C vrt...")
check_call('gdalbuildvrt mineral_soil_C.vrt *{}*'.format(
cn.pattern_mineral_soil_C_raw),
shell=True)
uu.print_log("Done making mineral soil C vrt")
# Creates mineral soil C density tiles
source_raster = 'mineral_soil_C.vrt'
out_pattern = cn.pattern_soil_C_full_extent_2000_non_mang
dt = 'Int16'
if cn.count == 96:
processes = 50 # 32 processors = 100 GB peak; 50 = XXX GB peak
else:
processes = int(cn.count / 2)
uu.print_log(
"Creating mineral soil C density tiles with {} processors...".format(
processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt,
no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in tile_id_list:
#
# create_soil_C.create_mineral_soil_C(tile_id)
uu.print_log("Done making non-mangrove soil C tiles", "\n")
output_pattern = cn.pattern_soil_C_full_extent_2000_non_mang
processes = 50 # 50 processors = 550 GB peak
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors...".
format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty, output_pattern=output_pattern),
tile_id_list)
pool.close()
pool.join()
# If no_upload flag is not activated, output is uploaded to s3
if not no_upload:
uu.print_log("Uploading non-mangrove soil C density tiles")
uu.upload_final_set(output_dir_list[0], output_pattern_list[0])
def mp_prep_other_inputs(tile_id_list, run_date, no_upload=None):
os.chdir(cn.docker_base_dir)
sensit_type = 'std'
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
### BUG: THIS SHOULD ALSO INCLUDE cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir IN ITS LIST
tile_id_list = uu.create_combined_tile_list(
cn.WHRC_biomass_2000_unmasked_dir,
cn.mangrove_biomass_2000_dir,
set3=cn.gain_dir)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
'''
Before processing the driver, it needs to be reprojected from Goode Homolosine to WGS84.
gdal_warp is producing a weird output, so I did it in ArcMap for the 2020 update,
with the output cell size being 0.01 x 0.01 degree and the method being nearest.
arcpy.ProjectRaster_management(in_raster="C:/GIS/Drivers of loss/2020_drivers__tif__from_Forrest_Follett_20210323/FinalClassification_2020_v2__from_Jimmy_MacCarthy_20210323.tif",
out_raster="C:/GIS/Drivers of loss/2020_drivers__tif__from_Forrest_Follett_20210323/Final_Classification_2020__reproj_nearest_0-005_0-005_deg__20210323.tif",
out_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]",
resampling_type="NEAREST", cell_size="0.005 0.005", geographic_transform="",
Registration_Point="",
in_coor_system="PROJCS['WGS_1984_Goode_Homolosine',GEOGCS['GCS_unknown',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Goode_Homolosine'],PARAMETER['False_Easting',0.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',0.0],PARAMETER['Option',1.0],UNIT['Meter',1.0]]",
vertical="NO_VERTICAL")
'''
# List of output directories and output file name patterns
output_dir_list = [
# cn.climate_zone_processed_dir, cn.plant_pre_2000_processed_dir,
cn.drivers_processed_dir
# cn.ifl_primary_processed_dir,
# cn.annual_gain_AGC_natrl_forest_young_dir,
# cn.stdev_annual_gain_AGC_natrl_forest_young_dir,
# cn.annual_gain_AGC_BGC_natrl_forest_Europe_dir,
# cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_dir,
# cn.FIA_forest_group_processed_dir,
# cn.age_cat_natrl_forest_US_dir,
# cn.FIA_regions_processed_dir
]
output_pattern_list = [
# cn.pattern_climate_zone, cn.pattern_plant_pre_2000,
cn.pattern_drivers
# cn.pattern_ifl_primary,
# cn.pattern_annual_gain_AGC_natrl_forest_young,
# cn.pattern_stdev_annual_gain_AGC_natrl_forest_young,
# cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe,
# cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe,
# cn.pattern_FIA_forest_group_processed,
# cn.pattern_age_cat_natrl_forest_US,
# cn.pattern_FIA_regions_processed
]
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log(
"Changing output directory and file name pattern based on sensitivity analysis"
)
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
output_pattern_list = uu.alter_patterns(sensit_type,
output_pattern_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# # Files to process: climate zone, IDN/MYS plantations before 2000, tree cover loss drivers, combine IFL and primary forest
# uu.s3_file_download(os.path.join(cn.climate_zone_raw_dir, cn.climate_zone_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.plant_pre_2000_raw_dir, '{}.zip'.format(cn.pattern_plant_pre_2000_raw)), cn.docker_base_dir, sensit_type)
uu.s3_file_download(
os.path.join(cn.drivers_raw_dir, cn.pattern_drivers_raw),
cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.age_cat_natrl_forest_US_raw_dir, cn.name_age_cat_natrl_forest_US_raw), cn.docker_base_dir, sensit_type)
# uu.s3_file_download(os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, sensit_type)
# # For some reason, using uu.s3_file_download or otherwise using AWSCLI as a subprocess doesn't work for this raster.
# # Thus, using wget instead.
# cmd = ['wget', '{}'.format(cn.annual_gain_AGC_natrl_forest_young_raw_URL), '-P', '{}'.format(cn.docker_base_dir)]
# process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
# with process.stdout:
# uu.log_subprocess_output(process.stdout)
# uu.s3_file_download(cn.stdev_annual_gain_AGC_natrl_forest_young_raw_URL, cn.docker_base_dir, sensit_type)
# cmd = ['aws', 's3', 'cp', cn.primary_raw_dir, cn.docker_base_dir, '--recursive']
# uu.log_subprocess_output_full(cmd)
#
# uu.s3_flexible_download(cn.ifl_dir, cn.pattern_ifl, cn.docker_base_dir, sensit_type, tile_id_list)
#
# uu.print_log("Unzipping pre-2000 plantations...")
# cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_plant_pre_2000_raw)]
# uu.log_subprocess_output_full(cmd)
# Creates tree cover loss driver tiles.
# The raw driver tile should have NoData for unassigned drivers as opposed to 0 for unassigned drivers.
# For the 2020 driver update, I reclassified the 0 values as NoData in ArcMap. I also unprojected the global drivers
# map to WGS84 because running the homolosine projection that Jimmy provided was giving incorrect processed results.
source_raster = cn.pattern_drivers_raw
out_pattern = cn.pattern_drivers
dt = 'Byte'
if cn.count == 96:
processes = 87 # 45 processors = 70 GB peak; 70 = 90 GB peak; 80 = 100 GB peak; 87 = 125 GB peak
else:
processes = int(cn.count / 2)
uu.print_log(
"Creating tree cover loss driver tiles with {} processors...".format(
processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt,
no_upload=no_upload), tile_id_list)
pool.close()
pool.join()
# # Creates young natural forest removal rate tiles
# source_raster = cn.name_annual_gain_AGC_natrl_forest_young_raw
# out_pattern = cn.pattern_annual_gain_AGC_natrl_forest_young
# dt = 'float32'
# if cn.count == 96:
# processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates young natural forest removal rate standard deviation tiles
# source_raster = cn.name_stdev_annual_gain_AGC_natrl_forest_young_raw
# out_pattern = cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
# dt = 'float32'
# if cn.count == 96:
# processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates pre-2000 oil palm plantation tiles
# if cn.count == 96:
# processes = 80 # 45 processors = 100 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating pre-2000 oil palm plantation tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(prep_other_inputs.rasterize_pre_2000_plantations, tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates climate zone tiles
# if cn.count == 96:
# processes = 80 # 45 processors = 230 GB peak (on second step); 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating climate zone tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(prep_other_inputs.create_climate_zone_tiles, tile_id_list)
# pool.close()
# pool.join()
#
# # Creates European natural forest removal rate tiles
# source_raster = cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw
# out_pattern = cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe
# dt = 'float32'
# if cn.count == 96:
# processes = 60 # 32 processors = 60 GB peak; 60 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates European natural forest standard deviation of removal rate tiles
# source_raster = cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw
# out_pattern = cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe
# dt = 'float32'
# if cn.count == 96:
# processes = 32 # 32 processors = 60 GB peak; 60 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates a vrt of the primary forests with nodata=0 from the continental primary forest rasters
# uu.print_log("Creating vrt of humid tropial primary forest...")
# primary_vrt = 'primary_2001.vrt'
# os.system('gdalbuildvrt -srcnodata 0 {} *2001_primary.tif'.format(primary_vrt))
# uu.print_log(" Humid tropical primary forest vrt created")
#
# # Creates primary forest tiles
# source_raster = primary_vrt
# out_pattern = 'primary_2001'
# dt = 'Byte'
# if cn.count == 96:
# processes = 45 # 45 processors = 650 GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating primary forest tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates a combined IFL/primary forest raster
# # Uses very little memory since it's just file renaming
# if cn.count == 96:
# processes = 60 # 60 processors = 10 GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Assigning each tile to ifl2000 or primary forest with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(prep_other_inputs.create_combined_ifl_primary, tile_id_list)
# pool.close()
# pool.join()
#
#
# # Creates forest age category tiles for US forests
# source_raster = cn.name_age_cat_natrl_forest_US_raw
# out_pattern = cn.pattern_age_cat_natrl_forest_US
# dt = 'Byte'
# if cn.count == 96:
# processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating US forest age category tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates forest groups for US forests
# source_raster = cn.name_FIA_forest_group_raw
# out_pattern = cn.pattern_FIA_forest_group_processed
# dt = 'Byte'
# if cn.count == 96:
# processes = 80 # 32 processors = 25 GB peak; 80 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating US forest group tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
# # Creates FIA regions for US forests
# source_raster = cn.name_FIA_regions_raw
# out_pattern = cn.pattern_FIA_regions_processed
# dt = 'Byte'
# if cn.count == 96:
# processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak
# else:
# processes = int(cn.count/2)
# uu.print_log("Creating US forest region tiles with {} processors...".format(processes))
# pool = multiprocessing.Pool(processes)
# pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list)
# pool.close()
# pool.join()
#
#
for output_pattern in [
cn.pattern_drivers
# ,cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
]:
# For some reason I can't figure out, the young forest rasters (rate and stdev) have NaN values in some places where 0 (NoData)
# should be. These NaN values show up as values when the check_and_delete_if_empty function runs, making the tiles not
# deleted even if they have no data. However, the light version (which uses gdalinfo rather than rasterio masks) doesn't
# have this problem. So I'm forcing the young forest rates to and stdev to have their emptiness checked by the gdalinfo version.
if output_pattern in [
cn.pattern_annual_gain_AGC_natrl_forest_young,
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young
]:
processes = int(cn.count / 2)
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors using light function..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty_light,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
if cn.count == 96:
processes = 50 # 60 processors = >730 GB peak (for European natural forest forest removal rates); 50 = XXX GB peak
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
elif cn.count <= 2: # For local tests
processes = 1
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors using light function..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty_light,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
else:
processes = int(cn.count / 2)
uu.print_log(
"Checking for empty tiles of {0} pattern with {1} processors..."
.format(output_pattern, processes))
pool = multiprocessing.Pool(processes)
pool.map(
partial(uu.check_and_delete_if_empty,
output_pattern=output_pattern), tile_id_list)
pool.close()
pool.join()
uu.print_log('\n')
# Uploads output tiles to s3
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
def mp_net_flux(sensit_type, tile_id_list, run_date=None):
os.chdir(cn.docker_base_dir)
# If a full model run is specified, the correct set of tiles for the particular script is listed
if tile_id_list == 'all':
# List of tiles to run in the model
tile_id_list = uu.create_combined_tile_list(
cn.gross_emis_all_gases_all_drivers_biomass_soil_dir,
cn.cumul_gain_AGCO2_BGCO2_all_types_dir,
sensit_type=sensit_type)
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))) + "\n")
# Files to download for this script
download_dict = {
cn.cumul_gain_AGCO2_BGCO2_all_types_dir:
[cn.pattern_cumul_gain_AGCO2_BGCO2_all_types],
cn.gross_emis_all_gases_all_drivers_biomass_soil_dir:
[cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil]
}
# List of output directories and output file name patterns
output_dir_list = [cn.net_flux_dir]
output_pattern_list = [cn.pattern_net_flux]
# Downloads input files or entire directories, depending on how many tiles are in the tile_id_list
for key, values in download_dict.items():
dir = key
pattern = values[0]
uu.s3_flexible_download(dir, pattern, cn.docker_base_dir, sensit_type,
tile_id_list)
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log(
"Changing output directory and file name pattern based on sensitivity analysis"
)
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
output_pattern_list = uu.alter_patterns(sensit_type,
output_pattern_list)
# A date can optionally be provided by the full model script or a run of this script.
# This replaces the date in constants_and_names.
if run_date is not None:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
# Creates a single filename pattern to pass to the multiprocessor call
pattern = output_pattern_list[0]
if cn.count == 96:
if sensit_type == 'biomass_swap':
processes = 32 # 32 processors = XXX GB peak
else:
processes = 40 # 38 = 690 GB peak; 40 = 715 GB peak
else:
processes = 9
uu.print_log('Net flux max processors=', processes)
pool = multiprocessing.Pool(processes)
pool.map(
partial(net_flux.net_calc, pattern=pattern, sensit_type=sensit_type),
tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in tile_id_list:
# net_flux.net_calc(tile_id, output_pattern_list[0], sensit_type)
# Uploads output tiles to s3
uu.upload_final_set(output_dir_list[0], output_pattern_list[0])