'{0}{1}_{2}.tif'.format(cn.fao_ecozone_processed_dir, tile,
cn.pattern_fao_ecozone_processed), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.precip_processed_dir, tile,
cn.pattern_precip), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.soil_C_processed_dir, tile,
cn.pattern_soil_C), '.')
uu.s3_file_download(
'{0}{1}_{2}.tif'.format(cn.srtm_processed_dir, tile, cn.pattern_srtm),
'.')
tile_count = []
for i, input in enumerate(input_files):
tile_count[i] = uu.count_tiles_s3(input)
print "The number of tiles for each input to the carbon pools is", tile_count
if len(set(tile_count)) > 0 & tile_count[0] == cn.biomass_tile_count:
print "Input tiles for carbon pool generation do not exist. You must create them. Exiting now."
exit()
print "Creating carbon pools"
count = multiprocessing.cpu_count()
pool = multiprocessing.Pool(processes=count / 3)
pool.map(create_carbon_pools.create_carbon_pools, tile_list)
def main():
os.chdir(cn.docker_base_dir)
# List of possible model stages to run (not including mangrove and planted forest stages)
model_stages = [
'all', 'model_extent', 'forest_age_category_IPCC',
'annual_removals_IPCC', 'annual_removals_all_forest_types',
'gain_year_count', 'gross_removals_all_forest_types', 'carbon_pools',
'gross_emissions', 'net_flux', 'aggregate'
]
# The argument for what kind of model run is being done: standard conditions or a sensitivity analysis run
parser = argparse.ArgumentParser(
description='Run the full carbon flux model')
parser.add_argument('--model-type',
'-t',
required=True,
help='{}'.format(cn.model_type_arg_help))
parser.add_argument(
'--stages',
'-s',
required=True,
help='Stages for running the flux model. Options are {}'.format(
model_stages))
parser.add_argument(
'--run-through',
'-r',
required=True,
help=
'Options: true or false. true: run named stage and following stages. false: run only named stage.'
)
parser.add_argument('--run-date',
'-d',
required=False,
help='Date of run. Must be format YYYYMMDD.')
parser.add_argument(
'--tile-id-list',
'-l',
required=True,
help=
'List of tile ids to use in the model. Should be of form 00N_110E or 00N_110E,00N_120E or all.'
)
parser.add_argument(
'--carbon-pool-extent',
'-ce',
required=False,
help=
'Time period for which carbon emitted_pools should be calculated: loss, 2000, loss,2000, or 2000,loss'
)
parser.add_argument(
'--emitted-pools-to-use',
'-p',
required=False,
help=
'Options are soil_only or biomass_soil. Former only considers emissions from soil. Latter considers emissions from biomass and soil.'
)
parser.add_argument(
'--tcd-threshold',
'-tcd',
required=False,
help=
'Tree cover density threshold above which pixels will be included in the aggregation.'
)
parser.add_argument(
'--std-net-flux-aggreg',
'-sagg',
required=False,
help=
'The s3 standard model net flux aggregated tif, for comparison with the sensitivity analysis map'
)
parser.add_argument(
'--mangroves',
'-ma',
required=False,
help=
'Include mangrove removal rate and standard deviation tile creation step (before model extent). true or false.'
)
parser.add_argument(
'--us-rates',
'-us',
required=False,
help=
'Include US removal rate and standard deviation tile creation step (before model extent). true or false.'
)
parser.add_argument(
'--per-pixel-results',
'-ppr',
required=False,
help=
'Include per pixel result calculations for gross emissions (all gases, all pools), gross removals, and net flux. true or false.'
)
parser.add_argument('--log-note',
'-ln',
required=False,
help='Note to include in log header about model run.')
args = parser.parse_args()
sensit_type = args.model_type
stage_input = args.stages
run_through = args.run_through
run_date = args.run_date
tile_id_list = args.tile_id_list
carbon_pool_extent = args.carbon_pool_extent
emitted_pools = args.emitted_pools_to_use
thresh = args.tcd_threshold
if thresh is not None:
thresh = int(thresh)
std_net_flux = args.std_net_flux_aggreg
include_mangroves = args.mangroves
include_us = args.us_rates
include_per_pixel = args.per_pixel_results
log_note = args.log_note
# Start time for script
script_start = datetime.datetime.now()
# Create the output log
uu.initiate_log(tile_id_list=tile_id_list,
sensit_type=sensit_type,
run_date=run_date,
stage_input=stage_input,
run_through=run_through,
carbon_pool_extent=carbon_pool_extent,
emitted_pools=emitted_pools,
thresh=thresh,
std_net_flux=std_net_flux,
include_mangroves=include_mangroves,
include_us=include_us,
include_per_pixel=include_per_pixel,
log_note=log_note)
# Checks the validity of the model stage arguments. If either one is invalid, the script ends.
if (stage_input not in model_stages):
uu.exception_log(
'Invalid stage selection. Please provide a stage from',
model_stages)
else:
pass
if (run_through not in ['true', 'false']):
uu.exception_log(
'Invalid run through option. Please enter true or false.')
else:
pass
# Generates the list of stages to run
actual_stages = uu.analysis_stages(model_stages,
stage_input,
run_through,
include_mangroves=include_mangroves,
include_us=include_us,
include_per_pixel=include_per_pixel)
uu.print_log("Analysis stages to run:", actual_stages)
# Reports how much storage is being used with files
uu.check_storage()
# Checks whether the sensitivity analysis argument is valid
uu.check_sensit_type(sensit_type)
# Checks if the carbon pool type is specified if the stages to run includes carbon pool generation.
# Does this up front so the user knows before the run begins that information is missing.
if ('carbon_pools' in actual_stages) & (carbon_pool_extent not in [
'loss', '2000', 'loss,2000', '2000,loss'
]):
uu.exception_log(
"Invalid carbon_pool_extent input. Please choose loss, 2000, loss,2000 or 2000,loss."
)
# Checks if the correct c++ script has been compiled for the pool option selected.
# Does this up front so that the user is prompted to compile the C++ before the script starts running, if necessary.
if 'gross_emissions' in actual_stages:
if emitted_pools == 'biomass_soil':
# Some sensitivity analyses have specific gross emissions scripts.
# The rest of the sensitivity analyses and the standard model can all use the same, generic gross emissions script.
if sensit_type in ['no_shifting_ag', 'convert_to_grassland']:
if os.path.exists('{0}/calc_gross_emissions_{1}.exe'.format(
cn.c_emis_compile_dst, sensit_type)):
uu.print_log(
"C++ for {} already compiled.".format(sensit_type))
else:
uu.exception_log(
'Must compile standard {} model C++...'.format(
sensit_type))
else:
if os.path.exists(
'{0}/calc_gross_emissions_generic.exe'.format(
cn.c_emis_compile_dst)):
uu.print_log("C++ for generic emissions already compiled.")
else:
uu.exception_log('Must compile generic emissions C++...')
elif (emitted_pools == 'soil_only') & (sensit_type == 'std'):
if os.path.exists('{0}/calc_gross_emissions_soil_only.exe'.format(
cn.c_emis_compile_dst)):
uu.print_log("C++ for generic emissions already compiled.")
else:
uu.exception_log('Must compile soil_only C++...')
else:
uu.exception_log(
'Pool and/or sensitivity analysis option not valid for gross emissions'
)
# Checks whether the canopy cover argument is valid up front.
if 'aggregate' in actual_stages:
if thresh < 0 or thresh > 99:
uu.exception_log(
'Invalid tcd. Please provide an integer between 0 and 99.')
else:
pass
# If the tile_list argument is an s3 folder, the list of tiles in it is created
if 's3://' in tile_id_list:
tile_id_list = uu.tile_list_s3(tile_id_list, 'std')
uu.print_log(tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(tile_id_list))),
"\n")
# Otherwise, check that the tile list argument is valid. "all" is the way to specify that all tiles should be processed
else:
tile_id_list = uu.tile_id_list_check(tile_id_list)
# List of output directories and output file name patterns.
# The directory list is only used for counting tiles in output folders at the end of the model
output_dir_list = [
cn.model_extent_dir, cn.age_cat_IPCC_dir,
cn.annual_gain_AGB_IPCC_defaults_dir,
cn.annual_gain_BGB_IPCC_defaults_dir,
cn.stdev_annual_gain_AGB_IPCC_defaults_dir, cn.removal_forest_type_dir,
cn.annual_gain_AGC_all_types_dir, cn.annual_gain_BGC_all_types_dir,
cn.annual_gain_AGC_BGC_all_types_dir,
cn.stdev_annual_gain_AGC_all_types_dir, cn.gain_year_count_dir,
cn.cumul_gain_AGCO2_all_types_dir, cn.cumul_gain_BGCO2_all_types_dir,
cn.cumul_gain_AGCO2_BGCO2_all_types_dir
]
# Prepends the mangrove and US output directories if mangroves are included
if 'annual_removals_mangrove' in actual_stages:
output_dir_list = [
cn.annual_gain_AGB_mangrove_dir, cn.annual_gain_BGB_mangrove_dir,
cn.stdev_annual_gain_AGB_mangrove_dir
] + output_dir_list
if 'annual_removals_us' in actual_stages:
output_dir_list = [
cn.annual_gain_AGC_BGC_natrl_forest_US_dir,
cn.stdev_annual_gain_AGC_BGC_natrl_forest_US_dir
] + output_dir_list
# Adds the carbon directories depending on which carbon emitted_pools are being generated: 2000 and/or emissions year
if 'carbon_pools' in actual_stages:
if 'loss' in carbon_pool_extent:
output_dir_list = output_dir_list + [
cn.AGC_emis_year_dir, cn.BGC_emis_year_dir,
cn.deadwood_emis_year_2000_dir, cn.litter_emis_year_2000_dir,
cn.soil_C_emis_year_2000_dir, cn.total_C_emis_year_dir
]
if '2000' in carbon_pool_extent:
output_dir_list = output_dir_list + [
cn.AGC_2000_dir, cn.BGC_2000_dir, cn.deadwood_2000_dir,
cn.litter_2000_dir, cn.soil_C_full_extent_2000_dir,
cn.total_C_2000_dir
]
# Adds the biomass_soil output directories or the soil_only output directories depending on the model run
if 'gross_emissions' in actual_stages:
if emitted_pools == 'biomass_soil':
output_dir_list = output_dir_list + [
cn.gross_emis_commod_biomass_soil_dir,
cn.gross_emis_shifting_ag_biomass_soil_dir,
cn.gross_emis_forestry_biomass_soil_dir,
cn.gross_emis_wildfire_biomass_soil_dir,
cn.gross_emis_urban_biomass_soil_dir,
cn.gross_emis_no_driver_biomass_soil_dir,
cn.gross_emis_all_gases_all_drivers_biomass_soil_dir,
cn.gross_emis_co2_only_all_drivers_biomass_soil_dir,
cn.gross_emis_non_co2_all_drivers_biomass_soil_dir,
cn.gross_emis_nodes_biomass_soil_dir
]
else:
output_dir_list = output_dir_list + [
cn.gross_emis_commod_soil_only_dir,
cn.gross_emis_shifting_ag_soil_only_dir,
cn.gross_emis_forestry_soil_only_dir,
cn.gross_emis_wildfire_soil_only_dir,
cn.gross_emis_urban_soil_only_dir,
cn.gross_emis_no_driver_soil_only_dir,
cn.gross_emis_all_gases_all_drivers_soil_only_dir,
cn.gross_emis_co2_only_all_drivers_soil_only_dir,
cn.gross_emis_non_co2_all_drivers_soil_only_dir,
cn.gross_emis_nodes_soil_only_dir
]
output_dir_list = output_dir_list + [
cn.net_flux_dir, cn.cumul_gain_AGCO2_BGCO2_all_types_per_pixel_dir,
cn.gross_emis_all_gases_all_drivers_biomass_soil_per_pixel_dir,
cn.net_flux_per_pixel_dir
]
# Output patterns aren't actually used in the script-- here just for reference.
output_pattern_list = [
cn.pattern_model_extent, cn.pattern_age_cat_IPCC,
cn.pattern_annual_gain_AGB_IPCC_defaults,
cn.pattern_annual_gain_BGB_IPCC_defaults,
cn.pattern_stdev_annual_gain_AGB_IPCC_defaults,
cn.pattern_removal_forest_type, cn.pattern_annual_gain_AGC_all_types,
cn.pattern_annual_gain_BGC_all_types,
cn.pattern_annual_gain_AGC_BGC_all_types,
cn.pattern_stdev_annual_gain_AGC_all_types, cn.pattern_gain_year_count,
cn.pattern_cumul_gain_AGCO2_all_types,
cn.pattern_cumul_gain_BGCO2_all_types,
cn.pattern_cumul_gain_AGCO2_BGCO2_all_types
]
# Prepends the mangrove and US output pattern if mangroves are included
if 'annual_removals_mangrove' in actual_stages:
output_pattern_list = [
cn.pattern_annual_gain_AGB_mangrove,
cn.pattern_annual_gain_BGB_mangrove,
cn.pattern_stdev_annual_gain_AGB_mangrove
] + output_pattern_list
if 'annual_removals_us' in actual_stages:
output_pattern_list = [
cn.pattern_annual_gain_AGC_BGC_natrl_forest_US,
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US
] + output_pattern_list
# Adds the soil carbon patterns depending on which carbon emitted_pools are being generated: 2000 and/or emissions year
if 'carbon_pools' in actual_stages:
if 'loss' in carbon_pool_extent:
output_pattern_list = output_pattern_list + [
cn.pattern_AGC_emis_year, cn.pattern_BGC_emis_year,
cn.pattern_deadwood_emis_year_2000,
cn.pattern_litter_emis_year_2000,
cn.pattern_soil_C_emis_year_2000, cn.pattern_total_C_emis_year
]
if '2000' in carbon_pool_extent:
output_pattern_list = output_pattern_list + [
cn.pattern_AGC_2000, cn.pattern_BGC_2000,
cn.pattern_deadwood_2000, cn.pattern_litter_2000,
cn.pattern_soil_C_full_extent_2000, cn.pattern_total_C_2000
]
# Adds the biomass_soil output patterns or the soil_only output directories depending on the model run
if 'gross_emissions' in actual_stages:
if emitted_pools == 'biomass_soil':
output_pattern_list = output_pattern_list + [
cn.pattern_gross_emis_commod_biomass_soil,
cn.pattern_gross_emis_shifting_ag_biomass_soil,
cn.pattern_gross_emis_forestry_biomass_soil,
cn.pattern_gross_emis_wildfire_biomass_soil,
cn.pattern_gross_emis_urban_biomass_soil,
cn.pattern_gross_emis_no_driver_biomass_soil,
cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil,
cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil,
cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil
]
else:
output_pattern_list = output_pattern_list + [
cn.pattern_gross_emis_commod_soil_only,
cn.pattern_gross_emis_shifting_ag_soil_only,
cn.pattern_gross_emis_forestry_soil_only,
cn.pattern_gross_emis_wildfire_soil_only,
cn.pattern_gross_emis_urban_soil_only,
cn.pattern_gross_emis_no_driver_soil_only,
cn.pattern_gross_emis_all_gases_all_drivers_soil_only,
cn.pattern_gross_emis_co2_only_all_drivers_soil_only,
cn.pattern_gross_emis_non_co2_all_drivers_soil_only,
cn.pattern_gross_emis_nodes_soil_only
]
output_pattern_list = output_pattern_list + [
cn.pattern_net_flux,
cn.pattern_cumul_gain_AGCO2_BGCO2_all_types_per_pixel,
cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil_per_pixel,
cn.pattern_net_flux_per_pixel
]
# Creates tiles of annual AGB and BGB gain rate and AGB stdev for mangroves using the standard model
# removal function
if 'annual_removals_mangrove' in actual_stages:
uu.print_log(":::::Creating tiles of annual removals for mangrove")
start = datetime.datetime.now()
mp_annual_gain_rate_mangrove(sensit_type,
tile_id_list,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for annual_gain_rate_mangrove:",
elapsed_time, "\n")
# Creates tiles of annual AGC+BGC gain rate and AGC stdev for US-specific removals using the standard model
# removal function
if 'annual_removals_us' in actual_stages:
uu.print_log(":::::Creating tiles of annual removals for US")
start = datetime.datetime.now()
mp_US_removal_rates(sensit_type, tile_id_list, run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for annual_gain_rate_us:",
elapsed_time, "\n")
# Creates model extent tiles
if 'model_extent' in actual_stages:
uu.print_log(":::::Creating tiles of model extent")
start = datetime.datetime.now()
mp_model_extent(sensit_type, tile_id_list, run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for model_extent:", elapsed_time,
"\n", "\n")
# Creates age category tiles for natural forests
if 'forest_age_category_IPCC' in actual_stages:
uu.print_log(
":::::Creating tiles of forest age categories for IPCC removal rates"
)
start = datetime.datetime.now()
mp_forest_age_category_IPCC(sensit_type,
tile_id_list,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for forest_age_category_IPCC:",
elapsed_time, "\n", "\n")
# Creates tiles of annual AGB and BGB gain rates using IPCC Table 4.9 defaults
if 'annual_removals_IPCC' in actual_stages:
uu.print_log(
":::::Creating tiles of annual aboveground and belowground removal rates using IPCC defaults"
)
start = datetime.datetime.now()
mp_annual_gain_rate_IPCC_defaults(sensit_type,
tile_id_list,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for annual_gain_rate_IPCC:",
elapsed_time, "\n", "\n")
# Creates tiles of annual AGC and BGC removal factors for the entire model, combining removal factors from all forest types
if 'annual_removals_all_forest_types' in actual_stages:
uu.print_log(
":::::Creating tiles of annual aboveground and belowground removal rates for all forest types"
)
start = datetime.datetime.now()
mp_annual_gain_rate_AGC_BGC_all_forest_types(sensit_type,
tile_id_list,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(
":::::Processing time for annual_gain_rate_AGC_BGC_all_forest_types:",
elapsed_time, "\n", "\n")
# Creates tiles of the number of years of removals for all model pixels (across all forest types)
if 'gain_year_count' in actual_stages:
uu.print_log(
":::::Freeing up memory for gain year count creation by deleting unneeded tiles"
)
tiles_to_delete = []
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_mangrove_biomass_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_WHRC_biomass_2000_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_mangrove)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_mangrove)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_natrl_forest_US)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_natrl_forest_young)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_age_cat_IPCC)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGB_IPCC_defaults)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_BGB_IPCC_defaults)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_all_types)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_ifl_primary)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_planted_forest_type_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGB_mangrove)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGB_IPCC_defaults)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_all_types)))
uu.print_log(" Deleting", len(tiles_to_delete), "tiles...")
for tile_to_delete in tiles_to_delete:
os.remove(tile_to_delete)
uu.print_log(":::::Deleted unneeded tiles")
uu.check_storage()
uu.print_log(
":::::Creating tiles of gain year count for all removal pixels")
start = datetime.datetime.now()
mp_gain_year_count_all_forest_types(sensit_type,
tile_id_list,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for gain_year_count:", elapsed_time,
"\n", "\n")
# Creates tiles of gross removals for all forest types (aboveground, belowground, and above+belowground)
if 'gross_removals_all_forest_types' in actual_stages:
uu.print_log(
":::::Creating gross removals for all forest types combined (above + belowground) tiles'"
)
start = datetime.datetime.now()
mp_gross_removals_all_forest_types(sensit_type,
tile_id_list,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(
":::::Processing time for gross_removals_all_forest_types:",
elapsed_time, "\n", "\n")
# Creates carbon emitted_pools in loss year
if 'carbon_pools' in actual_stages:
uu.print_log(
":::::Freeing up memory for carbon pool creation by deleting unneeded tiles"
)
tiles_to_delete = []
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_model_extent)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGB_mangrove)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGB_mangrove)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_natrl_forest_US)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_natrl_forest_young)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_age_cat_IPCC)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGB_IPCC_defaults)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_BGB_IPCC_defaults)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_annual_gain_BGC_all_types)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_annual_gain_AGC_BGC_all_types)))
tiles_to_delete.extend(glob.glob('*growth_years*tif'))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_gain_year_count)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_BGCO2_all_types)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_cumul_gain_AGCO2_BGCO2_all_types)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_ifl_primary)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_planted_forest_type_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGB_mangrove)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_BGC_planted_forest_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_US)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_natrl_forest_young)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGB_IPCC_defaults)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_stdev_annual_gain_AGC_all_types)))
uu.print_log(" Deleting", len(tiles_to_delete), "tiles...")
for tile_to_delete in tiles_to_delete:
os.remove(tile_to_delete)
uu.print_log(":::::Deleted unneeded tiles")
uu.check_storage()
uu.print_log(":::::Creating carbon pool tiles")
start = datetime.datetime.now()
mp_create_carbon_pools(sensit_type,
tile_id_list,
carbon_pool_extent,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for create_carbon_pools:",
elapsed_time, "\n", "\n")
# Creates gross emissions tiles by driver, gas, and all emissions combined
if 'gross_emissions' in actual_stages:
uu.print_log(
":::::Freeing up memory for gross emissions creation by deleting unneeded tiles"
)
tiles_to_delete = []
# tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_removal_forest_type)))
tiles_to_delete.extend(glob.glob('*{}*tif'.format(
cn.pattern_AGC_2000)))
tiles_to_delete.extend(glob.glob('*{}*tif'.format(
cn.pattern_BGC_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_deadwood_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_litter_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_total_C_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_elevation)))
tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_precip)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_annual_gain_AGC_all_types)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_cumul_gain_AGCO2_all_types)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_cont_eco_processed)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_WHRC_biomass_2000_unmasked)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_mangrove_biomass_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_removal_forest_type)))
uu.print_log(" Deleting", len(tiles_to_delete), "tiles...")
uu.print_log(tiles_to_delete)
for tile_to_delete in tiles_to_delete:
os.remove(tile_to_delete)
uu.print_log(":::::Deleted unneeded tiles")
uu.check_storage()
uu.print_log(":::::Creating gross emissions tiles")
start = datetime.datetime.now()
mp_calculate_gross_emissions(sensit_type,
tile_id_list,
emitted_pools,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for gross_emissions:", elapsed_time,
"\n", "\n")
# Creates net flux tiles (gross emissions - gross removals)
if 'net_flux' in actual_stages:
uu.print_log(
":::::Freeing up memory for net flux creation by deleting unneeded tiles"
)
tiles_to_delete = []
tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_loss)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_non_co2_all_drivers_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_co2_only_all_drivers_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_commod_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_shifting_ag_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_forestry_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_wildfire_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_urban_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_no_driver_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_gross_emis_nodes_biomass_soil)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_AGC_emis_year)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_BGC_emis_year)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_deadwood_emis_year_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_litter_emis_year_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_soil_C_emis_year_2000)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_total_C_emis_year)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_peat_mask)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_ifl_primary)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(
cn.pattern_planted_forest_type_unmasked)))
tiles_to_delete.extend(glob.glob('*{}*tif'.format(cn.pattern_drivers)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_climate_zone)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_bor_tem_trop_processed)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_burn_year)))
tiles_to_delete.extend(
glob.glob('*{}*tif'.format(cn.pattern_plant_pre_2000)))
uu.print_log(" Deleting", len(tiles_to_delete), "tiles...")
for tile_to_delete in tiles_to_delete:
os.remove(tile_to_delete)
uu.print_log(":::::Deleted unneeded tiles")
uu.check_storage()
uu.print_log(":::::Creating net flux tiles")
start = datetime.datetime.now()
mp_net_flux(sensit_type, tile_id_list, run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for net_flux:", elapsed_time, "\n",
"\n")
# Aggregates gross emissions, gross removals, and net flux to coarser resolution.
# For sensitivity analyses, creates percent difference and sign change maps compared to standard model net flux.
if 'aggregate' in actual_stages:
uu.print_log(":::::Creating 4x4 km aggregate maps")
start = datetime.datetime.now()
mp_aggregate_results_to_4_km(sensit_type,
thresh,
tile_id_list,
std_net_flux=std_net_flux,
run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for aggregate:", elapsed_time, "\n",
"\n")
# Converts gross emissions, gross removals and net flux from per hectare rasters to per pixel rasters
if 'per_pixel_results' in actual_stages:
uu.print_log(":::::Creating per pixel versions of main model outputs")
start = datetime.datetime.now()
mp_output_per_pixel(sensit_type, tile_id_list, run_date=run_date)
end = datetime.datetime.now()
elapsed_time = end - start
uu.check_storage()
uu.print_log(":::::Processing time for per pixel raster creation:",
elapsed_time, "\n", "\n")
uu.print_log(":::::Counting tiles output to each folder")
# Modifies output directory names to make them match those used during the model run.
# The tiles in each of these directories and counted and logged.
# If the model run isn't the standard one, the output directory and file names are changed
if sensit_type != 'std':
uu.print_log(
"Modifying output directory and file name pattern based on sensitivity analysis"
)
output_dir_list = uu.alter_dirs(sensit_type, output_dir_list)
# Changes the date in the output directories. This date was used during the model run.
# This replaces the date in constants_and_names.
if run_date:
output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date)
for output in output_dir_list:
tile_count = uu.count_tiles_s3(output)
uu.print_log("Total tiles in", output, ": ", tile_count)
script_end = datetime.datetime.now()
script_elapsed_time = script_end - script_start
uu.print_log(":::::Processing time for entire run:", script_elapsed_time,
"\n")
def main():
# Create the output log
uu.initiate_log()
os.chdir(cn.docker_base_dir)
# Files to download for this script.
download_dict = {
cn.gain_dir: [cn.pattern_gain],
cn.annual_gain_AGB_IPCC_defaults_dir:
[cn.pattern_annual_gain_AGB_IPCC_defaults]
}
# List of tiles that could be run. This list is only used to create the FIA region tiles if they don't already exist.
tile_id_list = uu.tile_list_s3(cn.annual_gain_AGB_IPCC_defaults_dir)
# tile_id_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles
# tile_id_list = ['50N_130W'] # test tiles
# List of output directories and output file name patterns
output_dir_list = [
cn.US_annual_gain_AGB_natrl_forest_dir,
cn.US_annual_gain_BGB_natrl_forest_dir
]
output_pattern_list = [
cn.pattern_US_annual_gain_AGB_natrl_forest,
cn.pattern_US_annual_gain_BGB_natrl_forest
]
# By definition, this script is for US-specific removals
sensit_type = 'US_removals'
# Counts how many processed FIA region tiles there are on s3 already. 16 tiles cover the continental US.
FIA_regions_tile_count = uu.count_tiles_s3(cn.FIA_regions_processed_dir)
# Only creates FIA region tiles if they don't already exist on s3.
if FIA_regions_tile_count == 16:
uu.print_log("FIA region tiles already created. Copying to s3 now...")
uu.s3_flexible_download(cn.FIA_regions_processed_dir,
cn.pattern_FIA_regions_processed,
cn.docker_base_dir, 'std', 'all')
else:
uu.print_log(
"FIA region tiles do not exist. Creating tiles, then copying to s3 for future use..."
)
uu.s3_file_download(
os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw),
cn.docker_base_dir, 'std')
cmd = ['unzip', '-o', '-j', cn.name_FIA_regions_raw]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Converts the region shapefile to Hansen tiles
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(US_removal_rates.prep_FIA_regions, tile_id_list)
# List of FIA region tiles on the spot machine. Only this list is used for the rest of the script.
US_tile_list = uu.tile_list_spot_machine(
cn.docker_base_dir, '{}.tif'.format(cn.pattern_FIA_regions_processed))
US_tile_id_list = [i[0:8] for i in US_tile_list]
# US_tile_id_list = ['50N_130W'] # For testing
uu.print_log(US_tile_id_list)
uu.print_log(
"There are {} tiles to process".format(str(len(US_tile_id_list))) +
"\n")
# Counts how many processed forest age category tiles there are on s3 already. 16 tiles cover the continental US.
US_age_tile_count = uu.count_tiles_s3(cn.US_forest_age_cat_processed_dir)
# Only creates FIA forest age category tiles if they don't already exist on s3.
if US_age_tile_count == 16:
uu.print_log(
"Forest age category tiles already created. Copying to spot machine now..."
)
uu.s3_flexible_download(cn.US_forest_age_cat_processed_dir,
cn.pattern_US_forest_age_cat_processed, '',
'std', US_tile_id_list)
else:
uu.print_log(
"Southern forest age category tiles do not exist. Creating tiles, then copying to s3 for future use..."
)
uu.s3_file_download(
os.path.join(cn.US_forest_age_cat_raw_dir,
cn.name_US_forest_age_cat_raw), cn.docker_base_dir,
'std')
# Converts the national forest age category raster to Hansen tiles
source_raster = cn.name_US_forest_age_cat_raw
out_pattern = cn.pattern_US_forest_age_cat_processed
dt = 'Int16'
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt), US_tile_id_list)
uu.upload_final_set(cn.US_forest_age_cat_processed_dir,
cn.pattern_US_forest_age_cat_processed)
# Counts how many processed FIA forest group tiles there are on s3 already. 16 tiles cover the continental US.
FIA_forest_group_tile_count = uu.count_tiles_s3(
cn.FIA_forest_group_processed_dir)
# Only creates FIA forest group tiles if they don't already exist on s3.
if FIA_forest_group_tile_count == 16:
uu.print_log(
"FIA forest group tiles already created. Copying to spot machine now..."
)
uu.s3_flexible_download(cn.FIA_forest_group_processed_dir,
cn.pattern_FIA_forest_group_processed, '',
'std', US_tile_id_list)
else:
uu.print_log(
"FIA forest group tiles do not exist. Creating tiles, then copying to s3 for future use..."
)
uu.s3_file_download(
os.path.join(cn.FIA_forest_group_raw_dir,
cn.name_FIA_forest_group_raw), cn.docker_base_dir,
'std')
# Converts the national forest group raster to Hansen forest group tiles
source_raster = cn.name_FIA_forest_group_raw
out_pattern = cn.pattern_FIA_forest_group_processed
dt = 'Byte'
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(
partial(uu.mp_warp_to_Hansen,
source_raster=source_raster,
out_pattern=out_pattern,
dt=dt), US_tile_id_list)
uu.upload_final_set(cn.FIA_forest_group_processed_dir,
cn.pattern_FIA_forest_group_processed)
# Downloads input files or entire directories, depending on how many tiles are in the tile_id_list
for key, values in download_dict.items():
dir = key
pattern = values[0]
uu.s3_flexible_download(dir, pattern, cn.docker_base_dir, sensit_type,
US_tile_id_list)
# Table with US-specific removal rates
cmd = [
'aws', 's3', 'cp',
os.path.join(cn.gain_spreadsheet_dir, cn.table_US_removal_rate),
cn.docker_base_dir
]
# Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
process = Popen(cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
uu.log_subprocess_output(process.stdout)
# Imports the table with the region-group-age AGB removal rates
gain_table = pd.read_excel("{}".format(cn.table_US_removal_rate),
sheet_name="US_rates_for_model")
# Converts gain table from wide to long, so each region-group-age category has its own row
gain_table_group_region_by_age = pd.melt(
gain_table,
id_vars=['FIA_region_code', 'forest_group_code'],
value_vars=['growth_young', 'growth_middle', 'growth_old'])
gain_table_group_region_by_age = gain_table_group_region_by_age.dropna()
# In the forest age category raster, each category has this value
age_dict = {
'growth_young': 1000,
'growth_middle': 2000,
'growth_old': 3000
}
# Creates a unique value for each forest group-region-age category in the table.
# Although these rates are applied to all standard gain model pixels at first, they are not ultimately used for
# pixels that have Hansen gain (see below).
gain_table_group_region_age = gain_table_group_region_by_age.replace(
{"variable": age_dict})
gain_table_group_region_age[
'age_cat'] = gain_table_group_region_age['variable'] * 10
gain_table_group_region_age['group_region_age_combined'] = gain_table_group_region_age['age_cat'] + \
gain_table_group_region_age['forest_group_code']*100 + \
gain_table_group_region_age['FIA_region_code']
# Converts the forest group-region-age codes and corresponding gain rates to a dictionary,
# where the key is the unique group-region-age code and the value is the AGB removal rate.
gain_table_group_region_age_dict = pd.Series(
gain_table_group_region_age.value.values,
index=gain_table_group_region_age.group_region_age_combined).to_dict()
uu.print_log(gain_table_group_region_age_dict)
# Creates a unique value for each forest group-region category using just young forest rates.
# These are assigned to Hansen gain pixels, which automatically get the young forest rate, regardless of the
# forest age category raster.
gain_table_group_region = gain_table_group_region_age.drop(
gain_table_group_region_age[
gain_table_group_region_age.age_cat != 10000].index)
gain_table_group_region['group_region_combined'] = gain_table_group_region['forest_group_code']*100 + \
gain_table_group_region['FIA_region_code']
# Converts the forest group-region codes and corresponding gain rates to a dictionary,
# where the key is the unique group-region code (youngest age category) and the value is the AGB removal rate.
gain_table_group_region_dict = pd.Series(
gain_table_group_region.value.values,
index=gain_table_group_region.group_region_combined).to_dict()
uu.print_log(gain_table_group_region_dict)
# count/2 on a m4.16xlarge maxes out at about 230 GB of memory (processing 16 tiles at once), so it's okay on an m4.16xlarge
pool = multiprocessing.Pool(int(cn.count / 2))
pool.map(
partial(
US_removal_rates.US_removal_rate_calc,
gain_table_group_region_age_dict=gain_table_group_region_age_dict,
gain_table_group_region_dict=gain_table_group_region_dict,
output_pattern_list=output_pattern_list,
sensit_type=sensit_type), US_tile_id_list)
pool.close()
pool.join()
# # For single processor use
# for tile_id in US_tile_id_list:
#
# US_removal_rates.US_removal_rate_calc(tile_id, gain_table_group_region_age_dict, gain_table_group_region_dict,
# output_pattern_list, sensit_type)
# Uploads output tiles to s3
for i in range(0, len(output_dir_list)):
uu.upload_final_set(output_dir_list[i], output_pattern_list[i])