def main (): no_upload = False # Create the output log uu.initiate_log() os.chdir(cn.docker_base_dir) # The list of tiles to iterate through tile_id_list = uu.tile_list_s3(cn.WHRC_biomass_2000_unmasked_dir) # tile_id_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles # tile_id_list = ['00N_110E'] # test tile uu.print_log(tile_id_list) uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # By definition, this script is for the biomass swap analysis (replacing WHRC AGB with Saatchi/JPL AGB) sensit_type = 'biomass_swap' # Downloads a pan-tropical raster that has the erroneous integer values in the oceans removed uu.s3_file_download(cn.JPL_raw_dir, cn.JPL_raw_name, sensit_type) # Converts the Saatchi AGB vrt to Hansen tiles source_raster = cn.JPL_raw_name out_pattern = cn.pattern_JPL_unmasked_processed dt = 'Float32' pool = multiprocessing.Pool(cn.count-5) # count-5 peaks at 320GB of memory pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # Checks if each tile has data in it. Only tiles with data are uploaded. upload_dir = cn.JPL_processed_dir pattern = cn.pattern_JPL_unmasked_processed pool = multiprocessing.Pool(cn.count - 5) # count-5 peaks at 410GB of memory pool.map(partial(uu.check_and_upload, upload_dir=upload_dir, pattern=pattern), tile_id_list)
def mp_mangrove_processing(tile_id_list, run_date=None, no_upload=None): os.chdir(cn.docker_base_dir) # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.tile_list_s3(cn.pixel_area_dir) uu.print_log(tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # Downloads zipped raw mangrove files uu.s3_file_download( os.path.join(cn.mangrove_biomass_raw_dir, cn.mangrove_biomass_raw_file), cn.docker_base_dir, 'std') # Unzips mangrove images into a flat structure (all tifs into main folder using -j argument) # NOTE: Unzipping some tifs (e.g., Australia, Indonesia) takes a very long time, so don't worry if the script appears to stop on that. cmd = ['unzip', '-o', '-j', cn.mangrove_biomass_raw_file] uu.log_subprocess_output_full(cmd) # Creates vrt for the Saatchi biomass rasters mangrove_vrt = 'mangrove_biomass.vrt' os.system('gdalbuildvrt {} *.tif'.format(mangrove_vrt)) # Converts the mangrove AGB vrt into Hansen tiles source_raster = mangrove_vrt out_pattern = cn.pattern_mangrove_biomass_2000 dt = 'float32' processes = int(cn.count / 4) uu.print_log('Mangrove preprocessing max processors=', processes) pool = multiprocessing.Pool(processes) pool.map( partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # # For single processor use, for testing purposes # for tile_id in tile_id_list: # # mangrove_processing.create_mangrove_tiles(tile_id, source_raster, out_pattern, no_upload) # Checks if each tile has data in it. Only tiles with data are uploaded. upload_dir = cn.mangrove_biomass_2000_dir pattern = cn.pattern_mangrove_biomass_2000 processes = int(cn.count - 5) uu.print_log('Mangrove check for data max processors=', processes) pool = multiprocessing.Pool(processes) pool.map( partial(uu.check_and_upload, upload_dir=upload_dir, pattern=pattern), tile_id_list)
def mp_continent_ecozone_tiles(tile_id_list, run_date=None): os.chdir(cn.docker_base_dir) # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.create_combined_tile_list( cn.pattern_WHRC_biomass_2000_non_mang_non_planted, cn.mangrove_biomass_2000_dir) uu.print_log(tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # if the continent-ecozone shapefile hasn't already been downloaded, it will be downloaded and unzipped uu.s3_file_download(cn.cont_eco_s3_zip, cn.docker_base_dir, 'std') # Unzips ecozone shapefile cmd = ['unzip', cn.cont_eco_zip] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # List of output directories and output file name patterns output_dir_list = [cn.cont_eco_raw_dir, cn.cont_eco_dir] output_pattern_list = [ cn.pattern_cont_eco_raw, cn.pattern_cont_eco_processed ] # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # For multiprocessor use processes = int(cn.count / 4) uu.print_log('Continent-ecozone tile creation max processors=', processes) pool.map(continent_ecozone_tiles.create_continent_ecozone_tiles, tile_id_list) # Uploads the continent-ecozone tile to s3 before the codes are expanded to pixels in 1024x1024 windows that don't have codes. # These are not used for the model. They are for reference and completeness. for i in range(0, len(output_dir_list)): uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
### mangrove data. ### Output tiles conform to the dimensions, resolution, and other properties of Hansen loss tiles. import multiprocessing import mangrove_processing import sys import os import subprocess import utilities sys.path.append('../') import constants_and_names as cn import universal_util as uu # Downloads zipped raw mangrove files uu.s3_file_download( os.path.join(cn.mangrove_biomass_raw_dir, cn.mangrove_biomass_raw_file), '.') # Unzips mangrove images into a flat structure (all tifs into main folder using -j argument) # NOTE: Unzipping some tifs (e.g., Australia, Indonesia) takes a very long time, so don't worry if the script appears to stop on that. cmd = ['unzip', '-j', cn.mangrove_biomass_raw_file] subprocess.check_call(cmd) # Creates vrt of all raw mangrove tifs utilities.build_vrt(utilities.mangrove_vrt) # Iterates through all possible tiles (not just WHRC biomass tiles) to create mangrove biomass tiles that don't have analogous WHRC tiles total_tile_list = uu.tile_list(cn.pixel_area_dir) # biomass_tile_list = ['00N_000E', '20S_120W', '00N_120E'] # test tile print total_tile_list
def mp_prep_other_inputs(tile_id_list, run_date): os.chdir(cn.docker_base_dir) sensit_type='std' # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.create_combined_tile_list(cn.WHRC_biomass_2000_unmasked_dir, cn.mangrove_biomass_2000_dir, set3=cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir ) uu.print_log(tile_id_list) uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # List of output directories and output file name patterns output_dir_list = [cn.climate_zone_processed_dir, cn.plant_pre_2000_processed_dir, cn.drivers_processed_dir, cn.ifl_primary_processed_dir, cn.annual_gain_AGC_natrl_forest_young_dir, cn.stdev_annual_gain_AGC_natrl_forest_young_dir, cn.annual_gain_AGC_BGC_natrl_forest_Europe_dir, cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_dir, cn.FIA_forest_group_processed_dir, cn.age_cat_natrl_forest_US_dir, cn.FIA_regions_processed_dir] output_pattern_list = [cn.pattern_climate_zone, cn.pattern_plant_pre_2000, cn.pattern_drivers, cn.pattern_ifl_primary, cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young, cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe, cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe, cn.pattern_FIA_forest_group_processed, cn.pattern_age_cat_natrl_forest_US, cn.pattern_FIA_regions_processed] # If the model run isn't the standard one, the output directory and file names are changed if sensit_type != 'std': uu.print_log("Changing output directory and file name pattern based on sensitivity analysis") output_dir_list = uu.alter_dirs(sensit_type, output_dir_list) output_pattern_list = uu.alter_patterns(sensit_type, output_pattern_list) # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # Files to process: climate zone, IDN/MYS plantations before 2000, tree cover loss drivers, combine IFL and primary forest uu.s3_file_download(os.path.join(cn.climate_zone_raw_dir, cn.climate_zone_raw), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.plant_pre_2000_raw_dir, '{}.zip'.format(cn.pattern_plant_pre_2000_raw)), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.drivers_raw_dir, '{}.zip'.format(cn.pattern_drivers_raw)), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.age_cat_natrl_forest_US_raw_dir, cn.name_age_cat_natrl_forest_US_raw), cn.docker_base_dir, sensit_type) uu.s3_file_download(os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, sensit_type) # For some reason, using uu.s3_file_download or otherwise using AWSCLI as a subprocess doesn't work for this raster. # Thus, using wget instead. cmd = ['wget', '{}'.format(cn.annual_gain_AGC_natrl_forest_young_raw_URL), '-P', '{}'.format(cn.docker_base_dir)] process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) uu.s3_file_download(cn.stdev_annual_gain_AGC_natrl_forest_young_raw_URL, cn.docker_base_dir, sensit_type) cmd = ['aws', 's3', 'cp', cn.primary_raw_dir, cn.docker_base_dir, '--recursive'] uu.log_subprocess_output_full(cmd) uu.s3_flexible_download(cn.ifl_dir, cn.pattern_ifl, cn.docker_base_dir, sensit_type, tile_id_list) uu.print_log("Unzipping pre-2000 plantations...") cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_plant_pre_2000_raw)] uu.log_subprocess_output_full(cmd) uu.print_log("Unzipping drivers...") cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_drivers_raw)] uu.log_subprocess_output_full(cmd) # Creates tree cover loss driver tiles source_raster = '{}.tif'.format(cn.pattern_drivers_raw) out_pattern = cn.pattern_drivers dt = 'Byte' if cn.count == 96: processes = 80 # 45 processors = 70 GB peak; 70 = 90 GB peak; 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating tree cover loss driver tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates young natural forest removal rate tiles source_raster = cn.name_annual_gain_AGC_natrl_forest_young_raw out_pattern = cn.pattern_annual_gain_AGC_natrl_forest_young dt = 'float32' if cn.count == 96: processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates young natural forest removal rate standard deviation tiles source_raster = cn.name_stdev_annual_gain_AGC_natrl_forest_young_raw out_pattern = cn.pattern_stdev_annual_gain_AGC_natrl_forest_young dt = 'float32' if cn.count == 96: processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates pre-2000 oil palm plantation tiles if cn.count == 96: processes = 80 # 45 processors = 100 GB peak; 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating pre-2000 oil palm plantation tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(prep_other_inputs.rasterize_pre_2000_plantations, tile_id_list) pool.close() pool.join() # Creates climate zone tiles if cn.count == 96: processes = 80 # 45 processors = 230 GB peak (on second step); 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating climate zone tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(prep_other_inputs.create_climate_zone_tiles, tile_id_list) pool.close() pool.join() # Creates European natural forest removal rate tiles source_raster = cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw out_pattern = cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe dt = 'float32' if cn.count == 96: processes = 60 # 32 processors = 60 GB peak; 60 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates European natural forest standard deviation of removal rate tiles source_raster = cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw out_pattern = cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe dt = 'float32' if cn.count == 96: processes = 32 # 32 processors = 60 GB peak; 60 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates a vrt of the primary forests with nodata=0 from the continental primary forest rasters uu.print_log("Creating vrt of humid tropial primary forest...") primary_vrt = 'primary_2001.vrt' os.system('gdalbuildvrt -srcnodata 0 {} *2001_primary.tif'.format(primary_vrt)) uu.print_log(" Humid tropical primary forest vrt created") # Creates primary forest tiles source_raster = primary_vrt out_pattern = 'primary_2001' dt = 'Byte' if cn.count == 96: processes = 45 # 45 processors = 650 GB peak else: processes = int(cn.count/2) uu.print_log("Creating primary forest tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates a combined IFL/primary forest raster # Uses very little memory since it's just file renaming if cn.count == 96: processes = 60 # 60 processors = 10 GB peak else: processes = int(cn.count/2) uu.print_log("Assigning each tile to ifl2000 or primary forest with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(prep_other_inputs.create_combined_ifl_primary, tile_id_list) pool.close() pool.join() # Creates forest age category tiles for US forests source_raster = cn.name_age_cat_natrl_forest_US_raw out_pattern = cn.pattern_age_cat_natrl_forest_US dt = 'Byte' if cn.count == 96: processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating US forest age category tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates forest groups for US forests source_raster = cn.name_FIA_forest_group_raw out_pattern = cn.pattern_FIA_forest_group_processed dt = 'Byte' if cn.count == 96: processes = 80 # 32 processors = 25 GB peak; 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating US forest group tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() # Creates FIA regions for US forests source_raster = cn.name_FIA_regions_raw out_pattern = cn.pattern_FIA_regions_processed dt = 'Byte' if cn.count == 96: processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating US forest region tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), tile_id_list) pool.close() pool.join() for output_pattern in [cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young]: # For some reason I can't figure out, the young forest rasters (rate and stdev) have NaN values in some places where 0 (NoData) # should be. These NaN values show up as values when the check_and_delete_if_empty function runs, making the tiles not # deleted even if they have no data. However, the light version (which uses gdalinfo rather than rasterio masks) doesn't # have this problem. So I'm forcing the young forest rates to and stdev to have their emptiness checked by the gdalinfo version. if output_pattern in [cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young]: processes = int(cn.count / 2) uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() if cn.count == 96: processes = 50 # 60 processors = >730 GB peak (for European natural forest forest removal rates); 50 = XXX GB peak uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() elif cn.count <= 2: # For local tests processes = 1 uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() else: processes = int(cn.count / 2) uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() uu.print_log('\n') # Uploads output tiles to s3 for i in range(0, len(output_dir_list)): uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
import sys sys.path.append('../') import constants_and_names as cn import universal_util as uu # tile_list = util.tile_list(cn.natrl_forest_biomass_2000_dir) tile_list = ['10N_080W', '40N_120E'] # test tiles print tile_list print "Downloading raw ecozone and precipitation files" # Downloads two of the raw input files for creating carbon pools input_files = [cn.fao_ecozone_raw_dir, cn.precip_raw_dir] for input in input_files: uu.s3_file_download('{}'.format(input), '.') print "Unzipping FAO ecozones" unzip_zones = ['unzip', '{}'.format(cn.pattern_fao_ecozone_raw), '-d', '.'] subprocess.check_call(unzip_zones) print "Copying srtm files" uu.s3_folder_download(cn.srtm_raw_dir, './srtm') print "Making srtm vrt" subprocess.check_call('gdalbuildvrt srtm.vrt srtm/*.tif', shell=True) # # Soil tiles are already processed, so there's no need to include them here. # # Leaving this in case I ever add in soil processing again. # print "Copying soil tiles" # uu.s3_folder_download(cn.soil_C_processed_dir)
def mp_peatland_processing(tile_id_list, run_date=None): os.chdir(cn.docker_base_dir) # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.tile_list_s3(cn.pixel_area_dir) uu.print_log(tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # List of output directories and output file name patterns output_dir_list = [cn.peat_mask_dir] output_pattern_list = [cn.pattern_peat_mask] # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # Download SoilGrids250 most probable soil class rasters. # There are 459 tiles and it takes about 20 minutes to download them cmd = [ 'wget', '--recursive', '--no-parent', '-nH', '--cut-dirs=7', '--accept', '*.geotiff', '{}'.format(cn.soilgrids250_peat_url) ] uu.log_subprocess_output_full(cmd) uu.print_log("Making SoilGrids250 most likely soil class vrt...") check_call('gdalbuildvrt most_likely_soil_class.vrt *{}*'.format( cn.pattern_soilgrids_most_likely_class), shell=True) uu.print_log("Done making SoilGrids250 most likely soil class vrt") # Downloads peat layers uu.s3_file_download( os.path.join(cn.peat_unprocessed_dir, cn.cifor_peat_file), cn.docker_base_dir, sensit_type) uu.s3_file_download( os.path.join(cn.peat_unprocessed_dir, cn.jukka_peat_zip), cn.docker_base_dir, sensit_type) # Unzips the Jukka peat shapefile (IDN and MYS) cmd = ['unzip', '-o', '-j', cn.jukka_peat_zip] uu.log_subprocess_output_full(cmd) jukka_tif = 'jukka_peat.tif' # Converts the Jukka peat shapefile to a raster uu.print_log('Rasterizing jukka peat...') cmd = [ 'gdal_rasterize', '-burn', '1', '-co', 'COMPRESS=LZW', '-tr', '{}'.format(cn.Hansen_res), '{}'.format(cn.Hansen_res), '-tap', '-ot', 'Byte', '-a_nodata', '0', cn.jukka_peat_shp, jukka_tif ] uu.log_subprocess_output_full(cmd) uu.print_log(' Jukka peat rasterized') # For multiprocessor use # count-10 maxes out at about 100 GB on an r5d.16xlarge processes = cn.count - 5 uu.print_log('Peatland preprocessing max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(peatland_processing.create_peat_mask_tiles, tile_id_list) pool.close() pool.join() # # For single processor use, for testing purposes # for tile_id in tile_id_list: # # peatland_processing.create_peat_mask_tiles(tile_id) output_pattern = output_pattern_list[0] processes = 50 # 50 processors = XXX GB peak uu.print_log( "Checking for empty tiles of {0} pattern with {1} processors...". format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map( partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() uu.print_log("Uploading output files") uu.upload_final_set(output_dir_list[0], output_pattern_list[0])
def mp_plantation_preparation(gadm_index_shp, planted_index_shp): os.chdir(cn.docker_base_dir) # ## Not actually using this but leaving it here in case I want to add this functionality eventually. This # # was to allow users to run plantations for a select (contiguous) area rather than for the whole planet. # # List of bounding box coordinates # bound_list = args.bounding_box # # Checks if bounding box coordinates are in multiples of 10 (10 degree tiles). If they're not, the script stops. # for bound in bound_list: # if bound%10: # uu.exception_log(bound, 'not a multiple of 10. Please make bounding box coordinates are multiples of 10.') # Checks the validity of the two arguments. If either one is invalid, the script ends. if (gadm_index_path not in cn.gadm_plant_1x1_index_dir or planted_index_path not in cn.gadm_plant_1x1_index_dir): uu.exception_log('Invalid inputs. Please provide None or s3 shapefile locations for both arguments.') # List of all possible 10x10 Hansen tiles except for those at very extreme latitudes (not just WHRC biomass tiles) total_tile_list = uu.tile_list_s3(cn.pixel_area_dir) uu.print_log("Number of possible 10x10 tiles to evaluate:", len(total_tile_list)) # Removes the latitude bands that don't have any planted forests in them according to Liz Goldman. # i.e., Liz Goldman said by Slack on 1/2/19 that the nothernmost planted forest is 69.5146 and the southernmost is -46.938968. # This creates a more focused list of 10x10 tiles to iterate through (removes ones that definitely don't have planted forest). # NOTE: If the planted forest gdb is updated, the list of latitudes to exclude below may need to be changed to not exclude certain latitude bands. planted_lat_tile_list = [tile for tile in total_tile_list if '90N' not in tile] planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '80N' not in tile] planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '50S' not in tile] planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '60S' not in tile] planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '70S' not in tile] planted_lat_tile_list = [tile for tile in planted_lat_tile_list if '80S' not in tile] # planted_lat_tile_list = ['10N_080W'] uu.print_log(planted_lat_tile_list) uu.print_log("Number of 10x10 tiles to evaluate after extreme latitudes have been removed:", len(planted_lat_tile_list)) # If a planted forest extent 1x1 tile index shapefile isn't supplied if 'None' in args.planted_tile_index: ### Entry point 1: # If no shapefile of 1x1 tiles for countries with planted forests is supplied, 1x1 tiles of country extents will be created. # This runs the process from the very beginning and will take a few days. if 'None' in args.gadm_tile_index: uu.print_log("No GADM 1x1 tile index shapefile provided. Creating 1x1 planted forest country tiles from scratch...") # Downloads and unzips the GADM shapefile, which will be used to create 1x1 tiles of land areas uu.s3_file_download(cn.gadm_path, cn.docker_base_dir) cmd = ['unzip', cn.gadm_zip] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Creates a new GADM shapefile with just the countries that have planted forests in them. # This limits creation of 1x1 rasters of land area on the countries that have planted forests rather than on all countries. # NOTE: If the planted forest gdb is updated and has new countries added to it, the planted forest country list # in constants_and_names.py must be updated, too. uu.print_log("Creating shapefile of countries with planted forests...") os.system('''ogr2ogr -sql "SELECT * FROM gadm_3_6_adm2_final WHERE iso IN ({0})" {1} gadm_3_6_adm2_final.shp'''.format(str(cn.plantation_countries)[1:-1], cn.gadm_iso)) # Creates 1x1 degree tiles of countries that have planted forests in them. # I think this can handle using 50 processors because it's not trying to upload files to s3 and the tiles are small. # This takes several days to run because it iterates through at least 250 10x10 tiles. # For multiprocessor use. processes = 50 uu.print_log('Rasterize GADM 1x1 max processors=', processes) pool = Pool(processes) pool.map(plantation_preparation.rasterize_gadm_1x1, planted_lat_tile_list) pool.close() pool.join() # # Creates 1x1 degree tiles of countries that have planted forests in them. # # For single processor use. # for tile in planted_lat_tile_list: # # plantation_preparation.rasterize_gadm_1x1(tile) # Creates a shapefile of the boundaries of the 1x1 GADM tiles in countries with planted forests os.system('''gdaltindex {0}_{1}.shp GADM_*.tif'''.format(cn.pattern_gadm_1x1_index, uu.date_time_today)) cmd = ['aws', 's3', 'cp', cn.docker_base_dir, cn.gadm_plant_1x1_index_dir, '--exclude', '*', '--include', '{}*'.format(cn.pattern_gadm_1x1_index), '--recursive'] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # # Saves the 1x1 country extent tiles to s3 # # Only use if the entire process can't run in one go on the spot machine # cmd = ['aws', 's3', 'cp', cn.docker_base_dir, 's3://gfw2-data/climate/carbon_model/temp_spotmachine_output/', '--exclude', '*', '--include', 'GADM_*.tif', '--recursive'] # # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging # process = Popen(cmd, stdout=PIPE, stderr=STDOUT) # with process.stdout: # uu.log_subprocess_output(process.stdout) # Delete the aux.xml files os.system('''rm GADM*.tif.*''') # List of all 1x1 degree countey extent tiles created gadm_list_1x1 = uu.tile_list_spot_machine(".", "GADM_") uu.print_log("List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1) uu.print_log(len(gadm_list_1x1)) ### Entry point 2: # If a shapefile of the boundaries of 1x1 degree tiles of countries with planted forests is supplied, # a list of the 1x1 tiles is created from the shapefile. # This avoids creating the 1x1 country extent tiles all over again because the relevant tile extent are supplied # in the shapefile. elif cn.gadm_plant_1x1_index_dir in args.gadm_tile_index: uu.print_log("Country extent 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest tiles...") uu.print_log('{}/'.format(gadm_index_path)) # Copies the shapefile of 1x1 tiles of extent of countries with planted forests cmd = ['aws', 's3', 'cp', '{}/'.format(gadm_index_path), cn.docker_base_dir, '--recursive', '--exclude', '*', '--include', '{}*'.format(gadm_index_shp)] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Gets the attribute table of the country extent 1x1 tile shapefile gadm = glob.glob('{}*.dbf'.format(cn.pattern_gadm_1x1_index))[0] # Converts the attribute table to a dataframe dbf = Dbf5(gadm) df = dbf.to_dataframe() # Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list gadm_list_1x1 = df['location'].tolist() gadm_list_1x1 = [str(y) for y in gadm_list_1x1] uu.print_log("List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1) uu.print_log("There are", len(gadm_list_1x1), "1x1 country extent tiles to iterate through.") # In case some other arguments are provided else: uu.exception_log('Invalid GADM tile index shapefile provided. Please provide a valid shapefile.') # Creates 1x1 degree tiles of plantation growth wherever there are plantations. # Because this is iterating through all 1x1 tiles in countries with planted forests, it first checks # whether each 1x1 tile intersects planted forests before creating a 1x1 planted forest tile for that # 1x1 country extent tile. # 55 processors seems to use about 350 GB of memory, which seems fine. But there was some error about "PQconnectdb failed-- sorry, too many clients already". # So, moved the number of processors down to 48. # For multiprocessor use processes = 48 uu.print_log('Create 1x1 plantation from 1x1 gadm max processors=', processes) pool = Pool(processes) pool.map(plantation_preparation.create_1x1_plantation_from_1x1_gadm, gadm_list_1x1) pool.close() pool.join() # # Creates 1x1 degree tiles of plantation growth wherever there are plantations # # For single processor use # for tile in gadm_list_1x1: # # plantation_preparation.create_1x1_plantation(tile) # Creates a shapefile in which each feature is the extent of a plantation extent tile. # This index shapefile can be used the next time this process is run if starting with Entry Point 3. os.system('''gdaltindex {0}_{1}.shp plant_gain_*.tif'''.format(cn.pattern_plant_1x1_index, uu.date_time_today)) cmd = ['aws', 's3', 'cp', cn.docker_base_dir, cn.gadm_plant_1x1_index_dir, '--exclude', '*', '--include', '{}*'.format(cn.pattern_plant_1x1_index), '--recursive'] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) ### Entry point 3 # If a shapefile of the extents of 1x1 planted forest tiles is provided. # This is the part that actually creates the sequestration rate and forest type tiles. if cn.pattern_plant_1x1_index in args.planted_tile_index: uu.print_log("Planted forest 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest growth rate and forest type tiles...") # Copies the shapefile of 1x1 tiles of extent of planted forests cmd = ['aws', 's3', 'cp', '{}/'.format(planted_index_path), cn.docker_base_dir, '--recursive', '--exclude', '*', '--include', '{}*'.format(planted_index_shp), '--recursive'] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Gets the attribute table of the planted forest extent 1x1 tile shapefile gadm = glob.glob('{}*.dbf'.format(cn.pattern_plant_1x1_index))[0] # Converts the attribute table to a dataframe dbf = Dbf5(gadm) df = dbf.to_dataframe() # Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list planted_list_1x1 = df['location'].tolist() planted_list_1x1 = [str(y) for y in planted_list_1x1] uu.print_log("List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", planted_list_1x1) uu.print_log("There are", len(planted_list_1x1), "1x1 planted forest extent tiles to iterate through.") # Creates 1x1 degree tiles of plantation growth and type wherever there are plantations. # Because this is iterating through only 1x1 tiles that are known to have planted forests (from a previous run # of this script), it does not need to check whether there are planted forests in this tile. It goes directly # to intersecting the planted forest table with the 1x1 tile. # For single processor use #for tile in planted_list_1x1: # plantation_preparation.create_1x1_plantation_growth_from_1x1_planted(tile) # For multiprocessor use # processes=40 uses about 360 GB of memory. Works on r4.16xlarge with space to spare # processes=52 uses about 465 GB of memory (quite stably), so this is basically the max. num_of_processes = 52 pool = Pool(num_of_processes) pool.map(plantation_preparation.create_1x1_plantation_growth_from_1x1_planted, planted_list_1x1) pool.close() pool.join() # This works with 50 processors on an r4.16xlarge marchine. Uses about 430 GB out of 480 GB. num_of_processes = 52 pool = Pool(num_of_processes) processes = 50 uu.print_log('Create 1x1 plantation type max processors=', processes) pool = Pool(processes) pool.map(plantation_preparation.create_1x1_plantation_type_from_1x1_planted, planted_list_1x1) pool.close() pool.join() # This rasterizes the plantation removal factor standard deviations # processes=50 peaks at about 450 GB num_of_processes = 50 pool = Pool(num_of_processes) pool.map(plantation_preparation.create_1x1_plantation_stdev_from_1x1_planted, planted_list_1x1) pool.close() pool.join()
def mp_create_inputs_for_C_pools(tile_id_list, run_date=None): os.chdir(cn.docker_base_dir) sensit_type = 'std' # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.tile_list_s3(cn.model_extent_dir, sensit_type) # List of output directories and output file name patterns output_dir_list = [ cn.bor_tem_trop_processed_dir, cn.elevation_processed_dir, cn.precip_processed_dir ] output_pattern_list = [ cn.pattern_bor_tem_trop_processed, cn.pattern_elevation, cn.pattern_precip ] # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # Downloads two of the raw input files for creating carbon emitted_pools input_files = [cn.fao_ecozone_raw_dir, cn.precip_raw_dir] for input in input_files: uu.s3_file_download('{}'.format(input), cn.docker_base_dir, sensit_type) uu.print_log( "Unzipping boreal/temperate/tropical file (from FAO ecozones)") cmd = [ 'unzip', '{}'.format(cn.pattern_fao_ecozone_raw), '-d', cn.docker_base_dir ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) uu.print_log("Copying elevation (srtm) files") uu.s3_folder_download(cn.srtm_raw_dir, './srtm', sensit_type) uu.print_log("Making elevation (srtm) vrt") check_call( 'gdalbuildvrt srtm.vrt srtm/*.tif', shell=True ) # I don't know how to convert this to output to the pipe, so just leaving as is # Worked with count/3 on an r4.16xlarge (140 out of 480 GB used). I think it should be fine with count/2 but didn't try it. processes = int(cn.count / 2) uu.print_log('Inputs for C emitted_pools max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(create_inputs_for_C_pools.create_input_files, tile_id_list) # # For single processor use # for tile_id in tile_id_list: # # create_inputs_for_C_pools.create_input_files(tile_id) uu.print_log("Uploading output files") for i in range(0, len(output_dir_list)): uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
# For downloading all tiles in the input folders. input_files = [ cn.natrl_forest_biomass_2000_dir, cn.mangrove_biomass_2000_dir, cn.fao_ecozone_processed_dir, cn.precip_processed_dir, cn.soil_C_processed_dir, cn.srtm_processed_dir ] # for input in input_files: # uu.s3_folder_download('{}'.format(input), '.') # For copying individual tiles to spot machine for testing. for tile in tile_list: uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.natrl_forest_biomass_emitted_dir, tile, cn.pattern_natrl_forest_biomass_emitted), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.mangrove_biomass_emitted_dir, tile, cn.pattern_mangrove_biomass_emitted), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.fao_ecozone_processed_dir, tile, cn.pattern_fao_ecozone_processed), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.precip_processed_dir, tile, cn.pattern_precip), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.soil_C_processed_dir, tile, cn.pattern_soil_C), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.srtm_processed_dir, tile, cn.pattern_srtm),
def main(): # Create the output log uu.initiate_log() os.chdir(cn.docker_base_dir) # Files to download for this script. download_dict = { cn.gain_dir: [cn.pattern_gain], cn.annual_gain_AGB_IPCC_defaults_dir: [cn.pattern_annual_gain_AGB_IPCC_defaults] } # List of tiles that could be run. This list is only used to create the FIA region tiles if they don't already exist. tile_id_list = uu.tile_list_s3(cn.annual_gain_AGB_IPCC_defaults_dir) # tile_id_list = ["00N_000E", "00N_050W", "00N_060W", "00N_010E", "00N_020E", "00N_030E", "00N_040E", "10N_000E", "10N_010E", "10N_010W", "10N_020E", "10N_020W"] # test tiles # tile_id_list = ['50N_130W'] # test tiles # List of output directories and output file name patterns output_dir_list = [ cn.US_annual_gain_AGB_natrl_forest_dir, cn.US_annual_gain_BGB_natrl_forest_dir ] output_pattern_list = [ cn.pattern_US_annual_gain_AGB_natrl_forest, cn.pattern_US_annual_gain_BGB_natrl_forest ] # By definition, this script is for US-specific removals sensit_type = 'US_removals' # Counts how many processed FIA region tiles there are on s3 already. 16 tiles cover the continental US. FIA_regions_tile_count = uu.count_tiles_s3(cn.FIA_regions_processed_dir) # Only creates FIA region tiles if they don't already exist on s3. if FIA_regions_tile_count == 16: uu.print_log("FIA region tiles already created. Copying to s3 now...") uu.s3_flexible_download(cn.FIA_regions_processed_dir, cn.pattern_FIA_regions_processed, cn.docker_base_dir, 'std', 'all') else: uu.print_log( "FIA region tiles do not exist. Creating tiles, then copying to s3 for future use..." ) uu.s3_file_download( os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, 'std') cmd = ['unzip', '-o', '-j', cn.name_FIA_regions_raw] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Converts the region shapefile to Hansen tiles pool = multiprocessing.Pool(int(cn.count / 2)) pool.map(US_removal_rates.prep_FIA_regions, tile_id_list) # List of FIA region tiles on the spot machine. Only this list is used for the rest of the script. US_tile_list = uu.tile_list_spot_machine( cn.docker_base_dir, '{}.tif'.format(cn.pattern_FIA_regions_processed)) US_tile_id_list = [i[0:8] for i in US_tile_list] # US_tile_id_list = ['50N_130W'] # For testing uu.print_log(US_tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(US_tile_id_list))) + "\n") # Counts how many processed forest age category tiles there are on s3 already. 16 tiles cover the continental US. US_age_tile_count = uu.count_tiles_s3(cn.US_forest_age_cat_processed_dir) # Only creates FIA forest age category tiles if they don't already exist on s3. if US_age_tile_count == 16: uu.print_log( "Forest age category tiles already created. Copying to spot machine now..." ) uu.s3_flexible_download(cn.US_forest_age_cat_processed_dir, cn.pattern_US_forest_age_cat_processed, '', 'std', US_tile_id_list) else: uu.print_log( "Southern forest age category tiles do not exist. Creating tiles, then copying to s3 for future use..." ) uu.s3_file_download( os.path.join(cn.US_forest_age_cat_raw_dir, cn.name_US_forest_age_cat_raw), cn.docker_base_dir, 'std') # Converts the national forest age category raster to Hansen tiles source_raster = cn.name_US_forest_age_cat_raw out_pattern = cn.pattern_US_forest_age_cat_processed dt = 'Int16' pool = multiprocessing.Pool(int(cn.count / 2)) pool.map( partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), US_tile_id_list) uu.upload_final_set(cn.US_forest_age_cat_processed_dir, cn.pattern_US_forest_age_cat_processed) # Counts how many processed FIA forest group tiles there are on s3 already. 16 tiles cover the continental US. FIA_forest_group_tile_count = uu.count_tiles_s3( cn.FIA_forest_group_processed_dir) # Only creates FIA forest group tiles if they don't already exist on s3. if FIA_forest_group_tile_count == 16: uu.print_log( "FIA forest group tiles already created. Copying to spot machine now..." ) uu.s3_flexible_download(cn.FIA_forest_group_processed_dir, cn.pattern_FIA_forest_group_processed, '', 'std', US_tile_id_list) else: uu.print_log( "FIA forest group tiles do not exist. Creating tiles, then copying to s3 for future use..." ) uu.s3_file_download( os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, 'std') # Converts the national forest group raster to Hansen forest group tiles source_raster = cn.name_FIA_forest_group_raw out_pattern = cn.pattern_FIA_forest_group_processed dt = 'Byte' pool = multiprocessing.Pool(int(cn.count / 2)) pool.map( partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt), US_tile_id_list) uu.upload_final_set(cn.FIA_forest_group_processed_dir, cn.pattern_FIA_forest_group_processed) # Downloads input files or entire directories, depending on how many tiles are in the tile_id_list for key, values in download_dict.items(): dir = key pattern = values[0] uu.s3_flexible_download(dir, pattern, cn.docker_base_dir, sensit_type, US_tile_id_list) # Table with US-specific removal rates cmd = [ 'aws', 's3', 'cp', os.path.join(cn.gain_spreadsheet_dir, cn.table_US_removal_rate), cn.docker_base_dir ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Imports the table with the region-group-age AGB removal rates gain_table = pd.read_excel("{}".format(cn.table_US_removal_rate), sheet_name="US_rates_for_model") # Converts gain table from wide to long, so each region-group-age category has its own row gain_table_group_region_by_age = pd.melt( gain_table, id_vars=['FIA_region_code', 'forest_group_code'], value_vars=['growth_young', 'growth_middle', 'growth_old']) gain_table_group_region_by_age = gain_table_group_region_by_age.dropna() # In the forest age category raster, each category has this value age_dict = { 'growth_young': 1000, 'growth_middle': 2000, 'growth_old': 3000 } # Creates a unique value for each forest group-region-age category in the table. # Although these rates are applied to all standard gain model pixels at first, they are not ultimately used for # pixels that have Hansen gain (see below). gain_table_group_region_age = gain_table_group_region_by_age.replace( {"variable": age_dict}) gain_table_group_region_age[ 'age_cat'] = gain_table_group_region_age['variable'] * 10 gain_table_group_region_age['group_region_age_combined'] = gain_table_group_region_age['age_cat'] + \ gain_table_group_region_age['forest_group_code']*100 + \ gain_table_group_region_age['FIA_region_code'] # Converts the forest group-region-age codes and corresponding gain rates to a dictionary, # where the key is the unique group-region-age code and the value is the AGB removal rate. gain_table_group_region_age_dict = pd.Series( gain_table_group_region_age.value.values, index=gain_table_group_region_age.group_region_age_combined).to_dict() uu.print_log(gain_table_group_region_age_dict) # Creates a unique value for each forest group-region category using just young forest rates. # These are assigned to Hansen gain pixels, which automatically get the young forest rate, regardless of the # forest age category raster. gain_table_group_region = gain_table_group_region_age.drop( gain_table_group_region_age[ gain_table_group_region_age.age_cat != 10000].index) gain_table_group_region['group_region_combined'] = gain_table_group_region['forest_group_code']*100 + \ gain_table_group_region['FIA_region_code'] # Converts the forest group-region codes and corresponding gain rates to a dictionary, # where the key is the unique group-region code (youngest age category) and the value is the AGB removal rate. gain_table_group_region_dict = pd.Series( gain_table_group_region.value.values, index=gain_table_group_region.group_region_combined).to_dict() uu.print_log(gain_table_group_region_dict) # count/2 on a m4.16xlarge maxes out at about 230 GB of memory (processing 16 tiles at once), so it's okay on an m4.16xlarge pool = multiprocessing.Pool(int(cn.count / 2)) pool.map( partial( US_removal_rates.US_removal_rate_calc, gain_table_group_region_age_dict=gain_table_group_region_age_dict, gain_table_group_region_dict=gain_table_group_region_dict, output_pattern_list=output_pattern_list, sensit_type=sensit_type), US_tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in US_tile_id_list: # # US_removal_rates.US_removal_rate_calc(tile_id, gain_table_group_region_age_dict, gain_table_group_region_dict, # output_pattern_list, sensit_type) # Uploads output tiles to s3 for i in range(0, len(output_dir_list)): uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
# For downloading all tiles in the input folders. input_files = [ cn.natrl_forest_biomass_2000_dir, cn.mangrove_biomass_2000_dir, cn.cumul_gain_AGC_mangrove_dir, cn.cumul_gain_AGC_natrl_forest_dir ] # for input in input_files: # uu.s3_folder_download('{}'.format(input), '.') # For copying individual tiles to spot machine for testing. # The cumulative carbon gain tiles are for adding to the biomass 2000 tiles to get AGC at the time of tree cover loss. for tile in tile_list: uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.natrl_forest_biomass_2000_dir, tile, cn.pattern_natrl_forest_biomass_2000), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.mangrove_biomass_raw_dir, tile, cn.pattern_mangrove_biomass_emitted), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.cumul_gain_AGC_mangrove_dir, cn.pattern_cumul_gain_AGC_mangrove, tile), '.') uu.s3_file_download( '{0}{1}_{2}.tif'.format(cn.cumul_gain_AGC_natrl_forest_dir, cn.pattern_cumul_gain_AGC_natrl_forest, tile), '.') print "Creating tiles of emitted biomass (biomass 2000 + biomass accumulation)" count = multiprocessing.cpu_count()
def mp_prep_other_inputs(tile_id_list, run_date, no_upload=None): os.chdir(cn.docker_base_dir) sensit_type = 'std' # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model ### BUG: THIS SHOULD ALSO INCLUDE cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir IN ITS LIST tile_id_list = uu.create_combined_tile_list( cn.WHRC_biomass_2000_unmasked_dir, cn.mangrove_biomass_2000_dir, set3=cn.gain_dir) uu.print_log(tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(tile_id_list))) + "\n") ''' Before processing the driver, it needs to be reprojected from Goode Homolosine to WGS84. gdal_warp is producing a weird output, so I did it in ArcMap for the 2020 update, with the output cell size being 0.01 x 0.01 degree and the method being nearest. arcpy.ProjectRaster_management(in_raster="C:/GIS/Drivers of loss/2020_drivers__tif__from_Forrest_Follett_20210323/FinalClassification_2020_v2__from_Jimmy_MacCarthy_20210323.tif", out_raster="C:/GIS/Drivers of loss/2020_drivers__tif__from_Forrest_Follett_20210323/Final_Classification_2020__reproj_nearest_0-005_0-005_deg__20210323.tif", out_coor_system="GEOGCS['GCS_WGS_1984',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]]", resampling_type="NEAREST", cell_size="0.005 0.005", geographic_transform="", Registration_Point="", in_coor_system="PROJCS['WGS_1984_Goode_Homolosine',GEOGCS['GCS_unknown',DATUM['D_WGS_1984',SPHEROID['WGS_1984',6378137.0,298.257223563]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Goode_Homolosine'],PARAMETER['False_Easting',0.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',0.0],PARAMETER['Option',1.0],UNIT['Meter',1.0]]", vertical="NO_VERTICAL") ''' # List of output directories and output file name patterns output_dir_list = [ # cn.climate_zone_processed_dir, cn.plant_pre_2000_processed_dir, cn.drivers_processed_dir # cn.ifl_primary_processed_dir, # cn.annual_gain_AGC_natrl_forest_young_dir, # cn.stdev_annual_gain_AGC_natrl_forest_young_dir, # cn.annual_gain_AGC_BGC_natrl_forest_Europe_dir, # cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_dir, # cn.FIA_forest_group_processed_dir, # cn.age_cat_natrl_forest_US_dir, # cn.FIA_regions_processed_dir ] output_pattern_list = [ # cn.pattern_climate_zone, cn.pattern_plant_pre_2000, cn.pattern_drivers # cn.pattern_ifl_primary, # cn.pattern_annual_gain_AGC_natrl_forest_young, # cn.pattern_stdev_annual_gain_AGC_natrl_forest_young, # cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe, # cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe, # cn.pattern_FIA_forest_group_processed, # cn.pattern_age_cat_natrl_forest_US, # cn.pattern_FIA_regions_processed ] # If the model run isn't the standard one, the output directory and file names are changed if sensit_type != 'std': uu.print_log( "Changing output directory and file name pattern based on sensitivity analysis" ) output_dir_list = uu.alter_dirs(sensit_type, output_dir_list) output_pattern_list = uu.alter_patterns(sensit_type, output_pattern_list) # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # # Files to process: climate zone, IDN/MYS plantations before 2000, tree cover loss drivers, combine IFL and primary forest # uu.s3_file_download(os.path.join(cn.climate_zone_raw_dir, cn.climate_zone_raw), cn.docker_base_dir, sensit_type) # uu.s3_file_download(os.path.join(cn.plant_pre_2000_raw_dir, '{}.zip'.format(cn.pattern_plant_pre_2000_raw)), cn.docker_base_dir, sensit_type) uu.s3_file_download( os.path.join(cn.drivers_raw_dir, cn.pattern_drivers_raw), cn.docker_base_dir, sensit_type) # uu.s3_file_download(os.path.join(cn.annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type) # uu.s3_file_download(os.path.join(cn.stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw_dir, cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw), cn.docker_base_dir, sensit_type) # uu.s3_file_download(os.path.join(cn.FIA_regions_raw_dir, cn.name_FIA_regions_raw), cn.docker_base_dir, sensit_type) # uu.s3_file_download(os.path.join(cn.age_cat_natrl_forest_US_raw_dir, cn.name_age_cat_natrl_forest_US_raw), cn.docker_base_dir, sensit_type) # uu.s3_file_download(os.path.join(cn.FIA_forest_group_raw_dir, cn.name_FIA_forest_group_raw), cn.docker_base_dir, sensit_type) # # For some reason, using uu.s3_file_download or otherwise using AWSCLI as a subprocess doesn't work for this raster. # # Thus, using wget instead. # cmd = ['wget', '{}'.format(cn.annual_gain_AGC_natrl_forest_young_raw_URL), '-P', '{}'.format(cn.docker_base_dir)] # process = Popen(cmd, stdout=PIPE, stderr=STDOUT) # with process.stdout: # uu.log_subprocess_output(process.stdout) # uu.s3_file_download(cn.stdev_annual_gain_AGC_natrl_forest_young_raw_URL, cn.docker_base_dir, sensit_type) # cmd = ['aws', 's3', 'cp', cn.primary_raw_dir, cn.docker_base_dir, '--recursive'] # uu.log_subprocess_output_full(cmd) # # uu.s3_flexible_download(cn.ifl_dir, cn.pattern_ifl, cn.docker_base_dir, sensit_type, tile_id_list) # # uu.print_log("Unzipping pre-2000 plantations...") # cmd = ['unzip', '-j', '{}.zip'.format(cn.pattern_plant_pre_2000_raw)] # uu.log_subprocess_output_full(cmd) # Creates tree cover loss driver tiles. # The raw driver tile should have NoData for unassigned drivers as opposed to 0 for unassigned drivers. # For the 2020 driver update, I reclassified the 0 values as NoData in ArcMap. I also unprojected the global drivers # map to WGS84 because running the homolosine projection that Jimmy provided was giving incorrect processed results. source_raster = cn.pattern_drivers_raw out_pattern = cn.pattern_drivers dt = 'Byte' if cn.count == 96: processes = 87 # 45 processors = 70 GB peak; 70 = 90 GB peak; 80 = 100 GB peak; 87 = 125 GB peak else: processes = int(cn.count / 2) uu.print_log( "Creating tree cover loss driver tiles with {} processors...".format( processes)) pool = multiprocessing.Pool(processes) pool.map( partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # Creates young natural forest removal rate tiles # source_raster = cn.name_annual_gain_AGC_natrl_forest_young_raw # out_pattern = cn.pattern_annual_gain_AGC_natrl_forest_young # dt = 'float32' # if cn.count == 96: # processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating young natural forest gain rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # Creates young natural forest removal rate standard deviation tiles # source_raster = cn.name_stdev_annual_gain_AGC_natrl_forest_young_raw # out_pattern = cn.pattern_stdev_annual_gain_AGC_natrl_forest_young # dt = 'float32' # if cn.count == 96: # processes = 80 # 32 processors = 210 GB peak; 60 = 370 GB peak; 80 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating standard deviation for young natural forest removal rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # # Creates pre-2000 oil palm plantation tiles # if cn.count == 96: # processes = 80 # 45 processors = 100 GB peak; 80 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating pre-2000 oil palm plantation tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(prep_other_inputs.rasterize_pre_2000_plantations, tile_id_list) # pool.close() # pool.join() # # # # Creates climate zone tiles # if cn.count == 96: # processes = 80 # 45 processors = 230 GB peak (on second step); 80 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating climate zone tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(prep_other_inputs.create_climate_zone_tiles, tile_id_list) # pool.close() # pool.join() # # # Creates European natural forest removal rate tiles # source_raster = cn.name_annual_gain_AGC_BGC_natrl_forest_Europe_raw # out_pattern = cn.pattern_annual_gain_AGC_BGC_natrl_forest_Europe # dt = 'float32' # if cn.count == 96: # processes = 60 # 32 processors = 60 GB peak; 60 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating European natural forest gain rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # Creates European natural forest standard deviation of removal rate tiles # source_raster = cn.name_stdev_annual_gain_AGC_BGC_natrl_forest_Europe_raw # out_pattern = cn.pattern_stdev_annual_gain_AGC_BGC_natrl_forest_Europe # dt = 'float32' # if cn.count == 96: # processes = 32 # 32 processors = 60 GB peak; 60 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating standard deviation for European natural forest gain rate tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # # Creates a vrt of the primary forests with nodata=0 from the continental primary forest rasters # uu.print_log("Creating vrt of humid tropial primary forest...") # primary_vrt = 'primary_2001.vrt' # os.system('gdalbuildvrt -srcnodata 0 {} *2001_primary.tif'.format(primary_vrt)) # uu.print_log(" Humid tropical primary forest vrt created") # # # Creates primary forest tiles # source_raster = primary_vrt # out_pattern = 'primary_2001' # dt = 'Byte' # if cn.count == 96: # processes = 45 # 45 processors = 650 GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating primary forest tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # # Creates a combined IFL/primary forest raster # # Uses very little memory since it's just file renaming # if cn.count == 96: # processes = 60 # 60 processors = 10 GB peak # else: # processes = int(cn.count/2) # uu.print_log("Assigning each tile to ifl2000 or primary forest with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(prep_other_inputs.create_combined_ifl_primary, tile_id_list) # pool.close() # pool.join() # # # # Creates forest age category tiles for US forests # source_raster = cn.name_age_cat_natrl_forest_US_raw # out_pattern = cn.pattern_age_cat_natrl_forest_US # dt = 'Byte' # if cn.count == 96: # processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating US forest age category tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # Creates forest groups for US forests # source_raster = cn.name_FIA_forest_group_raw # out_pattern = cn.pattern_FIA_forest_group_processed # dt = 'Byte' # if cn.count == 96: # processes = 80 # 32 processors = 25 GB peak; 80 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating US forest group tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # # Creates FIA regions for US forests # source_raster = cn.name_FIA_regions_raw # out_pattern = cn.pattern_FIA_regions_processed # dt = 'Byte' # if cn.count == 96: # processes = 70 # 32 processors = 35 GB peak; 70 = XXX GB peak # else: # processes = int(cn.count/2) # uu.print_log("Creating US forest region tiles with {} processors...".format(processes)) # pool = multiprocessing.Pool(processes) # pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) # pool.close() # pool.join() # # for output_pattern in [ cn.pattern_drivers # ,cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young ]: # For some reason I can't figure out, the young forest rasters (rate and stdev) have NaN values in some places where 0 (NoData) # should be. These NaN values show up as values when the check_and_delete_if_empty function runs, making the tiles not # deleted even if they have no data. However, the light version (which uses gdalinfo rather than rasterio masks) doesn't # have this problem. So I'm forcing the young forest rates to and stdev to have their emptiness checked by the gdalinfo version. if output_pattern in [ cn.pattern_annual_gain_AGC_natrl_forest_young, cn.pattern_stdev_annual_gain_AGC_natrl_forest_young ]: processes = int(cn.count / 2) uu.print_log( "Checking for empty tiles of {0} pattern with {1} processors using light function..." .format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map( partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() if cn.count == 96: processes = 50 # 60 processors = >730 GB peak (for European natural forest forest removal rates); 50 = XXX GB peak uu.print_log( "Checking for empty tiles of {0} pattern with {1} processors..." .format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map( partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() elif cn.count <= 2: # For local tests processes = 1 uu.print_log( "Checking for empty tiles of {0} pattern with {1} processors using light function..." .format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map( partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() else: processes = int(cn.count / 2) uu.print_log( "Checking for empty tiles of {0} pattern with {1} processors..." .format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map( partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() uu.print_log('\n') # Uploads output tiles to s3 for i in range(0, len(output_dir_list)): uu.upload_final_set(output_dir_list[i], output_pattern_list[i])
def mp_create_soil_C(tile_id_list, no_upload=None): os.chdir(cn.docker_base_dir) sensit_type = 'std' # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.create_combined_tile_list(cn.WHRC_biomass_2000_unmasked_dir, cn.mangrove_biomass_2000_dir, set3=cn.gain_dir ) uu.print_log(tile_id_list) uu.print_log("There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # List of output directories and output file name patterns output_dir_list = [cn.soil_C_full_extent_2000_non_mang_dir, cn.soil_C_full_extent_2000_dir, cn.stdev_soil_C_full_extent_2000_dir] output_pattern_list = [cn.pattern_soil_C_full_extent_2000_non_mang, cn.pattern_soil_C_full_extent_2000, cn.pattern_stdev_soil_C_full_extent] ### Soil carbon density uu.print_log("Downloading mangrove soil C rasters") uu.s3_file_download(os.path.join(cn.mangrove_soil_C_dir, cn.name_mangrove_soil_C), cn.docker_base_dir, sensit_type) # For downloading all tiles in the input folders. input_files = [cn.mangrove_biomass_2000_dir] for input in input_files: uu.s3_folder_download(input, cn.docker_base_dir, sensit_type) # Download raw mineral soil C density tiles. # First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder # Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it # There are 12951 tiles and it takes about 3 hours to download them! cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*', '--accept', '*.tif', '{}'.format(cn.mineral_soil_C_url)] uu.log_subprocess_output_full(cmd) uu.print_log("Unzipping mangrove soil C rasters...") cmd = ['unzip', '-j', cn.name_mangrove_soil_C, '-d', cn.docker_base_dir] uu.log_subprocess_output_full(cmd) # Mangrove soil receives precedence over mineral soil uu.print_log("Making mangrove soil C vrt...") check_call('gdalbuildvrt mangrove_soil_C.vrt *{}*.tif'.format(cn.pattern_mangrove_soil_C_raw), shell=True) uu.print_log("Done making mangrove soil C vrt") uu.print_log("Making mangrove soil C tiles...") if cn.count == 96: processes = 32 # 32 processors = 570 GB peak else: processes = int(cn.count/3) uu.print_log('Mangrove soil C max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(create_soil_C.create_mangrove_soil_C, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: # # create_soil_C.create_mangrove_soil_C(tile_id, no_Upload) uu.print_log('Done making mangrove soil C tiles', '\n') uu.print_log("Making mineral soil C vrt...") check_call('gdalbuildvrt mineral_soil_C.vrt *{}*'.format(cn.pattern_mineral_soil_C_raw), shell=True) uu.print_log("Done making mineral soil C vrt") # Creates mineral soil C density tiles source_raster = 'mineral_soil_C.vrt' out_pattern = cn.pattern_soil_C_full_extent_2000_non_mang dt = 'Int16' if cn.count == 96: processes = 80 # 32 processors = 100 GB peak; 50 = 160 GB peak; 80 = XXX GB peak else: processes = int(cn.count/2) uu.print_log("Creating mineral soil C density tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile_id in tile_id_list: # # create_soil_C.create_mineral_soil_C(tile_id) uu.print_log("Done making non-mangrove soil C tiles", "\n") output_pattern = cn.pattern_soil_C_full_extent_2000_non_mang processes = 60 # 50 processors = ~450 GB peak; 60 = XXX GB peak uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() # If no_upload flag is not activated, output is uploaded to s3 if not no_upload: uu.print_log("Uploading non-mangrove soil C density tiles") uu.upload_final_set(output_dir_list[0], output_pattern_list[0]) uu.print_log("Making combined (mangrove & non-mangrove) soil C tiles...") if cn.count == 96: processes = 45 # 45 processors = XXX GB peak else: processes = int(cn.count/2) uu.print_log('Combined soil C max processors=', processes) pool = multiprocessing.Pool(processes) pool.map(partial(create_soil_C.create_combined_soil_C, no_upload=no_upload), tile_id_list) pool.close() pool.join() # # For single processor use # for tile in tile_list: # # create_soil_C.create_combined_soil_C(tile_id, no_upload) uu.print_log("Done making combined soil C tiles") # If no_upload flag is not activated, output is uploaded if not no_upload: uu.print_log("Uploading combined soil C density tiles") uu.upload_final_set(output_dir_list[1], output_pattern_list[1]) # Need to delete soil c density rasters because they have the same pattern as the standard deviation rasters uu.print_log("Deleting raw soil C density rasters") c_stocks = glob.glob('*{}*'.format(cn.pattern_soil_C_full_extent_2000)) for c_stock in c_stocks: os.remove(c_stock) ### Soil carbon density uncertainty # Separate directories for the 5% CI and 95% CI dir_CI05 = '{0}{1}'.format(cn.docker_base_dir, 'CI05/') dir_CI95 = '{0}{1}'.format(cn.docker_base_dir, 'CI95/') vrt_CI05 = 'mineral_soil_C_CI05.vrt' vrt_CI95 = 'mineral_soil_C_CI95.vrt' soil_C_stdev_global = 'soil_C_stdev.tif' # Download raw mineral soil C density 5% CI tiles # First tries to download index.html.tmp from every folder, then goes back and downloads all the tifs in each folder # Based on https://stackoverflow.com/questions/273743/using-wget-to-recursively-fetch-a-directory-with-arbitrary-files-in-it # Like soil C density rasters, there are 12951 tifs and they take about 3 hours to download. os.mkdir(dir_CI05) cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*', '--directory-prefix={}'.format(dir_CI05), '--accept', '*.tif', '{}'.format(cn.CI5_mineral_soil_C_url)] uu.log_subprocess_output_full(cmd) uu.print_log("Making mineral soil C 5% CI vrt...") check_call('gdalbuildvrt {0} {1}*{2}*'.format(vrt_CI05, dir_CI05, cn.pattern_uncert_mineral_soil_C_raw), shell=True) uu.print_log("Done making mineral soil C CI05 vrt") # Download raw mineral soil C density 5% CI tiles # Like soil C density rasters, there are 12951 tifs and they take about 3 hours to download. os.mkdir(dir_CI95) cmd = ['wget', '--recursive', '-nH', '--cut-dirs=6', '--no-parent', '--reject', 'index.html*', '--directory-prefix={}'.format(dir_CI95), '--accept', '*.tif', '{}'.format(cn.CI95_mineral_soil_C_url)] uu.log_subprocess_output_full(cmd) uu.print_log("Making mineral soil C 95% CI vrt...") check_call('gdalbuildvrt {0} {1}*{2}*'.format(vrt_CI95, dir_CI95, cn.pattern_uncert_mineral_soil_C_raw), shell=True) uu.print_log("Done making mineral soil C CI95 vrt") uu.print_log("Creating raster of standard deviations in soil C at native SoilGrids250 resolution. This may take a while...") # global tif with approximation of the soil C stanard deviation (based on the 5% and 95% CIs) # This takes about 20 minutes. It doesn't show any progress until the last moment, when it quickly counts # up to 100. calc = '--calc=(A-B)/3' out_filearg = '--outfile={}'.format(soil_C_stdev_global) cmd = ['gdal_calc.py', '-A', vrt_CI95, '-B', vrt_CI05, calc, out_filearg, '--NoDataValue=0', '--overwrite', '--co', 'COMPRESS=LZW', '--type=Float32'] uu.log_subprocess_output_full(cmd) uu.print_log("{} created.".format(soil_C_stdev_global)) # Creates soil carbon 2000 density standard deviation tiles out_pattern = cn.pattern_stdev_soil_C_full_extent dt = 'Float32' source_raster = soil_C_stdev_global if cn.count == 96: processes = 56 # 32 processors = 290 GB peak; 56 = XXX GB peal else: processes = 2 uu.print_log("Creating mineral soil C stock stdev tiles with {} processors...".format(processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.mp_warp_to_Hansen, source_raster=source_raster, out_pattern=out_pattern, dt=dt, no_upload=no_upload), tile_id_list) pool.close() pool.join() output_pattern = cn.pattern_stdev_soil_C_full_extent processes = 50 # 50 processors = 550 GB peak uu.print_log("Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() # Checks the gross removals outputs for tiles with no data for output_pattern in output_pattern_list: if cn.count <= 2: # For local tests processes = 1 uu.print_log("Checking for empty tiles of {0} pattern with {1} processors using light function...".format( output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty_light, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() else: processes = 55 # 50 processors = XXX GB peak uu.print_log( "Checking for empty tiles of {0} pattern with {1} processors...".format(output_pattern, processes)) pool = multiprocessing.Pool(processes) pool.map(partial(uu.check_and_delete_if_empty, output_pattern=output_pattern), tile_id_list) pool.close() pool.join() # If no_upload flag is not activated, output is uploaded if not no_upload: uu.print_log("Uploading soil C density standard deviation tiles") uu.upload_final_set(output_dir_list[2], output_pattern_list[2])
def mp_aggregate_results_to_4_km(sensit_type, thresh, tile_id_list, std_net_flux=None, run_date=None, no_upload=None): os.chdir(cn.docker_base_dir) # If a full model run is specified, the correct set of tiles for the particular script is listed if tile_id_list == 'all': # List of tiles to run in the model tile_id_list = uu.tile_list_s3(cn.net_flux_dir, sensit_type) uu.print_log(tile_id_list) uu.print_log( "There are {} tiles to process".format(str(len(tile_id_list))) + "\n") # Files to download for this script download_dict = { cn.annual_gain_AGC_all_types_dir: [cn.pattern_annual_gain_AGC_all_types], cn.cumul_gain_AGCO2_BGCO2_all_types_dir: [cn.pattern_cumul_gain_AGCO2_BGCO2_all_types], cn.gross_emis_all_gases_all_drivers_biomass_soil_dir: [cn.pattern_gross_emis_all_gases_all_drivers_biomass_soil], cn.net_flux_dir: [cn.pattern_net_flux] } # Checks whether the canopy cover argument is valid if thresh < 0 or thresh > 99: uu.exception_log( no_upload, 'Invalid tcd. Please provide an integer between 0 and 99.') if uu.check_aws_creds(): # Pixel area tiles-- necessary for calculating sum of pixels for any set of tiles uu.s3_flexible_download(cn.pixel_area_dir, cn.pattern_pixel_area, cn.docker_base_dir, sensit_type, tile_id_list) # Tree cover density, Hansen gain, and mangrove biomass tiles-- necessary for filtering sums to model extent uu.s3_flexible_download(cn.tcd_dir, cn.pattern_tcd, cn.docker_base_dir, sensit_type, tile_id_list) uu.s3_flexible_download(cn.gain_dir, cn.pattern_gain, cn.docker_base_dir, sensit_type, tile_id_list) uu.s3_flexible_download(cn.mangrove_biomass_2000_dir, cn.pattern_mangrove_biomass_2000, cn.docker_base_dir, sensit_type, tile_id_list) uu.print_log("Model outputs to process are:", download_dict) # List of output directories. Modified later for sensitivity analysis. # Output pattern is determined later. output_dir_list = [cn.output_aggreg_dir] # If the model run isn't the standard one, the output directory is changed if sensit_type != 'std': uu.print_log( "Changing output directory and file name pattern based on sensitivity analysis" ) output_dir_list = uu.alter_dirs(sensit_type, output_dir_list) # A date can optionally be provided by the full model script or a run of this script. # This replaces the date in constants_and_names. if run_date is not None: output_dir_list = uu.replace_output_dir_date(output_dir_list, run_date) # Iterates through the types of tiles to be processed for dir, download_pattern in list(download_dict.items()): download_pattern_name = download_pattern[0] # Downloads input files or entire directories, depending on how many tiles are in the tile_id_list, if AWS credentials are found if uu.check_aws_creds(): uu.s3_flexible_download(dir, download_pattern_name, cn.docker_base_dir, sensit_type, tile_id_list) # Gets an actual tile id to use as a dummy in creating the actual tile pattern local_tile_list = uu.tile_list_spot_machine(cn.docker_base_dir, download_pattern_name) sample_tile_id = uu.get_tile_id(local_tile_list[0]) # Renames the tiles according to the sensitivity analysis before creating dummy tiles. # The renaming function requires a whole tile name, so this passes a dummy time name that is then stripped a few # lines later. tile_id = sample_tile_id # a dummy tile id (but it has to be a real tile id). It is removed later. output_pattern = uu.sensit_tile_rename(sensit_type, tile_id, download_pattern_name) pattern = output_pattern[9:-4] # For sensitivity analysis runs, only aggregates the tiles if they were created as part of the sensitivity analysis if (sensit_type != 'std') & (sensit_type not in pattern): uu.print_log( "{} not a sensitivity analysis output. Skipping aggregation..." .format(pattern)) uu.print_log("") continue # Lists the tiles of the particular type that is being iterates through. # Excludes all intermediate files tile_list = uu.tile_list_spot_machine(".", "{}.tif".format(pattern)) # from https://stackoverflow.com/questions/12666897/removing-an-item-from-list-matching-a-substring tile_list = [i for i in tile_list if not ('hanson_2013' in i)] tile_list = [i for i in tile_list if not ('rewindow' in i)] tile_list = [i for i in tile_list if not ('0_4deg' in i)] tile_list = [i for i in tile_list if not ('.ovr' in i)] # tile_list = ['00N_070W_cumul_gain_AGCO2_BGCO2_t_ha_all_forest_types_2001_15_biomass_swap.tif'] # test tiles uu.print_log("There are {0} tiles to process for pattern {1}".format( str(len(tile_list)), download_pattern) + "\n") uu.print_log("Processing:", dir, "; ", pattern) # Converts the 10x10 degree Hansen tiles that are in windows of 40000x1 pixels to windows of 400x400 pixels, # which is the resolution of the output tiles. This will allow the 30x30 m pixels in each window to be summed. # For multiprocessor use. count/2 used about 400 GB of memory on an r4.16xlarge machine, so that was okay. if cn.count == 96: if sensit_type == 'biomass_swap': processes = 12 # 12 processors = XXX GB peak else: processes = 16 # 12 processors = 140 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out) else: processes = 8 uu.print_log('Rewindow max processors=', processes) pool = multiprocessing.Pool(processes) pool.map( partial(aggregate_results_to_4_km.rewindow, no_upload=no_upload), tile_list) # Added these in response to error12: Cannot allocate memory error. # This fix was mentioned here: of https://stackoverflow.com/questions/26717120/python-cannot-allocate-memory-using-multiprocessing-pool # Could also try this: https://stackoverflow.com/questions/42584525/python-multiprocessing-debugging-oserror-errno-12-cannot-allocate-memory pool.close() pool.join() # # For single processor use # for tile in tile_list: # # aggregate_results_to_4_km.rewindow(til, no_upload) # Converts the existing (per ha) values to per pixel values (e.g., emissions/ha to emissions/pixel) # and sums those values in each 400x400 pixel window. # The sum for each 400x400 pixel window is stored in a 2D array, which is then converted back into a raster at # 0.1x0.1 degree resolution (approximately 10m in the tropics). # Each pixel in that raster is the sum of the 30m pixels converted to value/pixel (instead of value/ha). # The 0.1x0.1 degree tile is output. # For multiprocessor use. This used about 450 GB of memory with count/2, it's okay on an r4.16xlarge if cn.count == 96: if sensit_type == 'biomass_swap': processes = 10 # 10 processors = XXX GB peak else: processes = 12 # 16 processors = 180 GB peak; 16 = XXX GB peak; 20 = >750 GB (maxed out) else: processes = 8 uu.print_log('Conversion to per pixel and aggregate max processors=', processes) pool = multiprocessing.Pool(processes) pool.map( partial(aggregate_results_to_4_km.aggregate, thresh=thresh, sensit_type=sensit_type, no_upload=no_upload), tile_list) pool.close() pool.join() # # For single processor use # for tile in tile_list: # # aggregate_results_to_4_km.aggregate(tile, thresh, sensit_type, no_upload) # Makes a vrt of all the output 10x10 tiles (10 km resolution) out_vrt = "{}_0_4deg.vrt".format(pattern) os.system('gdalbuildvrt -tr 0.04 0.04 {0} *{1}_0_4deg*.tif'.format( out_vrt, pattern)) # Creates the output name for the 10km map out_pattern = uu.name_aggregated_output(download_pattern_name, thresh, sensit_type) uu.print_log(out_pattern) # Produces a single raster of all the 10x10 tiles (0.4 degree resolution) cmd = [ 'gdalwarp', '-t_srs', "EPSG:4326", '-overwrite', '-dstnodata', '0', '-co', 'COMPRESS=LZW', '-tr', '0.04', '0.04', out_vrt, '{}.tif'.format(out_pattern) ] uu.log_subprocess_output_full(cmd) # Adds metadata tags to output rasters uu.add_universal_metadata_tags('{0}.tif'.format(out_pattern), sensit_type) # Units are different for annual removal factor, so metadata has to reflect that if 'annual_removal_factor' in out_pattern: cmd = [ 'gdal_edit.py', '-mo', 'units=Mg aboveground carbon/yr/pixel, where pixels are 0.04x0.04 degrees', '-mo', 'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels', '-mo', 'extent=Global', '-mo', 'scale=negative values are removals', '-mo', 'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation' .format(thresh), '{0}.tif'.format(out_pattern) ] uu.log_subprocess_output_full(cmd) else: cmd = [ 'gdal_edit.py', '-mo', 'units=Mg CO2e/yr/pixel, where pixels are 0.04x0.04 degrees', '-mo', 'source=per hectare version of the same model output, aggregated from 0.00025x0.00025 degree pixels', '-mo', 'extent=Global', '-mo', 'treecover_density_threshold={0} (only model pixels with canopy cover > {0} are included in aggregation' .format(thresh), '{0}.tif'.format(out_pattern) ] uu.log_subprocess_output_full(cmd) # If no_upload flag is not activated, output is uploaded if not no_upload: uu.print_log("Tiles processed. Uploading to s3 now...") uu.upload_final_set(output_dir_list[0], out_pattern) # Cleans up the folder before starting on the next raster type vrtList = glob.glob('*vrt') for vrt in vrtList: os.remove(vrt) for tile_name in tile_list: tile_id = uu.get_tile_id(tile_name) # os.remove('{0}_{1}.tif'.format(tile_id, pattern)) os.remove('{0}_{1}_rewindow.tif'.format(tile_id, pattern)) os.remove('{0}_{1}_0_4deg.tif'.format(tile_id, pattern)) # Compares the net flux from the standard model and the sensitivity analysis in two ways. # This does not work for compariing the raw outputs of the biomass_swap and US_removals sensitivity models because their # extents are different from the standard model's extent (tropics and US tiles vs. global). # Thus, in order to do this comparison, you need to clip the standard model net flux and US_removals net flux to # the outline of the US and clip the standard model net flux to the extent of JPL AGB2000. # Then, manually upload the clipped US_removals and biomass_swap net flux rasters to the spot machine and the # code below should work. if sensit_type not in [ 'std', 'biomass_swap', 'US_removals', 'legal_Amazon_loss' ]: if std_net_flux: uu.print_log( "Standard aggregated flux results provided. Creating comparison maps." ) # Copies the standard model aggregation outputs to s3. Only net flux is used, though. uu.s3_file_download(std_net_flux, cn.docker_base_dir, sensit_type) # Identifies the standard model net flux map std_aggreg_flux = os.path.split(std_net_flux)[1] try: # Identifies the sensitivity model net flux map sensit_aggreg_flux = glob.glob( 'net_flux_Mt_CO2e_*{}*'.format(sensit_type))[0] uu.print_log("Standard model net flux:", std_aggreg_flux) uu.print_log("Sensitivity model net flux:", sensit_aggreg_flux) except: uu.print_log( 'Cannot do comparison. One of the input flux tiles is not valid. Verify that both net flux rasters are on the spot machine.' ) uu.print_log( "Creating map of percent difference between standard and {} net flux" .format(sensit_type)) aggregate_results_to_4_km.percent_diff(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload) uu.print_log( "Creating map of which pixels change sign and which stay the same between standard and {}" .format(sensit_type)) aggregate_results_to_4_km.sign_change(std_aggreg_flux, sensit_aggreg_flux, sensit_type, no_upload) # If no_upload flag is not activated, output is uploaded if not no_upload: uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_perc_diff) uu.upload_final_set(output_dir_list[0], cn.pattern_aggreg_sensit_sign_change) else: uu.print_log( "No standard aggregated flux results provided. Not creating comparison maps." )
def main(): parser = argparse.ArgumentParser( description='Create planted forest carbon gain rate tiles') parser.add_argument( '--gadm-tile-index', '-gi', required=True, help= 'Shapefile of 1x1 degree tiles of countries that contain planted forests (i.e. countries with planted forests rasterized to 1x1 deg). If no shapefile, write None.' ) parser.add_argument( '--planted-tile-index', '-pi', required=True, help= 'Shapefile of 1x1 degree tiles of that contain planted forests (i.e. planted forest extent rasterized to 1x1 deg). If no shapefile, write None.' ) args = parser.parse_args() # Creates the directory and shapefile names for the two possible arguments (index shapefiles) gadm_index = os.path.split(args.gadm_tile_index) gadm_index_path = gadm_index[0] gadm_index_shp = gadm_index[1] gadm_index_shp = gadm_index_shp[:-4] planted_index = os.path.split(args.planted_tile_index) planted_index_path = planted_index[0] planted_index_shp = planted_index[1] planted_index_shp = planted_index_shp[:-4] # Checks the validity of the two arguments. If either one is invalid, the script ends. if (gadm_index_path not in cn.gadm_plant_1x1_index_dir or planted_index_path not in cn.gadm_plant_1x1_index_dir): raise Exception( 'Invalid inputs. Please provide None or s3 shapefile locations for both arguments.' ) # List of all possible 10x10 Hansen tiles except for those at very extreme latitudes (not just WHRC biomass tiles) total_tile_list = uu.tile_list(cn.pixel_area_dir) print "Number of possible 10x10 tiles to evaluate:", len(total_tile_list) # Removes the latitude bands that don't have any planted forests in them according to Liz Goldman. # i.e., Liz Goldman said by Slack on 1/2/19 that the nothernmost planted forest is 69.5146 and the southernmost is -46.938968. # This creates a more focused list of 10x10 tiles to iterate through (removes ones that definitely don't have planted forest). # NOTE: If the planted forest gdb is updated, the list of latitudes to exclude below may need to be changed to not exclude certain latitude bands. planted_lat_tile_list = [ tile for tile in total_tile_list if '90N' not in tile ] planted_lat_tile_list = [ tile for tile in planted_lat_tile_list if '80N' not in tile ] planted_lat_tile_list = [ tile for tile in planted_lat_tile_list if '50S' not in tile ] planted_lat_tile_list = [ tile for tile in planted_lat_tile_list if '60S' not in tile ] planted_lat_tile_list = [ tile for tile in planted_lat_tile_list if '70S' not in tile ] planted_lat_tile_list = [ tile for tile in planted_lat_tile_list if '80S' not in tile ] # planted_lat_tile_list = ['10N_080W'] print planted_lat_tile_list print "Number of 10x10 tiles to evaluate after extreme latitudes have been removed:", len( planted_lat_tile_list) # If a planted forest extent 1x1 tile index shapefile isn't supplied if 'None' in args.planted_tile_index: ### Entry point 1: # If no shapefile of 1x1 tiles for countries with planted forests is supplied, 1x1 tiles of country extents will be created. # This runs the process from the very beginning and will take a few days. if 'None' in args.gadm_tile_index: print "No GADM 1x1 tile index shapefile provided. Creating 1x1 planted forest country tiles from scratch..." # Downloads and unzips the GADM shapefile, which will be used to create 1x1 tiles of land areas uu.s3_file_download(cn.gadm_path, '.') cmd = ['unzip', cn.gadm_zip] subprocess.check_call(cmd) # Creates a new GADM shapefile with just the countries that have planted forests in them. # This limits creation of 1x1 rasters of land area on the countries that have planted forests rather than on all countries. # NOTE: If the planted forest gdb is updated and has new countries added to it, the planted forest country list # in constants_and_names.py must be updated, too. print "Creating shapefile of countries with planted forests..." os.system( '''ogr2ogr -sql "SELECT * FROM gadm_3_6_adm2_final WHERE iso IN ({0})" {1} gadm_3_6_adm2_final.shp''' .format(str(cn.plantation_countries)[1:-1], cn.gadm_iso)) # Creates 1x1 degree tiles of countries that have planted forests in them. # I assume this can handle using 50 processors because it's not trying to upload files to s3 and the tiles are small. # This takes several days to run because it iterates through at least 250 10x10 tiles. # For multiprocessor use. num_of_processes = 50 pool = Pool(num_of_processes) pool.map(plantation_preparation.rasterize_gadm_1x1, planted_lat_tile_list) pool.close() pool.join() # # Creates 1x1 degree tiles of countries that have planted forests in them. # # For single processor use. # for tile in planted_lat_tile_list: # # plantation_preparation.rasterize_gadm_1x1(tile) # Creates a shapefile of the boundaries of the 1x1 GADM tiles in countries with planted forests os.system('''gdaltindex {0}_{1}.shp GADM_*.tif'''.format( cn.pattern_gadm_1x1_index, uu.date)) cmd = [ 'aws', 's3', 'cp', '.', cn.gadm_plant_1x1_index_dir, '--exclude', '*', '--include', '{}*'.format(cn.pattern_gadm_1x1_index), '--recursive' ] subprocess.check_call(cmd) # # Saves the 1x1 country extent tiles to s3 # # Only use if the entire process can't run in one go on the spot machine # cmd = ['aws', 's3', 'cp', '.', 's3://gfw2-data/climate/carbon_model/temp_spotmachine_output/', '--exclude', '*', '--include', 'GADM_*.tif', '--recursive'] # subprocess.check_call(cmd) # Delete the aux.xml files os.system('''rm GADM*.tif.*''') # List of all 1x1 degree countey extent tiles created gadm_list_1x1 = uu.tile_list_spot_machine(".", "GADM_") print "List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1 print len(gadm_list_1x1) ### Entry point 2: # If a shapefile of the boundaries of 1x1 degree tiles of countries with planted forests is supplied, # a list of the 1x1 tiles is created from the shapefile. # This avoids creating the 1x1 country extent tiles all over again because the relevant tile extent are supplied # in the shapefile. elif cn.gadm_plant_1x1_index_dir in args.gadm_tile_index: print "Country extent 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest tiles..." # Copies the shapefile of 1x1 tiles of extent of countries with planted forests cmd = [ 'aws', 's3', 'cp', '{}/'.format(gadm_index_path), '.', '--recursive', '--exclude', '*', '--include', '{}*'.format(gadm_index_shp), '--recursive' ] subprocess.check_call(cmd) # Gets the attribute table of the country extent 1x1 tile shapefile gadm = glob.glob('{}*.dbf'.format(cn.pattern_gadm_1x1_index))[0] # Converts the attribute table to a dataframe dbf = Dbf5(gadm) df = dbf.to_dataframe() # Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list gadm_list_1x1 = df['location'].tolist() gadm_list_1x1 = [str(y) for y in gadm_list_1x1] print "List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", gadm_list_1x1 print "There are", len( gadm_list_1x1), "1x1 country extent tiles to iterate through." # In case some other arguments are provided else: raise Exception( 'Invalid GADM tile index shapefile provided. Please provide a valid shapefile.' ) # Creates 1x1 degree tiles of plantation growth wherever there are plantations. # Because this is iterating through all 1x1 tiles in countries with planted forests, it first checks # whether each 1x1 tile intersects planted forests before creating a 1x1 planted forest tile for that # 1x1 country extent tile. # For multiprocessor use num_of_processes = 30 pool = Pool(num_of_processes) pool.map(plantation_preparation.create_1x1_plantation_from_1x1_gadm, gadm_list_1x1) pool.close() pool.join() # # Creates 1x1 degree tiles of plantation growth wherever there are plantations # # For single processor use # for tile in gadm_list_1x1: # # plantation_preparation.create_1x1_plantation(tile) # Creates a shapefile in which each feature is the extent of a plantation extent tile. # This index shapefile can be used the next time this process is run if starting with Entry Point 3. os.system('''gdaltindex {0}_{1}.shp plant_*.tif'''.format( cn.pattern_plant_1x1_index, uu.date)) cmd = [ 'aws', 's3', 'cp', '.', cn.gadm_plant_1x1_index_dir, '--exclude', '*', '--include', '{}*'.format(cn.pattern_plant_1x1_index), '--recursive' ] subprocess.check_call(cmd) ### Entry point 3 # If a shapefile of the extents of 1x1 planted forest tiles is provided if cn.pattern_plant_1x1_index in args.planted_tile_index: print "Planted forest 1x1 tile index shapefile supplied. Using that to create 1x1 planted forest growth tiles..." # Copies the shapefile of 1x1 tiles of extent of planted forests cmd = [ 'aws', 's3', 'cp', '{}/'.format(planted_index_path), '.', '--recursive', '--exclude', '*', '--include', '{}*'.format(planted_index_shp), '--recursive' ] subprocess.check_call(cmd) # Gets the attribute table of the planted forest extent 1x1 tile shapefile gadm = glob.glob('{}*.dbf'.format(cn.pattern_plant_1x1_index))[0] # Converts the attribute table to a dataframe dbf = Dbf5(gadm) df = dbf.to_dataframe() # Converts the column of the dataframe with the names of the tiles (which contain their coordinates) to a list planted_list_1x1 = df['location'].tolist() planted_list_1x1 = [str(y) for y in planted_list_1x1] print "List of 1x1 degree tiles in countries that have planted forests, with defining coordinate in the northwest corner:", planted_list_1x1 print "There are", len( planted_list_1x1 ), "1x1 planted forest extent tiles to iterate through." # Creates 1x1 degree tiles of plantation growth wherever there are plantations. # Because this is iterating through only 1x1 tiles that are known to have planted forests (from a previous run # of this script), it does not need to check whether there are planted forests in this tile. It goes directly # to intersecting the planted forest table with the 1x1 tile. # For multiprocessor use # This works with 30 processors on an r4.16xlarge. num_of_processes = 50 pool = Pool(num_of_processes) pool.map(plantation_preparation.create_1x1_plantation_from_1x1_planted, planted_list_1x1) pool.close() pool.join() ### All entry points meet here: creation of 10x10 degree planted forest tiles from 1x1 degree planted forest tiles # Name of the vrt of 1x1 planted forest tiles plant_1x1_vrt = 'plant_1x1.vrt' # Creates a mosaic of all the 1x1 plantation growth rate tiles print "Creating vrt of 1x1 plantation growth rate tiles" os.system('gdalbuildvrt {} plant_*.tif'.format(plant_1x1_vrt)) # Creates 10x10 degree tiles of plantation growth by iterating over the pixel area tiles that are in latitudes with planted forests # For multiprocessor use num_of_processes = 20 pool = Pool(num_of_processes) pool.map( partial(plantation_preparation.create_10x10_plantation, plant_1x1_vrt=plant_1x1_vrt), planted_lat_tile_list) pool.close() pool.join()