def create_10x10_plantation(tile_id, plant_1x1_vrt): print "Getting bounding coordinates for tile", tile_id xmin, ymin, xmax, ymax = uu.coords(tile_id) print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax tile_10x10 = '{0}_{1}.tif'.format( tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked) print "Rasterizing", tile_10x10 cmd = [ 'gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Float32', plant_1x1_vrt, tile_10x10 ] subprocess.check_call(cmd) print "Checking if {} contains any data...".format(tile_id) stats = uu.check_for_data(tile_10x10) if stats[0] > 0: print " Data found in {}. Copying tile to s3...".format(tile_id) uu.upload_final(cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir, tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked) print " Tile converted and copied to s3" else: print " No data found. Not copying {}.".format(tile_id)
def create_mangrove_tiles(tile_id): print "Getting bounding coordinates for tile", tile_id xmin, xmax, ymin, ymax = utilities.coords(tile_id) print " ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin print "Creating tile", tile_id out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000) cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', '0.00025', '0.00025', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-overwrite', utilities.mangrove_vrt, out_tile] subprocess.check_call(cmd) print " Tile created" print "Checking if {} contains any data...".format(tile_id) stats = uu.check_for_data(out_tile) if stats[0] > 0: print " Data found in {}. Copying tile to s3...".format(tile_id) uu.upload_final(cn.mangrove_biomass_2000_dir, tile_id, cn.pattern_mangrove_biomass_2000) print " Tile copied to s3" else: print " No data found. Not copying {}.".format(tile_id)
def create_10x10_plantation_type(tile_id, plant_type_1x1_vrt): uu.print_log("Getting bounding coordinates for tile", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax) tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_planted_forest_type_unmasked) uu.print_log("Rasterizing", tile_10x10) cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Byte', plant_type_1x1_vrt, tile_10x10] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) uu.print_log("Checking if {} contains any data...".format(tile_id)) stats = uu.check_for_data(tile_10x10) if stats[0] > 0: uu.print_log(" Data found in {}. Copying tile to s3...".format(tile_id)) uu.upload_final(cn.planted_forest_type_unmasked_dir, tile_id, cn.pattern_planted_forest_type_unmasked) uu.print_log(" Tile converted and copied to s3") else: print(" No data found. Not copying {}.".format(tile_id))
def prep_FIA_regions(tile_id): uu.print_log("Creating Hansen tile for FIA regions") # Start time start = datetime.datetime.now() uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log("Rasterizing FIA region shapefile", tile_id) blocksizex = 1024 blocksizey = 1024 uu.rasterize( '{}.shp'.format(cn.name_FIA_regions_raw[:-4]), "{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed), xmin, ymin, xmax, ymax, blocksizex, blocksizey, '.00025', 'Byte', 'regionCode', '0') uu.print_log("Checking if {} contains any data...".format(tile_id)) no_data = uu.check_for_data("{0}_{1}.tif".format( tile_id, cn.pattern_FIA_regions_processed)) if no_data: uu.print_log(" No data found. Deleting {}.".format(tile_id)) os.remove("{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed)) else: uu.print_log( " Data found in {}. Copying tile to s3...".format(tile_id)) uu.upload_final(cn.FIA_regions_processed_dir, tile_id, cn.pattern_FIA_regions_processed) uu.print_log(" Tile copied to s3") # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_FIA_regions_processed)
def create_emitted_biomass(tile_id): start = datetime.datetime.now() # Names of the input tiles natrl_forest_biomass_2000 = '{0}_{1}.tif'.format( tile_id, cn.pattern_natrl_forest_biomass_2000) mangrove_biomass_2000 = '{0}_{1}.tif'.format( tile_id, cn.pattern_mangrove_biomass_2000) natrl_forest_gain = '{0}_{1}.tif'.format( tile_id, cn.pattern_cumul_gain_AGC_natrl_forest) mangrove_gain = '{0}_{1}.tif'.format(tile_id, cn.pattern_cumul_gain_AGC_mangrove) natrl_forest_emitted = '{0}_{1}.tif'.format( tile_id, cn.pattern_natrl_forest_biomass_2000) mangrove_emitted = '{0}_{1}.tif'.format(tile_id, cn.pattern_cont_eco_processed) print " Reading input files and evaluating conditions" # Opens biomass tile with rasterio.open(loss) as loss_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = loss_src.meta # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = loss_src.block_windows(1) # Opens gain tile with rasterio.open(gain) as gain_src: # Opens ifl tile with rasterio.open(ifl) as ifl_src: # Opens continent-ecozone combinations tile with rasterio.open(cont_eco) as cont_eco_src: # Opens biomass 2000 tile with rasterio.open(biomass) as biomass_src: # Opens tree cover density tile with rasterio.open(tcd) as extent_src: # Updates kwargs for the output dataset kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) # Opens the output tile, giving it the arguments of the input tiles with rasterio.open( '{0}{1}.tif'.format( tile_id, cn.pattern_age_cat_natrl_forest), 'w', **kwargs) as dst: # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: # Creates windows for each input raster loss = loss_src.read(1, window=window) gain = gain_src.read(1, window=window) tcd = extent_src.read(1, window=window) ifl = ifl_src.read(1, window=window) cont_eco = cont_eco_src.read(1, window=window) biomass = biomass_src.read(1, window=window) # Creates a numpy array that has the <=20 year secondary forest growth rate x 20 # based on the continent-ecozone code of each pixel (the dictionary). # This is used to assign pixels to the correct age category. gain_20_years = np.vectorize( gain_table_dict.get)(cont_eco) * 20 # Create a 0s array for the output dst_data = np.zeros( (window.height, window.width), dtype='uint8') # Logic tree for assigning age categories begins here # No change pixels- no loss or gain if tropics == 0: dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0))] = 1 if tropics == 1: dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (ifl != 1))] = 2 dst_data[np.where((tcd > 0) & (gain == 0) & (loss == 0) & (ifl == 1))] = 3 # Loss-only pixels dst_data[np.where( (gain == 0) & (loss > 0) & (ifl != 1) & (biomass <= gain_20_years))] = 4 dst_data[np.where( (gain == 0) & (loss > 0) & (ifl != 1) & (biomass > gain_20_years))] = 5 dst_data[np.where((gain == 0) & (loss > 0) & (ifl == 1))] = 6 # Gain-only pixels dst_data[np.where((gain == 1) & (loss == 0))] = 7 # Pixels with loss and gain dst_data[np.where((gain == 1) & (loss >= 13))] = 8 dst_data[np.where((gain == 1) & (loss > 0) & (loss <= 6))] = 9 dst_data[np.where((gain == 1) & (loss > 6) & (loss < 13))] = 10 # Writes the output window to the output dst.write_band(1, dst_data, window=window) print 'Uploading tiles to s3' for type in cn.pool_types: uu.upload_final('{0}/{1}/'.format(cn.agc_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_agc, type)) uu.upload_final('{0}/{1}/'.format(cn.bgc_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_bgc, type)) uu.upload_final('{0}/{1}/'.format(cn.deadwood_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_deadwood, type)) uu.upload_final('{0}/{1}/'.format(cn.litter_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_litter, type)) uu.upload_final('{0}/{1}/'.format(cn.soil_C_pool_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_soil_pool, type)) uu.upload_final('{0}/{1}/'.format(cn.total_C_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_total_C, type)) print "Elapsed time: {}".format(datetime.datetime.now() - start)
def create_carbon_pools(tile_id): start = datetime.datetime.now() print 'Writing aboveground carbon, belowground carbon, deadwood, litter, soil, and total carbon tiles' calc_all_cmd = ['./calc_carbon_pools.exe', tile_id] subprocess.check_call(calc_all_cmd) print 'Uploading tiles to s3' for type in cn.pool_types: uu.upload_final('{0}/{1}/'.format(cn.agc_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_agc, type)) uu.upload_final('{0}/{1}/'.format(cn.bgc_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_bgc, type)) uu.upload_final('{0}/{1}/'.format(cn.deadwood_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_deadwood, type)) uu.upload_final('{0}/{1}/'.format(cn.litter_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_litter, type)) uu.upload_final('{0}/{1}/'.format(cn.soil_C_pool_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_soil_pool, type)) uu.upload_final('{0}/{1}/'.format(cn.total_C_dir, type), tile_id, '{0}_{1}'.format(cn.pattern_total_C, type)) print "Deleting intermediate tiles" tiles_to_remove = glob.glob('*{}*'.format(tile_id)) for tile in tiles_to_remove: try: os.remove(tile) except: pass print "elapsed time: {}".format(datetime.datetime.now() - start)