def create_10x10_plantation(tile_id, plant_1x1_vrt): print "Getting bounding coordinates for tile", tile_id xmin, ymin, xmax, ymax = uu.coords(tile_id) print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax tile_10x10 = '{0}_{1}.tif'.format( tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked) print "Rasterizing", tile_10x10 cmd = [ 'gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Float32', plant_1x1_vrt, tile_10x10 ] subprocess.check_call(cmd) print "Checking if {} contains any data...".format(tile_id) stats = uu.check_for_data(tile_10x10) if stats[0] > 0: print " Data found in {}. Copying tile to s3...".format(tile_id) uu.upload_final(cn.annual_gain_AGC_BGC_planted_forest_unmasked_dir, tile_id, cn.pattern_annual_gain_AGC_BGC_planted_forest_unmasked) print " Tile converted and copied to s3" else: print " No data found. Not copying {}.".format(tile_id)
def create_mangrove_tiles(tile_id): print "Getting bounding coordinates for tile", tile_id xmin, xmax, ymin, ymax = utilities.coords(tile_id) print " ymax:", ymax, "; ymin:", ymin, "; xmax", xmax, "; xmin:", xmin print "Creating tile", tile_id out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_mangrove_biomass_2000) cmd = ['gdalwarp', '-t_srs', 'EPSG:4326', '-co', 'COMPRESS=LZW', '-tr', '0.00025', '0.00025', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-overwrite', utilities.mangrove_vrt, out_tile] subprocess.check_call(cmd) print " Tile created" print "Checking if {} contains any data...".format(tile_id) stats = uu.check_for_data(out_tile) if stats[0] > 0: print " Data found in {}. Copying tile to s3...".format(tile_id) uu.upload_final(cn.mangrove_biomass_2000_dir, tile_id, cn.pattern_mangrove_biomass_2000) print " Tile copied to s3" else: print " No data found. Not copying {}.".format(tile_id)
def create_10x10_plantation_type(tile_id, plant_type_1x1_vrt): uu.print_log("Getting bounding coordinates for tile", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax) tile_10x10 = '{0}_{1}.tif'.format(tile_id, cn.pattern_planted_forest_type_unmasked) uu.print_log("Rasterizing", tile_10x10) cmd = ['gdalwarp', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-tap', '-te', str(xmin), str(ymin), str(xmax), str(ymax), '-dstnodata', '0', '-t_srs', 'EPSG:4326', '-overwrite', '-ot', 'Byte', plant_type_1x1_vrt, tile_10x10] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) uu.print_log("Checking if {} contains any data...".format(tile_id)) stats = uu.check_for_data(tile_10x10) if stats[0] > 0: uu.print_log(" Data found in {}. Copying tile to s3...".format(tile_id)) uu.upload_final(cn.planted_forest_type_unmasked_dir, tile_id, cn.pattern_planted_forest_type_unmasked) uu.print_log(" Tile converted and copied to s3") else: print(" No data found. Not copying {}.".format(tile_id))
def clip_year_tiles(tile_year_list, no_upload): # Start time start = datetime.datetime.now() tile_id = tile_year_list[0].strip('.tif') year = tile_year_list[1] vrt_name = "global_vrt_{}_wgs84.vrt".format(year) # Gets coordinates of hansen tile uu.print_log("Getting coordinates of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) # Clips vrt to tile extent uu.print_log("Clipping burn year vrt to {0} for {1}".format(tile_id, year)) clipped_raster = "ba_clipped_{0}_{1}.tif".format(year, tile_id) cmd = [ 'gdal_translate', '-ot', 'Byte', '-co', 'COMPRESS=LZW', '-a_nodata', '0' ] cmd += [vrt_name, clipped_raster, '-tr', '.00025', '.00025'] cmd += ['-projwin', str(xmin), str(ymax), str(xmax), str(ymin)] uu.log_subprocess_output_full(cmd) # Calculates year tile values to be equal to year. ex: 17*1 calc = '--calc={}*(A>0)'.format(int(year) - 2000) recoded_output = "ba_{0}_{1}.tif".format(year, tile_id) outfile = '--outfile={}'.format(recoded_output) cmd = [ 'gdal_calc.py', '-A', clipped_raster, calc, outfile, '--NoDataValue=0', '--co', 'COMPRESS=LZW', '--quiet' ] uu.log_subprocess_output_full(cmd) # Only copies to s3 if the tile has data. # No tiles for 2000 have data because the burn year is coded as 0, which is NoData. uu.print_log("Checking if {} contains any data...".format(tile_id)) empty = uu.check_for_data(recoded_output) if empty: uu.print_log(" No data found. Not copying {}.".format(tile_id)) else: uu.print_log( " Data found in {}. Copying tile to s3...".format(tile_id)) cmd = [ 'aws', 's3', 'cp', recoded_output, cn.burn_year_warped_to_Hansen_dir ] uu.log_subprocess_output_full(cmd) uu.print_log(" Tile copied to", cn.burn_year_warped_to_Hansen_dir) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, "ba_{}".format(year), no_upload)
def rasterize_gadm_1x1(tile_id): uu.print_log("Getting bounding coordinates for tile", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log(" xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax) # Degrees of tile in x and y dimensions x_size = abs(int(xmin) - int(xmax)) y_size = abs(int(ymin) - int(ymax)) # Iterates through input 10x10 tile by 1x1 degree for x in range(x_size): xmin_1x1 = int(xmin) + x xmax_1x1 = int(xmin) + x + 1 for y in range(y_size): ymin_1x1 = int(ymin) + y ymax_1x1 = int(ymin) + y + 1 uu.print_log(" xmin_1x1:", xmin_1x1, "; xmax_1x1:", xmax_1x1, "; ymin_1x1", ymin_1x1, "; ymax_1x1:", ymax_1x1) tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1) uu.print_log("Rasterizing", tile_1x1) cmd = [ 'gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-burn', '1', '-a_nodata', '0', cn.gadm_iso, tile_1x1 ] # Solution for adding subprocess output to log is from https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging process = Popen(cmd, stdout=PIPE, stderr=STDOUT) with process.stdout: uu.log_subprocess_output(process.stdout) # Only keeps 1x1 GADM tiles if they actually include a country; many 1x1 tiles created out of 10x10 tiles # don't actually include a country. uu.print_log( "Checking if {} contains any data...".format(tile_1x1)) stats = uu.check_for_data(tile_1x1) if stats[1] > 0: uu.print_log( " Data found in {}. Keeping tile".format(tile_1x1)) else: uu.print_log( " No data found in {}. Deleting.".format(tile_1x1)) os.remove(tile_1x1)
def rasterize_gadm_1x1(tile_id): print "Getting bounding coordinates for tile", tile_id xmin, ymin, xmax, ymax = uu.coords(tile_id) print " xmin:", xmin, "; xmax:", xmax, "; ymin", ymin, "; ymax:", ymax # Degrees of tile in x and y dimensions x_size = abs(int(xmin) - int(xmax)) y_size = abs(int(ymin) - int(ymax)) # Iterates through input 10x10 tile by 1x1 degree for x in range(x_size): xmin_1x1 = int(xmin) + x xmax_1x1 = int(xmin) + x + 1 for y in range(y_size): ymin_1x1 = int(ymin) + y ymax_1x1 = int(ymin) + y + 1 print " xmin_1x1:", xmin_1x1, "; xmax_1x1:", xmax_1x1, "; ymin_1x1", ymin_1x1, "; ymax_1x1:", ymax_1x1 tile_1x1 = 'GADM_{0}_{1}.tif'.format(ymax_1x1, xmin_1x1) print "Rasterizing", tile_1x1 cmd = [ 'gdal_rasterize', '-tr', '{}'.format(str(cn.Hansen_res)), '{}'.format(str(cn.Hansen_res)), '-co', 'COMPRESS=LZW', '-te', str(xmin_1x1), str(ymin_1x1), str(xmax_1x1), str(ymax_1x1), '-burn', '1', '-a_nodata', '0', cn.gadm_iso, tile_1x1 ] subprocess.check_call(cmd) # Only keeps 1x1 GADM tiles if they actually include a country; many 1x1 tiles created out of 10x10 tiles # don't actually include a country. print "Checking if {} contains any data...".format(tile_1x1) stats = uu.check_for_data(tile_1x1) if stats[1] > 0: print " Data found in {}. Keeping tile".format(tile_1x1) else: print " No data found in {}. Deleting.".format(tile_1x1) os.remove(tile_1x1)
def prep_FIA_regions(tile_id): uu.print_log("Creating Hansen tile for FIA regions") # Start time start = datetime.datetime.now() uu.print_log("Getting extent of", tile_id) xmin, ymin, xmax, ymax = uu.coords(tile_id) uu.print_log("Rasterizing FIA region shapefile", tile_id) blocksizex = 1024 blocksizey = 1024 uu.rasterize( '{}.shp'.format(cn.name_FIA_regions_raw[:-4]), "{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed), xmin, ymin, xmax, ymax, blocksizex, blocksizey, '.00025', 'Byte', 'regionCode', '0') uu.print_log("Checking if {} contains any data...".format(tile_id)) no_data = uu.check_for_data("{0}_{1}.tif".format( tile_id, cn.pattern_FIA_regions_processed)) if no_data: uu.print_log(" No data found. Deleting {}.".format(tile_id)) os.remove("{0}_{1}.tif".format(tile_id, cn.pattern_FIA_regions_processed)) else: uu.print_log( " Data found in {}. Copying tile to s3...".format(tile_id)) uu.upload_final(cn.FIA_regions_processed_dir, tile_id, cn.pattern_FIA_regions_processed) uu.print_log(" Tile copied to s3") # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_FIA_regions_processed)
def hansen_burnyear(tile_id, no_upload): # Start time start = datetime.datetime.now() uu.print_log("Processing", tile_id) # The tiles that are used. out_tile_no_tag is the output before metadata tags are added. out_tile is the output # once metadata tags have been added. out_tile_no_tag = '{0}_{1}_no_tag.tif'.format(tile_id, cn.pattern_burn_year) out_tile = '{0}_{1}.tif'.format(tile_id, cn.pattern_burn_year) loss = '{0}.tif'.format(tile_id) # Does not continue processing tile if no loss (because there will not be any output) if not os.path.exists(loss): uu.print_log("No loss tile for", tile_id) return else: uu.print_log("Loss tile exists for", tile_id) # Downloads the burned area tiles for each year include = 'ba_*_{}.tif'.format(tile_id) burn_tiles_dir = 'burn_tiles' if not os.path.exists(burn_tiles_dir): os.mkdir(burn_tiles_dir) cmd = [ 'aws', 's3', 'cp', cn.burn_year_warped_to_Hansen_dir, burn_tiles_dir, '--recursive', '--exclude', "*", '--include', include ] uu.log_subprocess_output_full(cmd) # For each year tile, converts to array and stacks them array_list = [] ba_tifs = glob.glob(burn_tiles_dir + '/*{}*'.format(tile_id)) # Skips the tile if it has no burned area data for any year uu.print_log("There are {0} tiles to stack for {1}".format( len(ba_tifs), tile_id)) if len(ba_tifs) == 0: uu.print_log( "Skipping {} because there are no tiles to stack".format(tile_id)) return # NOTE: All of this could pretty easily be done in rasterio. However, Sam's use of GDAL for this still works fine, # so I've left it using GDAL. for ba_tif in ba_tifs: uu.print_log("Creating array with {}".format(ba_tif)) array = utilities.raster_to_array(ba_tif) array_list.append(array) # Stacks arrays from each year uu.print_log("Stacking arrays for", tile_id) stacked_year_array = utilities.stack_arrays(array_list) # Converts Hansen tile to array uu.print_log("Creating loss year array for", tile_id) loss_array = utilities.raster_to_array(loss) # Determines what year to assign burned area lossarray_min1 = np.subtract(loss_array, 1) stack_con = (stacked_year_array >= lossarray_min1) & (stacked_year_array <= loss_array) stack_con2 = stack_con * stacked_year_array lossyear_burn_array = stack_con2.max(0) utilities.array_to_raster_simple(lossyear_burn_array, out_tile_no_tag, loss) # Only copies to s3 if the tile has data uu.print_log("Checking if {} contains any data...".format(tile_id)) empty = uu.check_for_data(out_tile_no_tag) # Checks output for data. There could be burned area but none of it coincides with tree cover loss, # so this is the final check for whether there is any data. if empty: uu.print_log(" No data found. Not copying {}.".format(tile_id)) # Without this, the untagged version is counted and eventually copied to s3 if it has data in it os.remove(out_tile_no_tag) return else: uu.print_log( " Data found in {}. Adding metadata tags...".format(tile_id)) ### Thomas suggested these on 8/19/2020 but they didn't work. The first one wrote the tags but erased all the ### data in the tiles (everything became 0 according to gdalinfo). The second one had some other error. # with rasterio.open(out_tile_no_tag, 'r') as src: # # profile = src.profile # # with rasterio.open(out_tile_no_tag, 'w', **profile) as dst: # # dst.update_tags(units='year (2001, 2002, 2003...)', # source='MODIS collection 6 burned area', # extent='global') # # with rasterio.open(out_tile_no_tag, 'w+') as src: # # dst.update_tags(units='year (2001, 2002, 2003...)', # source='MODIS collection 6 burned area', # extent='global') # All of the below is to add metadata tags to the output burn year masks. # For some reason, just doing what's at https://rasterio.readthedocs.io/en/latest/topics/tags.html # results in the data getting removed. # I found it necessary to copy the desired output and read its windows into a new copy of the file, to which the # metadata tags are added. I'm sure there's an easier way to do this but I couldn't figure out how. # I know it's very convoluted but I really couldn't figure out how to add the tags without erasing the data. copyfile(out_tile_no_tag, out_tile) with rasterio.open(out_tile_no_tag) as out_tile_no_tag_src: # Grabs metadata about the tif, like its location/projection/cellsize kwargs = out_tile_no_tag_src.meta #### Use profile instead # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory windows = out_tile_no_tag_src.block_windows(1) # Updates kwargs for the output dataset kwargs.update(driver='GTiff', count=1, compress='lzw', nodata=0) out_tile_tagged = rasterio.open(out_tile, 'w', **kwargs) # Adds metadata tags to the output raster uu.add_rasterio_tags(out_tile_tagged, 'std') out_tile_tagged.update_tags(units='year (2001, 2002, 2003...)') out_tile_tagged.update_tags( source= 'MODIS collection 6 burned area, https://modis-fire.umd.edu/files/MODIS_C6_BA_User_Guide_1.3.pdf' ) out_tile_tagged.update_tags(extent='global') # Iterates across the windows (1 pixel strips) of the input tile for idx, window in windows: in_window = out_tile_no_tag_src.read(1, window=window) # Writes the output window to the output out_tile_tagged.write_band(1, in_window, window=window) # Without this, the untagged version is counted and eventually copied to s3 if it has data in it os.remove(out_tile_no_tag) # Prints information about the tile that was just processed uu.end_of_fx_summary(start, tile_id, cn.pattern_burn_year, no_upload)