def main(image_ws, ini_path, bs=2048, stats_flag=False, overwrite_flag=False): """Prep a Landsat scene for METRIC Parameters ---------- image_ws : str Landsat scene folder that will be prepped. ini_path : str File path of the input parameters file. bs : int, optional Processing block size (the default is 2048). stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). Returns ------- True is successful """ # Open config file config = dripy.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') calc_refl_toa_flag = dripy.read_param( 'calc_refl_toa_flag', True, config, 'INPUTS') calc_refl_toa_qa_flag = dripy.read_param( 'calc_refl_toa_qa_flag', True, config, 'INPUTS') # calc_refl_sur_ledaps_flag = dripy.read_param( # 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS') # calc_refl_sur_qa_flag = dripy.read_param( # 'calc_refl_sur_qa_flag', False, config, 'INPUTS') calc_ts_bt_flag = dripy.read_param( 'calc_ts_bt_flag', True, config, 'INPUTS') # Use QA band to set common area # Fmask cloud, shadow, & snow pixels will be removed from common area calc_fmask_common_flag = dripy.read_param( 'calc_fmask_common_flag', True, config, 'INPUTS') fmask_smooth_flag = dripy.read_param( 'fmask_smooth_flag', False, config, 'INPUTS') fmask_buffer_flag = dripy.read_param( 'fmask_buffer_flag', False, config, 'INPUTS') fmask_erode_flag = dripy.read_param( 'fmask_erode_flag', False, config, 'INPUTS') if fmask_smooth_flag: fmask_smooth_cells = int(dripy.read_param( 'fmask_smooth_cells', 1, config, 'INPUTS')) if fmask_smooth_cells == 0 and fmask_smooth_flag: fmask_smooth_flag = False if fmask_erode_flag: fmask_erode_cells = int(dripy.read_param( 'fmask_erode_cells', 1, config, 'INPUTS')) if fmask_erode_cells == 0 and fmask_erode_flag: fmask_erode_flag = False if fmask_buffer_flag: fmask_buffer_cells = int(dripy.read_param( 'fmask_buffer_cells', 1, config, 'INPUTS')) if fmask_buffer_cells == 0 and fmask_buffer_flag: fmask_buffer_flag = False # Remove edge (fringe) cells edge_smooth_flag = dripy.read_param( 'edge_smooth_flag', True, config, 'INPUTS') # Include hand made cloud masks cloud_mask_flag = dripy.read_param( 'cloud_mask_flag', False, config, 'INPUTS') cloud_mask_ws = "" if cloud_mask_flag: cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws') # Extract separate Fmask rasters calc_fmask_flag = dripy.read_param( 'calc_fmask_flag', True, config, 'INPUTS') calc_fmask_cloud_flag = dripy.read_param( 'calc_fmask_cloud_flag', True, config, 'INPUTS') calc_fmask_snow_flag = dripy.read_param( 'calc_fmask_snow_flag', True, config, 'INPUTS') calc_fmask_water_flag = dripy.read_param( 'calc_fmask_water_flag', True, config, 'INPUTS') # Keep Landsat DN, LEDAPS, and Fmask rasters keep_dn_flag = dripy.read_param( 'keep_dn_flag', True, config, 'INPUTS') # keep_sr_flag = dripy.read_param( # 'keep_sr_flag', True, config, 'INPUTS') # For this to work I would need to pass in the metric input file # calc_elev_flag = dripy.read_param( # 'calc_elev_flag', False, config, 'INPUTS') # calc_landuse_flag = dripy.read_param( # 'calc_landuse_flag', False, config, 'INPUTS') # calc_acca_cloud_flag = dripy.read_param( # 'calc_acca_cloud_flag', True, config, 'INPUTS') # calc_acca_snow_flag = dripy.read_param( # 'calc_acca_snow_flag', True, config, 'INPUTS') # calc_ledaps_dem_land_flag = dripy.read_param( # 'calc_ledaps_dem_land_flag', False, config, 'INPUTS') # calc_ledaps_veg_flag = dripy.read_param( # 'calc_ledaps_veg_flag', False, config, 'INPUTS') # calc_ledaps_snow_flag = dripy.read_param( # 'calc_ledaps_snow_flag', False, config, 'INPUTS') # calc_ledaps_land_flag = dripy.read_param( # 'calc_ledaps_land_flag', False, config, 'INPUTS') # calc_ledaps_cloud_flag = dripy.read_param( # 'calc_ledaps_cloud_flag', False, config, 'INPUTS') # Interpolate/clip/project hourly rasters for each Landsat scene # calc_metric_flag = dripy.read_param( # 'calc_metric_flag', False, config, 'INPUTS') calc_metric_ea_flag = dripy.read_param( 'calc_metric_ea_flag', False, config, 'INPUTS') calc_metric_wind_flag = dripy.read_param( 'calc_metric_wind_flag', False, config, 'INPUTS') calc_metric_etr_flag = dripy.read_param( 'calc_metric_etr_flag', False, config, 'INPUTS') calc_metric_tair_flag = dripy.read_param( 'calc_metric_tair_flag', False, config, 'INPUTS') # Interpolate/clip/project AWC and daily ETr/PPT rasters # to compute SWB Ke for each Landsat scene calc_swb_ke_flag = dripy.read_param( 'calc_swb_ke_flag', False, config, 'INPUTS') if cloud_mask_flag: spinup_days = dripy.read_param( 'swb_spinup_days', 30, config, 'INPUTS') min_spinup_days = dripy.read_param( 'swb_min_spinup_days', 5, config, 'INPUTS') # Round ea raster to N digits to save space rounding_digits = dripy.read_param( 'rounding_digits', 3, config, 'INPUTS') env = drigo.env image = et_image.Image(image_ws, env) np.seterr(invalid='ignore', divide='ignore') gdal.UseExceptions() # Input file paths dn_image_dict = et_common.landsat_band_image_dict( image.orig_data_ws, image.image_name_re) # # Open METRIC config file # if config_file: # logging.info( # log_f.format('METRIC INI File:', os.path.basename(config_file))) # config = configparser.ConfigParser() # try: # config.read(config_file) # except: # logging.error('\nERROR: Config file could not be read, ' + # 'is not an input file, or does not exist\n' + # 'ERROR: config_file = {}\n').format(config_file) # sys.exit() # # Overwrite # overwrite_flag = dripy.read_param('overwrite_flag', True, config) # # # Elevation and landuse parameters/flags from METRIC input file # calc_elev_flag = dripy.read_param('save_dem_raster_flag', True, config) # calc_landuse_flag = dripy.read_param( # 'save_landuse_raster_flag', True, config) # if calc_elev_flag: # elev_pr_path = config.get('INPUTS','dem_raster') # if calc_landuse_flag: # landuse_pr_path = config.get('INPUTS', 'landuse_raster') # else: # overwrite_flag = False # calc_elev_flag = False # calc_landuse_flag = False # # Elev raster must exist # if calc_elev_flag and not os.path.isfile(elev_pr_path): # logging.error('\nERROR: Elevation raster {} does not exist\n'.format( # elev_pr_path)) # return False # Landuse raster must exist # if calc_landuse_flag and not os.path.isfile(landuse_pr_path): # logging.error('\nERROR: Landuse raster {} does not exist\n'.format( # landuse_pr_path)) # return False # Removing ancillary files before checking for inputs if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')): shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask')) for item in os.listdir(image.orig_data_ws): if (image.type == 'Landsat7' and (item.endswith('_B8.TIF') or item.endswith('_B6_VCID_2.TIF'))): os.remove(os.path.join(image.orig_data_ws, item)) elif (image.type == 'Landsat8' and (item.endswith('_B1.TIF') or item.endswith('_B8.TIF') or item.endswith('_B9.TIF') or item.endswith('_B11.TIF'))): os.remove(os.path.join(image.orig_data_ws, item)) elif (item.endswith('_VER.jpg') or item.endswith('_VER.txt') or item.endswith('_GCP.txt') or item == 'README.GTF'): os.remove(os.path.join(image.orig_data_ws, item)) # Check correction level (image must be L1T to process) if image.correction != 'L1TP': logging.debug(' Image is not L1TP corrected, skipping') return False # calc_fmask_common_flag = False # calc_refl_toa_flag = False # calc_ts_bt_flag = False # calc_metric_ea_flag = False # calc_metric_wind_flag = False # calc_metric_etr_flag = False # overwrite_flag = False # QA band must exist if calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys(): logging.warning( '\nQA band does not exist but calc_fmask_common_flag=True' '\n Setting calc_fmask_common_flag=False\n {}'.format( image.qa_band)) calc_fmask_common_flag = False if cloud_mask_flag and not os.path.isdir(cloud_mask_ws): logging.warning( '\ncloud_mask_ws is not a directory but cloud_mask_flag=True.' '\n Setting cloud_mask_flag=False\n {}'.format(cloud_mask_ws)) cloud_mask_flag = False # Check for Landsat TOA images if (calc_refl_toa_flag and (set(list(image.band_toa_dict.keys()) + [image.thermal_band, image.qa_band]) != set(dn_image_dict.keys()))): logging.warning( '\nMissing Landsat images but calc_refl_toa_flag=True' '\n Setting calc_refl_toa_flag=False') calc_refl_toa_flag = False # Check for Landsat brightness temperature image if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys(): logging.warning( '\nThermal band image does not exist but calc_ts_bt_flag=True' '\n Setting calc_ts_bt_flag=False') calc_ts_bt_flag = False # DEADBEEF - Should the function return False if Ts doesn't exist? # return False # Check for METRIC hourly/daily input folders if calc_metric_ea_flag: metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder') if not os.path.isdir(metric_ea_input_ws): logging.warning( '\nHourly Ea folder does not exist but calc_metric_ea_flag=True' '\n Setting calc_metric_ea_flag=False\n {}'.format( metric_ea_input_ws)) calc_metric_ea_flag = False if calc_metric_wind_flag: metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder') if not os.path.isdir(metric_wind_input_ws): logging.warning( '\nHourly wind folder does not exist but calc_metric_wind_flag=True' '\n Setting calc_metric_wind_flag=False\n {}'.format( metric_wind_input_ws)) calc_metric_wind_flag = False if calc_metric_etr_flag: metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder') if not os.path.isdir(metric_etr_input_ws): logging.warning( '\nHourly ETr folder does not exist but calc_metric_etr_flag=True' '\n Setting calc_metric_etr_flag=False\n {}'.format( metric_etr_input_ws)) calc_metric_etr_flag = False if calc_metric_tair_flag: metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder') if not os.path.isdir(metric_tair_input_ws): logging.warning( '\nHourly Tair folder does not exist but calc_metric_tair_flag=True' '\n Setting calc_metric_tair_flag=False\n {}'.format( metric_tair_input_ws)) calc_metric_tair_flag = False if (calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag or calc_metric_tair_flag): metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re')) metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re')) if calc_swb_ke_flag: awc_input_path = config.get('INPUTS', 'awc_input_path') etr_input_ws = config.get('INPUTS', 'etr_input_folder') ppt_input_ws = config.get('INPUTS', 'ppt_input_folder') etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re')) ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re')) if not os.path.isfile(awc_input_path): logging.warning( '\nAWC raster does not exist but calc_swb_ke_flag=True' '\n Setting calc_swb_ke_flag=False\n {}'.format( awc_input_path)) calc_swb_ke_flag = False if not os.path.isdir(etr_input_ws): logging.warning( '\nDaily ETr folder does not exist but calc_swb_ke_flag=True' '\n Setting calc_swb_ke_flag=False\n {}'.format( etr_input_ws)) calc_swb_ke_flag = False if not os.path.isdir(ppt_input_ws): logging.warning( '\nDaily PPT folder does not exist but calc_swb_ke_flag=True' '\n Setting calc_swb_ke_flag=False\n {}'.format( ppt_input_ws)) calc_swb_ke_flag = False # Build folders for support rasters if ((calc_fmask_common_flag or calc_refl_toa_flag or # calc_refl_sur_ledaps_flag or calc_ts_bt_flag or calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag or calc_metric_tair_flag or calc_swb_ke_flag) and not os.path.isdir(image.support_ws)): os.makedirs(image.support_ws) if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws): os.makedirs(image.refl_toa_ws) # if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws): # os.makedirs(image.refl_sur_ws) # DEADBEEF - This is being further down just for the Fmask images # # Apply overwrite flag # if overwrite_flag: # overwrite_list = [ # image.fmask_cloud_raster, image.fmask_snow_raster, # image.fmask_water_raster # # image.elev_raster, image.landuse_raster # # image.common_area_raster # ] # for overwrite_path in overwrite_list: # try: # dripy.remove_file(image.fmask_cloud_raster) # except: # pass # Use QA band to build common area rasters logging.info('\nCommon Area Raster') qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0) common_geo = drigo.raster_ds_geo(qa_ds) common_extent = drigo.raster_ds_extent(qa_ds) common_proj = drigo.raster_ds_proj(qa_ds) common_osr = drigo.raster_ds_osr(qa_ds) # Initialize common_area as all non-fill QA values qa_array = drigo.raster_ds_to_array(qa_ds, return_nodata=False) common_array = qa_array != 1 common_rows, common_cols = common_array.shape del qa_ds # Erode and dilate to remove fringe on edge # Default is to not smooth, but user can force smoothing # This needs to be applied before Fmask if edge_smooth_flag and image.prefix in ['LT05', 'LE07']: struct = ndimage.generate_binary_structure(2, 2).astype(np.uint8) if image.prefix == 'LT05': cells = 8 elif image.prefix == 'LE07': cells = 2 else: cells = 0 common_array = ndimage.binary_dilation( ndimage.binary_erosion(common_array, struct, cells), struct, cells) # Try applying user defined cloud masks to common_area cloud_mask_path = os.path.join( cloud_mask_ws, image.folder_id + '_mask.shp') if cloud_mask_flag and os.path.isfile(cloud_mask_path): logging.info(' Applying cloud mask shapefile') feature_path = os.path.join( cloud_mask_ws, (image.folder_id + '_mask.shp')) logging.info(' {}'.format(feature_path)) cloud_mask_memory_ds = drigo.polygon_to_raster_ds( feature_path, nodata_value=0, burn_value=1, output_osr=common_osr, output_cs=30, output_extent=common_extent) cloud_array = drigo.raster_ds_to_array( cloud_mask_memory_ds, return_nodata=False) # DEADBEEF - If user sets a cloud mask, # it is probably better than Fmask # Eventually change "if" calc_fmask_common_flag: to "elif" common_array[cloud_array == 1] = 0 del cloud_mask_memory_ds, cloud_array # Remove Fmask cloud, shadow, and snow pixels from common_area if calc_fmask_common_flag: logging.info(' Applying Fmask to common area') fmask_array = et_numpy.bqa_fmask_func(qa_array) fmask_mask = (fmask_array >= 2) & (fmask_array <= 4) if fmask_smooth_flag: logging.debug( ' Smoothing (dilate/erode/erode/dilate) Fmask clouds, shadows,' ' and snow pixels by {} cells'.format(fmask_smooth_cells)) # ArcGIS smoothing procedure fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_smooth_cells, structure=ndimage.generate_binary_structure(2, 2)) if fmask_erode_flag: logging.debug( ' Eroding Fmask clouds, shadows, and snow pixels by ' '{} cells'.format(fmask_erode_cells)) fmask_mask = ndimage.binary_erosion( fmask_mask, iterations=fmask_erode_cells, structure=ndimage.generate_binary_structure(2, 2)) if fmask_buffer_flag: logging.debug( ' Dilating (buffering) Fmask clouds, shadows, and snow pixels ' 'by {} cells'.format(fmask_buffer_cells)) fmask_mask = ndimage.binary_dilation( fmask_mask, iterations=fmask_buffer_cells, structure=ndimage.generate_binary_structure(2, 2)) # Reset common_array for buffered cells common_array[fmask_mask] = 0 del fmask_array, fmask_mask # Check that there are some cloud free pixels if not np.any(common_array): logging.error( ' ERROR: There are no cloud/snow free pixels, returning False') return False # Always overwrite common area raster # if not os.path.isfile(image.common_area_raster): drigo.array_to_raster( common_array, image.common_area_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) # Print common geo/extent logging.debug(' Common geo: {}'.format(common_geo)) logging.debug(' Common extent: {}'.format(common_extent)) # Extract Fmask components as separate rasters if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or calc_fmask_water_flag): logging.info('\nFmask') fmask_array = et_numpy.bqa_fmask_func(qa_array) # Remove existing Fmask rasters if (calc_fmask_flag and overwrite_flag and os.path.isfile(image.fmask_output_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_output_raster)) dripy.remove_file(image.fmask_output_raster) if (calc_fmask_cloud_flag and overwrite_flag and os.path.isfile(image.fmask_cloud_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_cloud_raster)) dripy.remove_file(image.fmask_cloud_raster) if (calc_fmask_snow_flag and overwrite_flag and os.path.isfile(image.fmask_snow_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_snow_raster)) dripy.remove_file(image.fmask_snow_raster) if (calc_fmask_water_flag and overwrite_flag and os.path.isfile(image.fmask_water_raster)): logging.debug(' Overwriting: {}'.format( image.fmask_water_raster)) dripy.remove_file(image.fmask_water_raster) # Save Fmask data as separate rasters if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)): logging.debug(' Saving Fmask raster') drigo.array_to_raster( fmask_array.astype(np.uint8), image.fmask_output_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) if (calc_fmask_cloud_flag and not os.path.isfile(image.fmask_cloud_raster)): logging.debug(' Saving Fmask cloud raster') fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4) drigo.array_to_raster( fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_cloud_array if (calc_fmask_snow_flag and not os.path.isfile(image.fmask_snow_raster)): logging.debug(' Saving Fmask snow raster') fmask_snow_array = (fmask_array == 3) drigo.array_to_raster( fmask_snow_array.astype(np.uint8), image.fmask_snow_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_snow_array if (calc_fmask_water_flag and not os.path.isfile(image.fmask_water_raster)): logging.debug(' Saving Fmask water raster') fmask_water_array = (fmask_array == 1) drigo.array_to_raster( fmask_water_array.astype(np.uint8), image.fmask_water_raster, output_geo=common_geo, output_proj=common_proj, mask_array=None, output_nodata=255, stats_flag=stats_flag) del fmask_water_array del fmask_array # # Calculate elevation # if calc_elev_flag and not os.path.isfile(elev_path): # logging.info('Elevation') # elev_array, elev_nodata = drigo.raster_to_array( # elev_pr_path, 1, common_extent) # drigo.array_to_raster( # elev_array, elev_raster, # output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del elev_array, elev_nodata, elev_path # # # Calculate landuse # if calc_landuse_flag and not os.path.isfile(landuse_raster): # logging.info('Landuse') # landuse_array, landuse_nodata = drigo.raster_to_array( # landuse_pr_path, 1, common_extent) # drigo.array_to_raster( # landuse_array, landuse_raster, # output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del landuse_array, landuse_nodata, landuse_raster # Calculate toa reflectance # f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32) if calc_refl_toa_flag: logging.info('Top-of-Atmosphere Reflectance') if os.path.isfile(image.refl_toa_raster) and overwrite_flag: logging.debug(' Overwriting: {}'.format( image.refl_toa_raster)) dripy.remove_file(image.refl_toa_raster) if not os.path.isfile(image.refl_toa_raster): # First build empty composite raster drigo.build_empty_raster( image.refl_toa_raster, image.band_toa_cnt, np.float32, None, env.snap_proj, env.cellsize, common_extent) # cos_theta_solar_flt = et_common.cos_theta_solar_func( # image.sun_elevation) # Process by block logging.info('Processing by block') logging.debug(' Mask cols/rows: {}/{}'.format( common_cols, common_rows)) for b_i, b_j in drigo.block_gen(common_rows, common_cols, bs): logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j)) block_data_mask = drigo.array_to_block( common_array, b_i, b_j, bs).astype(np.bool) block_rows, block_cols = block_data_mask.shape block_geo = drigo.array_offset_geo(common_geo, b_j, b_i) block_extent = drigo.geo_extent( block_geo, block_rows, block_cols) logging.debug(' Block rows: {} cols: {}'.format( block_rows, block_cols)) logging.debug(' Block extent: {}'.format(block_extent)) logging.debug(' Block geo: {}'.format(block_geo)) # Process each TOA band # for band, band_i in sorted(image.band_toa_dict.items()): for band, dn_image in sorted(dn_image_dict.items()): if band not in image.band_toa_dict.keys(): continue # thermal_band_flag = (band == image.thermal_band) # Set 0 as nodata value drigo.raster_path_set_nodata(dn_image, 0) # Calculate TOA reflectance dn_array, dn_nodata = drigo.raster_to_array( dn_image, 1, block_extent) dn_array = dn_array.astype(np.float64) # dn_array = dn_array.astype(np.float32) dn_array[dn_array == 0] = np.nan # if image.type in ['Landsat4', 'Landsat5', 'Landsat7']: refl_toa_array = et_numpy.l457_refl_toa_band_func( dn_array, image.cos_theta_solar, image.dr, image.esun_dict[band], image.lmin_dict[band], image.lmax_dict[band], image.qcalmin_dict[band], image.qcalmax_dict[band]) elif image.type in ['Landsat8']: refl_toa_array = et_numpy.l8_refl_toa_band_func( dn_array, image.cos_theta_solar, image.refl_mult_dict[band], image.refl_add_dict[band]) # if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l457_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], # image.qcalmax_dict[band]) # # image.rad_mult_dict[band], # # image.rad_add_dict[band]) # elif (image.type in ['Landsat8'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l8_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.refl_mult_dict[band], # image.refl_add_dict[band]) # elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # thermal_band_flag): # refl_toa_array = et_numpy.l457_ts_bt_band_func( # dn_array, # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], # image.qcalmax_dict[band], # # image.rad_mult_dict[band], # # image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # elif (image.type in ['Landsat8'] and # thermal_band_flag): # refl_toa_array = et_numpy.l8_ts_bt_band_func( # dn_array, # image.rad_mult_dict[band], # image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # refl_toa_array = et_numpy.refl_toa_band_func( # dn_array, cos_theta_solar_flt, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # thermal_band_flag) drigo.block_to_raster( refl_toa_array.astype(np.float32), image.refl_toa_raster, b_i, b_j, band=image.band_toa_dict[band]) # drigo.array_to_comp_raster( # refl_toa_array.astype(np.float32), # image.refl_toa_raster, # image.band_toa_dict[band], common_array) del refl_toa_array, dn_array if stats_flag: drigo.raster_statistics(image.refl_toa_raster) # # Process each TOA band # # for band, band_i in sorted(image.band_toa_dict.items()): # for band, dn_image in sorted(dn_image_dict.items()): # thermal_band_flag = (band == image.thermal_band) # # Set 0 as nodata value # drigo.raster_path_set_nodata(dn_image, 0) # # Calculate TOA reflectance # dn_array, dn_nodata = drigo.raster_to_array( # dn_image, 1, common_extent) # dn_array = dn_array.astype(np.float64) # # dn_array = dn_array.astype(np.float32) # dn_array[dn_array == 0] = np.nan # # # if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l457_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band]) # # image.rad_mult_dict[band], image.rad_add_dict[band]) # elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and # thermal_band_flag): # refl_toa_array = et_numpy.l457_ts_bt_band_func( # dn_array, image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # # image.rad_mult_dict[band], image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # elif (image.type in ['Landsat8'] and # not thermal_band_flag): # refl_toa_array = et_numpy.l8_refl_toa_band_func( # dn_array, image.cos_theta_solar, # image.refl_mult_dict[band], image.refl_add_dict[band]) # elif (image.type in ['Landsat8'] and # thermal_band_flag): # refl_toa_array = et_numpy.l8_ts_bt_band_func( # dn_array, # image.rad_mult_dict[band], image.rad_add_dict[band], # image.k1_dict[band], image.k2_dict[band]) # # refl_toa_array = et_numpy.refl_toa_band_func( # # dn_array, cos_theta_solar_flt, # # image.dr, image.esun_dict[band], # # image.lmin_dict[band], image.lmax_dict[band], # # image.qcalmin_dict[band], image.qcalmax_dict[band], # # thermal_band_flag) # drigo.array_to_comp_raster( # refl_toa_array.astype(np.float32), image.refl_toa_raster, # image.band_toa_dict[band], common_array) # del refl_toa_array, dn_array # Calculate brightness temperature if calc_ts_bt_flag: logging.info('Brightness Temperature') if os.path.isfile(image.ts_bt_raster) and overwrite_flag: logging.debug(' Overwriting: {}'.format(image.ts_bt_raster)) dripy.remove_file(image.ts_bt_raster) if not os.path.isfile(image.ts_bt_raster): band = image.thermal_band thermal_dn_path = dn_image_dict[band] drigo.raster_path_set_nodata(thermal_dn_path, 0) thermal_dn_array, thermal_dn_nodata = drigo.raster_to_array( thermal_dn_path, 1, common_extent, return_nodata=True) thermal_dn_mask = thermal_dn_array != thermal_dn_nodata if image.type in ['Landsat4', 'Landsat5', 'Landsat7']: ts_bt_array = et_numpy.l457_ts_bt_band_func( thermal_dn_array, image.lmin_dict[band], image.lmax_dict[band], image.qcalmin_dict[band], image.qcalmax_dict[band], # image.rad_mult_dict[band], image.rad_add_dict[band], image.k1_dict[band], image.k2_dict[band]) elif image.type in ['Landsat8']: ts_bt_array = et_numpy.l8_ts_bt_band_func( thermal_dn_array, image.rad_mult_dict[band], image.rad_add_dict[band], image.k1_dict[band], image.k2_dict[band]) # thermal_rad_array = et_numpy.refl_toa_band_func( # thermal_dn_array, image.cos_theta_solar, # image.dr, image.esun_dict[band], # image.lmin_dict[band], image.lmax_dict[band], # image.qcalmin_dict[band], image.qcalmax_dict[band], # thermal_band_flag=True) # ts_bt_array = et_numpy.ts_bt_func( # thermal_rad_array, image.k1_dict[image.thermal_band], # image.k2_dict[image.thermal_band]) ts_bt_array[~thermal_dn_mask] = np.nan drigo.array_to_raster( ts_bt_array, image.ts_bt_raster, output_geo=common_geo, output_proj=env.snap_proj, # mask_array=common_array, stats_flag=stats_flag) # del thermal_dn_array, thermal_rad_array del thermal_dn_path, thermal_dn_array, ts_bt_array # Interpolate/project/clip METRIC hourly/daily rasters if (calc_metric_ea_flag or calc_metric_wind_flag or calc_metric_etr_flag): logging.info('METRIC hourly/daily rasters') # Get bracketing hours from image acquisition time image_prev_dt = image.acq_datetime.replace( minute=0, second=0, microsecond=0) image_next_dt = image_prev_dt + timedelta(seconds=3600) # Get NLDAS properties from one of the images input_ws = os.path.join( metric_etr_input_ws, str(image_prev_dt.year)) try: input_path = [ os.path.join(input_ws, file_name) for file_name in os.listdir(input_ws) for match in [metric_hourly_re.match(file_name)] if (match and (image_prev_dt.strftime('%Y%m%d') == match.group('YYYYMMDD')))][0] except IndexError: logging.error(' No hourly file for {}'.format( image_prev_dt.strftime('%Y-%m-%d %H00'))) return False try: input_ds = gdal.Open(input_path) input_osr = drigo.raster_ds_osr(input_ds) # input_proj = drigo.osr_proj(input_osr) input_extent = drigo.raster_ds_extent(input_ds) input_cs = drigo.raster_ds_cellsize(input_ds, x_only=True) # input_geo = input_extent.geo(input_cs) input_x, input_y = input_extent.origin() input_ds = None except: logging.error(' Could not get default input image properties') logging.error(' {}'.format(input_path)) return False # Project Landsat scene extent to NLDAS GCS common_gcs_osr = common_osr.CloneGeogCS() common_gcs_extent = drigo.project_extent( common_extent, common_osr, common_gcs_osr, cellsize=env.cellsize) common_gcs_extent.buffer_extent(0.1) common_gcs_extent.adjust_to_snap( 'EXPAND', input_x, input_y, input_cs) # common_gcs_geo = common_gcs_extent.geo(input_cs) def metric_weather_func(output_raster, input_ws, input_re, prev_dt, next_dt, resample_method=gdal.GRA_NearestNeighbour, rounding_flag=False): """Interpolate/project/clip METRIC hourly rasters""" logging.debug(' Output: {}'.format(output_raster)) if os.path.isfile(output_raster): if overwrite_flag: logging.debug(' Overwriting output') dripy.remove_file(output_raster) else: logging.debug(' Skipping, file already exists ' + 'and overwrite is False') return False prev_ws = os.path.join(input_ws, str(prev_dt.year)) next_ws = os.path.join(input_ws, str(next_dt.year)) # Technically previous and next could come from different days # or even years, although this won't happen in the U.S. try: prev_path = [ os.path.join(prev_ws, input_name) for input_name in os.listdir(prev_ws) for input_match in [input_re.match(input_name)] if (input_match and (prev_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input prev: {}'.format(prev_path)) except IndexError: logging.error(' No previous hourly file') logging.error(' {}'.format(prev_dt)) return False try: next_path = [ os.path.join(next_ws, input_name) for input_name in os.listdir(next_ws) for input_match in [input_re.match(input_name)] if (input_match and (next_dt.strftime('%Y%m%d') == input_match.group('YYYYMMDD')))][0] logging.debug(' Input next: {}'.format(next_path)) except IndexError: logging.error(' No next hourly file') logging.error(' {}'.format(next_dt)) return False # Band numbers are 1's based prev_band = int(prev_dt.strftime('%H')) + 1 next_band = int(next_dt.strftime('%H')) + 1 logging.debug(' Input prev band: {}'.format(prev_band)) logging.debug(' Input next band: {}'.format(next_band)) # Read arrays prev_array = drigo.raster_to_array( prev_path, band=prev_band, mask_extent=common_gcs_extent, return_nodata=False) next_array = drigo.raster_to_array( next_path, band=next_band, mask_extent=common_gcs_extent, return_nodata=False) if not np.any(prev_array) or not np.any(next_array): logging.warning('\nWARNING: Input NLDAS array is all nodata\n') return None output_array = hourly_interpolate_func( prev_array, next_array, prev_dt, next_dt, image.acq_datetime) output_array = drigo.project_array( output_array, resample_method, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits if rounding_flag: output_array = np.around(output_array, rounding_digits) # Force output to 32-bit float drigo.array_to_raster( output_array.astype(np.float32), output_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array return True # Ea - Project to Landsat scene after clipping if calc_metric_ea_flag: logging.info(' Hourly vapor pressure (Ea)') metric_weather_func( image.metric_ea_raster, metric_ea_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_Bilinear, rounding_flag=True) # Wind - Project to Landsat scene after clipping if calc_metric_wind_flag: logging.info(' Hourly windspeed') metric_weather_func( image.metric_wind_raster, metric_wind_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # ETr - Project to Landsat scene after clipping if calc_metric_etr_flag: logging.info(' Hourly reference ET (ETr)') metric_weather_func( image.metric_etr_raster, metric_etr_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # ETr 24hr - Project to Landsat scene after clipping if calc_metric_etr_flag: logging.info(' Daily reference ET (ETr)') logging.debug(' Output: {}'.format( image.metric_etr_24hr_raster)) if (os.path.isfile(image.metric_etr_24hr_raster) and overwrite_flag): logging.debug(' Overwriting output') os.remove(image.metric_etr_24hr_raster) if not os.path.isfile(image.metric_etr_24hr_raster): etr_prev_ws = os.path.join( metric_etr_input_ws, str(image_prev_dt.year)) try: input_path = [ os.path.join(etr_prev_ws, file_name) for file_name in os.listdir(etr_prev_ws) for match in [metric_daily_re.match(file_name)] if (match and (image_prev_dt.strftime('%Y%m%d') == match.group('YYYYMMDD')))][0] logging.debug(' Input: {}'.format(input_path)) except IndexError: logging.error(' No daily file for {}'.format( image_prev_dt.strftime('%Y-%m-%d'))) return False output_array = drigo.raster_to_array( input_path, mask_extent=common_gcs_extent, return_nodata=False) output_array = drigo.project_array( output_array, gdal.GRA_NearestNeighbour, input_osr, input_cs, common_gcs_extent, common_osr, env.cellsize, common_extent, output_nodata=None) # Apply common area mask output_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest n digits # output_array = np.around(output_array, rounding_digits) drigo.array_to_raster( output_array, image.metric_etr_24hr_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) del output_array del input_path # Tair - Project to Landsat scene after clipping if calc_metric_tair_flag: logging.info(' Hourly air temperature (Tair)') metric_weather_func( image.metric_tair_raster, metric_tair_input_ws, metric_hourly_re, image_prev_dt, image_next_dt, gdal.GRA_NearestNeighbour, rounding_flag=False) # Cleanup del image_prev_dt, image_next_dt # Soil Water Balance if calc_swb_ke_flag: logging.info('Daily soil water balance') # Check if output file already exists logging.debug(' Ke: {}'.format(image.ke_raster)) if os.path.isfile(image.ke_raster): if overwrite_flag: logging.debug(' Overwriting output') dripy.remove_file(image.ke_raster) else: logging.debug(' Skipping, file already ' 'exists and overwrite is False') return False ke_array = et_common.raster_swb_func( image.acq_datetime, common_osr, env.cellsize, common_extent, awc_input_path, etr_input_ws, etr_input_re, ppt_input_ws, ppt_input_re, spinup_days=spinup_days, min_spinup_days=min_spinup_days) # Apply common area mask ke_array[~common_array] = np.nan # Reduce the file size by rounding to the nearest 2 digits np.around(ke_array, 2, out=ke_array) # Force output to 32-bit float drigo.array_to_raster( ke_array.astype(np.float32), image.ke_raster, output_geo=common_geo, output_proj=common_proj, stats_flag=stats_flag) # Remove Landsat TOA rasters if not keep_dn_flag: for landsat_item in dripy.build_file_list( image.orig_data_ws, image.image_name_re): os.remove(os.path.join(image.orig_data_ws, landsat_item)) return True
def main(ini_path, tile_list=None, stats_flag=True, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, new_window_flag=False): """Run METRIC Model 1 for all images Args: ini_path (str): file path of the input parameters file tile_list (list): list of Landsat path/row (i.e. [p45r43, p45r33]) This will override the tile list in the INI file stats_flag (bool): if True, compute raster statistics. Default is True. overwrite_flag (bool): if True, overwrite existing files mp_procs (int): number of cores to use delay: max random delay starting function in seconds debug_flag (bool): if True, enable debug level logging new_window_flag (bool): if True, open each process in new terminal. Microsoft Windows only. Returns: None """ logging.info('\nRunning METRIC model 1 for all images') # Open config file config = open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) func_path = config.get('INPUTS', 'metric_model1_func') # For now build INI file name from template INI names ini_name = os.path.basename(config.get('INPUTS', 'metric_ini')) ini_name = os.path.splitext(os.path.basename(ini_name))[0] # INI file is built as a function of year and tile_name ini_fmt = '{}_{}_{}.ini' # Calculate pixel regions/rating immediatly after running Model 1 # pixel_regions_flag = True # Only allow new terminal windows on Windows if os.name is not 'nt': new_window_flag = False # Regular expressions # For now assume tiles are two digit numbers tile_re = re.compile('p\d{3}r\d{3}', re.IGNORECASE) image_re = re.compile( '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\n Folder {} does not exist'.format(project_ws)) sys.exit() mp_list = [] for tile_name in sorted(tile_list): tile_ws = os.path.join(project_ws, str(year), tile_name) if not os.path.isdir(tile_ws) and not tile_re.match(tile_name): continue # Check that there are scene folders scene_id_list = [ scene_id for scene_id in sorted(os.listdir(tile_ws)) if (os.path.isdir(os.path.join(tile_ws, scene_id)) or image_re.match(scene_id)) ] if not scene_id_list: continue logging.debug(' {} {}'.format(year, tile_name)) # Check that there is an input file for the path/row ini_path = os.path.join(tile_ws, ini_fmt.format(ini_name, year, tile_name)) if not os.path.join(ini_path): logging.warning( ' METRIC input file {} does not exist'.format(ini_path)) continue # Setup command line argument call_args = [sys.executable, func_path, '-i', ini_path] if stats_flag: call_args.append('--stats') if overwrite_flag: call_args.append('--overwrite') if debug_flag: call_args.append('--debug') # Run METRIC Model 1 for scene_id in scene_id_list: logging.debug(' {}'.format(scene_id)) scene_ws = os.path.join(tile_ws, scene_id) if mp_procs > 1: mp_list.append([call_args, scene_ws, delay, new_window_flag]) else: subprocess.call(call_args, cwd=scene_ws) if mp_list: pool = mp.Pool(mp_procs) results = pool.map(call_mp, mp_list, chunksize=1) pool.close() pool.join() del results, pool logging.debug('\nScript complete')
def main(ini_path, tile_list=None, blocksize=2048, smooth_flag=True, stats_flag=True, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, new_window_flag=False): """Prep Landsat scenes Parameters ---------- ini_path : str File path of the input parameters file. tile_list : list, optional Landsat path/rows to process (i.e. [p045r043, p045r033]). This will override the tile list in the INI file. blocksize : int, optional Processing block size (the default is 2048). smooth_flag : bool, optional If True, dilate/erode image to remove fringe/edge pixels (the Default is True). stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). mp_procs : int, optional Number of cores to use (the default is 1). delay : float, optional max random delay starting function in seconds (the default is 0). debug_flag : bool, optional If True, enable debug level logging (the default is False). new_window_flag : bool, optional If True, open each process in new terminal window (the default is False). Microsoft Windows only. Returns ------- None """ logging.info('\nPreparing Landsat scenes') # Open config file config = open_ini(ini_path) logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) func_path = config.get('INPUTS', 'prep_scene_func') skip_list_path = read_param('skip_list_path', '', config, 'INPUTS') # Only allow new terminal windows on Windows if os.name is not 'nt': new_window_flag = False # Regular expressions # For now assume path/row are two digit numbers tile_re = re.compile('p\d{3}r\d{3}', re.IGNORECASE) image_re = re.compile( '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\nFolder {} does not exist'.format(project_ws)) sys.exit() # Setup command line argument call_args = [sys.executable, func_path, '-i', ini_path] if blocksize: call_args.extend(['--blocksize', str(blocksize)]) if stats_flag: call_args.append('--stats') if overwrite_flag: call_args.append('--overwrite') if debug_flag: call_args.append('--debug') if smooth_flag: call_args.append('--smooth') # Read skip list if skip_list_path: logging.debug('\nReading scene skip list') with open(skip_list_path) as skip_list_f: skip_list = skip_list_f.readlines() skip_list = [ image_id.strip() for image_id in skip_list if image_re.match(image_id.strip()) ] else: logging.debug('\nSkip list not set in INI') skip_list = [] # Process each image mp_list = [] for tile_name in sorted(tile_list): logging.debug('\nTile: {}'.format(tile_name)) tile_ws = os.path.join(project_ws, str(year), tile_name) if not os.path.isdir(tile_ws) and not tile_re.match(tile_name): logging.debug(' No image folder, skipping') continue # Check that there are scene folders image_id_list = [ image_id for image_id in sorted(os.listdir(tile_ws)) if (os.path.isdir(os.path.join(tile_ws, image_id)) and image_re.match(image_id) and image_id not in skip_list) ] if not image_id_list: logging.debug(' No available images, skipping') continue logging.info(' {} {}'.format(year, tile_name)) # Prep each Landsat scene for image_id in image_id_list: image_ws = os.path.join(tile_ws, image_id) if mp_procs > 1: mp_list.append([call_args, image_ws, delay, new_window_flag]) else: logging.debug(' {}'.format(image_id)) subprocess.call(call_args, cwd=image_ws) if mp_list: pool = mp.Pool(mp_procs) results = pool.map(call_mp, mp_list, chunksize=1) pool.close() pool.join() del results, pool logging.debug('\nScript complete')
def main(ini_path, mc_iter_str='', tile_list=None, cold_tgt_pct=None, hot_tgt_pct=None, groupsize=64, blocksize=2048, multipoint_flag=True, shapefile_flag=True, stats_flag=True, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, new_window_flag=False, no_file_logging=False, no_final_plots=None, no_temp_plots=None): """Run METRIC Monte Carlo for all Landsat scenes Parameters ---------- ini_path : str File path of the input parameters file. mc_iter_str : str MonteCarlo iteration list and/or range. tile_list : list, optional Landsat path/rows to process (i.e. [p045r043, p045r033]). This will override the tile list in the INI file. cold_tgt_pct : float, optional Target percentage of pixels with ETrF greater than cold Kc. hot_tgt_pct : float, optional Target percentage of pixels with ETrF less than hot Kc. groupsize : int, optional Script will try to place calibration point randomly into a labeled group of clustered values with at least n pixels (the default is 64). -1 = In the largest group 0 = Anywhere in the image (not currently implemented) 1 >= In any group with a pixel count greater or equal to n blocksize : int, optional Processing block size (the default is 2048). multipoint_flag : bool, optional If True, save cal. points to multipoint shapefile (the default is True). shapefile_flag : bool, optional If True, save calibration points to shapefile (the default False). stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). mp_procs : int, optional Number of cores to use (the default is 1). delay : float, optional Max random delay starting function in seconds (the default is 0). debug_flag : bool, optional If True, enable debug level logging (the default is False). new_window_flag : bool, optional If True, open each process in new terminal window (the default is False). Microsoft Windows only. no_file_logging : bool If True, don't write logging to file (the default is False). no_final_plots : bool If True, don't save final ETrF histograms (the default is None). This will override the flag in the INI file no_temp_plots : bool If True, don't save temp ETrF histograms (the default is None). This will override the flag in the INI file Returns ------- None """ logging.info('\nRunning METRIC Monte Carlo') # Open config file config = dripy.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = dripy.read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) func_path = config.get('INPUTS', 'monte_carlo_func') keep_list_path = dripy.read_param('keep_list_path', '', config, 'INPUTS') # skip_list_path = dripy.read_param('skip_list_path', '', config, 'INPUTS') # For now, get mc_iter list from command line, not from project file # mc_iter_list = config.get('INPUTS', 'mc_iter_list') mc_iter_list = list(dripy.parse_int_set(mc_iter_str)) # Need soemthing in mc_iter_list to iterate over if not mc_iter_list: mc_iter_list = [None] # For now build INI file name from template INI names metric_ini_name = os.path.basename(config.get('INPUTS', 'metric_ini')) metric_ini_name = os.path.splitext(os.path.basename(metric_ini_name))[0] mc_ini_name = os.path.basename(config.get('INPUTS', 'monte_carlo_ini')) mc_ini_name = os.path.splitext(os.path.basename(mc_ini_name))[0] # INI file is built as a function of year and tile_name metric_ini_fmt = '{}_{}_{}.ini' mc_ini_fmt = '{}_{}_{}.ini' # Only allow new terminal windows on Windows if os.name is not 'nt': new_window_flag = False # if len(tile_list) == 1: # devel_flag = True # else: # devel_flag = False # # devel_flag = True # Regular expressions # For now assume path/row are two digit numbers tile_re = re.compile('p\d{3}r\d{3}', re.IGNORECASE) image_id_re = re.compile( '^(LT04|LT05|LE07|LC08)_(?:\w{4})_(\d{3})(\d{3})_' '(\d{4})(\d{2})(\d{2})_(?:\d{8})_(?:\d{2})_(?:\w{2})$') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\n Folder {} does not exist'.format(project_ws)) sys.exit() # Read keep/skip lists if keep_list_path: logging.debug('\nReading scene keep list') with open(keep_list_path) as keep_list_f: image_keep_list = keep_list_f.readlines() image_keep_list = [image_id.strip() for image_id in image_keep_list if image_id_re.match(image_id.strip())] else: logging.debug('\nScene keep list not set in INI') image_keep_list = [] # if skip_list_path: # logging.debug('\nReading scene skip list') # with open(skip_list_path) as skip_list_f: # image_skip_list = skip_list_f.readlines() # image_skip_list = [image_id.strip() for image_id in image_skip_list # if image_re.match(image_id.strip())] # else: # logging.debug('\nScene skip list not set in INI') # image_skip_list = [] mp_list = [] for tile_name in sorted(tile_list): logging.debug('\nTile: {}'.format(tile_name)) tile_ws = os.path.join(project_ws, str(year), tile_name) if not os.path.isdir(tile_ws) and not tile_re.match(tile_name): logging.debug(' {} {} - invalid tile, skipping'.format( year, tile_name)) continue # Check that there are image folders image_id_list = [ image_id for image_id in sorted(os.listdir(tile_ws)) if (image_id_re.match(image_id) and os.path.isdir(os.path.join(tile_ws, image_id)) and (image_keep_list and image_id in image_keep_list))] # (image_skip_list and image_id not in image_skip_list))] if not image_id_list: logging.debug(' {} {} - no available images, skipping'.format( year, tile_name)) continue else: logging.debug(' {} {}'.format(year, tile_name)) for image_id in image_id_list: image_ws = os.path.join(tile_ws, image_id) pixel_ws = os.path.join(image_ws, 'PIXELS') if not os.path.isdir(pixel_ws): os.mkdir(pixel_ws) # Since the multipoint shapefile will be appended, delete it # in the wrapper script if multipoint_flag and os.path.isdir(pixel_ws): for pixel_file in os.listdir(pixel_ws): if re.match('\w+_\w+.shp$', pixel_file): logging.info('\n Removing {}'.format(pixel_file)) os.remove(os.path.join(pixel_ws, pixel_file)) logging.debug(' {} {}'.format(year, tile_name)) # Check that there is an input file for the path/row metric_ini_path = os.path.join( tile_ws, metric_ini_fmt.format(metric_ini_name, year, tile_name)) mc_ini_path = os.path.join( tile_ws, mc_ini_fmt.format(mc_ini_name, year, tile_name)) if not os.path.join(metric_ini_path): logging.warning(' METRIC Input file {} does not exist'.format( metric_ini_path)) continue elif not os.path.join(mc_ini_path): logging.warning( ' Monte Carlo Input file {} does not exist'.format( mc_ini_path)) continue # Setup command line argument # call_args = [sys.executable, mc_func_path, '-i', ini_path] call_args = [sys.executable, func_path, '--metric_ini', metric_ini_path, '--mc_ini', mc_ini_path, '--groupsize', str(groupsize)] if cold_tgt_pct is not None and hot_tgt_pct is not None: call_args.extend(['-t', str(cold_tgt_pct), str(hot_tgt_pct)]) if blocksize: call_args.extend(['--blocksize', str(blocksize)]) if shapefile_flag: call_args.append('--shapefile') if multipoint_flag: call_args.append('--multipoint') if stats_flag: call_args.append('--stats') if overwrite_flag: call_args.append('--overwrite') if debug_flag: call_args.append('--debug') if no_file_logging: call_args.append('--no_file_logging') if no_final_plots: call_args.append('--no_final_plots') if no_temp_plots: call_args.append('--no_temp_plots') # Run all scenes for each Monte Carlo iteration for mc_iter in mc_iter_list: if mc_iter is not None: mc_args = ['-mc', str(mc_iter)] else: mc_args = [] for image_id in image_id_list: image_folder = os.path.join(tile_ws, image_id) logging.debug(' {}'.format(os.path.basename(image_folder))) if mp_procs > 1: mp_list.append([ call_args + mc_args, image_folder, delay, new_window_flag]) else: subprocess.call(call_args + mc_args, cwd=image_folder) if mp_list: pool = mp.Pool(mp_procs) results = pool.map(dripy.call_mp, mp_list, chunksize=1) pool.close() pool.join() del results, pool logging.debug('\nScript complete')
def pixel_rating(image_ws, ini_path, stats_flag=False, overwrite_flag=None): """Calculate pixel rating Args: image_ws (str): Image folder path ini_path (str): Pixel regions config file path stats_flag (bool): if True, compute raster statistics ovewrite_flag (bool): if True, overwrite existing files Returns: None """ logging.info('Generating suggested hot/cold pixel regions') log_fmt = ' {:<18s} {}' env = gdc.env image = et_image.Image(image_ws, env) np.seterr(invalid='ignore') # # Check that image_ws is valid # image_re = re.compile( # '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})') # if not os.path.isdir(image_ws) or not image_re.match(scene_id): # logging.error('\nERROR: Image folder is invalid or does not exist\n') # return False # Folder Paths region_ws = os.path.join(image_ws, 'PIXEL_REGIONS') # Open config file config = open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') # Arrays are processed by block bs = read_param('block_size', 1024, config) logging.info(' {:<18s} {}'.format('Block Size:', bs)) # Raster pyramids/statistics pyramids_flag = read_param('pyramids_flag', False, config) if pyramids_flag: gdal.SetConfigOption('HFA_USE_RRD', 'YES') if stats_flag is None: stats_flag = read_param('statistics_flag', False, config) # Overwrite if overwrite_flag is None: overwrite_flag = read_param('overwrite_flag', True, config) # Check that common_area raster exists if not os.path.isfile(image.common_area_raster): logging.error( '\nERROR: A common area raster was not found.' + '\nERROR: Please rerun prep tool to build these files.\n' + ' {}\n'.format(image.common_area_raster)) sys.exit() # Use common_area to set mask parameters common_ds = gdal.Open(image.common_area_raster) # env.mask_proj = raster_ds_proj(common_ds) env.mask_geo = gdc.raster_ds_geo(common_ds) env.mask_rows, env.mask_cols = gdc.raster_ds_shape(common_ds) env.mask_extent = gdc.geo_extent(env.mask_geo, env.mask_rows, env.mask_cols) env.mask_array = gdc.raster_ds_to_array(common_ds)[0] env.mask_path = image.common_area_raster env.snap_osr = gdc.raster_path_osr(image.common_area_raster) env.snap_proj = env.snap_osr.ExportToWkt() env.cellsize = gdc.raster_path_cellsize(image.common_area_raster)[0] common_ds = None logging.debug(' {:<18s} {}'.format('Mask Extent:', env.mask_extent)) # Read Pixel Regions config file # Currently there is no code to support applying an NLCD mask apply_nlcd_mask = False # apply_nlcd_mask = read_param('apply_nlcd_mask', False, config) apply_cdl_ag_mask = read_param('apply_cdl_ag_mask', False, config) apply_field_mask = read_param('apply_field_mask', False, config) apply_ndwi_mask = read_param('apply_ndwi_mask', True, config) apply_ndvi_mask = read_param('apply_ndvi_mask', True, config) # Currently the code to apply a study area mask is commented out # apply_study_area_mask = read_param( # 'apply_study_area_mask', False, config) albedo_rating_flag = read_param('albedo_rating_flag', True, config) nlcd_rating_flag = read_param('nlcd_rating_flag', True, config) ndvi_rating_flag = read_param('ndvi_rating_flag', True, config) ts_rating_flag = read_param('ts_rating_flag', True, config) ke_rating_flag = read_param('ke_rating_flag', False, config) # if apply_study_area_mask: # study_area_path = config.get('INPUTS', 'study_area_path') if apply_nlcd_mask or nlcd_rating_flag: nlcd_raster = config.get('INPUTS', 'landuse_raster') if apply_cdl_ag_mask: cdl_ag_raster = config.get('INPUTS', 'cdl_ag_raster') cdl_buffer_cells = read_param('cdl_buffer_cells', 0, config) cdl_ag_eroded_name = read_param('cdl_ag_eroded_name', 'cdl_ag_eroded_{}.img', config) if apply_field_mask: field_raster = config.get('INPUTS', 'fields_raster') cold_rating_pct = read_param('cold_percentile', 99, config) hot_rating_pct = read_param('hot_percentile', 99, config) # min_cold_rating_score = read_param('min_cold_rating_score', 0.3, config) # min_hot_rating_score = read_param('min_hot_rating_score', 0.3, config) ts_bin_count = int(read_param('ts_bin_count', 10, config)) if 100 % ts_bin_count != 0: logging.warning( 'WARNING: ts_bins_count of {} is not a divisor ' + 'of 100. Using default ts_bins_count = 4'.format(ts_bin_count)) ts_bin_count = 10 bin_size = 1. / (ts_bin_count - 1) hot_rating_values = np.arange(0., 1. + bin_size, step=bin_size) cold_rating_values = hot_rating_values[::-1] # Input raster paths r_fmt = '.img' if 'Landsat' in image.type: albedo_raster = image.albedo_sur_raster ndvi_raster = image.ndvi_toa_raster ndwi_raster = image.ndwi_toa_raster ts_raster = image.ts_raster ke_raster = image.ke_raster # Check config file input paths # if apply_study_area_mask and not os.path.isfile(study_area_path): # logging.error( # ('\nERROR: The study area shapefile {} does ' + # 'not exist\n').format(study_area_path)) # sys.exit() if ((apply_nlcd_mask or nlcd_rating_flag) and not os.path.isfile(nlcd_raster)): logging.error(('\nERROR: The NLCD raster {} does ' + 'not exist\n').format(nlcd_raster)) sys.exit() if apply_cdl_ag_mask and not os.path.isfile(cdl_ag_raster): logging.error(('\nERROR: The CDL Ag raster {} does ' + 'not exist\n').format(cdl_ag_raster)) sys.exit() if apply_field_mask and not os.path.isfile(field_raster): logging.error(('\nERROR: The field raster {} does ' + 'not exist\n').format(field_raster)) sys.exit() if (not (isinstance(cold_rating_pct, (int, float)) and (0 <= cold_rating_pct <= 100))): logging.error( '\nERROR: cold_percentile must be a value between 0 and 100\n') sys.exit() if (not (isinstance(hot_rating_pct, (int, float)) and (0 <= hot_rating_pct <= 100))): logging.error( '\nERROR: hot_percentile must be a value between 0 and 100\n') sys.exit() # Set raster names raster_dict = dict() # Output Rasters raster_dict['region_mask'] = os.path.join(region_ws, 'region_mask' + r_fmt) raster_dict['cold_rating'] = os.path.join(region_ws, 'cold_pixel_rating' + r_fmt) raster_dict['hot_rating'] = os.path.join(region_ws, 'hot_pixel_rating' + r_fmt) raster_dict['cold_sugg'] = os.path.join(region_ws, 'cold_pixel_suggestion' + r_fmt) raster_dict['hot_sugg'] = os.path.join(region_ws, 'hot_pixel_suggestion' + r_fmt) # Read pixel region raster flags save_dict = dict() save_dict['region_mask'] = read_param('save_region_mask_flag', False, config) save_dict['cold_rating'] = read_param('save_rating_rasters_flag', False, config) save_dict['hot_rating'] = read_param('save_rating_rasters_flag', False, config) save_dict['cold_sugg'] = read_param('save_suggestion_rasters_flag', True, config) save_dict['hot_sugg'] = read_param('save_suggestion_rasters_flag', True, config) # Output folder if not os.path.isdir(region_ws): os.mkdir(region_ws) # Remove existing files if necessary region_ws_file_list = [ os.path.join(region_ws, item) for item in os.listdir(region_ws) ] if overwrite_flag and region_ws_file_list: for raster_path in raster_dict.values(): if raster_path in region_ws_file_list: remove_file(raster_path) # Check scene specific input paths if apply_ndwi_mask and not os.path.isfile(ndwi_raster): logging.error( 'ERROR: NDWI raster does not exist\n {}'.format(ndwi_raster)) sys.exit() elif apply_ndvi_mask and not os.path.isfile(ndvi_raster): logging.error( 'ERROR: NDVI raster does not exist\n {}'.format(ndvi_raster)) sys.exit() elif ke_rating_flag and not os.path.isfile(ke_raster): logging.error( ('ERROR: The Ke raster does not exist\n {}').format(ke_raster)) sys.exit() # Remove existing and build new empty rasters if necessary # If processing by block, rating rasters must be built logging.debug('\nBuilding empty rasters') for name, save_flag in sorted(save_dict.items()): if save_flag and 'rating' in name: gdc.build_empty_raster(raster_dict[name], 1, np.float32) elif save_flag: gdc.build_empty_raster(raster_dict[name], 1, np.uint8, output_nodata=0) if apply_cdl_ag_mask: logging.info('Building CDL ag mask') cdl_array = gdc.raster_to_array(cdl_ag_raster, mask_extent=env.mask_extent, return_nodata=False) if cdl_buffer_cells > 0: logging.info(' Eroding CDL by {} cells'.format(cdl_buffer_cells)) structure_array = np.ones((cdl_buffer_cells, cdl_buffer_cells), dtype=np.int) # Deadbeef - This could blow up in memory on bigger rasters cdl_array = ndimage.binary_erosion( cdl_array, structure_array).astype(structure_array.dtype) cdl_ag_eroded_raster = os.path.join( image.support_ws, cdl_ag_eroded_name.format(cdl_buffer_cells)) gdc.array_to_raster(cdl_array, cdl_ag_eroded_raster, output_geo=env.mask_geo, output_proj=env.snap_proj, mask_array=env.mask_array, output_nodata=0, stats_flag=False) cdl_array = None del cdl_array # Build region mask logging.debug('Building region mask') region_mask = np.copy(env.mask_array).astype(np.bool) if apply_field_mask: field_mask, field_nodata = gdc.raster_to_array( field_raster, mask_extent=env.mask_extent, return_nodata=True) region_mask &= field_mask != field_nodata del field_mask, field_nodata if apply_ndwi_mask: ndwi_array = gdc.raster_to_array(ndwi_raster, 1, mask_extent=env.mask_extent, return_nodata=False) region_mask &= ndwi_array > 0.0 del ndwi_array if apply_ndvi_mask: ndvi_array = gdc.raster_to_array(ndvi_raster, 1, mask_extent=env.mask_extent, return_nodata=False) region_mask &= ndvi_array > 0.12 del ndvi_array if apply_cdl_ag_mask: cdl_array, cdl_nodata = gdc.raster_to_array( cdl_ag_eroded_raster, mask_extent=env.mask_extent, return_nodata=True) region_mask &= cdl_array != cdl_nodata del cdl_array, cdl_nodata if save_dict['region_mask']: gdc.array_to_raster(region_mask, raster_dict['region_mask'], stats_flag=False) # Initialize rating arrays # This needs to be done before the ts_rating if block cold_rating_array = np.ones(env.mask_array.shape, dtype=np.float32) hot_rating_array = np.ones(env.mask_array.shape, dtype=np.float32) cold_rating_array[~region_mask] = np.nan hot_rating_array[~region_mask] = np.nan # Temperature pixel rating - grab the max and min value for the entire # Ts image in a memory safe way by using gdal_common blocks # The following is a percentile based approach if ts_rating_flag: logging.debug('Computing Ts percentile rating') ts_array = gdc.raster_to_array(ts_raster, mask_extent=env.mask_extent, return_nodata=False) ts_array[~region_mask] = np.nan percentiles = range(0, (100 + ts_bin_count), int(100 / ts_bin_count)) ts_score_value = 1. / (ts_bin_count - 1) hot_rating_values = np.arange(0, (1. + ts_score_value), step=ts_score_value)[:ts_bin_count] cold_rating_values = hot_rating_values[::-1] ts_percentile_array = stats.scoreatpercentile( ts_array[np.isfinite(ts_array)], percentiles) for bins_i in range(len(ts_percentile_array))[:-1]: bool_array = ((ts_array > ts_percentile_array[bins_i]) & (ts_array <= ts_percentile_array[bins_i + 1])) cold_rating_array[bool_array] = cold_rating_values[bins_i] hot_rating_array[bool_array] = hot_rating_values[bins_i] # gdc.array_to_raster(cold_rating_array, raster_dict['cold_rating']) # gdc.array_to_raster(hot_rating_array, raster_dict['hot_rating']) # Cleanup del ts_array, ts_percentile_array del cold_rating_values, hot_rating_values del ts_score_value, percentiles # Process by block logging.info('\nProcessing by block') logging.debug(' Mask cols/rows: {}/{}'.format(env.mask_cols, env.mask_rows)) for b_i, b_j in gdc.block_gen(env.mask_rows, env.mask_cols, bs): logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j)) block_data_mask = gdc.array_to_block(env.mask_array, b_i, b_j, bs).astype(np.bool) # block_nodata_mask = ~block_data_mask block_rows, block_cols = block_data_mask.shape block_geo = gdc.array_offset_geo(env.mask_geo, b_j, b_i) block_extent = gdc.geo_extent(block_geo, block_rows, block_cols) logging.debug(' Block rows: {} cols: {}'.format( block_rows, block_cols)) # logging.debug(' Block extent: {}'.format(block_extent)) # logging.debug(' Block geo: {}'.format(block_geo)) # Don't skip empty blocks since block rating needs to be written # back to the array at the end of the block loop block_region_mask = gdc.array_to_block(region_mask, b_i, b_j, bs) if not np.any(block_region_mask): logging.debug(' Empty block') block_empty_flag = True else: block_empty_flag = False # New style continuous pixel weighting cold_rating_block = gdc.array_to_block(cold_rating_array, b_i, b_j, bs) hot_rating_block = gdc.array_to_block(hot_rating_array, b_i, b_j, bs) # Rating arrays already have region_mask set # cold_rating_block = np.ones(block_region_mask.shape, dtype=np.float32) # hot_rating_block = np.ones(block_region_mask.shape, dtype=np.float32) # cold_rating_block[~block_region_mask] = np.nan # hot_rating_block[~block_region_mask] = np.nan # del block_region_mask if ndvi_rating_flag and not block_empty_flag: # NDVI based rating ndvi_array = gdc.raster_to_array(ndvi_raster, 1, mask_extent=block_extent, return_nodata=False) # Don't let NDVI be negative ndvi_array.clip(0., 0.833, out=ndvi_array) # ndvi_array.clip(0.001, 0.833, out=ndvi_array) cold_rating_block *= ndvi_array cold_rating_block *= 1.20 ndvi_mask = (ndvi_array > 0) # DEADBEEF - Can this calculation be masked to only NDVI > 0? ndvi_mask = ndvi_array > 0 hot_rating_block[ndvi_mask] *= stats.norm.pdf( np.log(ndvi_array[ndvi_mask]), math.log(0.15), 0.5) hot_rating_block[ndvi_mask] *= 1.25 del ndvi_mask # hot_rating_block *= stats.norm.pdf( # np.log(ndvi_array), math.log(0.15), 0.5) # hot_rating_block *= 1.25 # cold_rating_block.clip(0., 1., out=cold_rating_block) # hot_rating_block.clip(0., 1., out=hot_rating_block) del ndvi_array if albedo_rating_flag and not block_empty_flag: # Albdo based rating albedo_array = gdc.raster_to_array(albedo_raster, 1, mask_extent=block_extent, return_nodata=False) albedo_cold_pdf = stats.norm.pdf(albedo_array, 0.21, 0.03) albedo_hot_pdf = stats.norm.pdf(albedo_array, 0.21, 0.06) del albedo_array cold_rating_block *= albedo_cold_pdf cold_rating_block *= 0.07 hot_rating_block *= albedo_hot_pdf hot_rating_block *= 0.15 # cold_rating_block.clip(0., 1., out=cold_rating_block) # hot_rating_block.clip(0., 1., out=hot_rating_block) del albedo_cold_pdf, albedo_hot_pdf if nlcd_rating_flag and not block_empty_flag: # NLCD based weighting, this could be CDL instead? nlcd_array = nlcd_rating( gdc.raster_to_array(nlcd_raster, 1, mask_extent=block_extent, return_nodata=False)) cold_rating_block *= nlcd_array hot_rating_block *= nlcd_array del nlcd_array if ke_rating_flag and not block_empty_flag: # SWB Ke based rating ke_array = gdc.raster_to_array(ke_raster, 1, mask_extent=block_extent, return_nodata=False) # Don't let NDVI be negative ke_array.clip(0., 1., out=ke_array) # Assumption, lower Ke is better for selecting the hot pixel # As the power (2) decreases and approaches 1, # the relationship gets more linear # cold_rating_block *= (1 - ke_array ** 2) # hot_rating_block *= (1 - ke_array ** 1.5) # Linear inverse # cold_rating_block *= (1. - ke_array) hot_rating_block *= (1. - ke_array) # cold_rating_block.clip(0., 1., out=cold_rating_block) # hot_rating_block.clip(0., 1., out=hot_rating_block) del ke_array # Clearness # clearness = 1.0 # cold_rating *= clearness # hot_rating *= clearness # Reset nan values # cold_rating_block[~region_mask] = np.nan # hot_rating_block[~region_mask] = np.nan # Save rating values cold_rating_array = gdc.block_to_array(cold_rating_block, cold_rating_array, b_i, b_j, bs) hot_rating_array = gdc.block_to_array(hot_rating_block, hot_rating_array, b_i, b_j, bs) # Save rating rasters if save_dict['cold_rating']: gdc.block_to_raster(cold_rating_block, raster_dict['cold_rating'], b_i, b_j, bs) if save_dict['hot_rating']: gdc.block_to_raster(hot_rating_block, raster_dict['hot_rating'], b_i, b_j, bs) # Save rating values cold_rating_array = gdc.block_to_array(cold_rating_block, cold_rating_array, b_i, b_j, bs) hot_rating_array = gdc.block_to_array(hot_rating_block, hot_rating_array, b_i, b_j, bs) del cold_rating_block, hot_rating_block # Select pixels above target percentile # Only build suggestion arrays if saving logging.debug('Building suggested pixel rasters') if save_dict['cold_sugg']: cold_rating_score = float( stats.scoreatpercentile( cold_rating_array[np.isfinite(cold_rating_array)], cold_rating_pct)) # cold_rating_array, cold_rating_nodata = gdc.raster_to_array( # raster_dict['cold_rating'], 1, mask_extent=env.mask_extent) # if cold_rating_score < float(min_cold_rating_score): # logging.error(('ERROR: The cold_rating_score ({}) is less ' + # 'than the min_cold_rating_score ({})').format( # cold_rating_score, min_cold_rating_score)) # sys.exit() cold_sugg_mask = cold_rating_array >= cold_rating_score gdc.array_to_raster(cold_sugg_mask, raster_dict['cold_sugg'], stats_flag=stats_flag) logging.debug(' Cold Percentile: {}'.format(cold_rating_pct)) logging.debug(' Cold Score: {:.6f}'.format(cold_rating_score)) logging.debug(' Cold Pixels: {}'.format(np.sum(cold_sugg_mask))) del cold_sugg_mask, cold_rating_array if save_dict['hot_sugg']: hot_rating_score = float( stats.scoreatpercentile( hot_rating_array[np.isfinite(hot_rating_array)], hot_rating_pct)) # hot_rating_array, hot_rating_nodata = gdc.raster_to_array( # raster_dict['hot_rating'], 1, mask_extent=env.mask_extent) # if hot_rating_score < float(min_hot_rating_score): # logging.error(('ERROR: The hot_rating_array ({}) is less ' + # 'than the min_hot_rating_score ({})').format( # hot_rating_array, min_hot_rating_score)) # sys.exit() hot_sugg_mask = hot_rating_array >= hot_rating_score gdc.array_to_raster(hot_sugg_mask, raster_dict['hot_sugg'], stats_flag=stats_flag) logging.debug(' Hot Percentile: {}'.format(hot_rating_pct)) logging.debug(' Hot Score: {:.6f}'.format(hot_rating_score)) logging.debug(' Hot Pixels: {}'.format(np.sum(hot_sugg_mask))) del hot_sugg_mask, hot_rating_array # Raster Statistics if stats_flag: logging.info('Calculating Statistics') for name, save_flag in save_dict.items(): if save_flag: gdc.raster_statistics(raster_dict[name]) # Raster Pyramids if pyramids_flag: logging.info('Building Pyramids') for name, save_flag in save_dict.items(): if save_flag: gdc.raster_pyramids(raster_dict[name])
def main(ini_path, tile_list=None, overwrite_flag=False): """Prep Landsat path/row specific data Args: ini_path (str): file path of the input parameters file tile_list (list): list of Landsat path/row (i.e. [p45r43, p45r33]) This will override the tile list in the INI file overwrite_flag (bool): boolean, overwrite existing files mp_procs (int): number of cores to use Returns: None """ logging.info('\nPrepare path/row INI files') # Open config file config = python_common.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = python_common.read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) ini_file_flag = python_common.read_param('ini_file_flag', True, config, 'INPUTS') landsat_flag = python_common.read_param('landsat_flag', True, config, 'INPUTS') ledaps_flag = python_common.read_param('ledaps_flag', False, config, 'INPUTS') dem_flag = python_common.read_param('dem_flag', True, config, 'INPUTS') nlcd_flag = python_common.read_param('nlcd_flag', True, config, 'INPUTS') cdl_flag = python_common.read_param('cdl_flag', True, config, 'INPUTS') landfire_flag = python_common.read_param('landfire_flag', False, config, 'INPUTS') field_flag = python_common.read_param('field_flag', False, config, 'INPUTS') metric_flag = python_common.read_param('metric_flag', True, config, 'INPUTS') monte_carlo_flag = python_common.read_param('monte_carlo_flag', False, config, 'INPUTS') interp_rasters_flag = python_common.read_param('interpolate_rasters_flag', False, config, 'INPUTS') interp_tables_flag = python_common.read_param('interpolate_tables_flag', False, config, 'INPUTS') metric_hourly_weather = python_common.read_param('metric_hourly_weather', 'NLDAS', config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') footprint_path = config.get('INPUTS', 'footprint_path') # For now, assume the UTM zone file is colocated with the footprints shapefile utm_path = python_common.read_param( 'utm_path', os.path.join(os.path.dirname(footprint_path), 'wrs2_tile_utm_zones.json'), config, 'INPUTS') skip_list_path = python_common.read_param('skip_list_path', '', config, 'INPUTS') # Ts and albedo corrections ts_correction_flag = python_common.read_param('Ts_correction_flag', True, config, 'INPUTS') k_value = python_common.read_param('K_value', 2, config, 'INPUTS') albedo_correction_flag = python_common.read_param( 'albedo_correction_flag ', True, config, 'INPUTS') dense_veg_min_albedo = python_common.read_param('dense_veg_min_albedo ', 0.18, config, 'INPUTS') # tile_gcs_buffer = read_param('tile_buffer', 0.1, config) # Template input files for scripts if metric_flag: metric_ini = config.get('INPUTS', 'metric_ini') pixel_rating_ini = config.get('INPUTS', 'pixel_rating_ini') if monte_carlo_flag: monte_carlo_ini = config.get('INPUTS', 'monte_carlo_ini') if interp_rasters_flag or interp_tables_flag: interpolate_folder = python_common.read_param('interpolate_folder', 'ET', config) interpolate_ini = config.get('INPUTS', 'interpolate_ini') if interp_rasters_flag: study_area_path = config.get('INPUTS', 'study_area_path') study_area_mask_flag = python_common.read_param( 'study_area_mask_flag', True, config) study_area_snap = python_common.read_param('study_area_snap', (0, 0), config) study_area_cellsize = python_common.read_param('study_area_cellsize', 30, config) study_area_buffer = python_common.read_param('study_area_buffer', 0, config) study_area_proj = python_common.read_param('study_area_proj', '', config) if interp_tables_flag: zones_path = config.get('INPUTS', 'zones_path') zones_name_field = python_common.read_param('zones_name_field', 'FID', config) # zones_buffer = read_param('zones_buffer', 0, config) zones_snap = python_common.read_param('zones_snap', (0, 0), config) zones_cellsize = python_common.read_param('zones_cellsize', 30, config) # zones_proj = read_param('zones_proj', '', config) zones_mask = python_common.read_param('zones_mask', None, config) zones_buffer = None zones_proj = None # Input/output folder and file paths if landsat_flag: landsat_input_ws = config.get('INPUTS', 'landsat_input_folder') else: landsat_input_ws = None if ledaps_flag: ledaps_input_ws = config.get('INPUTS', 'ledaps_input_folder') else: ledaps_input_ws = None if dem_flag: dem_input_ws = config.get('INPUTS', 'dem_input_folder') dem_tile_fmt = config.get('INPUTS', 'dem_tile_fmt') dem_output_ws = config.get('INPUTS', 'dem_output_folder') dem_output_name = python_common.read_param('dem_output_name', 'dem.img', config) # dem_output_name = config.get('INPUTS', 'dem_output_name') else: dem_input_ws, dem_tile_fmt = None, None dem_output_ws, dem_output_name = None, None if nlcd_flag: nlcd_input_path = config.get('INPUTS', 'nlcd_input_path') nlcd_output_ws = config.get('INPUTS', 'nlcd_output_folder') nlcd_output_fmt = python_common.read_param('nlcd_output_fmt', 'nlcd_{:04d}.img', config) else: nlcd_input_path, nlcd_output_ws, nlcd_output_fmt = None, None, None if cdl_flag: cdl_input_path = config.get('INPUTS', 'cdl_input_path') cdl_ag_list = config.get('INPUTS', 'cdl_ag_list') cdl_ag_list = list(python_common.parse_int_set(cdl_ag_list)) # default_cdl_ag_list = range(1,62) + range(66,78) + range(204,255) # cdl_ag_list = read_param( # 'cdl_ag_list', default_cdl_ag_list, config) # cdl_ag_list = list(map(int, cdl_ag_list)) # cdl_non_ag_list = read_param( # 'cdl_non_ag_list', [], config) cdl_output_ws = config.get('INPUTS', 'cdl_output_folder') cdl_output_fmt = python_common.read_param('cdl_output_fmt', 'cdl_{:04d}.img', config) cdl_ag_output_fmt = python_common.read_param('cdl_ag_output_fmt', 'cdl_ag_{:04d}.img', config) else: cdl_input_path, cdl_ag_list = None, None cdl_output_ws, cdl_output_fmt, cdl_ag_output_fmt = None, None, None if landfire_flag: landfire_input_path = config.get('INPUTS', 'landfire_input_path') landfire_ag_list = config.get('INPUTS', 'landfire_ag_list') landfire_ag_list = list(python_common.parse_int_set(landfire_ag_list)) # default_landfire_ag_list = range(3960,4000) # landfire_ag_list = read_param( # 'landfire_ag_list', default_landfire_ag_list, config) # landfire_ag_list = list(map(int, landfire_ag_list)) landfire_output_ws = config.get('INPUTS', 'landfire_output_folder') landfire_output_fmt = python_common.read_param('landfire_output_fmt', 'landfire_{:04d}.img', config) landfire_ag_output_fmt = python_common.read_param( 'landfire_ag_output_fmt', 'landfire_ag_{:04d}.img', config) else: landfire_input_path, landfire_ag_list = None, None landfire_output_ws = None landfire_output_fmt, landfire_ag_output_fmt = None, None if field_flag: field_input_path = config.get('INPUTS', 'field_input_path') field_output_ws = config.get('INPUTS', 'field_output_folder') field_output_fmt = python_common.read_param('field_output_fmt', 'fields_{:04d}.img', config) else: field_input_path = None field_output_ws, field_output_fmt = None, None if monte_carlo_flag: etrf_training_path = config.get('INPUTS', 'etrf_training_path') # mc_iter_list = config.get('INPUTS', 'mc_iter_list') # mc_iter_list = list(python_common.parse_int_set(mc_iter_list)) if monte_carlo_flag or interp_rasters_flag or interp_tables_flag: etrf_input_ws = python_common.read_param('etrf_input_folder', None, config) # if etrf_input_ws is None: # etrf_input_ws = os.path.join(project_ws, year) etr_input_ws = config.get('INPUTS', 'etr_input_folder') ppt_input_ws = config.get('INPUTS', 'ppt_input_folder') etr_input_re = config.get('INPUTS', 'etr_input_re') ppt_input_re = config.get('INPUTS', 'ppt_input_re') if monte_carlo_flag or interp_rasters_flag or interp_tables_flag: awc_input_path = config.get('INPUTS', 'awc_input_path') spinup_days = python_common.read_param('swb_spinup_days', 30, config, 'INPUTS') min_spinup_days = python_common.read_param('swb_min_spinup_days', 5, config, 'INPUTS') # Weather data parameters if metric_flag: metric_hourly_weather_list = ['NLDAS', 'REFET'] metric_hourly_weather = config.get('INPUTS', 'metric_hourly_weather').upper() if metric_hourly_weather not in metric_hourly_weather_list: logging.error( ('\nERROR: The METRIC hourly weather type {} is invalid.' + '\nERROR: Set metric_hourly_weather to {}').format( metric_hourly_weather, ','.join(metric_hourly_weather_list))) sys.exit() elif metric_hourly_weather == 'REFET': refet_params_path = os.path.normpath( config.get('INPUTS', 'refet_params_path')) elif metric_hourly_weather == 'NLDAS': # metric_hourly_re = config.get('INPUTS', 'metric_hourly_re') # metric_daily_re = config.get('INPUTS', 'metric_daily_re') metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder') metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder') metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder') try: calc_metric_tair_flag = config.getboolean( 'INPUTS', 'calc_metric_tair_flag') metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder') except: calc_metric_tair_flag = False metric_tair_input_ws = '' # Check inputs folders/paths logging.info('\nChecking input folders/files') file_check(footprint_path) if landsat_flag: folder_check(landsat_input_ws) if ledaps_flag: folder_check(ledaps_input_ws) if dem_flag: folder_check(dem_input_ws) if nlcd_flag: file_check(nlcd_input_path) if cdl_flag: file_check(cdl_input_path) if landfire_flag: # Landfire will likely be an ESRI grid (set as a folder) if not (os.path.isdir(landfire_input_path) or os.path.isfile(landfire_input_path)): logging.error(' {} does not exist.'.format(landfire_input_path)) sys.exit() if field_flag: file_check(field_input_path) if metric_flag: file_check(metric_ini) file_check(pixel_rating_ini) if interp_rasters_flag or interp_tables_flag or monte_carlo_flag: if etrf_input_ws is not None: folder_check(etrf_input_ws) folder_check(etr_input_ws) folder_check(ppt_input_ws) file_check(awc_input_path) if monte_carlo_flag: file_check(monte_carlo_ini) file_check(etrf_training_path) if metric_flag: if metric_hourly_weather == 'REFET': file_check(refet_params_path) elif metric_hourly_weather == 'NLDAS': folder_check(metric_ea_input_ws) folder_check(metric_wind_input_ws) folder_check(metric_etr_input_ws) if calc_metric_tair_flag: folder_check(metric_tair_input_ws) if skip_list_path: file_check(skip_list_path) # Build output folders if not os.path.isdir(project_ws): os.makedirs(project_ws) # For now assume path/row are two digit numbers tile_fmt = 'p{:03d}r{:03d}' # Set snap environment parameters snap_cs = 30 snap_xmin, snap_ymin = (15, 15) env = gdc.env env.cellsize = snap_cs env.snap_xmin, env.snap_ymin = snap_xmin, snap_ymin # Use WGSS84 (EPSG 4326) for GCS spatial reference # Could also use NAD83 (EPSG 4269) # gcs_epsg = 4326 # gcs_osr = epsg_osr(4326) # gcs_proj = osr_proj(gcs_osr) # Landsat Footprints (WRS2 Descending Polygons) logging.debug('\nFootprint (WRS2 descending should be GCS84):') tile_gcs_osr = gdc.feature_path_osr(footprint_path) logging.debug(' OSR: {}'.format(tile_gcs_osr)) # Doublecheck that WRS2 descending shapefile is GCS84 # if tile_gcs_osr != epsg_osr(4326): # logging.error(' WRS2 is not GCS84') # sys.exit() # Get geometry for each path/row tile_gcs_wkt_dict = path_row_wkt_func(footprint_path, path_field='PATH', row_field='ROW') # Get UTM zone for each path/row # DEADBEEF - Using "eval" is considered unsafe and should be changed tile_utm_zone_dict = eval(open(utm_path, 'r').read()) # Check that each path/row extent and UTM zone exist logging.info('\nChecking path/row list against footprint shapefile') for tile_name in sorted(tile_list): if tile_name not in tile_gcs_wkt_dict.keys(): logging.error( ' {} feature not in footprint shapefile'.format(tile_name)) continue elif tile_name not in tile_utm_zone_dict.keys(): logging.error( ' {} UTM zone not in footprint shapefile'.format(tile_name)) continue elif tile_utm_zone_dict[tile_name] == 0: logging.error((' UTM zone is not set for {} in ' + 'footprint shapefile').format(tile_name)) continue # Read RefET parameters if metric_hourly_weather == 'REFET': refet_ws = os.path.dirname(refet_params_path) with open(refet_params_path, 'r') as input_f: lines = input_f.readlines() lines = [line.strip() for line in lines] lines = [line.split(',') for line in lines if line] columns = lines.pop(0) refet_params_dict = defaultdict(dict) for line in lines: tile_name = tile_fmt.format(int(line[columns.index('PATH')]), int(line[columns.index('ROW')])) yr_tile_name = '{}_{}'.format(line[columns.index('YEAR')], tile_name) for i, column in enumerate(columns): if column not in ['YEAR', 'PATH', 'ROW']: refet_params_dict[yr_tile_name][column.lower()] = line[i] # Process input files for each year and path/row logging.info('\nBuilding path/row specific input files') for tile_name in tile_list: tile_output_ws = os.path.join(project_ws, str(year), tile_name) logging.info('{} {}'.format(year, tile_name)) yr_tile_name = '{}_{}'.format(year, tile_name) if not os.path.isdir(tile_output_ws): os.makedirs(tile_output_ws) # File paths if metric_flag: tile_metric_ini = os.path.join( tile_output_ws, os.path.basename(metric_ini).replace( '.ini', '_{}_{}.ini'.format(year, tile_name))) tile_pixel_rating_ini = os.path.join( tile_output_ws, os.path.basename(pixel_rating_ini).replace( '.ini', '_{}_{}.ini'.format(year, tile_name))) if overwrite_flag and os.path.isfile(tile_metric_ini): os.remove(tile_metric_ini) if overwrite_flag and os.path.isfile(tile_pixel_rating_ini): os.remove(tile_pixel_rating_ini) # Monte Carlo is independent of tile and year, but process # with METRIC input file if monte_carlo_flag: tile_monte_carlo_ini = os.path.join( tile_output_ws, os.path.basename(monte_carlo_ini).replace( '.ini', '_{}_{}.ini'.format(year, tile_name))) if overwrite_flag and os.path.isfile(tile_monte_carlo_ini): os.remove(tile_monte_carlo_ini) if dem_flag: dem_output_path = os.path.join(dem_output_ws, tile_name, dem_output_name) if nlcd_flag: nlcd_output_path = os.path.join(nlcd_output_ws, tile_name, nlcd_output_fmt.format(year)) if cdl_flag: cdl_ag_output_path = os.path.join(cdl_output_ws, tile_name, cdl_ag_output_fmt.format(year)) if landfire_flag: landfire_ag_output_path = os.path.join( landfire_output_ws, tile_name, landfire_output_fmt.format(year)) if field_flag: field_output_path = os.path.join(field_output_ws, tile_name, field_output_fmt.format(year)) # Check that the path/row was in the RefET parameters file if (metric_flag and metric_hourly_weather == 'REFET' and yr_tile_name not in refet_params_dict.keys()): logging.error( (' The year {} & path/row {} is not in the ' + 'RefET parameters csv, skipping').format(year, tile_name)) continue if metric_flag and not os.path.isfile(tile_metric_ini): # DEADBEEF - This approach removes all formatting and comments config = configparser.RawConfigParser() config.read(metric_ini) # shutil.copy(metric_ini, tile_metric_ini) # config.read(tile_metric_ini) if metric_hourly_weather == 'REFET': # Add RefET options config.set('INPUTS', 'weather_data_source', 'REFET') config.set( 'INPUTS', 'refet_file', os.path.join( refet_ws, os.path.normpath( refet_params_dict[yr_tile_name]['refet_file']))) config.set('INPUTS', 'gmt_offset', refet_params_dict[yr_tile_name]['gmt_offset']) config.set('INPUTS', 'datum', refet_params_dict[yr_tile_name]['datum']) elif metric_hourly_weather == 'NLDAS': # Add NLDAS options config.set('INPUTS', 'weather_data_source', 'NLDAS') # Remove RefET options try: config.remove_option('INPUTS', 'refet_file') except: pass try: config.remove_option('INPUTS', 'gmt_offset') except: pass # try: # config.remove_option('INPUTS', 'datum') # except: # pass if dem_flag: config.set('INPUTS', 'dem_raster', dem_output_path) else: try: config.remove_option('INPUTS', 'dem_raster') except: pass # config.set('INPUTS', 'dem_raster', 'None') if nlcd_flag: config.set('INPUTS', 'landuse_raster', nlcd_output_path) else: try: config.remove_option('INPUTS', 'landuse_raster') except: pass # config.set('INPUTS', 'landuse_raster', 'None') logging.debug(' {}'.format(tile_metric_ini)) with open(tile_metric_ini, 'w') as config_f: config.write(config_f) if metric_flag and not os.path.isfile(tile_pixel_rating_ini): config = configparser.RawConfigParser() config.read(pixel_rating_ini) if nlcd_flag: config.set('INPUTS', 'landuse_raster', nlcd_output_path) else: try: config.remove_option('INPUTS', 'landuse_raster') except: pass # config.set('INPUTS', 'landuse_raster', 'None') if cdl_flag: config.set('INPUTS', 'apply_cdl_ag_mask', True) config.set('INPUTS', 'cdl_ag_raster', cdl_ag_output_path) else: config.set('INPUTS', 'apply_cdl_ag_mask', False) try: config.remove_option('INPUTS', 'cdl_ag_raster') except: pass # config.set('INPUTS', 'cdl_ag_raster', 'None') if field_flag: config.set('INPUTS', 'apply_field_mask', True) config.set('INPUTS', 'fields_raster', field_output_path) else: config.set('INPUTS', 'apply_field_mask', False) try: config.remove_option('INPUTS', 'fields_raster') except: pass # config.set('INPUTS', 'fields_raster', 'None') # if landfire_flag: # config.set('INPUTS', 'apply_landfire_ag_mask', True) # config.set('INPUTS', 'landfire_ag_raster', cdl_ag_output_path) # else: # config.set('INPUTS', 'apply_landfire_ag_mask', False) # try: config.remove_option('INPUTS', 'landfire_ag_raster') # except: pass # # config.set('INPUTS', 'landfire_ag_raster', 'None') logging.debug(' {}'.format(tile_pixel_rating_ini)) with open(tile_pixel_rating_ini, 'w') as config_f: config.write(config_f) if monte_carlo_flag and not os.path.isfile(tile_monte_carlo_ini): config = configparser.RawConfigParser() config.read(monte_carlo_ini) config.set('INPUTS', 'etrf_training_path', etrf_training_path) config.set('INPUTS', 'etr_ws', etr_input_ws) config.set('INPUTS', 'ppt_ws', ppt_input_ws) config.set('INPUTS', 'etr_re', etr_input_re) config.set('INPUTS', 'ppt_re', ppt_input_re) config.set('INPUTS', 'awc_path', awc_input_path) config.set('INPUTS', 'swb_spinup_days', spinup_days) config.set('INPUTS', 'swb_min_spinup_days', min_spinup_days) logging.debug(' {}'.format(tile_monte_carlo_ini)) with open(tile_monte_carlo_ini, 'w') as config_f: config.write(config_f) # Cleanup del tile_output_ws, yr_tile_name # Interpolator input file if interp_rasters_flag or interp_tables_flag: logging.info('\nBuilding interpolator input files') year_interpolator_name = os.path.basename(interpolate_ini).replace( '.ini', '_{}_{}.ini'.format(year, interpolate_folder.lower())) year_interpolator_ini = os.path.join(project_ws, str(year), year_interpolator_name) if overwrite_flag and os.path.isfile(year_interpolator_ini): os.remove(year_interpolator_ini) if not os.path.isfile(year_interpolator_ini): # First copy the template config file to the year folder shutil.copy(interpolate_ini, year_interpolator_ini) # Open the existing config file and update the values # DEADBEEF - This approach removes all formatting and comments config = configparser.RawConfigParser() config.read(year_interpolator_ini) config.set('INPUTS', 'folder_name', interpolate_folder) config.set('INPUTS', 'tile_list', ', '.join(tile_list)) if interp_rasters_flag: config.set('INPUTS', 'study_area_path', study_area_path) config.set('INPUTS', 'study_area_mask_flag', study_area_mask_flag) config.set('INPUTS', 'study_area_snap', ', '.join(map(str, study_area_snap))) config.set('INPUTS', 'study_area_cellsize', study_area_cellsize) config.set('INPUTS', 'study_area_buffer', study_area_buffer) if study_area_proj: config.set('INPUTS', 'study_area_proj', study_area_proj) else: try: config.remove_option('INPUTS', 'study_area_proj', study_area_proj) except: pass if interp_tables_flag: config.set('INPUTS', 'zones_path', zones_path) config.set('INPUTS', 'zones_snap', ', '.join(map(str, zones_snap))) config.set('INPUTS', 'zones_cellsize', zones_cellsize) config.set('INPUTS', 'zones_name_field', zones_name_field) # zones_buffer is not currently implemented if zones_buffer: config.set('INPUTS', 'zones_buffer', zones_buffer) else: try: config.remove_option('INPUTS', 'zones_buffer', zones_buffer) except: pass # zones proj., cellsize, and snap are not needed or # read in if zones_mask is set # zones_proj is not currently implemented if zones_mask: config.set('INPUTS', 'zones_mask', zones_mask) try: config.remove_option('INPUTS', 'zones_proj') except: pass try: config.remove_option('INPUTS', 'zones_cellsize') except: pass try: config.remove_option('INPUTS', 'zones_snap') except: pass # elif zones_proj: # config.set('INPUTS', 'zones_proj', zones_proj) # try: # config.remove_option('INPUTS', 'zones_mask') # except: # pass else: try: config.remove_option('INPUTS', 'zones_proj') except: pass try: config.remove_option('INPUTS', 'zones_mask') except: pass config.set('INPUTS', 'year', year) config.set('INPUTS', 'footprint_path', footprint_path) if etrf_input_ws is not None: config.set('INPUTS', 'etrf_input_folder', etrf_input_ws) config.set('INPUTS', 'etr_input_folder', etr_input_ws) config.set('INPUTS', 'etr_input_re', etr_input_re) config.set('INPUTS', 'ppt_input_folder', ppt_input_ws) config.set('INPUTS', 'ppt_input_re', ppt_input_re) # DEADBEEF - add check for SWB flag config.set('INPUTS', 'awc_input_path', awc_input_path) config.set('INPUTS', 'swb_spinup_days', spinup_days) config.set('INPUTS', 'swb_min_spinup_days', min_spinup_days) # Albdeo and Ts correction config.set('INPUTS', 'Ts_correction_flag', ts_correction_flag) config.set('INPUTS', 'K_value ', k_value) config.set('INPUTS', 'albedo_correction_flag', albedo_correction_flag) config.set('INPUTS', 'dense_veg_min_albedo', dense_veg_min_albedo) logging.debug(' {}'.format(year_interpolator_ini)) with open(year_interpolator_ini, 'w') as config_f: config.write(config_f) logging.debug('\nScript complete')
def main(ini_path, tile_list=None, overwrite_flag=False, mp_procs=1): """Prep Landsat path/row specific data Parameters ---------- ini_path : str File path of the input parameters file. tile_list : list, optional Landsat path/rows to process (i.e. [p045r043, p045r033]). This will override the tile list in the INI file. overwrite_flag : bool, optional If True, overwrite existing files (the default is False). mp_procs : int, optional Number of cores to use (the default is 1). Returns ------- None """ logging.info('\nPrepare path/row data') # Open config file config = python_common.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = python_common.read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) # study_area_path = config.get('INPUTS', 'study_area_path') footprint_path = config.get('INPUTS', 'footprint_path') # For now, assume the UTM zone file is colocated with the footprints shapefile utm_path = python_common.read_param( 'utm_path', os.path.join(os.path.dirname(footprint_path), 'wrs2_tile_utm_zones.json'), config, 'INPUTS') skip_list_path = python_common.read_param('skip_list_path', '', config, 'INPUTS') landsat_flag = python_common.read_param('landsat_flag', True, config, 'INPUTS') ledaps_flag = False dem_flag = python_common.read_param('dem_flag', True, config, 'INPUTS') nlcd_flag = python_common.read_param('nlcd_flag', True, config, 'INPUTS') cdl_flag = python_common.read_param('cdl_flag', False, config, 'INPUTS') landfire_flag = python_common.read_param('landfire_flag', False, config, 'INPUTS') field_flag = python_common.read_param('field_flag', False, config, 'INPUTS') tile_gcs_buffer = python_common.read_param('tile_buffer', 0.25, config) # Input/output folder and file paths if landsat_flag: landsat_input_ws = config.get('INPUTS', 'landsat_input_folder') else: landsat_input_ws = None # if ledaps_flag: # ledaps_input_ws = config.get('INPUTS', 'ledaps_input_folder') # else: # ledaps_input_ws = None if dem_flag: dem_input_ws = config.get('INPUTS', 'dem_input_folder') dem_tile_fmt = config.get('INPUTS', 'dem_tile_fmt') dem_output_ws = config.get('INPUTS', 'dem_output_folder') dem_output_name = python_common.read_param('dem_output_name', 'dem.img', config) # dem_output_name = config.get('INPUTS', 'dem_output_name') else: dem_input_ws, dem_tile_fmt = None, None dem_output_ws, dem_output_name = None, None if nlcd_flag: nlcd_input_path = config.get('INPUTS', 'nlcd_input_path') nlcd_output_ws = config.get('INPUTS', 'nlcd_output_folder') nlcd_output_fmt = python_common.read_param('nlcd_output_fmt', 'nlcd_{:04d}.img', config) else: nlcd_input_path, nlcd_output_ws, nlcd_output_fmt = None, None, None if cdl_flag: cdl_input_path = config.get('INPUTS', 'cdl_input_path') cdl_ag_list = config.get('INPUTS', 'cdl_ag_list') cdl_ag_list = list(python_common.parse_int_set(cdl_ag_list)) # default_cdl_ag_list = range(1,62) + range(66,78) + range(204,255) # cdl_ag_list = python_common.read_param( # 'cdl_ag_list', default_cdl_ag_list, config) # cdl_ag_list = list(map(int, cdl_ag_list)) # cdl_non_ag_list = python_common.read_param( # 'cdl_non_ag_list', [], config) cdl_output_ws = config.get('INPUTS', 'cdl_output_folder') cdl_output_fmt = python_common.read_param('cdl_output_fmt', 'cdl_{:04d}.img', config) cdl_ag_output_fmt = python_common.read_param('cdl_ag_output_fmt', 'cdl_ag_{:04d}.img', config) else: cdl_input_path, cdl_ag_list = None, None cdl_output_ws, cdl_output_fmt, cdl_ag_output_fmt = None, None, None if landfire_flag: landfire_input_path = config.get('INPUTS', 'landfire_input_path') landfire_ag_list = config.get('INPUTS', 'landfire_ag_list') landfire_ag_list = list(python_common.parse_int_set(landfire_ag_list)) # default_landfire_ag_list = range(3960,4000) # landfire_ag_list = python_common.read_param( # 'landfire_ag_list', default_landfire_ag_list, config) # landfire_ag_list = list(map(int, landfire_ag_list)) landfire_output_ws = config.get('INPUTS', 'landfire_output_folder') landfire_output_fmt = python_common.read_param('landfire_output_fmt', 'landfire_{:04d}.img', config) landfire_ag_output_fmt = python_common.read_param( 'landfire_ag_output_fmt', 'landfire_ag_{:04d}.img', config) else: landfire_input_path, landfire_ag_list = None, None landfire_output_ws = None landfire_output_fmt, landfire_ag_output_fmt = None, None if field_flag: field_input_path = config.get('INPUTS', 'field_input_path') field_output_ws = config.get('INPUTS', 'field_output_folder') field_output_fmt = python_common.read_param('field_output_fmt', 'fields_{:04d}.img', config) else: field_input_path = None field_output_ws, field_output_fmt = None, None # File/folder names orig_data_folder_name = 'ORIGINAL_DATA' # Check inputs folders/paths logging.info('\nChecking input folders/files') file_check(footprint_path) file_check(utm_path) if landsat_flag: folder_check(landsat_input_ws) # if ledaps_flag: # folder_check(ledaps_input_ws) if dem_flag: folder_check(dem_input_ws) if nlcd_flag: file_check(nlcd_input_path) if cdl_flag: file_check(cdl_input_path) if landfire_flag: # Landfire will likely be an ESRI grid (set as a folder) if not (os.path.isdir(landfire_input_path) or os.path.isfile(landfire_input_path)): logging.error('\n {} does not exist'.format(landfire_input_path)) if field_flag: file_check(field_input_path) if skip_list_path: file_check(skip_list_path) # Build output folders if not os.path.isdir(project_ws): os.makedirs(project_ws) if dem_flag and not os.path.isdir(dem_output_ws): os.makedirs(dem_output_ws) if nlcd_flag and not os.path.isdir(nlcd_output_ws): os.makedirs(nlcd_output_ws) if cdl_flag and not os.path.isdir(cdl_output_ws): os.makedirs(cdl_output_ws) if landfire_flag and not os.path.isdir(landfire_output_ws): os.makedirs(landfire_output_ws) if field_flag and not os.path.isdir(field_output_ws): os.makedirs(field_output_ws) # For now assume path/row are two digit numbers tile_fmt = 'p{:03d}r{:03d}' tile_re = re.compile('p(\d{3})r(\d{3})') image_re = re.compile( '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})') snap_cs = 30 snap_xmin, snap_ymin = (15, 15) # Set snap environment parameters env = drigo.env env.cellsize = snap_cs env.snap_xmin, env.snap_ymin = snap_xmin, snap_ymin # Use WGSS84 (EPSG 4326) for GCS spatial reference # Could also use NAD83 (EPSG 4269) # gcs_epsg = 4326 # gcs_osr = epsg_osr(4326) # gcs_proj = osr_proj(gcs_osr) # Landsat Footprints (WRS2 Descending Polygons) logging.debug('\nFootprint (WRS2 descending should be GCS84):') tile_gcs_osr = drigo.feature_path_osr(footprint_path) logging.debug(' OSR: {}'.format(tile_gcs_osr)) # Doublecheck that WRS2 descending shapefile is GCS84 # if tile_gcs_osr != epsg_osr(4326): # logging.error(' WRS2 is not GCS84') # sys.exit() # Get geometry for each path/row tile_gcs_wkt_dict = path_row_wkt_func(footprint_path, path_field='PATH', row_field='ROW') # Get UTM zone for each path/row # DEADBEEF - Using "eval" is considered unsafe and should be changed tile_utm_zone_dict = eval(open(utm_path, 'r').read()) # Project study area geometry to GCS coordinates # logging.debug('\nStudy area') # study_area_geom = feature_path_geom_union(study_area_path) # study_area_gcs_geom = study_area_geom.Clone() # study_area_gcs_geom.TransformTo(tile_gcs_osr) # Get list of all intersecting Landsat path/rows # logging.info('\nLandsat path/rows') # tile_list = [] # for tile_name, tile_gcs_wkt in tile_gcs_wkt_dict.items(): # tile_gcs_geom = ogr.CreateGeometryFromWkt(tile_gcs_wkt) # if tile_gcs_geom.Intersects(study_area_gcs_geom): # tile_list.append(tile_name) # for tile_name in sorted(tile_list): # logging.debug(' {}'.format(tile_name)) # Check that each path/row extent and UTM zone exist logging.info('\nChecking path/row list against footprint shapefile') for tile_name in sorted(tile_list): if tile_name not in tile_gcs_wkt_dict.keys(): logging.error( ' {} feature not in footprint shapefile'.format(tile_name)) continue elif tile_name not in tile_utm_zone_dict.keys(): logging.error( ' {} UTM zone not in footprint shapefile'.format(tile_name)) continue elif tile_utm_zone_dict[tile_name] == 0: logging.error((' UTM zone is not set for {} in ' + 'footprint shapefile').format(tile_name)) continue # Build output folders for each path/row logging.info('\nBuilding path/row folders') for tile_name in tile_list: logging.debug(' {} {}'.format(year, tile_name)) tile_output_ws = os.path.join(project_ws, str(year), tile_name) if ((landsat_flag or ledaps_flag) and not os.path.isdir(tile_output_ws)): os.makedirs(tile_output_ws) if (dem_flag and not os.path.isdir(os.path.join(dem_output_ws, tile_name))): os.makedirs(os.path.join(dem_output_ws, tile_name)) if (nlcd_flag and not os.path.isdir(os.path.join(nlcd_output_ws, tile_name))): os.makedirs(os.path.join(nlcd_output_ws, tile_name)) if (cdl_flag and not os.path.isdir(os.path.join(cdl_output_ws, tile_name))): os.makedirs(os.path.join(cdl_output_ws, tile_name)) if (landfire_flag and not os.path.isdir( os.path.join(landfire_output_ws, tile_name))): os.makedirs(os.path.join(landfire_output_ws, tile_name)) if (field_flag and not os.path.isdir(os.path.join(field_output_ws, tile_name))): os.makedirs(os.path.join(field_output_ws, tile_name)) # Read skip list if (landsat_flag or ledaps_flag) and skip_list_path: logging.debug('\nReading scene skiplist') with open(skip_list_path) as skip_list_f: skip_list = skip_list_f.readlines() skip_list = [ scene.strip() for scene in skip_list if image_re.match(scene.strip()) ] else: logging.debug('\nSkip list not set in INI') skip_list = [] # Copy and unzip raw Landsat scenes # Use these for thermal band, MTL file (scene time), and to run FMask if landsat_flag: logging.info('\nExtract raw Landsat scenes') # Process each path/row extract_targz_list = [] for tile_name in tile_list: tile_output_ws = os.path.join(project_ws, str(year), tile_name) # path/row as strings with leading zeros path, row = map(str, tile_re.match(tile_name).groups()) tile_input_ws = os.path.join(landsat_input_ws, path, row, str(year)) if not os.path.isdir(tile_input_ws): continue logging.info(' {} {}'.format(year, tile_name)) # Process each tar.gz file for input_name in sorted(os.listdir(tile_input_ws)): if (not image_re.match(input_name) and not input_name.endswith('.tar.gz')): continue # Get Landsat scene ID from tar.gz file name # DEADBEEF - For now this is the EE scene ID, but it could be # changed to the full collection 1 ID scene_id = input_name.split('.')[0] # Output workspace image_output_ws = os.path.join(tile_output_ws, scene_id) orig_data_ws = os.path.join(image_output_ws, orig_data_folder_name) if skip_list and scene_id in skip_list: logging.debug(' {} - Skipping scene'.format(scene_id)) # DEADBEEF - Should the script always remove the scene # if it is in the skip list? # Maybe only if overwrite is set? if os.path.isdir(image_output_ws): # input('Press ENTER to delete {}'.format(scene_id)) shutil.rmtree(image_output_ws) continue # If orig_data_ws doesn't exist, don't check images if not os.path.isdir(orig_data_ws): os.makedirs(orig_data_ws) elif (not overwrite_flag and landsat_files_check(image_output_ws)): continue # Extract Landsat tar.gz file input_path = os.path.join(tile_input_ws, input_name) print(orig_data_ws) # sys.exit() if mp_procs > 1: extract_targz_list.append([input_path, orig_data_ws]) else: python_common.extract_targz_func(input_path, orig_data_ws) # # Use a command line call # input_path = os.path.join(tile_input_ws, input_name) # if job_i % pbs_jobs != 0: # job_list.append('tar -zxvf {} -C {} &\n'.format( # input_path, orig_data_ws)) # else: # job_list.append('tar -zxvf {} -C {}\n'.format( # input_path, orig_data_ws)) # # job_list.append('tar -zxvf {} -C {} &\n'.format( # # input_path, orig_data_ws)) # # job_list.append('wait\n') # job_i += 1 # Extract Landsat tar.gz files using multiprocessing if extract_targz_list: pool = mp.Pool(mp_procs) results = pool.map(python_common.extract_targz_mp, extract_targz_list, chunksize=1) pool.close() pool.join() del results, pool # Get projected extent for each path/row # This should probably be in a function if (dem_flag or nlcd_flag or cdl_flag or landfire_flag or field_flag): tile_utm_extent_dict = gcs_to_utm_dict(tile_list, tile_utm_zone_dict, tile_gcs_osr, tile_gcs_wkt_dict, tile_gcs_buffer, snap_xmin, snap_ymin, snap_cs) # Mosaic DEM tiles for each path/row if dem_flag: logging.info('\nBuild DEM for each path/row') mosaic_mp_list = [] for tile_name in tile_list: # Output folder and path tile_output_path = os.path.join(dem_output_ws, tile_name, dem_output_name) if not overwrite_flag and os.path.isfile(tile_output_path): logging.debug(' {} already exists, skipping'.format( os.path.basename(tile_output_path))) continue logging.info(' {}'.format(tile_name)) # Get the path/row geometry in GCS for selecting intersecting tiles tile_gcs_geom = ogr.CreateGeometryFromWkt( tile_gcs_wkt_dict[tile_name]) # Apply a small buffer (in degrees) to the extent # DEADBEEF - Buffer fails if GDAL is not built with GEOS support # tile_gcs_geom = tile_gcs_geom.Buffer(tile_gcs_buffer) tile_gcs_extent = drigo.Extent(tile_gcs_geom.GetEnvelope()) tile_gcs_extent = tile_gcs_extent.ogrenv_swap() tile_gcs_extent.buffer_extent(tile_gcs_buffer) # tile_gcs_extent.ymin, tile_gcs_extent.xmax = tile_gcs_extent.xmax, tile_gcs_extent.ymin # Offsets are needed since tile name is upper left corner of tile # Tile n36w120 spans -120 <-> -119 and 35 <-> 36 lon_list = range( int(tile_gcs_extent.xmin) - 1, int(tile_gcs_extent.xmax)) lat_list = range( int(tile_gcs_extent.ymin) + 1, int(tile_gcs_extent.ymax) + 2) # Get list of DEM tile rasters dem_tile_list = [] for lat, lon in itertools.product(lat_list, lon_list): # Convert sign of lat/lon to letter lat = ('n' + '{:02d}'.format(abs(lat)) if lat >= 0 else 's' + '{:02d}'.format(abs(lat))) lon = ('w' + '{:03d}'.format(abs(lon)) if lon < 0 else 'e' + '{:03d}'.format(abs(lon))) dem_tile_path = os.path.join(dem_input_ws, dem_tile_fmt.format(lat, lon)) if os.path.isfile(dem_tile_path): dem_tile_list.append(dem_tile_path) if not dem_tile_list: logging.warning(' WARNING: No DEM tiles were selected') continue # Mosaic tiles using mosaic function tile_utm_osr = drigo.epsg_osr(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_proj = drigo.epsg_proj(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_extent = tile_utm_extent_dict[tile_name] tile_utm_ullr = tile_utm_extent.ul_lr_swap() # Mosaic, clip, project using custom function if mp_procs > 1: mosaic_mp_list.append([ dem_tile_list, tile_output_path, tile_utm_proj, snap_cs, tile_utm_extent ]) else: drigo.mosaic_tiles(dem_tile_list, tile_output_path, tile_utm_osr, snap_cs, tile_utm_extent) # Cleanup del tile_output_path del tile_gcs_geom, tile_gcs_extent, tile_utm_extent del tile_utm_osr, tile_utm_proj del lon_list, lat_list, dem_tile_list # Mosaic DEM rasters using multiprocessing if mosaic_mp_list: pool = mp.Pool(mp_procs) results = pool.map(mosaic_tiles_mp, mosaic_mp_list, chunksize=1) pool.close() pool.join() del results, pool # Project/clip NLCD for each path/row if nlcd_flag: logging.info('\nBuild NLCD for each path/row') project_mp_list = [] for tile_name in tile_list: nlcd_output_path = os.path.join(nlcd_output_ws, tile_name, nlcd_output_fmt.format(year)) if not overwrite_flag and os.path.isfile(nlcd_output_path): logging.debug(' {} already exists, skipping'.format( os.path.basename(nlcd_output_path))) continue logging.info(' {}'.format(tile_name)) # Set the nodata value on the NLCD raster if it is not set nlcd_ds = gdal.Open(nlcd_input_path, 0) nlcd_band = nlcd_ds.GetRasterBand(1) nlcd_nodata = nlcd_band.GetNoDataValue() nlcd_ds = None if nlcd_nodata is None: nlcd_nodata = 255 # Clip and project tile_utm_osr = drigo.epsg_osr(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_proj = drigo.epsg_proj(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_extent = tile_utm_extent_dict[tile_name] tile_utm_ullr = tile_utm_extent.ul_lr_swap() if mp_procs > 1: project_mp_list.append([ nlcd_input_path, nlcd_output_path, gdal.GRA_NearestNeighbour, tile_utm_proj, snap_cs, tile_utm_extent, nlcd_nodata ]) else: drigo.project_raster(nlcd_input_path, nlcd_output_path, gdal.GRA_NearestNeighbour, tile_utm_osr, snap_cs, tile_utm_extent, nlcd_nodata) # Cleanup del nlcd_output_path del nlcd_ds, nlcd_band, nlcd_nodata del tile_utm_osr, tile_utm_proj, tile_utm_extent # Project NLCD rasters using multiprocessing if project_mp_list: pool = mp.Pool(mp_procs) results = pool.map(drigo.project_raster_mp, project_mp_list, chunksize=1) pool.close() pool.join() del results, pool # Project/clip CDL for each path/row if cdl_flag: logging.info('\nBuild CDL for each path/row') project_mp_list, remap_mp_list = [], [] for tile_name in tile_list: cdl_output_path = os.path.join(cdl_output_ws, tile_name, cdl_output_fmt.format(year)) cdl_ag_output_path = os.path.join(cdl_output_ws, tile_name, cdl_ag_output_fmt.format(year)) if not os.path.isfile(cdl_input_path): logging.error('\n\n {} does not exist'.format(cdl_input_path)) sys.exit() if not overwrite_flag and os.path.isfile(cdl_output_path): logging.debug(' {} already exists, skipping'.format( os.path.basename(cdl_output_path))) continue logging.info(' {}'.format(tile_name)) # Set the nodata value on the CDL raster if it is not set cdl_ds = gdal.Open(cdl_input_path, 0) cdl_band = cdl_ds.GetRasterBand(1) cdl_nodata = cdl_band.GetNoDataValue() cdl_ds = None if cdl_nodata is None: cdl_nodata = 255 # Clip and project tile_utm_osr = drigo.epsg_osr(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_proj = drigo.epsg_proj(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_extent = tile_utm_extent_dict[tile_name] if mp_procs > 1: project_mp_list.append([ cdl_input_path, cdl_output_path, gdal.GRA_NearestNeighbour, tile_utm_proj, snap_cs, tile_utm_extent, cdl_nodata ]) remap_mp_list.append( [cdl_output_path, cdl_ag_output_path, cdl_ag_list]) else: drigo.project_raster(cdl_input_path, cdl_output_path, gdal.GRA_NearestNeighbour, tile_utm_osr, snap_cs, tile_utm_extent, cdl_nodata) # Build a mask of CDL ag lands remap_mask_func(cdl_output_path, cdl_ag_output_path, cdl_ag_list) # Cleanup del cdl_output_path del cdl_ds, cdl_band, cdl_nodata del tile_utm_osr, tile_utm_proj, tile_utm_extent # Project CDL rasters using multiprocessing if project_mp_list: pool = mp.Pool(mp_procs) results = pool.map(drigo.project_raster_mp, project_mp_list, chunksize=1) pool.close() pool.join() del results, pool if remap_mp_list: pool = mp.Pool(mp_procs) results = pool.map(remap_mask_mp, remap_mp_list, chunksize=1) pool.close() pool.join() del results, pool # Project/clip LANDFIRE for each path/row if landfire_flag: logging.info('\nBuild LANDFIRE for each path/row') project_mp_list, remap_mp_list = [], [] for tile_name in tile_list: landfire_output_path = os.path.join( landfire_output_ws, tile_name, landfire_output_fmt.format(year)) landfire_ag_output_path = os.path.join( landfire_output_ws, tile_name, landfire_ag_output_fmt.format(year)) if not overwrite_flag and os.path.isfile(landfire_output_path): logging.debug(' {} already exists, skipping'.format( os.path.basename(landfire_output_path))) continue logging.info(' {}'.format(tile_name)) # Set the nodata value on the LANDFIRE raster if it is not set # landfire_ds = gdal.Open(landfire_input_path, 0) # landfire_band = landfire_ds.GetRasterBand(1) # landfire_nodata = landfire_band.GetNoDataValue() # landfire_ds = None # if landfire_nodata is None: # landfire_nodata = 32767 # del landfire_ds, landfire_band landfire_nodata = 32767 # Clip and project tile_utm_osr = drigo.epsg_osr(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_proj = drigo.epsg_proj(32600 + int(tile_utm_zone_dict[tile_name])) tile_utm_extent = tile_utm_extent_dict[tile_name] if mp_procs > 1: project_mp_list.append([ landfire_input_path, landfire_output_path, gdal.GRA_NearestNeighbour, tile_utm_proj, snap_cs, tile_utm_extent, landfire_nodata ]) remap_mp_list.append([ landfire_output_path, landfire_ag_output_path, landfire_ag_list ]) else: drigo.project_raster(landfire_input_path, landfire_output_path, gdal.GRA_NearestNeighbour, tile_utm_osr, snap_cs, tile_utm_extent, landfire_nodata) # Build a mask of LANDFIRE ag lands remap_mask_func(landfire_output_path, landfire_ag_output_path, landfire_ag_list) # Cleanup del landfire_output_path del tile_utm_osr, tile_utm_proj, tile_utm_extent # Project LANDFIRE rasters using multiprocessing if project_mp_list: pool = mp.Pool(mp_procs) results = pool.map(drigo.project_raster_mp, project_mp_list, chunksize=1) pool.close() pool.join() del results, pool if remap_mp_list: pool = mp.Pool(mp_procs) results = pool.map(remap_mask_mp, remap_mp_list, chunksize=1) pool.close() pool.join() del results, pool # Convert field shapefiles to raster if field_flag: logging.info('\nBuild field rasters for each path/row') for tile_name in tile_list: logging.info(' {}'.format(tile_name)) tile_output_ws = os.path.join(field_output_ws, tile_name) # Shapefile paths field_proj_name = ( os.path.splitext(field_output_fmt.format(year))[0] + "_wgs84z{}.shp".format(tile_utm_zone_dict[tile_name])) field_proj_path = os.path.join(tile_output_ws, field_proj_name) field_output_path = os.path.join(tile_output_ws, field_output_fmt.format(year)) if not overwrite_flag and os.path.isfile(field_output_path): logging.debug(' {} already exists, skipping'.format( os.path.basename(field_output_path))) continue # The ogr2ogr spatial query is in the input spatial reference # Project the path/row extent to the field osr/proj field_input_osr = drigo.feature_path_osr(field_input_path) tile_utm_osr = drigo.epsg_osr(32600 + int(tile_utm_zone_dict[tile_name])) # field_input_proj = drigo.osr_proj(field_input_osr) # tile_utm_proj = drigo.osr_proj(tile_utm_osr) field_tile_extent = drigo.project_extent( tile_utm_extent_dict[tile_name], tile_utm_osr, field_input_osr, 30) # Project shapefile to the path/row zone # Clipping requires GDAL to be built with GEOS support subprocess.call( [ 'ogr2ogr', '-t_srs', 'EPSG:326{}'.format( tile_utm_zone_dict[tile_name]), '-f', 'ESRI Shapefile', '-overwrite' ] + ['-spat'] + list(map(str, field_tile_extent)) + ['-clipdst'] + list(map(str, tile_utm_extent_dict[tile_name])) + # ['-clipdst'] + list(map(str, tile_utm_extent_dict[tile_name])) + # ['-clipsrc'] + list(map(str, field_tile_extent)) + # ['-clipsrc'] + list(map(str, field_tile_extent)) + [field_proj_path, field_input_path]) # Convert shapefile to raster field_mem_ds = drigo.polygon_to_raster_ds( field_proj_path, nodata_value=0, burn_value=1, output_osr=tile_utm_osr, output_extent=tile_utm_extent_dict[tile_name]) field_output_driver = drigo.raster_driver(field_output_path) if field_output_path.lower().endswith('.img'): field_output_ds = field_output_driver.CreateCopy( field_output_path, field_mem_ds, 0, ['COMPRESS=YES']) else: field_output_ds = field_output_driver.CreateCopy( field_output_path, field_mem_ds, 0) field_output_ds, field_mem_ds = None, None # Remove field shapefile # try: # remove_file(field_proj_path) # except: # pass # Cleanup del tile_utm_osr, field_tile_extent, field_input_osr # del tile_utm_proj, field_input_proj del field_proj_name, field_proj_path, field_output_path logging.debug('\nScript complete')
def monte_carlo(image_ws, metric_ini_path, mc_ini_path, mc_iter=None, cold_tgt_pct=None, hot_tgt_pct=None, groupsize=64, blocksize=4096, multipoint_flag=False, shapefile_flag=False, stats_flag=False, overwrite_flag=False, debug_flag=False, no_etrf_final_plots=None, no_etrf_temp_plots=None): """METRIC Monte Carlo Parameters ---------- image_ws : str The workspace (path) of the landsat scene folder. metric_ini_path : str The METRIC config file (path). mc_ini_path : str The Monte Carlo config file (path). mc_iter : int, optional Iteration number for Monte Carlo processing. cold_tgt_pct : float, optional Target percentage of pixels with ETrF > than cold Kc. hot_tgt_pct : float, optional Target percentage of pixels with ETrF < than hot Kc. groupsize : int, optional Script will try to place calibration point randomly into a labeled group of clustered values with at least n pixels. -1 = In the largest group 0 = Anywhere in the image (not currently implemented) 1 >= In any group with a pixel count greater or equal to n blocksize : int, optional Processing block size (the default is 4096). shapefile_flag : bool, optional If True, save calibration points to shapefile (the default is False). multipoint_flag : bool, optional If True, save cal. points to multipoint shapefile (the default is False). stats_flag : bool, optional If True, compute raster statistics (the default is False). ovewrite_flag : bool, optional If True, overwrite existing files (the default is False). debug_flag : bool, optional If True, enable debug level logging (the default is False). no_final_plots : bool, optional If True, don't save final ETrF histograms (the default is None). This will override the flag in the INI file. no_temp_plots : bool If True, don't save temp ETrF histogram (the default is None). This will override the flag in the INI file. Returns ------- None """ logging.info('METRIC Automated Calibration') # Open config file config = open_ini(mc_ini_path) # Get input parameters logging.debug(' Reading Input File') etrf_training_path = config.get('INPUTS', 'etrf_training_path') # Adjust Kc cold target value based on day of year # etrf_doy_adj_path = read_param( # 'etrf_doy_adj_path', None, config, 'INPUTS') # Intentionally set default to None, to trigger error in eval call kc_cold_doy_dict = read_param('kc_cold_doy_dict', None, config, 'INPUTS') kc_hot_doy_dict = read_param('kc_hot_doy_dict', None, config, 'INPUTS') # If the "no_" flags were set True, honor them and set the flag False # If the "no_" flags were not set by the user, use the INI flag values # If not set in the INI, default to False (don't save any plots) if no_etrf_temp_plots: save_etrf_temp_plots = False else: save_etrf_temp_plots = read_param('save_etrf_temp_plots', False, config, 'INPUTS') if no_etrf_final_plots: save_etrf_final_plots = False else: save_etrf_final_plots = read_param('save_etrf_final_plots', False, config, 'INPUTS') save_ndvi_plots = read_param('save_ndvi_plots', False, config, 'INPUTS') max_cal_iter = read_param('max_cal_iterations', 5, config, 'INPUTS') max_point_iter = read_param('max_point_iterations', 10, config, 'INPUTS') ts_diff_threshold = read_param('ts_diff_threshold', 4, config, 'INPUTS') etr_ws = config.get('INPUTS', 'etr_ws') ppt_ws = config.get('INPUTS', 'ppt_ws') etr_re = re.compile(config.get('INPUTS', 'etr_re')) ppt_re = re.compile(config.get('INPUTS', 'ppt_re')) awc_path = config.get('INPUTS', 'awc_path') spinup_days = read_param('swb_spinup_days', 5, config, 'INPUTS') min_spinup_days = read_param('swb_min_spinup_days', 30, config, 'INPUTS') log_fmt = ' {:<18s} {}' break_line = '\n{}'.format('#' * 80) env = drigo.env image = et_image.Image(image_ws, env) logging.info(log_fmt.format('Image:', image.folder_id)) # Check inputs for file_path in [awc_path]: if not os.path.isfile(file_path): logging.error('\nERROR: File {} does not exist'.format(file_path)) sys.exit() for folder in [etr_ws, ppt_ws]: if not os.path.isdir(folder): logging.error('\nERROR: Folder {} does not exist'.format(folder)) sys.exit() # if (etrf_doy_adj_path and not # os.path.isfile(etrf_doy_adj_path)): # logging.error( # '\nERROR: File {} does not exist.'.format( # etrf_doy_adj_path)) # sys.exit() # Use iteration number to file iteration string if mc_iter is None: mc_str = '' mc_fmt = '.img' elif int(mc_iter) < 0: logging.error('\nERROR: Iteration number must be a positive integer') return False else: mc_str = 'MC{:02d}_'.format(int(mc_iter)) mc_fmt = '_{:02d}.img'.format(int(mc_iter)) logging.info(' {:<18s} {}'.format('Iteration:', mc_iter)) # Folder names etrf_ws = os.path.join(image_ws, 'ETRF') # indices_ws = image.indices_ws region_ws = os.path.join(image_ws, 'PIXEL_REGIONS') pixels_ws = os.path.join(image_ws, 'PIXELS') plots_ws = os.path.join(image_ws, 'PLOTS') if shapefile_flag and not os.path.isdir(pixels_ws): os.mkdir(pixels_ws) if not os.path.isdir(plots_ws): os.mkdir(plots_ws) # File names r_fmt = '.img' etrf_path = os.path.join(etrf_ws, 'et_rf' + mc_fmt) region_path = os.path.join(region_ws, 'region_mask' + r_fmt) # Initialize calibration parameters dictionary logging.info(break_line) logging.info('Calibration Parameters') cal_dict = dict() logging.debug(' Reading target cold/hot Kc from INI') # Using eval is potentially a really bad way of reading this in try: kc_cold_doy_dict = eval('{' + kc_cold_doy_dict + '}') except: kc_cold_doy_dict = {1: 1.05, 366: 1.05} logging.info( ' ERROR: kc_cold_doy_dict was not parsed, using default values') try: kc_hot_doy_dict = eval('{' + kc_hot_doy_dict + '}') except: kc_hot_doy_dict = {1: 0.1, 366: 0.1} logging.info( ' ERROR: kc_hot_doy_dict was not parsed, using default values') logging.debug(' Kc cold dict: {}'.format(kc_cold_doy_dict)) logging.debug(' Kc hot dict: {}\n'.format(kc_hot_doy_dict)) # doy_cold, kc_cold = zip(*sorted(kc_cold_doy_dict.items())) cal_dict['cold_tgt_kc'] = np.interp( image.acq_doy, *zip(*sorted(kc_cold_doy_dict.items())), left=1.05, right=1.05) # doy_hot, kc_hot = zip(*sorted(kc_hot_doy_dict.items())) cal_dict['hot_tgt_kc'] = np.interp(image.acq_doy, *zip(*sorted(kc_hot_doy_dict.items())), left=0.1, right=0.1) # if etrf_doy_adj_path: # doy_adj_df = pd.read_csv(etrf_doy_adj_path) # doy_adj = float( # doy_adj_df[doy_adj_df['DOY'] == image.acq_doy]['ETRF_ADJ']) # cal_dict['cold_tgt_kc'] = cal_dict['cold_tgt_kc'] + doy_adj # Get hot/cold etrf fraction sizes if cold_tgt_pct is None or hot_tgt_pct is None: logging.info('ETrF Tail Size Percentages') logging.info(' Reading target tail size from file') cold_tgt_pct, hot_tgt_pct = auto_calibration.etrf_fractions( etrf_training_path) if cold_tgt_pct is None or hot_tgt_pct is None: logging.error('\nERROR: Tail sizes were not mannually set or ' 'read from the the file\n') return False cal_dict['cold_tgt_pct'] = cold_tgt_pct cal_dict['hot_tgt_pct'] = hot_tgt_pct logging.info(pixel_str_fmt.format('', 'Cold Pixel', 'Hot Pixel')) logging.info( pixel_flt_fmt.format('Target kc:', cal_dict['cold_tgt_kc'], cal_dict['hot_tgt_kc'])) logging.info( pixel_pct_fmt.format('Tail Size:', cal_dict['cold_tgt_pct'], cal_dict['hot_tgt_pct'])) # # Create calibration database # # Set overwrite false to use existing database if it exists # cal_ws = os.path.join(image_ws, cal_folder) # if not os.path.isdir(cal_ws): # os.mkdir(cal_ws) # cal_path = os.path.join(cal_ws, cal_name) # logging.info('{:<20s} {}\{}'.format( # 'Calibration DB:', cal_folder, cal_name)) # calibration_database.create_calibration_database( # image_ws, cal_path, overwrite_db_flag) # del cal_ws # Remove previous calibrations from database # logging.info(break_line) # calibration_database.remove_calibration_points( # image_ws, cal_path, cal_initials, mc_iter) # Get ETrF and region mask (from pixel rating) # Assume they have identical extents try: region_mask = drigo.raster_to_array(region_path, return_nodata=False) region_mask = region_mask.astype(np.bool) except: logging.error( '\nERROR: Pixel regions mask does not exist or could not be read.\n' ' Please try re-running the METRIC Pixel Rating tool.') logging.debug(' {} '.format(region_path)) return False # Remove previous plots logging.info(break_line) auto_calibration.remove_histograms(plots_ws, mc_iter) # Generate the NDVI histogram if save_ndvi_plots: logging.info(break_line) logging.info('NDVI Histograms') if os.path.isfile(image.ndvi_toa_raster): ndvi_array = drigo.raster_to_array(image.ndvi_toa_raster, return_nodata=False) else: logging.error( '\nERROR: NDVI raster does not exist. METRIC Model 1 may not ' 'have run successfully.') logging.debug(' {} '.format(image.ndvi_toa_raster)) # Only process ag. ETrF pixels ndvi_array[~region_mask] = np.nan ndvi_sub_array = ndvi_array[region_mask] if np.any(ndvi_sub_array): auto_calibration.save_ndvi_histograms(ndvi_sub_array, plots_ws, mc_iter) else: logging.error( '\nERROR: Empty NDVI array, histogram was not built\n') # Place points in suggested region allowing for a number of iterations # dependent on whether or not Ts meets certain criteria logging.info(break_line) pixel_point_iters = 0 while pixel_point_iters <= max_point_iter: if pixel_point_iters == max_point_iter: logging.error('\nERROR: Suitable hot and cold pixels could not be ' 'determined. The scene will not calibrate.\n') return False cold_xy, hot_xy = pixel_points.pixel_points( image_ws, groupsize=groupsize, blocksize=blocksize, mc_iter=mc_iter, shapefile_flag=shapefile_flag, multipoint_flag=multipoint_flag, overwrite_flag=overwrite_flag, pixel_point_iters=pixel_point_iters) if any(x is None for x in cold_xy) or any(x is None for x in hot_xy): logging.error(('\nPixel points coordinates are invalid. ' 'The scene will not calibrate.' '\n Cold: {}\n Hot: {}').format(cold_xy, hot_xy)) return False cold_ts = drigo.raster_value_at_xy(image.ts_raster, cold_xy) hot_ts = drigo.raster_value_at_xy(image.ts_raster, hot_xy) if cold_ts > hot_ts: logging.info( '\nThe cold pixel is hotter than the hot pixel. Placing ' 'the points again.\n') logging.info(break_line) pixel_point_iters += 1 elif abs(hot_ts - cold_ts) < ts_diff_threshold: logging.info(( '\nThere is less than a {} degree difference in Ts hot and cold. ' 'Placing the points again.\n').format(ts_diff_threshold)) logging.info(break_line) pixel_point_iters += 1 # raise et_common.TemperatureError else: break # Adjust Kc hot for soil water balance logging.info(break_line) cal_dict = auto_calibration.hot_kc_swb_adjust(cal_dict, hot_xy, env.snap_osr, image.acq_date, awc_path, etr_ws, etr_re, ppt_ws, ppt_re, spinup_days, min_spinup_days) # Adjust Kc cold based on NDVI # cal_dict['tgt_c_kc'] = auto_calibration.kc_ndvi_adjust( # cal_dict['tgt_c_kc'], cold_xy, ndvi_path, 'Cold') # Check that Kc hot (Ke) is not too high? if cal_dict['hot_tgt_kc'] >= 1.0: logging.error('\nERROR: Target Kc hot is too high for automated ' 'calibration\n ETrF will not be computed') return False elif (cal_dict['cold_tgt_kc'] - cal_dict['hot_tgt_kc']) <= 0.05: logging.error('\nERROR: Target Kc hot and Kc cold are too close for ' 'automated calibration\n ETrF will not be computed') return False # Initialize Kc values at targets cal_dict['kc_cold'] = cal_dict['cold_tgt_kc'] cal_dict['kc_hot'] = cal_dict['hot_tgt_kc'] # Iterate until max calibrations is reached or error is small cal_flag = False cal_iter = 1 while not cal_flag: logging.info(break_line) logging.info('Calibration Iteration: {}'.format(cal_iter)) # Run METRIC Model2 for initial ETrF map logging.info(break_line) metric_model2.metric_model2(image_ws, metric_ini_path, mc_iter=mc_iter, kc_cold=cal_dict['kc_cold'], kc_hot=cal_dict['kc_hot'], cold_xy=cold_xy, hot_xy=hot_xy, overwrite_flag=overwrite_flag) # Read in ETrF array if os.path.isfile(etrf_path): etrf_array = drigo.raster_to_array(etrf_path, return_nodata=False) else: logging.warning( ('WARNING: ETrF raster does not exist. METRIC Model 2 ' 'may not have run successfully.\n {}').format(etrf_path)) break etrf_geo = drigo.raster_path_geo(etrf_path) # Only process ag. ETrF pixels etrf_array[~region_mask] = np.nan etrf_sub_array = etrf_array[np.isfinite(etrf_array)] if not np.any(etrf_sub_array): logging.error( '\nERROR: Empty ETrF array, scene cannot be calibrated\n') break # Calculate calibration parameters logging.debug(break_line) cal_dict = auto_calibration.calibration_params(cal_dict, etrf_sub_array) # Plot intermediates calibration histograms if save_etrf_temp_plots: logging.debug(break_line) auto_calibration.save_etrf_histograms(etrf_sub_array, plots_ws, cal_dict, mc_iter, cal_iter) # Check calibration logging.debug(break_line) cal_flag = auto_calibration.check_calibration(cal_dict) # Don't re-calibrate if initial calibration was suitable if cal_flag: break # Limit calibration attempts # cal_iter index is 1 based for Monte Carlo # cal_iter is 0 for stand alone mode elif cal_iter >= max_cal_iter: logging.info(break_line) logging.info( ('All {} iteration attempts were made, ' 'the scene will not calibrate.').format(max_cal_iter)) if os.path.isfile(etrf_path): os.remove(etrf_path) return False # break # Adjust Kc value of calibration points (instead of moving them) cal_dict = auto_calibration.kc_calibration_adjust( cal_dict, etrf_sub_array) # # Select new calibration points based on ETrF distribution # logging.info(break_line) # cold_xy, hot_xy = auto_calibration.build_pixel_points( # etrf_array, etrf_geo, cal_dict, # shapefile_flag=shapefile_flag, pixels_ws=pixels_ws) del etrf_array, etrf_geo # Increment calibration iteration counter cal_iter += 1 # Only save 'final' results if the scene was calibrated if cal_flag and save_etrf_final_plots: # Plot final ETrF distribution # logging.info(break_line) auto_calibration.save_etrf_histograms(etrf_sub_array, plots_ws, cal_dict, mc_iter, None) # Save final calibration points to database # logging.info(break_line) # calibration_database.save_calibration_points( # image_ws, cal_path, cal_dict, mc_iter, 0) return True
def main(ini_path, tile_list=None, blocksize=None, stats_flag=True, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, new_window_flag=False): """Run METRIC Model 2 for all images Parameters ---------- ini_path : str File path of the input parameters file. tile_list : list, optional Landsat path/rows to process (i.e. [p045r043, p045r033]). This will override the tile list in the INI file. blocksize : int, optional Processing block size (the default is None). If set, this blocksize parameter will be used instead of the value in the INI file. stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). mp_procs : int, optional Number of cores to use (the default is 1). delay : float, optional max random delay starting function in seconds (the default is 0). debug_flag : bool, optional If True, enable debug level logging (the default is False). new_window_flag : bool, optional If True, open each process in new terminal window (the default is False). Microsoft Windows only. Returns ------- None """ logging.info('\nRunning METRIC model 2 for all images') # Open config file config = dripy.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = dripy.read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) func_path = config.get('INPUTS', 'metric_model2_func') keep_list_path = dripy.read_param('keep_list_path', '', config, 'INPUTS') # skip_list_path = dripy.read_param('skip_list_path', '', config, 'INPUTS') # For now build INI file name from template INI names ini_name = os.path.basename(config.get('INPUTS', 'metric_ini')) ini_name = os.path.splitext(os.path.basename(ini_name))[0] # INI file is built as a function of year and tile_name ini_fmt = '{}_{}_{}.ini' # Calculate pixel points before running Model 2 # pixel_points_flag = True # Only allow new terminal windows on Windows if os.name is not 'nt': new_window_flag = False # Regular expressions # For now assume path/row are two digit numbers tile_re = re.compile('p\d{3}r\d{3}', re.IGNORECASE) image_id_re = re.compile( '^(LT04|LT05|LE07|LC08)_(?:\w{4})_(\d{3})(\d{3})_' '(\d{4})(\d{2})(\d{2})_(?:\d{8})_(?:\d{2})_(?:\w{2})$') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\n Folder {} does not exist'.format(project_ws)) sys.exit() # Read keep/skip lists if keep_list_path: logging.debug('\nReading scene keep list') with open(keep_list_path) as keep_list_f: image_keep_list = keep_list_f.readlines() image_keep_list = [ image_id.strip() for image_id in image_keep_list if image_id_re.match(image_id.strip()) ] else: logging.debug('\nScene keep list not set in INI') image_keep_list = [] # # DEADBEEF # if skip_list_path: # logging.debug('\nReading scene skip list') # with open(skip_list_path) as skip_list_f: # image_skip_list = skip_list_f.readlines() # image_skip_list = [image_id.strip() for image_id in image_skip_list # if image_id_re.match(image_id.strip())] # else: # logging.debug('\nScene skip list not set in INI') # image_skip_list = [] mp_list = [] for tile_name in sorted(tile_list): tile_ws = os.path.join(project_ws, str(year), tile_name) if not os.path.isdir(tile_ws) and not tile_re.match(tile_name): logging.debug(' {} {} - invalid tile, skipping'.format( year, tile_name)) continue # Check that there are image folders image_id_list = [ image_id for image_id in sorted(os.listdir(tile_ws)) if (image_id_re.match(image_id) and os.path.isdir(os.path.join(tile_ws, image_id)) and ( image_keep_list and image_id in image_keep_list)) ] # (image_skip_list and image_id not in image_skip_list))] if not image_id_list: logging.debug(' {} {} - no available images, skipping'.format( year, tile_name)) continue else: logging.debug(' {} {}'.format(year, tile_name)) # Check that there is an input file for the path/row ini_path = os.path.join(tile_ws, ini_fmt.format(ini_name, year, tile_name)) if not os.path.join(ini_path): logging.warning(' METRIC input file {} does not exist, ' 'skipping'.format(ini_path)) continue # Setup command line argument call_args = [sys.executable, func_path, '-i', ini_path] if blocksize is not None: call_args.extend(['--blocksize', str(blocksize)]) if stats_flag: call_args.append('--stats') if overwrite_flag: call_args.append('--overwrite') if debug_flag: call_args.append('--debug') # Run METRIC Model 2 for each Monte Carlo iteration for image_id in image_id_list: logging.debug(' {}'.format(image_id)) scene_ws = os.path.join(tile_ws, image_id) if mp_procs > 1: mp_list.append([call_args, scene_ws, delay, new_window_flag]) else: subprocess.call(call_args, cwd=scene_ws) if mp_list: pool = mp.Pool(mp_procs) results = pool.map(dripy.call_mp, mp_list, chunksize=1) pool.close() pool.join() del results, pool logging.debug('\nScript complete')
def main(ini_path, tile_list=None, groupsize=1, blocksize=2046, shapefile_flag=True, geojson_flag=False, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, new_window_flag=False): """Run pixel points for all images Args: ini_path (str): file path of the input parameters file tile_list (list): list of Landsat path/row (i.e. [p45r43, p45r33]) This will override the tile list in the INI file groupsize (int): Script will try to place calibration point randomly into a labeled group of clustered values with at least n pixels. -1 = In the largest group 0 = Anywhere in the image (not currently implemented) 1 >= In any group with a pixel count greater or equal to n blocksize (int): shapefile_flag (bool): if True, save calibration points to shapefile (default False) geojson_flag (bool): if True, save calibration points to GeoJSON overwrite_flag (bool): if True, overwrite existing files mp_procs (int): number of cores to use delay (float): max random delay starting function in seconds debug_flag (bool): if True, enable debug level logging new_window_flag (bool): if True, open each process in a new terminal. Microsoft Windows only. Returns: None """ logging.info('\nRunning METRIC Pixel Points for all images') log_fmt = ' {:<18s} {}' # Open config file config = open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) func_path = config.get('INPUTS', 'pixel_points_func') # DEADBEEF - seems like this is passed in at the command line # groupsize = config.getint('INPUTS', 'groupsize') # Only allow new terminal windows on Windows if os.name is not 'nt': new_window_flag = False # Regular expressions tile_re = re.compile('p\d{3}r\d{3}', re.IGNORECASE) image_re = re.compile( '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\n Folder {} does not exist'.format(project_ws)) sys.exit() # Setup command line argument call_args = [sys.executable, func_path] call_args.extend(['--groupsize', str(groupsize)]) if blocksize: call_args.extend(['--blocksize', str(blocksize)]) if shapefile_flag: call_args.append('--shapefile') if geojson_flag: call_args.append('--geojson') if overwrite_flag: call_args.append('--overwrite') if debug_flag: call_args.append('--debug') mp_list = [] for tile_name in sorted(tile_list): tile_ws = os.path.join(project_ws, str(year), tile_name) if not os.path.isdir(tile_ws) and not tile_re.match(tile_name): continue # Check that there are scene folders scene_id_list = [ scene_id for scene_id in sorted(os.listdir(tile_ws)) if (os.path.isdir(os.path.join(tile_ws, scene_id)) or image_re.match(scene_id)) ] if not scene_id_list: continue logging.debug(' {} {}'.format(year, tile_name)) # Run METRIC Pixel Points for scene_id in scene_id_list: logging.debug(' {}'.format(scene_id)) scene_ws = os.path.join(tile_ws, scene_id) pixel_ws = os.path.join(scene_ws, 'PIXELS') # Since the GeoJSON will be appended, delete it in the wrapper # script if the overwrite_flag=True if geojson_flag and os.path.isdir(pixel_ws): for pixel_file in os.listdir(pixel_ws): if re.match('\w+.geojson$', pixel_file): os.remove(os.path.join(pixel_ws, pixel_file)) if mp_procs > 1: mp_list.append([call_args, scene_ws, delay, new_window_flag]) else: subprocess.call(call_args, cwd=scene_ws) if mp_list: pool = mp.Pool(mp_procs) results = pool.map(call_mp, mp_list, chunksize=1) pool.close() pool.join() del results, pool
def main(ini_path, rasters_flag=None, tables_flag=None, mc_iter_str='', tile_list=None, pyramids_flag=True, stats_flag=True, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, no_file_logging=False): """Run interpolater for all Landsat scenes Parameters ---------- ini_path : str File path of the input parameters file. rasters_flag : bool, optional If True, override INI and interpolate rasters. tables_flag : bool, optional If True, override INI and interpolate zone tables. mc_iter_str : str, optional MonteCarlo iteration list and/or range. tile_list : list, optional Landsat path/rows to process (i.e. [p045r043, p045r033]). This will override the tile list in the INI file. pyramids_flag : bool, optional If True, compute raster pyramids (the default is True). stats_flag : bool, optional If True, compute raster statistics (the default is True). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). mp_procs : int, optional Number of cpu cores to use (the default is 1). delay : float, optional Max random delay starting function in seconds (the default is 0). debug_flag : bool, optional If True, enable debug level logging (the default is False). no_file_logging : bool, optional If True, don't write logging to file (the default is False). Returns ------- None """ logging.info('\nRunning Interpolator') # Open config file config = open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) interpolate_folder = config.get('INPUTS', 'interpolate_folder') logging.debug(' Folder: {}'.format(interpolate_folder)) # If both flags were not set, read from INI if rasters_flag is None and tables_flag is None: logging.info(' Reading interpolator flags from INI file') if rasters_flag is None: rasters_flag = read_param('interpolate_rasters_flag', True, config, 'INPUTS') if tables_flag is None: tables_flag = read_param('interpolate_tables_flag', True, config, 'INPUTS') # If both flags were set false, for now, exit the script # It may make more sense to assumethe user wants to interpolate something elif rasters_flag is False and tables_flag is False: logging.error('Raster and table interpolator flags are both False\n') logging.error(' Exiting the script') return False # sys.exit() # logging.info('Raster and table interpolator flags are both False\n') # logging.info(' Defaulting to rasters_flag=True') # rasters_flag = True if rasters_flag: rasters_func_path = config.get('INPUTS', 'interpolate_rasters_func') if tables_flag: tables_func_path = config.get('INPUTS', 'interpolate_tables_func') # For now, get mc_iter list from command line, not from project file # mc_iter_list = config.get('INPUTS', 'mc_iter_list') mc_iter_list = list(parse_int_set(mc_iter_str)) # Need soemthing in mc_iter_list to iterate over if not mc_iter_list: mc_iter_list = [None] # For now build INI file name from template INI names ini_name = os.path.basename(config.get('INPUTS', 'interpolate_ini')) ini_name = os.path.splitext(os.path.basename(ini_name))[0] # INI file is built as a function of year ini_fmt = '{}_{}_{}.ini' # Regular expressions # For now assume path/row are two digit numbers # tile_re = re.compile('p(\d{3})r(\d{3})', re.IGNORECASE) # image_re = re.compile( # '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\n Folder {} does not exist'.format(project_ws)) sys.exit() # Check that there is an input file for the year and folder year_ws = os.path.join(project_ws, str(year)) ini_path = os.path.join( year_ws, ini_fmt.format(ini_name, str(year), interpolate_folder.lower())) if not os.path.join(ini_path): logging.warning(' Input file does not exist\n {}'.format(ini_path)) return False # Run Interpolater for each Monte Carlo iteration # mp_list = [] for mc_iter in sorted(mc_iter_list): logging.debug(' Year: {} Iteration: {}'.format(str(year), mc_iter)) rasters_args = [] tables_args = [] if rasters_flag: rasters_args = [ 'python', rasters_func_path, year_ws, '-i', ini_path ] if tables_flag: tables_args = ['python', tables_func_path, year_ws, '-i', ini_path] if mc_iter is not None: rasters_args.extend(['-mc', str(mc_iter)]) tables_args.extend(['-mc', str(mc_iter)]) if pyramids_flag: rasters_args.append('--pyramids') if stats_flag: rasters_args.append('--stats') if overwrite_flag: rasters_args.append('--overwrite') tables_args.append('--overwrite') if debug_flag: rasters_args.append('--debug') tables_args.append('--debug') if delay > 0: rasters_args.extend(['--delay', str(delay)]) tables_args.extend(['--delay', str(delay)]) if no_file_logging: rasters_args.append('--no_file_logging') tables_args.append('--no_file_logging') if mp_procs > 1: rasters_args.extend(['-mp', str(mp_procs)]) tables_args.extend(['-mp', str(mp_procs)]) if rasters_flag: subprocess.call(rasters_args, cwd=year_ws) if tables_flag: subprocess.call(tables_args, cwd=year_ws) logging.debug('\nScript complete')
def main(ini_path, tile_list=None, groupsize=1, blocksize=2048, shapefile_flag=True, geojson_flag=False, overwrite_flag=False, mp_procs=1, delay=0, debug_flag=False, new_window_flag=False): """Run pixel points for all images Parameters ---------- ini_path : str File path of the input parameters file. tile_list : list, optional Landsat path/rows to process (i.e. [p045r043, p045r033]). This will override the tile list in the INI file. groupsize : int, optional Script will try to place calibration point randomly into a labeled group of clustered values with at least n pixels (the default is 64). -1 = In the largest group 0 = Anywhere in the image (not currently implemented) 1 >= In any group with a pixel count greater or equal to n blocksize : int, optional Processing block size (the default is 2048). shapefile_flag : bool, optional If True, save calibration points to shapefile (the default False). geojson_flag : bool, optional If True, save calibration points to GeoJSON (the default is False). overwrite_flag : bool, optional If True, overwrite existing files (the default is False). mp_procs : int, optional Number of cores to use (the default is 1). delay : float, optional Max random delay starting function in seconds (the default is 0). debug_flag : bool, optional If True, enable debug level logging (the default is False). new_window_flag : bool, optional If True, open each process in new terminal window (the default is False). Microsoft Windows only. Returns ------- None """ logging.info('\nRunning METRIC Pixel Points for all images') log_fmt = ' {:<18s} {}' # Open config file config = dripy.open_ini(ini_path) # Get input parameters logging.debug(' Reading Input File') year = config.getint('INPUTS', 'year') if tile_list is None: tile_list = dripy.read_param('tile_list', [], config, 'INPUTS') project_ws = config.get('INPUTS', 'project_folder') logging.debug(' Year: {}'.format(year)) logging.debug(' Path/rows: {}'.format(', '.join(tile_list))) logging.debug(' Project: {}'.format(project_ws)) func_path = config.get('INPUTS', 'pixel_points_func') keep_list_path = dripy.read_param('keep_list_path', '', config, 'INPUTS') # skip_list_path = dripy.read_param('skip_list_path', '', config, 'INPUTS') # DEADBEEF - seems like this is passed in at the command line # groupsize = config.getint('INPUTS', 'groupsize') # Only allow new terminal windows on Windows if os.name is not 'nt': new_window_flag = False # Regular expressions tile_re = re.compile('p\d{3}r\d{3}', re.IGNORECASE) image_id_re = re.compile( '^(LT04|LT05|LE07|LC08)_(?:\w{4})_(\d{3})(\d{3})_' '(\d{4})(\d{2})(\d{2})_(?:\d{8})_(?:\d{2})_(?:\w{2})$') # Check inputs folders/paths if not os.path.isdir(project_ws): logging.error('\n Folder {} does not exist'.format(project_ws)) sys.exit() # Setup command line argument call_args = [sys.executable, func_path] call_args.extend(['--groupsize', str(groupsize)]) if blocksize: call_args.extend(['--blocksize', str(blocksize)]) if shapefile_flag: call_args.append('--shapefile') if geojson_flag: call_args.append('--geojson') if overwrite_flag: call_args.append('--overwrite') if debug_flag: call_args.append('--debug') # Read keep/skip lists if keep_list_path: logging.debug('\nReading scene keep list') with open(keep_list_path) as keep_list_f: image_keep_list = keep_list_f.readlines() image_keep_list = [ image_id.strip() for image_id in image_keep_list if image_id_re.match(image_id.strip()) ] else: logging.debug('\nScene keep list not set in INI') image_keep_list = [] # if skip_list_path: # logging.debug('\nReading scene skip list') # with open(skip_list_path) as skip_list_f: # image_skip_list = skip_list_f.readlines() # image_skip_list = [image_id.strip() for image_id in image_skip_list # if image_id_re.match(image_id.strip())] # else: # logging.debug('\nScene skip list not set in INI') # image_skip_list = [] mp_list = [] for tile_name in sorted(tile_list): tile_ws = os.path.join(project_ws, str(year), tile_name) if not os.path.isdir(tile_ws) and not tile_re.match(tile_name): logging.debug(' {} {} - invalid tile, skipping'.format( year, tile_name)) continue # Check that there are image folders image_id_list = [ image_id for image_id in sorted(os.listdir(tile_ws)) if (image_id_re.match(image_id) and os.path.isdir(os.path.join(tile_ws, image_id)) and ( image_keep_list and image_id in image_keep_list)) ] # (image_skip_list and image_id not in image_skip_list))] if not image_id_list: logging.debug(' {} {} - no available images, skipping'.format( year, tile_name)) continue else: logging.debug(' {} {}'.format(year, tile_name)) # Run METRIC Pixel Points for image_id in image_id_list: logging.debug(' {}'.format(image_id)) image_ws = os.path.join(tile_ws, image_id) pixel_ws = os.path.join(image_ws, 'PIXELS') # Since the GeoJSON will be appended, delete it in the wrapper # script if the overwrite_flag=True if geojson_flag and os.path.isdir(pixel_ws): for pixel_file in os.listdir(pixel_ws): if re.match('\w+.geojson$', pixel_file): os.remove(os.path.join(pixel_ws, pixel_file)) if mp_procs > 1: mp_list.append([call_args, image_ws, delay, new_window_flag]) else: subprocess.call(call_args, cwd=image_ws) if mp_list: pool = mp.Pool(mp_procs) results = pool.map(dripy.call_mp, mp_list, chunksize=1) pool.close() pool.join() del results, pool