def landsat_files_check(image_ws):
"""Check if Landsat folder needs to be rebuilt from tar.gz
Parameters
----------
image_ws : str
Landsat image folder.
Returns
-------
True if there are sufficient files to run METRIC
False if files are missing and image should be extracted from tar.gz
"""
try:
image = et_image.Image(image_ws)
except et_image.InvalidImage:
return False
if image.mtl_path is None:
return False
# Get list of digital number (DN) images from ORIGINAL_DATA folder
dn_image_dict = et_common.landsat_band_image_dict(image.orig_data_ws,
image.image_re)
# Check if sets of rasters are present
# Output from metric_model1
if (os.path.isfile(image.albedo_sur_raster)
and os.path.isfile(image.ts_raster)
and (os.path.isfile(image.ndvi_toa_raster)
or os.path.isfile(image.ndvi_sur_raster))
and (os.path.isfile(image.lai_toa_raster)
or os.path.isfile(image.lai_sur_raster))):
return True
# Output from prep_scene
elif (os.path.isfile(image.refl_toa_raster)
and os.path.isfile(image.ts_bt_raster)):
return True
# Output from prep_path_row
elif (dn_image_dict
and set(image.band_toa_dict.keys() + [image.thermal_band]) == set(
dn_image_dict.keys())):
return True
else:
return False
def main(image_ws, ini_path, bs=2048, stats_flag=False, overwrite_flag=False):
"""Prep a Landsat scene for METRIC
Parameters
----------
image_ws : str
Landsat scene folder that will be prepped.
ini_path : str
File path of the input parameters file.
bs : int, optional
Processing block size (the default is 2048).
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
True is successful
"""
# Open config file
config = dripy.open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
calc_refl_toa_flag = dripy.read_param(
'calc_refl_toa_flag', True, config, 'INPUTS')
calc_refl_toa_qa_flag = dripy.read_param(
'calc_refl_toa_qa_flag', True, config, 'INPUTS')
# calc_refl_sur_ledaps_flag = dripy.read_param(
# 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS')
# calc_refl_sur_qa_flag = dripy.read_param(
# 'calc_refl_sur_qa_flag', False, config, 'INPUTS')
calc_ts_bt_flag = dripy.read_param(
'calc_ts_bt_flag', True, config, 'INPUTS')
# Use QA band to set common area
# Fmask cloud, shadow, & snow pixels will be removed from common area
calc_fmask_common_flag = dripy.read_param(
'calc_fmask_common_flag', True, config, 'INPUTS')
fmask_smooth_flag = dripy.read_param(
'fmask_smooth_flag', False, config, 'INPUTS')
fmask_buffer_flag = dripy.read_param(
'fmask_buffer_flag', False, config, 'INPUTS')
fmask_erode_flag = dripy.read_param(
'fmask_erode_flag', False, config, 'INPUTS')
if fmask_smooth_flag:
fmask_smooth_cells = int(dripy.read_param(
'fmask_smooth_cells', 1, config, 'INPUTS'))
if fmask_smooth_cells == 0 and fmask_smooth_flag:
fmask_smooth_flag = False
if fmask_erode_flag:
fmask_erode_cells = int(dripy.read_param(
'fmask_erode_cells', 1, config, 'INPUTS'))
if fmask_erode_cells == 0 and fmask_erode_flag:
fmask_erode_flag = False
if fmask_buffer_flag:
fmask_buffer_cells = int(dripy.read_param(
'fmask_buffer_cells', 1, config, 'INPUTS'))
if fmask_buffer_cells == 0 and fmask_buffer_flag:
fmask_buffer_flag = False
# Remove edge (fringe) cells
edge_smooth_flag = dripy.read_param(
'edge_smooth_flag', True, config, 'INPUTS')
# Include hand made cloud masks
cloud_mask_flag = dripy.read_param(
'cloud_mask_flag', False, config, 'INPUTS')
cloud_mask_ws = ""
if cloud_mask_flag:
cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws')
# Extract separate Fmask rasters
calc_fmask_flag = dripy.read_param(
'calc_fmask_flag', True, config, 'INPUTS')
calc_fmask_cloud_flag = dripy.read_param(
'calc_fmask_cloud_flag', True, config, 'INPUTS')
calc_fmask_snow_flag = dripy.read_param(
'calc_fmask_snow_flag', True, config, 'INPUTS')
calc_fmask_water_flag = dripy.read_param(
'calc_fmask_water_flag', True, config, 'INPUTS')
# Keep Landsat DN, LEDAPS, and Fmask rasters
keep_dn_flag = dripy.read_param(
'keep_dn_flag', True, config, 'INPUTS')
# keep_sr_flag = dripy.read_param(
# 'keep_sr_flag', True, config, 'INPUTS')
# For this to work I would need to pass in the metric input file
# calc_elev_flag = dripy.read_param(
# 'calc_elev_flag', False, config, 'INPUTS')
# calc_landuse_flag = dripy.read_param(
# 'calc_landuse_flag', False, config, 'INPUTS')
# calc_acca_cloud_flag = dripy.read_param(
# 'calc_acca_cloud_flag', True, config, 'INPUTS')
# calc_acca_snow_flag = dripy.read_param(
# 'calc_acca_snow_flag', True, config, 'INPUTS')
# calc_ledaps_dem_land_flag = dripy.read_param(
# 'calc_ledaps_dem_land_flag', False, config, 'INPUTS')
# calc_ledaps_veg_flag = dripy.read_param(
# 'calc_ledaps_veg_flag', False, config, 'INPUTS')
# calc_ledaps_snow_flag = dripy.read_param(
# 'calc_ledaps_snow_flag', False, config, 'INPUTS')
# calc_ledaps_land_flag = dripy.read_param(
# 'calc_ledaps_land_flag', False, config, 'INPUTS')
# calc_ledaps_cloud_flag = dripy.read_param(
# 'calc_ledaps_cloud_flag', False, config, 'INPUTS')
# Interpolate/clip/project hourly rasters for each Landsat scene
# calc_metric_flag = dripy.read_param(
# 'calc_metric_flag', False, config, 'INPUTS')
calc_metric_ea_flag = dripy.read_param(
'calc_metric_ea_flag', False, config, 'INPUTS')
calc_metric_wind_flag = dripy.read_param(
'calc_metric_wind_flag', False, config, 'INPUTS')
calc_metric_etr_flag = dripy.read_param(
'calc_metric_etr_flag', False, config, 'INPUTS')
calc_metric_tair_flag = dripy.read_param(
'calc_metric_tair_flag', False, config, 'INPUTS')
# Interpolate/clip/project AWC and daily ETr/PPT rasters
# to compute SWB Ke for each Landsat scene
calc_swb_ke_flag = dripy.read_param(
'calc_swb_ke_flag', False, config, 'INPUTS')
if cloud_mask_flag:
spinup_days = dripy.read_param(
'swb_spinup_days', 30, config, 'INPUTS')
min_spinup_days = dripy.read_param(
'swb_min_spinup_days', 5, config, 'INPUTS')
# Round ea raster to N digits to save space
rounding_digits = dripy.read_param(
'rounding_digits', 3, config, 'INPUTS')
env = drigo.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore', divide='ignore')
gdal.UseExceptions()
# Input file paths
dn_image_dict = et_common.landsat_band_image_dict(
image.orig_data_ws, image.image_name_re)
# # Open METRIC config file
# if config_file:
# logging.info(
# log_f.format('METRIC INI File:', os.path.basename(config_file)))
# config = configparser.ConfigParser()
# try:
# config.read(config_file)
# except:
# logging.error('\nERROR: Config file could not be read, ' +
# 'is not an input file, or does not exist\n' +
# 'ERROR: config_file = {}\n').format(config_file)
# sys.exit()
# # Overwrite
# overwrite_flag = dripy.read_param('overwrite_flag', True, config)
#
# # Elevation and landuse parameters/flags from METRIC input file
# calc_elev_flag = dripy.read_param('save_dem_raster_flag', True, config)
# calc_landuse_flag = dripy.read_param(
# 'save_landuse_raster_flag', True, config)
# if calc_elev_flag:
# elev_pr_path = config.get('INPUTS','dem_raster')
# if calc_landuse_flag:
# landuse_pr_path = config.get('INPUTS', 'landuse_raster')
# else:
# overwrite_flag = False
# calc_elev_flag = False
# calc_landuse_flag = False
#
# Elev raster must exist
# if calc_elev_flag and not os.path.isfile(elev_pr_path):
# logging.error('\nERROR: Elevation raster {} does not exist\n'.format(
# elev_pr_path))
# return False
# Landuse raster must exist
# if calc_landuse_flag and not os.path.isfile(landuse_pr_path):
# logging.error('\nERROR: Landuse raster {} does not exist\n'.format(
# landuse_pr_path))
# return False
# Removing ancillary files before checking for inputs
if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')):
shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask'))
for item in os.listdir(image.orig_data_ws):
if (image.type == 'Landsat7' and
(item.endswith('_B8.TIF') or
item.endswith('_B6_VCID_2.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (image.type == 'Landsat8' and
(item.endswith('_B1.TIF') or
item.endswith('_B8.TIF') or
item.endswith('_B9.TIF') or
item.endswith('_B11.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (item.endswith('_VER.jpg') or
item.endswith('_VER.txt') or
item.endswith('_GCP.txt') or
item == 'README.GTF'):
os.remove(os.path.join(image.orig_data_ws, item))
# Check correction level (image must be L1T to process)
if image.correction != 'L1TP':
logging.debug(' Image is not L1TP corrected, skipping')
return False
# calc_fmask_common_flag = False
# calc_refl_toa_flag = False
# calc_ts_bt_flag = False
# calc_metric_ea_flag = False
# calc_metric_wind_flag = False
# calc_metric_etr_flag = False
# overwrite_flag = False
# QA band must exist
if calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys():
logging.warning(
'\nQA band does not exist but calc_fmask_common_flag=True'
'\n Setting calc_fmask_common_flag=False\n {}'.format(
image.qa_band))
calc_fmask_common_flag = False
if cloud_mask_flag and not os.path.isdir(cloud_mask_ws):
logging.warning(
'\ncloud_mask_ws is not a directory but cloud_mask_flag=True.'
'\n Setting cloud_mask_flag=False\n {}'.format(cloud_mask_ws))
cloud_mask_flag = False
# Check for Landsat TOA images
if (calc_refl_toa_flag and
(set(list(image.band_toa_dict.keys()) +
[image.thermal_band, image.qa_band]) !=
set(dn_image_dict.keys()))):
logging.warning(
'\nMissing Landsat images but calc_refl_toa_flag=True'
'\n Setting calc_refl_toa_flag=False')
calc_refl_toa_flag = False
# Check for Landsat brightness temperature image
if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys():
logging.warning(
'\nThermal band image does not exist but calc_ts_bt_flag=True'
'\n Setting calc_ts_bt_flag=False')
calc_ts_bt_flag = False
# DEADBEEF - Should the function return False if Ts doesn't exist?
# return False
# Check for METRIC hourly/daily input folders
if calc_metric_ea_flag:
metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder')
if not os.path.isdir(metric_ea_input_ws):
logging.warning(
'\nHourly Ea folder does not exist but calc_metric_ea_flag=True'
'\n Setting calc_metric_ea_flag=False\n {}'.format(
metric_ea_input_ws))
calc_metric_ea_flag = False
if calc_metric_wind_flag:
metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder')
if not os.path.isdir(metric_wind_input_ws):
logging.warning(
'\nHourly wind folder does not exist but calc_metric_wind_flag=True'
'\n Setting calc_metric_wind_flag=False\n {}'.format(
metric_wind_input_ws))
calc_metric_wind_flag = False
if calc_metric_etr_flag:
metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder')
if not os.path.isdir(metric_etr_input_ws):
logging.warning(
'\nHourly ETr folder does not exist but calc_metric_etr_flag=True'
'\n Setting calc_metric_etr_flag=False\n {}'.format(
metric_etr_input_ws))
calc_metric_etr_flag = False
if calc_metric_tair_flag:
metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder')
if not os.path.isdir(metric_tair_input_ws):
logging.warning(
'\nHourly Tair folder does not exist but calc_metric_tair_flag=True'
'\n Setting calc_metric_tair_flag=False\n {}'.format(
metric_tair_input_ws))
calc_metric_tair_flag = False
if (calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag):
metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re'))
metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re'))
if calc_swb_ke_flag:
awc_input_path = config.get('INPUTS', 'awc_input_path')
etr_input_ws = config.get('INPUTS', 'etr_input_folder')
ppt_input_ws = config.get('INPUTS', 'ppt_input_folder')
etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re'))
ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re'))
if not os.path.isfile(awc_input_path):
logging.warning(
'\nAWC raster does not exist but calc_swb_ke_flag=True'
'\n Setting calc_swb_ke_flag=False\n {}'.format(
awc_input_path))
calc_swb_ke_flag = False
if not os.path.isdir(etr_input_ws):
logging.warning(
'\nDaily ETr folder does not exist but calc_swb_ke_flag=True'
'\n Setting calc_swb_ke_flag=False\n {}'.format(
etr_input_ws))
calc_swb_ke_flag = False
if not os.path.isdir(ppt_input_ws):
logging.warning(
'\nDaily PPT folder does not exist but calc_swb_ke_flag=True'
'\n Setting calc_swb_ke_flag=False\n {}'.format(
ppt_input_ws))
calc_swb_ke_flag = False
# Build folders for support rasters
if ((calc_fmask_common_flag or calc_refl_toa_flag or
# calc_refl_sur_ledaps_flag or
calc_ts_bt_flag or
calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag or
calc_swb_ke_flag) and
not os.path.isdir(image.support_ws)):
os.makedirs(image.support_ws)
if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws):
os.makedirs(image.refl_toa_ws)
# if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws):
# os.makedirs(image.refl_sur_ws)
# DEADBEEF - This is being further down just for the Fmask images
# # Apply overwrite flag
# if overwrite_flag:
# overwrite_list = [
# image.fmask_cloud_raster, image.fmask_snow_raster,
# image.fmask_water_raster
# # image.elev_raster, image.landuse_raster
# # image.common_area_raster
# ]
# for overwrite_path in overwrite_list:
# try:
# dripy.remove_file(image.fmask_cloud_raster)
# except:
# pass
# Use QA band to build common area rasters
logging.info('\nCommon Area Raster')
qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0)
common_geo = drigo.raster_ds_geo(qa_ds)
common_extent = drigo.raster_ds_extent(qa_ds)
common_proj = drigo.raster_ds_proj(qa_ds)
common_osr = drigo.raster_ds_osr(qa_ds)
# Initialize common_area as all non-fill QA values
qa_array = drigo.raster_ds_to_array(qa_ds, return_nodata=False)
common_array = qa_array != 1
common_rows, common_cols = common_array.shape
del qa_ds
# Erode and dilate to remove fringe on edge
# Default is to not smooth, but user can force smoothing
# This needs to be applied before Fmask
if edge_smooth_flag and image.prefix in ['LT05', 'LE07']:
struct = ndimage.generate_binary_structure(2, 2).astype(np.uint8)
if image.prefix == 'LT05':
cells = 8
elif image.prefix == 'LE07':
cells = 2
else:
cells = 0
common_array = ndimage.binary_dilation(
ndimage.binary_erosion(common_array, struct, cells),
struct, cells)
# Try applying user defined cloud masks to common_area
cloud_mask_path = os.path.join(
cloud_mask_ws, image.folder_id + '_mask.shp')
if cloud_mask_flag and os.path.isfile(cloud_mask_path):
logging.info(' Applying cloud mask shapefile')
feature_path = os.path.join(
cloud_mask_ws, (image.folder_id + '_mask.shp'))
logging.info(' {}'.format(feature_path))
cloud_mask_memory_ds = drigo.polygon_to_raster_ds(
feature_path, nodata_value=0, burn_value=1,
output_osr=common_osr, output_cs=30,
output_extent=common_extent)
cloud_array = drigo.raster_ds_to_array(
cloud_mask_memory_ds, return_nodata=False)
# DEADBEEF - If user sets a cloud mask,
# it is probably better than Fmask
# Eventually change "if" calc_fmask_common_flag: to "elif"
common_array[cloud_array == 1] = 0
del cloud_mask_memory_ds, cloud_array
# Remove Fmask cloud, shadow, and snow pixels from common_area
if calc_fmask_common_flag:
logging.info(' Applying Fmask to common area')
fmask_array = et_numpy.bqa_fmask_func(qa_array)
fmask_mask = (fmask_array >= 2) & (fmask_array <= 4)
if fmask_smooth_flag:
logging.debug(
' Smoothing (dilate/erode/erode/dilate) Fmask clouds, shadows,'
' and snow pixels by {} cells'.format(fmask_smooth_cells))
# ArcGIS smoothing procedure
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
if fmask_erode_flag:
logging.debug(
' Eroding Fmask clouds, shadows, and snow pixels by '
'{} cells'.format(fmask_erode_cells))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_erode_cells,
structure=ndimage.generate_binary_structure(2, 2))
if fmask_buffer_flag:
logging.debug(
' Dilating (buffering) Fmask clouds, shadows, and snow pixels '
'by {} cells'.format(fmask_buffer_cells))
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_buffer_cells,
structure=ndimage.generate_binary_structure(2, 2))
# Reset common_array for buffered cells
common_array[fmask_mask] = 0
del fmask_array, fmask_mask
# Check that there are some cloud free pixels
if not np.any(common_array):
logging.error(
' ERROR: There are no cloud/snow free pixels, returning False')
return False
# Always overwrite common area raster
# if not os.path.isfile(image.common_area_raster):
drigo.array_to_raster(
common_array, image.common_area_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Print common geo/extent
logging.debug(' Common geo: {}'.format(common_geo))
logging.debug(' Common extent: {}'.format(common_extent))
# Extract Fmask components as separate rasters
if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or
calc_fmask_water_flag):
logging.info('\nFmask')
fmask_array = et_numpy.bqa_fmask_func(qa_array)
# Remove existing Fmask rasters
if (calc_fmask_flag and overwrite_flag and
os.path.isfile(image.fmask_output_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_output_raster))
dripy.remove_file(image.fmask_output_raster)
if (calc_fmask_cloud_flag and overwrite_flag and
os.path.isfile(image.fmask_cloud_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_cloud_raster))
dripy.remove_file(image.fmask_cloud_raster)
if (calc_fmask_snow_flag and overwrite_flag and
os.path.isfile(image.fmask_snow_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_snow_raster))
dripy.remove_file(image.fmask_snow_raster)
if (calc_fmask_water_flag and overwrite_flag and
os.path.isfile(image.fmask_water_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_water_raster))
dripy.remove_file(image.fmask_water_raster)
# Save Fmask data as separate rasters
if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)):
logging.debug(' Saving Fmask raster')
drigo.array_to_raster(
fmask_array.astype(np.uint8), image.fmask_output_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
if (calc_fmask_cloud_flag and
not os.path.isfile(image.fmask_cloud_raster)):
logging.debug(' Saving Fmask cloud raster')
fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4)
drigo.array_to_raster(
fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_cloud_array
if (calc_fmask_snow_flag and
not os.path.isfile(image.fmask_snow_raster)):
logging.debug(' Saving Fmask snow raster')
fmask_snow_array = (fmask_array == 3)
drigo.array_to_raster(
fmask_snow_array.astype(np.uint8), image.fmask_snow_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_snow_array
if (calc_fmask_water_flag and
not os.path.isfile(image.fmask_water_raster)):
logging.debug(' Saving Fmask water raster')
fmask_water_array = (fmask_array == 1)
drigo.array_to_raster(
fmask_water_array.astype(np.uint8), image.fmask_water_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_water_array
del fmask_array
# # Calculate elevation
# if calc_elev_flag and not os.path.isfile(elev_path):
# logging.info('Elevation')
# elev_array, elev_nodata = drigo.raster_to_array(
# elev_pr_path, 1, common_extent)
# drigo.array_to_raster(
# elev_array, elev_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del elev_array, elev_nodata, elev_path
#
# # Calculate landuse
# if calc_landuse_flag and not os.path.isfile(landuse_raster):
# logging.info('Landuse')
# landuse_array, landuse_nodata = drigo.raster_to_array(
# landuse_pr_path, 1, common_extent)
# drigo.array_to_raster(
# landuse_array, landuse_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del landuse_array, landuse_nodata, landuse_raster
# Calculate toa reflectance
# f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32)
if calc_refl_toa_flag:
logging.info('Top-of-Atmosphere Reflectance')
if os.path.isfile(image.refl_toa_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(
image.refl_toa_raster))
dripy.remove_file(image.refl_toa_raster)
if not os.path.isfile(image.refl_toa_raster):
# First build empty composite raster
drigo.build_empty_raster(
image.refl_toa_raster, image.band_toa_cnt, np.float32, None,
env.snap_proj, env.cellsize, common_extent)
# cos_theta_solar_flt = et_common.cos_theta_solar_func(
# image.sun_elevation)
# Process by block
logging.info('Processing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(
common_cols, common_rows))
for b_i, b_j in drigo.block_gen(common_rows, common_cols, bs):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = drigo.array_to_block(
common_array, b_i, b_j, bs).astype(np.bool)
block_rows, block_cols = block_data_mask.shape
block_geo = drigo.array_offset_geo(common_geo, b_j, b_i)
block_extent = drigo.geo_extent(
block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
logging.debug(' Block extent: {}'.format(block_extent))
logging.debug(' Block geo: {}'.format(block_geo))
# Process each TOA band
# for band, band_i in sorted(image.band_toa_dict.items()):
for band, dn_image in sorted(dn_image_dict.items()):
if band not in image.band_toa_dict.keys():
continue
# thermal_band_flag = (band == image.thermal_band)
# Set 0 as nodata value
drigo.raster_path_set_nodata(dn_image, 0)
# Calculate TOA reflectance
dn_array, dn_nodata = drigo.raster_to_array(
dn_image, 1, block_extent)
dn_array = dn_array.astype(np.float64)
# dn_array = dn_array.astype(np.float32)
dn_array[dn_array == 0] = np.nan
#
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
refl_toa_array = et_numpy.l457_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.dr, image.esun_dict[band],
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band])
elif image.type in ['Landsat8']:
refl_toa_array = et_numpy.l8_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.refl_mult_dict[band],
image.refl_add_dict[band])
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band])
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band],
# image.refl_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array,
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band],
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band],
# image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# refl_toa_array = et_numpy.refl_toa_band_func(
# dn_array, cos_theta_solar_flt,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag)
drigo.block_to_raster(
refl_toa_array.astype(np.float32),
image.refl_toa_raster,
b_i, b_j, band=image.band_toa_dict[band])
# drigo.array_to_comp_raster(
# refl_toa_array.astype(np.float32),
# image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
del refl_toa_array, dn_array
if stats_flag:
drigo.raster_statistics(image.refl_toa_raster)
# # Process each TOA band
# # for band, band_i in sorted(image.band_toa_dict.items()):
# for band, dn_image in sorted(dn_image_dict.items()):
# thermal_band_flag = (band == image.thermal_band)
# # Set 0 as nodata value
# drigo.raster_path_set_nodata(dn_image, 0)
# # Calculate TOA reflectance
# dn_array, dn_nodata = drigo.raster_to_array(
# dn_image, 1, common_extent)
# dn_array = dn_array.astype(np.float64)
# # dn_array = dn_array.astype(np.float32)
# dn_array[dn_array == 0] = np.nan
# #
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band])
# # image.rad_mult_dict[band], image.rad_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array, image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# # image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band], image.refl_add_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# # refl_toa_array = et_numpy.refl_toa_band_func(
# # dn_array, cos_theta_solar_flt,
# # image.dr, image.esun_dict[band],
# # image.lmin_dict[band], image.lmax_dict[band],
# # image.qcalmin_dict[band], image.qcalmax_dict[band],
# # thermal_band_flag)
# drigo.array_to_comp_raster(
# refl_toa_array.astype(np.float32), image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
# del refl_toa_array, dn_array
# Calculate brightness temperature
if calc_ts_bt_flag:
logging.info('Brightness Temperature')
if os.path.isfile(image.ts_bt_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(image.ts_bt_raster))
dripy.remove_file(image.ts_bt_raster)
if not os.path.isfile(image.ts_bt_raster):
band = image.thermal_band
thermal_dn_path = dn_image_dict[band]
drigo.raster_path_set_nodata(thermal_dn_path, 0)
thermal_dn_array, thermal_dn_nodata = drigo.raster_to_array(
thermal_dn_path, 1, common_extent, return_nodata=True)
thermal_dn_mask = thermal_dn_array != thermal_dn_nodata
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
ts_bt_array = et_numpy.l457_ts_bt_band_func(
thermal_dn_array,
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band],
# image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
elif image.type in ['Landsat8']:
ts_bt_array = et_numpy.l8_ts_bt_band_func(
thermal_dn_array,
image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
# thermal_rad_array = et_numpy.refl_toa_band_func(
# thermal_dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag=True)
# ts_bt_array = et_numpy.ts_bt_func(
# thermal_rad_array, image.k1_dict[image.thermal_band],
# image.k2_dict[image.thermal_band])
ts_bt_array[~thermal_dn_mask] = np.nan
drigo.array_to_raster(
ts_bt_array, image.ts_bt_raster,
output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array,
stats_flag=stats_flag)
# del thermal_dn_array, thermal_rad_array
del thermal_dn_path, thermal_dn_array, ts_bt_array
# Interpolate/project/clip METRIC hourly/daily rasters
if (calc_metric_ea_flag or
calc_metric_wind_flag or
calc_metric_etr_flag):
logging.info('METRIC hourly/daily rasters')
# Get bracketing hours from image acquisition time
image_prev_dt = image.acq_datetime.replace(
minute=0, second=0, microsecond=0)
image_next_dt = image_prev_dt + timedelta(seconds=3600)
# Get NLDAS properties from one of the images
input_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(input_ws, file_name)
for file_name in os.listdir(input_ws)
for match in [metric_hourly_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
except IndexError:
logging.error(' No hourly file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d %H00')))
return False
try:
input_ds = gdal.Open(input_path)
input_osr = drigo.raster_ds_osr(input_ds)
# input_proj = drigo.osr_proj(input_osr)
input_extent = drigo.raster_ds_extent(input_ds)
input_cs = drigo.raster_ds_cellsize(input_ds, x_only=True)
# input_geo = input_extent.geo(input_cs)
input_x, input_y = input_extent.origin()
input_ds = None
except:
logging.error(' Could not get default input image properties')
logging.error(' {}'.format(input_path))
return False
# Project Landsat scene extent to NLDAS GCS
common_gcs_osr = common_osr.CloneGeogCS()
common_gcs_extent = drigo.project_extent(
common_extent, common_osr, common_gcs_osr,
cellsize=env.cellsize)
common_gcs_extent.buffer_extent(0.1)
common_gcs_extent.adjust_to_snap(
'EXPAND', input_x, input_y, input_cs)
# common_gcs_geo = common_gcs_extent.geo(input_cs)
def metric_weather_func(output_raster, input_ws, input_re,
prev_dt, next_dt,
resample_method=gdal.GRA_NearestNeighbour,
rounding_flag=False):
"""Interpolate/project/clip METRIC hourly rasters"""
logging.debug(' Output: {}'.format(output_raster))
if os.path.isfile(output_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
dripy.remove_file(output_raster)
else:
logging.debug(' Skipping, file already exists ' +
'and overwrite is False')
return False
prev_ws = os.path.join(input_ws, str(prev_dt.year))
next_ws = os.path.join(input_ws, str(next_dt.year))
# Technically previous and next could come from different days
# or even years, although this won't happen in the U.S.
try:
prev_path = [
os.path.join(prev_ws, input_name)
for input_name in os.listdir(prev_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(prev_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input prev: {}'.format(prev_path))
except IndexError:
logging.error(' No previous hourly file')
logging.error(' {}'.format(prev_dt))
return False
try:
next_path = [
os.path.join(next_ws, input_name)
for input_name in os.listdir(next_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(next_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input next: {}'.format(next_path))
except IndexError:
logging.error(' No next hourly file')
logging.error(' {}'.format(next_dt))
return False
# Band numbers are 1's based
prev_band = int(prev_dt.strftime('%H')) + 1
next_band = int(next_dt.strftime('%H')) + 1
logging.debug(' Input prev band: {}'.format(prev_band))
logging.debug(' Input next band: {}'.format(next_band))
# Read arrays
prev_array = drigo.raster_to_array(
prev_path, band=prev_band, mask_extent=common_gcs_extent,
return_nodata=False)
next_array = drigo.raster_to_array(
next_path, band=next_band, mask_extent=common_gcs_extent,
return_nodata=False)
if not np.any(prev_array) or not np.any(next_array):
logging.warning('\nWARNING: Input NLDAS array is all nodata\n')
return None
output_array = hourly_interpolate_func(
prev_array, next_array,
prev_dt, next_dt, image.acq_datetime)
output_array = drigo.project_array(
output_array, resample_method,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent, output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
if rounding_flag:
output_array = np.around(output_array, rounding_digits)
# Force output to 32-bit float
drigo.array_to_raster(
output_array.astype(np.float32), output_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
return True
# Ea - Project to Landsat scene after clipping
if calc_metric_ea_flag:
logging.info(' Hourly vapor pressure (Ea)')
metric_weather_func(
image.metric_ea_raster, metric_ea_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_Bilinear, rounding_flag=True)
# Wind - Project to Landsat scene after clipping
if calc_metric_wind_flag:
logging.info(' Hourly windspeed')
metric_weather_func(
image.metric_wind_raster, metric_wind_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Hourly reference ET (ETr)')
metric_weather_func(
image.metric_etr_raster, metric_etr_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr 24hr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Daily reference ET (ETr)')
logging.debug(' Output: {}'.format(
image.metric_etr_24hr_raster))
if (os.path.isfile(image.metric_etr_24hr_raster) and
overwrite_flag):
logging.debug(' Overwriting output')
os.remove(image.metric_etr_24hr_raster)
if not os.path.isfile(image.metric_etr_24hr_raster):
etr_prev_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(etr_prev_ws, file_name)
for file_name in os.listdir(etr_prev_ws)
for match in [metric_daily_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
logging.debug(' Input: {}'.format(input_path))
except IndexError:
logging.error(' No daily file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d')))
return False
output_array = drigo.raster_to_array(
input_path, mask_extent=common_gcs_extent,
return_nodata=False)
output_array = drigo.project_array(
output_array, gdal.GRA_NearestNeighbour,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent,
output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
# output_array = np.around(output_array, rounding_digits)
drigo.array_to_raster(
output_array, image.metric_etr_24hr_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
del input_path
# Tair - Project to Landsat scene after clipping
if calc_metric_tair_flag:
logging.info(' Hourly air temperature (Tair)')
metric_weather_func(
image.metric_tair_raster, metric_tair_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# Cleanup
del image_prev_dt, image_next_dt
# Soil Water Balance
if calc_swb_ke_flag:
logging.info('Daily soil water balance')
# Check if output file already exists
logging.debug(' Ke: {}'.format(image.ke_raster))
if os.path.isfile(image.ke_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
dripy.remove_file(image.ke_raster)
else:
logging.debug(' Skipping, file already '
'exists and overwrite is False')
return False
ke_array = et_common.raster_swb_func(
image.acq_datetime, common_osr, env.cellsize, common_extent,
awc_input_path, etr_input_ws, etr_input_re,
ppt_input_ws, ppt_input_re,
spinup_days=spinup_days, min_spinup_days=min_spinup_days)
# Apply common area mask
ke_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest 2 digits
np.around(ke_array, 2, out=ke_array)
# Force output to 32-bit float
drigo.array_to_raster(
ke_array.astype(np.float32), image.ke_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Remove Landsat TOA rasters
if not keep_dn_flag:
for landsat_item in dripy.build_file_list(
image.orig_data_ws, image.image_name_re):
os.remove(os.path.join(image.orig_data_ws, landsat_item))
return True
def pixel_rating(image_ws, ini_path, stats_flag=False, overwrite_flag=None):
"""Calculate pixel rating
Args:
image_ws (str): Image folder path
ini_path (str): Pixel regions config file path
stats_flag (bool): if True, compute raster statistics
ovewrite_flag (bool): if True, overwrite existing files
Returns:
None
"""
logging.info('Generating suggested hot/cold pixel regions')
log_fmt = ' {:<18s} {}'
env = gdc.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore')
# # Check that image_ws is valid
# image_re = re.compile(
# '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})')
# if not os.path.isdir(image_ws) or not image_re.match(scene_id):
# logging.error('\nERROR: Image folder is invalid or does not exist\n')
# return False
# Folder Paths
region_ws = os.path.join(image_ws, 'PIXEL_REGIONS')
# Open config file
config = open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
# Arrays are processed by block
bs = read_param('block_size', 1024, config)
logging.info(' {:<18s} {}'.format('Block Size:', bs))
# Raster pyramids/statistics
pyramids_flag = read_param('pyramids_flag', False, config)
if pyramids_flag:
gdal.SetConfigOption('HFA_USE_RRD', 'YES')
if stats_flag is None:
stats_flag = read_param('statistics_flag', False, config)
# Overwrite
if overwrite_flag is None:
overwrite_flag = read_param('overwrite_flag', True, config)
# Check that common_area raster exists
if not os.path.isfile(image.common_area_raster):
logging.error(
'\nERROR: A common area raster was not found.' +
'\nERROR: Please rerun prep tool to build these files.\n' +
' {}\n'.format(image.common_area_raster))
sys.exit()
# Use common_area to set mask parameters
common_ds = gdal.Open(image.common_area_raster)
# env.mask_proj = raster_ds_proj(common_ds)
env.mask_geo = gdc.raster_ds_geo(common_ds)
env.mask_rows, env.mask_cols = gdc.raster_ds_shape(common_ds)
env.mask_extent = gdc.geo_extent(env.mask_geo, env.mask_rows,
env.mask_cols)
env.mask_array = gdc.raster_ds_to_array(common_ds)[0]
env.mask_path = image.common_area_raster
env.snap_osr = gdc.raster_path_osr(image.common_area_raster)
env.snap_proj = env.snap_osr.ExportToWkt()
env.cellsize = gdc.raster_path_cellsize(image.common_area_raster)[0]
common_ds = None
logging.debug(' {:<18s} {}'.format('Mask Extent:', env.mask_extent))
# Read Pixel Regions config file
# Currently there is no code to support applying an NLCD mask
apply_nlcd_mask = False
# apply_nlcd_mask = read_param('apply_nlcd_mask', False, config)
apply_cdl_ag_mask = read_param('apply_cdl_ag_mask', False, config)
apply_field_mask = read_param('apply_field_mask', False, config)
apply_ndwi_mask = read_param('apply_ndwi_mask', True, config)
apply_ndvi_mask = read_param('apply_ndvi_mask', True, config)
# Currently the code to apply a study area mask is commented out
# apply_study_area_mask = read_param(
# 'apply_study_area_mask', False, config)
albedo_rating_flag = read_param('albedo_rating_flag', True, config)
nlcd_rating_flag = read_param('nlcd_rating_flag', True, config)
ndvi_rating_flag = read_param('ndvi_rating_flag', True, config)
ts_rating_flag = read_param('ts_rating_flag', True, config)
ke_rating_flag = read_param('ke_rating_flag', False, config)
# if apply_study_area_mask:
# study_area_path = config.get('INPUTS', 'study_area_path')
if apply_nlcd_mask or nlcd_rating_flag:
nlcd_raster = config.get('INPUTS', 'landuse_raster')
if apply_cdl_ag_mask:
cdl_ag_raster = config.get('INPUTS', 'cdl_ag_raster')
cdl_buffer_cells = read_param('cdl_buffer_cells', 0, config)
cdl_ag_eroded_name = read_param('cdl_ag_eroded_name',
'cdl_ag_eroded_{}.img', config)
if apply_field_mask:
field_raster = config.get('INPUTS', 'fields_raster')
cold_rating_pct = read_param('cold_percentile', 99, config)
hot_rating_pct = read_param('hot_percentile', 99, config)
# min_cold_rating_score = read_param('min_cold_rating_score', 0.3, config)
# min_hot_rating_score = read_param('min_hot_rating_score', 0.3, config)
ts_bin_count = int(read_param('ts_bin_count', 10, config))
if 100 % ts_bin_count != 0:
logging.warning(
'WARNING: ts_bins_count of {} is not a divisor ' +
'of 100. Using default ts_bins_count = 4'.format(ts_bin_count))
ts_bin_count = 10
bin_size = 1. / (ts_bin_count - 1)
hot_rating_values = np.arange(0., 1. + bin_size, step=bin_size)
cold_rating_values = hot_rating_values[::-1]
# Input raster paths
r_fmt = '.img'
if 'Landsat' in image.type:
albedo_raster = image.albedo_sur_raster
ndvi_raster = image.ndvi_toa_raster
ndwi_raster = image.ndwi_toa_raster
ts_raster = image.ts_raster
ke_raster = image.ke_raster
# Check config file input paths
# if apply_study_area_mask and not os.path.isfile(study_area_path):
# logging.error(
# ('\nERROR: The study area shapefile {} does ' +
# 'not exist\n').format(study_area_path))
# sys.exit()
if ((apply_nlcd_mask or nlcd_rating_flag)
and not os.path.isfile(nlcd_raster)):
logging.error(('\nERROR: The NLCD raster {} does ' +
'not exist\n').format(nlcd_raster))
sys.exit()
if apply_cdl_ag_mask and not os.path.isfile(cdl_ag_raster):
logging.error(('\nERROR: The CDL Ag raster {} does ' +
'not exist\n').format(cdl_ag_raster))
sys.exit()
if apply_field_mask and not os.path.isfile(field_raster):
logging.error(('\nERROR: The field raster {} does ' +
'not exist\n').format(field_raster))
sys.exit()
if (not (isinstance(cold_rating_pct,
(int, float)) and (0 <= cold_rating_pct <= 100))):
logging.error(
'\nERROR: cold_percentile must be a value between 0 and 100\n')
sys.exit()
if (not (isinstance(hot_rating_pct,
(int, float)) and (0 <= hot_rating_pct <= 100))):
logging.error(
'\nERROR: hot_percentile must be a value between 0 and 100\n')
sys.exit()
# Set raster names
raster_dict = dict()
# Output Rasters
raster_dict['region_mask'] = os.path.join(region_ws, 'region_mask' + r_fmt)
raster_dict['cold_rating'] = os.path.join(region_ws,
'cold_pixel_rating' + r_fmt)
raster_dict['hot_rating'] = os.path.join(region_ws,
'hot_pixel_rating' + r_fmt)
raster_dict['cold_sugg'] = os.path.join(region_ws,
'cold_pixel_suggestion' + r_fmt)
raster_dict['hot_sugg'] = os.path.join(region_ws,
'hot_pixel_suggestion' + r_fmt)
# Read pixel region raster flags
save_dict = dict()
save_dict['region_mask'] = read_param('save_region_mask_flag', False,
config)
save_dict['cold_rating'] = read_param('save_rating_rasters_flag', False,
config)
save_dict['hot_rating'] = read_param('save_rating_rasters_flag', False,
config)
save_dict['cold_sugg'] = read_param('save_suggestion_rasters_flag', True,
config)
save_dict['hot_sugg'] = read_param('save_suggestion_rasters_flag', True,
config)
# Output folder
if not os.path.isdir(region_ws):
os.mkdir(region_ws)
# Remove existing files if necessary
region_ws_file_list = [
os.path.join(region_ws, item) for item in os.listdir(region_ws)
]
if overwrite_flag and region_ws_file_list:
for raster_path in raster_dict.values():
if raster_path in region_ws_file_list:
remove_file(raster_path)
# Check scene specific input paths
if apply_ndwi_mask and not os.path.isfile(ndwi_raster):
logging.error(
'ERROR: NDWI raster does not exist\n {}'.format(ndwi_raster))
sys.exit()
elif apply_ndvi_mask and not os.path.isfile(ndvi_raster):
logging.error(
'ERROR: NDVI raster does not exist\n {}'.format(ndvi_raster))
sys.exit()
elif ke_rating_flag and not os.path.isfile(ke_raster):
logging.error(
('ERROR: The Ke raster does not exist\n {}').format(ke_raster))
sys.exit()
# Remove existing and build new empty rasters if necessary
# If processing by block, rating rasters must be built
logging.debug('\nBuilding empty rasters')
for name, save_flag in sorted(save_dict.items()):
if save_flag and 'rating' in name:
gdc.build_empty_raster(raster_dict[name], 1, np.float32)
elif save_flag:
gdc.build_empty_raster(raster_dict[name],
1,
np.uint8,
output_nodata=0)
if apply_cdl_ag_mask:
logging.info('Building CDL ag mask')
cdl_array = gdc.raster_to_array(cdl_ag_raster,
mask_extent=env.mask_extent,
return_nodata=False)
if cdl_buffer_cells > 0:
logging.info(' Eroding CDL by {} cells'.format(cdl_buffer_cells))
structure_array = np.ones((cdl_buffer_cells, cdl_buffer_cells),
dtype=np.int)
# Deadbeef - This could blow up in memory on bigger rasters
cdl_array = ndimage.binary_erosion(
cdl_array, structure_array).astype(structure_array.dtype)
cdl_ag_eroded_raster = os.path.join(
image.support_ws, cdl_ag_eroded_name.format(cdl_buffer_cells))
gdc.array_to_raster(cdl_array,
cdl_ag_eroded_raster,
output_geo=env.mask_geo,
output_proj=env.snap_proj,
mask_array=env.mask_array,
output_nodata=0,
stats_flag=False)
cdl_array = None
del cdl_array
# Build region mask
logging.debug('Building region mask')
region_mask = np.copy(env.mask_array).astype(np.bool)
if apply_field_mask:
field_mask, field_nodata = gdc.raster_to_array(
field_raster, mask_extent=env.mask_extent, return_nodata=True)
region_mask &= field_mask != field_nodata
del field_mask, field_nodata
if apply_ndwi_mask:
ndwi_array = gdc.raster_to_array(ndwi_raster,
1,
mask_extent=env.mask_extent,
return_nodata=False)
region_mask &= ndwi_array > 0.0
del ndwi_array
if apply_ndvi_mask:
ndvi_array = gdc.raster_to_array(ndvi_raster,
1,
mask_extent=env.mask_extent,
return_nodata=False)
region_mask &= ndvi_array > 0.12
del ndvi_array
if apply_cdl_ag_mask:
cdl_array, cdl_nodata = gdc.raster_to_array(
cdl_ag_eroded_raster,
mask_extent=env.mask_extent,
return_nodata=True)
region_mask &= cdl_array != cdl_nodata
del cdl_array, cdl_nodata
if save_dict['region_mask']:
gdc.array_to_raster(region_mask,
raster_dict['region_mask'],
stats_flag=False)
# Initialize rating arrays
# This needs to be done before the ts_rating if block
cold_rating_array = np.ones(env.mask_array.shape, dtype=np.float32)
hot_rating_array = np.ones(env.mask_array.shape, dtype=np.float32)
cold_rating_array[~region_mask] = np.nan
hot_rating_array[~region_mask] = np.nan
# Temperature pixel rating - grab the max and min value for the entire
# Ts image in a memory safe way by using gdal_common blocks
# The following is a percentile based approach
if ts_rating_flag:
logging.debug('Computing Ts percentile rating')
ts_array = gdc.raster_to_array(ts_raster,
mask_extent=env.mask_extent,
return_nodata=False)
ts_array[~region_mask] = np.nan
percentiles = range(0, (100 + ts_bin_count), int(100 / ts_bin_count))
ts_score_value = 1. / (ts_bin_count - 1)
hot_rating_values = np.arange(0, (1. + ts_score_value),
step=ts_score_value)[:ts_bin_count]
cold_rating_values = hot_rating_values[::-1]
ts_percentile_array = stats.scoreatpercentile(
ts_array[np.isfinite(ts_array)], percentiles)
for bins_i in range(len(ts_percentile_array))[:-1]:
bool_array = ((ts_array > ts_percentile_array[bins_i]) &
(ts_array <= ts_percentile_array[bins_i + 1]))
cold_rating_array[bool_array] = cold_rating_values[bins_i]
hot_rating_array[bool_array] = hot_rating_values[bins_i]
# gdc.array_to_raster(cold_rating_array, raster_dict['cold_rating'])
# gdc.array_to_raster(hot_rating_array, raster_dict['hot_rating'])
# Cleanup
del ts_array, ts_percentile_array
del cold_rating_values, hot_rating_values
del ts_score_value, percentiles
# Process by block
logging.info('\nProcessing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(env.mask_cols,
env.mask_rows))
for b_i, b_j in gdc.block_gen(env.mask_rows, env.mask_cols, bs):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = gdc.array_to_block(env.mask_array, b_i, b_j,
bs).astype(np.bool)
# block_nodata_mask = ~block_data_mask
block_rows, block_cols = block_data_mask.shape
block_geo = gdc.array_offset_geo(env.mask_geo, b_j, b_i)
block_extent = gdc.geo_extent(block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
# logging.debug(' Block extent: {}'.format(block_extent))
# logging.debug(' Block geo: {}'.format(block_geo))
# Don't skip empty blocks since block rating needs to be written
# back to the array at the end of the block loop
block_region_mask = gdc.array_to_block(region_mask, b_i, b_j, bs)
if not np.any(block_region_mask):
logging.debug(' Empty block')
block_empty_flag = True
else:
block_empty_flag = False
# New style continuous pixel weighting
cold_rating_block = gdc.array_to_block(cold_rating_array, b_i, b_j, bs)
hot_rating_block = gdc.array_to_block(hot_rating_array, b_i, b_j, bs)
# Rating arrays already have region_mask set
# cold_rating_block = np.ones(block_region_mask.shape, dtype=np.float32)
# hot_rating_block = np.ones(block_region_mask.shape, dtype=np.float32)
# cold_rating_block[~block_region_mask] = np.nan
# hot_rating_block[~block_region_mask] = np.nan
# del block_region_mask
if ndvi_rating_flag and not block_empty_flag:
# NDVI based rating
ndvi_array = gdc.raster_to_array(ndvi_raster,
1,
mask_extent=block_extent,
return_nodata=False)
# Don't let NDVI be negative
ndvi_array.clip(0., 0.833, out=ndvi_array)
# ndvi_array.clip(0.001, 0.833, out=ndvi_array)
cold_rating_block *= ndvi_array
cold_rating_block *= 1.20
ndvi_mask = (ndvi_array > 0)
# DEADBEEF - Can this calculation be masked to only NDVI > 0?
ndvi_mask = ndvi_array > 0
hot_rating_block[ndvi_mask] *= stats.norm.pdf(
np.log(ndvi_array[ndvi_mask]), math.log(0.15), 0.5)
hot_rating_block[ndvi_mask] *= 1.25
del ndvi_mask
# hot_rating_block *= stats.norm.pdf(
# np.log(ndvi_array), math.log(0.15), 0.5)
# hot_rating_block *= 1.25
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del ndvi_array
if albedo_rating_flag and not block_empty_flag:
# Albdo based rating
albedo_array = gdc.raster_to_array(albedo_raster,
1,
mask_extent=block_extent,
return_nodata=False)
albedo_cold_pdf = stats.norm.pdf(albedo_array, 0.21, 0.03)
albedo_hot_pdf = stats.norm.pdf(albedo_array, 0.21, 0.06)
del albedo_array
cold_rating_block *= albedo_cold_pdf
cold_rating_block *= 0.07
hot_rating_block *= albedo_hot_pdf
hot_rating_block *= 0.15
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del albedo_cold_pdf, albedo_hot_pdf
if nlcd_rating_flag and not block_empty_flag:
# NLCD based weighting, this could be CDL instead?
nlcd_array = nlcd_rating(
gdc.raster_to_array(nlcd_raster,
1,
mask_extent=block_extent,
return_nodata=False))
cold_rating_block *= nlcd_array
hot_rating_block *= nlcd_array
del nlcd_array
if ke_rating_flag and not block_empty_flag:
# SWB Ke based rating
ke_array = gdc.raster_to_array(ke_raster,
1,
mask_extent=block_extent,
return_nodata=False)
# Don't let NDVI be negative
ke_array.clip(0., 1., out=ke_array)
# Assumption, lower Ke is better for selecting the hot pixel
# As the power (2) decreases and approaches 1,
# the relationship gets more linear
# cold_rating_block *= (1 - ke_array ** 2)
# hot_rating_block *= (1 - ke_array ** 1.5)
# Linear inverse
# cold_rating_block *= (1. - ke_array)
hot_rating_block *= (1. - ke_array)
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del ke_array
# Clearness
# clearness = 1.0
# cold_rating *= clearness
# hot_rating *= clearness
# Reset nan values
# cold_rating_block[~region_mask] = np.nan
# hot_rating_block[~region_mask] = np.nan
# Save rating values
cold_rating_array = gdc.block_to_array(cold_rating_block,
cold_rating_array, b_i, b_j, bs)
hot_rating_array = gdc.block_to_array(hot_rating_block,
hot_rating_array, b_i, b_j, bs)
# Save rating rasters
if save_dict['cold_rating']:
gdc.block_to_raster(cold_rating_block, raster_dict['cold_rating'],
b_i, b_j, bs)
if save_dict['hot_rating']:
gdc.block_to_raster(hot_rating_block, raster_dict['hot_rating'],
b_i, b_j, bs)
# Save rating values
cold_rating_array = gdc.block_to_array(cold_rating_block,
cold_rating_array, b_i, b_j, bs)
hot_rating_array = gdc.block_to_array(hot_rating_block,
hot_rating_array, b_i, b_j, bs)
del cold_rating_block, hot_rating_block
# Select pixels above target percentile
# Only build suggestion arrays if saving
logging.debug('Building suggested pixel rasters')
if save_dict['cold_sugg']:
cold_rating_score = float(
stats.scoreatpercentile(
cold_rating_array[np.isfinite(cold_rating_array)],
cold_rating_pct))
# cold_rating_array, cold_rating_nodata = gdc.raster_to_array(
# raster_dict['cold_rating'], 1, mask_extent=env.mask_extent)
# if cold_rating_score < float(min_cold_rating_score):
# logging.error(('ERROR: The cold_rating_score ({}) is less ' +
# 'than the min_cold_rating_score ({})').format(
# cold_rating_score, min_cold_rating_score))
# sys.exit()
cold_sugg_mask = cold_rating_array >= cold_rating_score
gdc.array_to_raster(cold_sugg_mask,
raster_dict['cold_sugg'],
stats_flag=stats_flag)
logging.debug(' Cold Percentile: {}'.format(cold_rating_pct))
logging.debug(' Cold Score: {:.6f}'.format(cold_rating_score))
logging.debug(' Cold Pixels: {}'.format(np.sum(cold_sugg_mask)))
del cold_sugg_mask, cold_rating_array
if save_dict['hot_sugg']:
hot_rating_score = float(
stats.scoreatpercentile(
hot_rating_array[np.isfinite(hot_rating_array)],
hot_rating_pct))
# hot_rating_array, hot_rating_nodata = gdc.raster_to_array(
# raster_dict['hot_rating'], 1, mask_extent=env.mask_extent)
# if hot_rating_score < float(min_hot_rating_score):
# logging.error(('ERROR: The hot_rating_array ({}) is less ' +
# 'than the min_hot_rating_score ({})').format(
# hot_rating_array, min_hot_rating_score))
# sys.exit()
hot_sugg_mask = hot_rating_array >= hot_rating_score
gdc.array_to_raster(hot_sugg_mask,
raster_dict['hot_sugg'],
stats_flag=stats_flag)
logging.debug(' Hot Percentile: {}'.format(hot_rating_pct))
logging.debug(' Hot Score: {:.6f}'.format(hot_rating_score))
logging.debug(' Hot Pixels: {}'.format(np.sum(hot_sugg_mask)))
del hot_sugg_mask, hot_rating_array
# Raster Statistics
if stats_flag:
logging.info('Calculating Statistics')
for name, save_flag in save_dict.items():
if save_flag:
gdc.raster_statistics(raster_dict[name])
# Raster Pyramids
if pyramids_flag:
logging.info('Building Pyramids')
for name, save_flag in save_dict.items():
if save_flag:
gdc.raster_pyramids(raster_dict[name])
def monte_carlo(image_ws,
metric_ini_path,
mc_ini_path,
mc_iter=None,
cold_tgt_pct=None,
hot_tgt_pct=None,
groupsize=64,
blocksize=4096,
multipoint_flag=False,
shapefile_flag=False,
stats_flag=False,
overwrite_flag=False,
debug_flag=False,
no_etrf_final_plots=None,
no_etrf_temp_plots=None):
"""METRIC Monte Carlo
Parameters
----------
image_ws : str
The workspace (path) of the landsat scene folder.
metric_ini_path : str
The METRIC config file (path).
mc_ini_path : str
The Monte Carlo config file (path).
mc_iter : int, optional
Iteration number for Monte Carlo processing.
cold_tgt_pct : float, optional
Target percentage of pixels with ETrF > than cold Kc.
hot_tgt_pct : float, optional
Target percentage of pixels with ETrF < than hot Kc.
groupsize : int, optional
Script will try to place calibration point randomly
into a labeled group of clustered values with at least n pixels.
-1 = In the largest group
0 = Anywhere in the image (not currently implemented)
1 >= In any group with a pixel count greater or equal to n
blocksize : int, optional
Processing block size (the default is 4096).
shapefile_flag : bool, optional
If True, save calibration points to shapefile (the default is False).
multipoint_flag : bool, optional
If True, save cal. points to multipoint shapefile (the default is False).
stats_flag : bool, optional
If True, compute raster statistics (the default is False).
ovewrite_flag : bool, optional
If True, overwrite existing files (the default is False).
debug_flag : bool, optional
If True, enable debug level logging (the default is False).
no_final_plots : bool, optional
If True, don't save final ETrF histograms (the default is None).
This will override the flag in the INI file.
no_temp_plots : bool
If True, don't save temp ETrF histogram (the default is None).
This will override the flag in the INI file.
Returns
-------
None
"""
logging.info('METRIC Automated Calibration')
# Open config file
config = open_ini(mc_ini_path)
# Get input parameters
logging.debug(' Reading Input File')
etrf_training_path = config.get('INPUTS', 'etrf_training_path')
# Adjust Kc cold target value based on day of year
# etrf_doy_adj_path = read_param(
# 'etrf_doy_adj_path', None, config, 'INPUTS')
# Intentionally set default to None, to trigger error in eval call
kc_cold_doy_dict = read_param('kc_cold_doy_dict', None, config, 'INPUTS')
kc_hot_doy_dict = read_param('kc_hot_doy_dict', None, config, 'INPUTS')
# If the "no_" flags were set True, honor them and set the flag False
# If the "no_" flags were not set by the user, use the INI flag values
# If not set in the INI, default to False (don't save any plots)
if no_etrf_temp_plots:
save_etrf_temp_plots = False
else:
save_etrf_temp_plots = read_param('save_etrf_temp_plots', False,
config, 'INPUTS')
if no_etrf_final_plots:
save_etrf_final_plots = False
else:
save_etrf_final_plots = read_param('save_etrf_final_plots', False,
config, 'INPUTS')
save_ndvi_plots = read_param('save_ndvi_plots', False, config, 'INPUTS')
max_cal_iter = read_param('max_cal_iterations', 5, config, 'INPUTS')
max_point_iter = read_param('max_point_iterations', 10, config, 'INPUTS')
ts_diff_threshold = read_param('ts_diff_threshold', 4, config, 'INPUTS')
etr_ws = config.get('INPUTS', 'etr_ws')
ppt_ws = config.get('INPUTS', 'ppt_ws')
etr_re = re.compile(config.get('INPUTS', 'etr_re'))
ppt_re = re.compile(config.get('INPUTS', 'ppt_re'))
awc_path = config.get('INPUTS', 'awc_path')
spinup_days = read_param('swb_spinup_days', 5, config, 'INPUTS')
min_spinup_days = read_param('swb_min_spinup_days', 30, config, 'INPUTS')
log_fmt = ' {:<18s} {}'
break_line = '\n{}'.format('#' * 80)
env = drigo.env
image = et_image.Image(image_ws, env)
logging.info(log_fmt.format('Image:', image.folder_id))
# Check inputs
for file_path in [awc_path]:
if not os.path.isfile(file_path):
logging.error('\nERROR: File {} does not exist'.format(file_path))
sys.exit()
for folder in [etr_ws, ppt_ws]:
if not os.path.isdir(folder):
logging.error('\nERROR: Folder {} does not exist'.format(folder))
sys.exit()
# if (etrf_doy_adj_path and not
# os.path.isfile(etrf_doy_adj_path)):
# logging.error(
# '\nERROR: File {} does not exist.'.format(
# etrf_doy_adj_path))
# sys.exit()
# Use iteration number to file iteration string
if mc_iter is None:
mc_str = ''
mc_fmt = '.img'
elif int(mc_iter) < 0:
logging.error('\nERROR: Iteration number must be a positive integer')
return False
else:
mc_str = 'MC{:02d}_'.format(int(mc_iter))
mc_fmt = '_{:02d}.img'.format(int(mc_iter))
logging.info(' {:<18s} {}'.format('Iteration:', mc_iter))
# Folder names
etrf_ws = os.path.join(image_ws, 'ETRF')
# indices_ws = image.indices_ws
region_ws = os.path.join(image_ws, 'PIXEL_REGIONS')
pixels_ws = os.path.join(image_ws, 'PIXELS')
plots_ws = os.path.join(image_ws, 'PLOTS')
if shapefile_flag and not os.path.isdir(pixels_ws):
os.mkdir(pixels_ws)
if not os.path.isdir(plots_ws):
os.mkdir(plots_ws)
# File names
r_fmt = '.img'
etrf_path = os.path.join(etrf_ws, 'et_rf' + mc_fmt)
region_path = os.path.join(region_ws, 'region_mask' + r_fmt)
# Initialize calibration parameters dictionary
logging.info(break_line)
logging.info('Calibration Parameters')
cal_dict = dict()
logging.debug(' Reading target cold/hot Kc from INI')
# Using eval is potentially a really bad way of reading this in
try:
kc_cold_doy_dict = eval('{' + kc_cold_doy_dict + '}')
except:
kc_cold_doy_dict = {1: 1.05, 366: 1.05}
logging.info(
' ERROR: kc_cold_doy_dict was not parsed, using default values')
try:
kc_hot_doy_dict = eval('{' + kc_hot_doy_dict + '}')
except:
kc_hot_doy_dict = {1: 0.1, 366: 0.1}
logging.info(
' ERROR: kc_hot_doy_dict was not parsed, using default values')
logging.debug(' Kc cold dict: {}'.format(kc_cold_doy_dict))
logging.debug(' Kc hot dict: {}\n'.format(kc_hot_doy_dict))
# doy_cold, kc_cold = zip(*sorted(kc_cold_doy_dict.items()))
cal_dict['cold_tgt_kc'] = np.interp(
image.acq_doy,
*zip(*sorted(kc_cold_doy_dict.items())),
left=1.05,
right=1.05)
# doy_hot, kc_hot = zip(*sorted(kc_hot_doy_dict.items()))
cal_dict['hot_tgt_kc'] = np.interp(image.acq_doy,
*zip(*sorted(kc_hot_doy_dict.items())),
left=0.1,
right=0.1)
# if etrf_doy_adj_path:
# doy_adj_df = pd.read_csv(etrf_doy_adj_path)
# doy_adj = float(
# doy_adj_df[doy_adj_df['DOY'] == image.acq_doy]['ETRF_ADJ'])
# cal_dict['cold_tgt_kc'] = cal_dict['cold_tgt_kc'] + doy_adj
# Get hot/cold etrf fraction sizes
if cold_tgt_pct is None or hot_tgt_pct is None:
logging.info('ETrF Tail Size Percentages')
logging.info(' Reading target tail size from file')
cold_tgt_pct, hot_tgt_pct = auto_calibration.etrf_fractions(
etrf_training_path)
if cold_tgt_pct is None or hot_tgt_pct is None:
logging.error('\nERROR: Tail sizes were not mannually set or '
'read from the the file\n')
return False
cal_dict['cold_tgt_pct'] = cold_tgt_pct
cal_dict['hot_tgt_pct'] = hot_tgt_pct
logging.info(pixel_str_fmt.format('', 'Cold Pixel', 'Hot Pixel'))
logging.info(
pixel_flt_fmt.format('Target kc:', cal_dict['cold_tgt_kc'],
cal_dict['hot_tgt_kc']))
logging.info(
pixel_pct_fmt.format('Tail Size:', cal_dict['cold_tgt_pct'],
cal_dict['hot_tgt_pct']))
# # Create calibration database
# # Set overwrite false to use existing database if it exists
# cal_ws = os.path.join(image_ws, cal_folder)
# if not os.path.isdir(cal_ws):
# os.mkdir(cal_ws)
# cal_path = os.path.join(cal_ws, cal_name)
# logging.info('{:<20s} {}\{}'.format(
# 'Calibration DB:', cal_folder, cal_name))
# calibration_database.create_calibration_database(
# image_ws, cal_path, overwrite_db_flag)
# del cal_ws
# Remove previous calibrations from database
# logging.info(break_line)
# calibration_database.remove_calibration_points(
# image_ws, cal_path, cal_initials, mc_iter)
# Get ETrF and region mask (from pixel rating)
# Assume they have identical extents
try:
region_mask = drigo.raster_to_array(region_path, return_nodata=False)
region_mask = region_mask.astype(np.bool)
except:
logging.error(
'\nERROR: Pixel regions mask does not exist or could not be read.\n'
' Please try re-running the METRIC Pixel Rating tool.')
logging.debug(' {} '.format(region_path))
return False
# Remove previous plots
logging.info(break_line)
auto_calibration.remove_histograms(plots_ws, mc_iter)
# Generate the NDVI histogram
if save_ndvi_plots:
logging.info(break_line)
logging.info('NDVI Histograms')
if os.path.isfile(image.ndvi_toa_raster):
ndvi_array = drigo.raster_to_array(image.ndvi_toa_raster,
return_nodata=False)
else:
logging.error(
'\nERROR: NDVI raster does not exist. METRIC Model 1 may not '
'have run successfully.')
logging.debug(' {} '.format(image.ndvi_toa_raster))
# Only process ag. ETrF pixels
ndvi_array[~region_mask] = np.nan
ndvi_sub_array = ndvi_array[region_mask]
if np.any(ndvi_sub_array):
auto_calibration.save_ndvi_histograms(ndvi_sub_array, plots_ws,
mc_iter)
else:
logging.error(
'\nERROR: Empty NDVI array, histogram was not built\n')
# Place points in suggested region allowing for a number of iterations
# dependent on whether or not Ts meets certain criteria
logging.info(break_line)
pixel_point_iters = 0
while pixel_point_iters <= max_point_iter:
if pixel_point_iters == max_point_iter:
logging.error('\nERROR: Suitable hot and cold pixels could not be '
'determined. The scene will not calibrate.\n')
return False
cold_xy, hot_xy = pixel_points.pixel_points(
image_ws,
groupsize=groupsize,
blocksize=blocksize,
mc_iter=mc_iter,
shapefile_flag=shapefile_flag,
multipoint_flag=multipoint_flag,
overwrite_flag=overwrite_flag,
pixel_point_iters=pixel_point_iters)
if any(x is None for x in cold_xy) or any(x is None for x in hot_xy):
logging.error(('\nPixel points coordinates are invalid. '
'The scene will not calibrate.'
'\n Cold: {}\n Hot: {}').format(cold_xy, hot_xy))
return False
cold_ts = drigo.raster_value_at_xy(image.ts_raster, cold_xy)
hot_ts = drigo.raster_value_at_xy(image.ts_raster, hot_xy)
if cold_ts > hot_ts:
logging.info(
'\nThe cold pixel is hotter than the hot pixel. Placing '
'the points again.\n')
logging.info(break_line)
pixel_point_iters += 1
elif abs(hot_ts - cold_ts) < ts_diff_threshold:
logging.info((
'\nThere is less than a {} degree difference in Ts hot and cold. '
'Placing the points again.\n').format(ts_diff_threshold))
logging.info(break_line)
pixel_point_iters += 1
# raise et_common.TemperatureError
else:
break
# Adjust Kc hot for soil water balance
logging.info(break_line)
cal_dict = auto_calibration.hot_kc_swb_adjust(cal_dict, hot_xy,
env.snap_osr, image.acq_date,
awc_path, etr_ws, etr_re,
ppt_ws, ppt_re, spinup_days,
min_spinup_days)
# Adjust Kc cold based on NDVI
# cal_dict['tgt_c_kc'] = auto_calibration.kc_ndvi_adjust(
# cal_dict['tgt_c_kc'], cold_xy, ndvi_path, 'Cold')
# Check that Kc hot (Ke) is not too high?
if cal_dict['hot_tgt_kc'] >= 1.0:
logging.error('\nERROR: Target Kc hot is too high for automated '
'calibration\n ETrF will not be computed')
return False
elif (cal_dict['cold_tgt_kc'] - cal_dict['hot_tgt_kc']) <= 0.05:
logging.error('\nERROR: Target Kc hot and Kc cold are too close for '
'automated calibration\n ETrF will not be computed')
return False
# Initialize Kc values at targets
cal_dict['kc_cold'] = cal_dict['cold_tgt_kc']
cal_dict['kc_hot'] = cal_dict['hot_tgt_kc']
# Iterate until max calibrations is reached or error is small
cal_flag = False
cal_iter = 1
while not cal_flag:
logging.info(break_line)
logging.info('Calibration Iteration: {}'.format(cal_iter))
# Run METRIC Model2 for initial ETrF map
logging.info(break_line)
metric_model2.metric_model2(image_ws,
metric_ini_path,
mc_iter=mc_iter,
kc_cold=cal_dict['kc_cold'],
kc_hot=cal_dict['kc_hot'],
cold_xy=cold_xy,
hot_xy=hot_xy,
overwrite_flag=overwrite_flag)
# Read in ETrF array
if os.path.isfile(etrf_path):
etrf_array = drigo.raster_to_array(etrf_path, return_nodata=False)
else:
logging.warning(
('WARNING: ETrF raster does not exist. METRIC Model 2 '
'may not have run successfully.\n {}').format(etrf_path))
break
etrf_geo = drigo.raster_path_geo(etrf_path)
# Only process ag. ETrF pixels
etrf_array[~region_mask] = np.nan
etrf_sub_array = etrf_array[np.isfinite(etrf_array)]
if not np.any(etrf_sub_array):
logging.error(
'\nERROR: Empty ETrF array, scene cannot be calibrated\n')
break
# Calculate calibration parameters
logging.debug(break_line)
cal_dict = auto_calibration.calibration_params(cal_dict,
etrf_sub_array)
# Plot intermediates calibration histograms
if save_etrf_temp_plots:
logging.debug(break_line)
auto_calibration.save_etrf_histograms(etrf_sub_array, plots_ws,
cal_dict, mc_iter, cal_iter)
# Check calibration
logging.debug(break_line)
cal_flag = auto_calibration.check_calibration(cal_dict)
# Don't re-calibrate if initial calibration was suitable
if cal_flag:
break
# Limit calibration attempts
# cal_iter index is 1 based for Monte Carlo
# cal_iter is 0 for stand alone mode
elif cal_iter >= max_cal_iter:
logging.info(break_line)
logging.info(
('All {} iteration attempts were made, '
'the scene will not calibrate.').format(max_cal_iter))
if os.path.isfile(etrf_path):
os.remove(etrf_path)
return False
# break
# Adjust Kc value of calibration points (instead of moving them)
cal_dict = auto_calibration.kc_calibration_adjust(
cal_dict, etrf_sub_array)
# # Select new calibration points based on ETrF distribution
# logging.info(break_line)
# cold_xy, hot_xy = auto_calibration.build_pixel_points(
# etrf_array, etrf_geo, cal_dict,
# shapefile_flag=shapefile_flag, pixels_ws=pixels_ws)
del etrf_array, etrf_geo
# Increment calibration iteration counter
cal_iter += 1
# Only save 'final' results if the scene was calibrated
if cal_flag and save_etrf_final_plots:
# Plot final ETrF distribution
# logging.info(break_line)
auto_calibration.save_etrf_histograms(etrf_sub_array, plots_ws,
cal_dict, mc_iter, None)
# Save final calibration points to database
# logging.info(break_line)
# calibration_database.save_calibration_points(
# image_ws, cal_path, cal_dict, mc_iter, 0)
return True
def pixel_points(image_ws,
groupsize=1,
blocksize=2048,
mc_iter=None,
shapefile_flag=False,
multipoint_flag=False,
pixel_point_iters=0,
overwrite_flag=None):
"""Place hot/cold pixels within .\PIXEL_REGIONS\pixel_suggestion.shp
Parameters
----------
image_ws : str
Image folder path.
groupsize : int, optional
Script will try to place calibration point randomly into a
labeled group of clustered values with at least n pixels.
-1 = In the largest group
0 = Anywhere in the image (not currently implemented)
1 >= In any group with a pixel count greater or equal to n
blocksize : int, optional
Processing block size (the default is 2048).
shapefile_flag : bool, optional
If True, save calibration points to shapefile (the default is False).
multipoing_flag : bool, optional
If True, save calibration points to multipoint shapefile
(the default is False).
pixel_point_iters : int, optional
Number of iterations (the default is 0).
ovewrite_flag : bool, optional
If True, overwrite existing files (the default is None).
Returns
-------
tuple of cold coordinates, tuple of hot coordinates
"""
logging.info('Placing hot/cold pixels in suggested regions')
env = drigo.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore')
common_ds = drigo.raster_path_ds(image.common_area_raster, read_only=True)
env.snap_proj = drigo.raster_ds_proj(common_ds)
common_ds = None
# Open config file
# config = dripy.open_ini(ini_path)
# Get input parameters
# logging.debug(' Reading Input File')
pixels_folder = 'PIXELS'
if mc_iter:
cold_pixel_name = 'cold_{:02d}'.format(int(mc_iter))
hot_pixel_name = 'hot_{:02d}'.format(int(mc_iter))
else:
cold_pixel_name = 'cold'
hot_pixel_name = 'hot'
hot_cold_pixels_name = 'hot_cold'
r_fmt = '.img'
s_fmt = '.shp'
# json_format = '.geojson'
# pixels_folder = dripy.read_param(pixels_folder, 'PIXELS', config)
# cold_pixel_name = dripy.read_param(cold_pixel_name, 'cold', config)
# hot_pixel_name = dripy.read_param(hot_pixel_name, 'hot', config)
# Create Pixels Folder and Scratch Folder
pixels_ws = os.path.join(image_ws, pixels_folder)
region_ws = os.path.join(image_ws, 'PIXEL_REGIONS')
if not os.path.isdir(pixels_ws):
os.mkdir(pixels_ws)
# Generate pixels shapefiles if they don't exist
cold_pixel_path = os.path.join(pixels_ws, cold_pixel_name + s_fmt)
hot_pixel_path = os.path.join(pixels_ws, hot_pixel_name + s_fmt)
hot_cold_pixel_path = os.path.join(pixels_ws, hot_cold_pixels_name + s_fmt)
if shapefile_flag and overwrite_flag:
dripy.remove_file(cold_pixel_path)
dripy.remove_file(hot_pixel_path)
# dripy.remove_file(hot_cold_pixel_path)
# Place points in the suggested pixel location raster
cold_region_raster = os.path.join(region_ws,
'cold_pixel_suggestion' + r_fmt)
hot_region_raster = os.path.join(region_ws, 'hot_pixel_suggestion' + r_fmt)
if not os.path.isfile(cold_region_raster):
logging.error('\nERROR: The cold pixel suggestion raster {} does '
'not exist\n'.format(
os.path.basename(cold_region_raster)))
sys.exit()
if not os.path.isfile(hot_region_raster):
logging.error('\nERROR: The hot pixel suggestion raster {} does '
'not exist\n'.format(
os.path.basename(hot_region_raster)))
sys.exit()
# Check placement_mode value
# Make sure it is between -1 and max pixelcount?
# Pixel placement mode
logging.info('Calibration point placement method:')
if type(groupsize) is not int:
logging.error('\nGroupsize must be an integer\n')
sys.exit()
elif groupsize <= -1:
logging.info(' Randomly within largest suggested region polygon')
elif groupsize == 0:
logging.info(' Randomly anywhere in the image')
sys.exit()
elif groupsize >= 1:
logging.info(' Randomly within suggested region polygons with '
'more than {} pixels'.format(groupsize))
# Select random pixel
cold_x, cold_y = get_random_point_in_raster(cold_region_raster, groupsize,
blocksize)
hot_x, hot_y = get_random_point_in_raster(hot_region_raster, groupsize,
blocksize)
# Save pixels
if shapefile_flag and cold_x and cold_y:
drigo.save_point_to_shapefile(cold_pixel_path, cold_x, cold_y,
env.snap_proj)
if shapefile_flag and hot_x and hot_y:
drigo.save_point_to_shapefile(hot_pixel_path, hot_x, hot_y,
env.snap_proj)
if multipoint_flag and cold_x and cold_y:
drigo.multipoint_shapefile(hot_cold_pixel_path,
cold_x,
cold_y,
'COLD_{:02d}_{:02d}'.format(
int(mc_iter), int(pixel_point_iters)),
id_=mc_iter,
input_proj=env.snap_proj)
if multipoint_flag and hot_x and hot_y:
drigo.multipoint_shapefile(hot_cold_pixel_path,
hot_x,
hot_y,
'HOT_{:02d}_{:02d}'.format(
int(mc_iter), int(pixel_point_iters)),
id_=mc_iter,
input_proj=env.snap_proj)
# Eventually don't save a shapefile and just return the coordinates
return (cold_x, cold_y), (hot_x, hot_y)