def remap_mask_func(input_path, output_path, value_list):
"""Remap the input raster to all 1's and return a mask raster
Args:
input_path = path to raw cdl data
output_path = filepath for output mask
value_list = list defining which values will be masked True
Returns:
None
"""
input_ds = gdal.Open(input_path)
input_geo = gdc.raster_ds_geo(input_ds)
input_proj = gdc.raster_ds_proj(input_ds)
input_array = gdc.raster_ds_to_array(input_ds)[0]
input_mask = np.zeros(input_array.shape, dtype=np.bool)
for input_value in value_list:
input_mask[input_array == input_value] = 1
gdc.array_to_raster(input_mask, output_path, input_geo, input_proj)
return True
def pixel_rating(image_ws, ini_path, stats_flag=False, overwrite_flag=None):
"""Calculate pixel rating
Args:
image_ws (str): Image folder path
ini_path (str): Pixel regions config file path
stats_flag (bool): if True, compute raster statistics
ovewrite_flag (bool): if True, overwrite existing files
Returns:
None
"""
logging.info('Generating suggested hot/cold pixel regions')
log_fmt = ' {:<18s} {}'
env = gdc.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore')
# # Check that image_ws is valid
# image_re = re.compile(
# '^(LT04|LT05|LE07|LC08)_(\d{3})(\d{3})_(\d{4})(\d{2})(\d{2})')
# if not os.path.isdir(image_ws) or not image_re.match(scene_id):
# logging.error('\nERROR: Image folder is invalid or does not exist\n')
# return False
# Folder Paths
region_ws = os.path.join(image_ws, 'PIXEL_REGIONS')
# Open config file
config = open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
# Arrays are processed by block
bs = read_param('block_size', 1024, config)
logging.info(' {:<18s} {}'.format('Block Size:', bs))
# Raster pyramids/statistics
pyramids_flag = read_param('pyramids_flag', False, config)
if pyramids_flag:
gdal.SetConfigOption('HFA_USE_RRD', 'YES')
if stats_flag is None:
stats_flag = read_param('statistics_flag', False, config)
# Overwrite
if overwrite_flag is None:
overwrite_flag = read_param('overwrite_flag', True, config)
# Check that common_area raster exists
if not os.path.isfile(image.common_area_raster):
logging.error(
'\nERROR: A common area raster was not found.' +
'\nERROR: Please rerun prep tool to build these files.\n' +
' {}\n'.format(image.common_area_raster))
sys.exit()
# Use common_area to set mask parameters
common_ds = gdal.Open(image.common_area_raster)
# env.mask_proj = raster_ds_proj(common_ds)
env.mask_geo = gdc.raster_ds_geo(common_ds)
env.mask_rows, env.mask_cols = gdc.raster_ds_shape(common_ds)
env.mask_extent = gdc.geo_extent(env.mask_geo, env.mask_rows,
env.mask_cols)
env.mask_array = gdc.raster_ds_to_array(common_ds)[0]
env.mask_path = image.common_area_raster
env.snap_osr = gdc.raster_path_osr(image.common_area_raster)
env.snap_proj = env.snap_osr.ExportToWkt()
env.cellsize = gdc.raster_path_cellsize(image.common_area_raster)[0]
common_ds = None
logging.debug(' {:<18s} {}'.format('Mask Extent:', env.mask_extent))
# Read Pixel Regions config file
# Currently there is no code to support applying an NLCD mask
apply_nlcd_mask = False
# apply_nlcd_mask = read_param('apply_nlcd_mask', False, config)
apply_cdl_ag_mask = read_param('apply_cdl_ag_mask', False, config)
apply_field_mask = read_param('apply_field_mask', False, config)
apply_ndwi_mask = read_param('apply_ndwi_mask', True, config)
apply_ndvi_mask = read_param('apply_ndvi_mask', True, config)
# Currently the code to apply a study area mask is commented out
# apply_study_area_mask = read_param(
# 'apply_study_area_mask', False, config)
albedo_rating_flag = read_param('albedo_rating_flag', True, config)
nlcd_rating_flag = read_param('nlcd_rating_flag', True, config)
ndvi_rating_flag = read_param('ndvi_rating_flag', True, config)
ts_rating_flag = read_param('ts_rating_flag', True, config)
ke_rating_flag = read_param('ke_rating_flag', False, config)
# if apply_study_area_mask:
# study_area_path = config.get('INPUTS', 'study_area_path')
if apply_nlcd_mask or nlcd_rating_flag:
nlcd_raster = config.get('INPUTS', 'landuse_raster')
if apply_cdl_ag_mask:
cdl_ag_raster = config.get('INPUTS', 'cdl_ag_raster')
cdl_buffer_cells = read_param('cdl_buffer_cells', 0, config)
cdl_ag_eroded_name = read_param('cdl_ag_eroded_name',
'cdl_ag_eroded_{}.img', config)
if apply_field_mask:
field_raster = config.get('INPUTS', 'fields_raster')
cold_rating_pct = read_param('cold_percentile', 99, config)
hot_rating_pct = read_param('hot_percentile', 99, config)
# min_cold_rating_score = read_param('min_cold_rating_score', 0.3, config)
# min_hot_rating_score = read_param('min_hot_rating_score', 0.3, config)
ts_bin_count = int(read_param('ts_bin_count', 10, config))
if 100 % ts_bin_count != 0:
logging.warning(
'WARNING: ts_bins_count of {} is not a divisor ' +
'of 100. Using default ts_bins_count = 4'.format(ts_bin_count))
ts_bin_count = 10
bin_size = 1. / (ts_bin_count - 1)
hot_rating_values = np.arange(0., 1. + bin_size, step=bin_size)
cold_rating_values = hot_rating_values[::-1]
# Input raster paths
r_fmt = '.img'
if 'Landsat' in image.type:
albedo_raster = image.albedo_sur_raster
ndvi_raster = image.ndvi_toa_raster
ndwi_raster = image.ndwi_toa_raster
ts_raster = image.ts_raster
ke_raster = image.ke_raster
# Check config file input paths
# if apply_study_area_mask and not os.path.isfile(study_area_path):
# logging.error(
# ('\nERROR: The study area shapefile {} does ' +
# 'not exist\n').format(study_area_path))
# sys.exit()
if ((apply_nlcd_mask or nlcd_rating_flag)
and not os.path.isfile(nlcd_raster)):
logging.error(('\nERROR: The NLCD raster {} does ' +
'not exist\n').format(nlcd_raster))
sys.exit()
if apply_cdl_ag_mask and not os.path.isfile(cdl_ag_raster):
logging.error(('\nERROR: The CDL Ag raster {} does ' +
'not exist\n').format(cdl_ag_raster))
sys.exit()
if apply_field_mask and not os.path.isfile(field_raster):
logging.error(('\nERROR: The field raster {} does ' +
'not exist\n').format(field_raster))
sys.exit()
if (not (isinstance(cold_rating_pct,
(int, float)) and (0 <= cold_rating_pct <= 100))):
logging.error(
'\nERROR: cold_percentile must be a value between 0 and 100\n')
sys.exit()
if (not (isinstance(hot_rating_pct,
(int, float)) and (0 <= hot_rating_pct <= 100))):
logging.error(
'\nERROR: hot_percentile must be a value between 0 and 100\n')
sys.exit()
# Set raster names
raster_dict = dict()
# Output Rasters
raster_dict['region_mask'] = os.path.join(region_ws, 'region_mask' + r_fmt)
raster_dict['cold_rating'] = os.path.join(region_ws,
'cold_pixel_rating' + r_fmt)
raster_dict['hot_rating'] = os.path.join(region_ws,
'hot_pixel_rating' + r_fmt)
raster_dict['cold_sugg'] = os.path.join(region_ws,
'cold_pixel_suggestion' + r_fmt)
raster_dict['hot_sugg'] = os.path.join(region_ws,
'hot_pixel_suggestion' + r_fmt)
# Read pixel region raster flags
save_dict = dict()
save_dict['region_mask'] = read_param('save_region_mask_flag', False,
config)
save_dict['cold_rating'] = read_param('save_rating_rasters_flag', False,
config)
save_dict['hot_rating'] = read_param('save_rating_rasters_flag', False,
config)
save_dict['cold_sugg'] = read_param('save_suggestion_rasters_flag', True,
config)
save_dict['hot_sugg'] = read_param('save_suggestion_rasters_flag', True,
config)
# Output folder
if not os.path.isdir(region_ws):
os.mkdir(region_ws)
# Remove existing files if necessary
region_ws_file_list = [
os.path.join(region_ws, item) for item in os.listdir(region_ws)
]
if overwrite_flag and region_ws_file_list:
for raster_path in raster_dict.values():
if raster_path in region_ws_file_list:
remove_file(raster_path)
# Check scene specific input paths
if apply_ndwi_mask and not os.path.isfile(ndwi_raster):
logging.error(
'ERROR: NDWI raster does not exist\n {}'.format(ndwi_raster))
sys.exit()
elif apply_ndvi_mask and not os.path.isfile(ndvi_raster):
logging.error(
'ERROR: NDVI raster does not exist\n {}'.format(ndvi_raster))
sys.exit()
elif ke_rating_flag and not os.path.isfile(ke_raster):
logging.error(
('ERROR: The Ke raster does not exist\n {}').format(ke_raster))
sys.exit()
# Remove existing and build new empty rasters if necessary
# If processing by block, rating rasters must be built
logging.debug('\nBuilding empty rasters')
for name, save_flag in sorted(save_dict.items()):
if save_flag and 'rating' in name:
gdc.build_empty_raster(raster_dict[name], 1, np.float32)
elif save_flag:
gdc.build_empty_raster(raster_dict[name],
1,
np.uint8,
output_nodata=0)
if apply_cdl_ag_mask:
logging.info('Building CDL ag mask')
cdl_array = gdc.raster_to_array(cdl_ag_raster,
mask_extent=env.mask_extent,
return_nodata=False)
if cdl_buffer_cells > 0:
logging.info(' Eroding CDL by {} cells'.format(cdl_buffer_cells))
structure_array = np.ones((cdl_buffer_cells, cdl_buffer_cells),
dtype=np.int)
# Deadbeef - This could blow up in memory on bigger rasters
cdl_array = ndimage.binary_erosion(
cdl_array, structure_array).astype(structure_array.dtype)
cdl_ag_eroded_raster = os.path.join(
image.support_ws, cdl_ag_eroded_name.format(cdl_buffer_cells))
gdc.array_to_raster(cdl_array,
cdl_ag_eroded_raster,
output_geo=env.mask_geo,
output_proj=env.snap_proj,
mask_array=env.mask_array,
output_nodata=0,
stats_flag=False)
cdl_array = None
del cdl_array
# Build region mask
logging.debug('Building region mask')
region_mask = np.copy(env.mask_array).astype(np.bool)
if apply_field_mask:
field_mask, field_nodata = gdc.raster_to_array(
field_raster, mask_extent=env.mask_extent, return_nodata=True)
region_mask &= field_mask != field_nodata
del field_mask, field_nodata
if apply_ndwi_mask:
ndwi_array = gdc.raster_to_array(ndwi_raster,
1,
mask_extent=env.mask_extent,
return_nodata=False)
region_mask &= ndwi_array > 0.0
del ndwi_array
if apply_ndvi_mask:
ndvi_array = gdc.raster_to_array(ndvi_raster,
1,
mask_extent=env.mask_extent,
return_nodata=False)
region_mask &= ndvi_array > 0.12
del ndvi_array
if apply_cdl_ag_mask:
cdl_array, cdl_nodata = gdc.raster_to_array(
cdl_ag_eroded_raster,
mask_extent=env.mask_extent,
return_nodata=True)
region_mask &= cdl_array != cdl_nodata
del cdl_array, cdl_nodata
if save_dict['region_mask']:
gdc.array_to_raster(region_mask,
raster_dict['region_mask'],
stats_flag=False)
# Initialize rating arrays
# This needs to be done before the ts_rating if block
cold_rating_array = np.ones(env.mask_array.shape, dtype=np.float32)
hot_rating_array = np.ones(env.mask_array.shape, dtype=np.float32)
cold_rating_array[~region_mask] = np.nan
hot_rating_array[~region_mask] = np.nan
# Temperature pixel rating - grab the max and min value for the entire
# Ts image in a memory safe way by using gdal_common blocks
# The following is a percentile based approach
if ts_rating_flag:
logging.debug('Computing Ts percentile rating')
ts_array = gdc.raster_to_array(ts_raster,
mask_extent=env.mask_extent,
return_nodata=False)
ts_array[~region_mask] = np.nan
percentiles = range(0, (100 + ts_bin_count), int(100 / ts_bin_count))
ts_score_value = 1. / (ts_bin_count - 1)
hot_rating_values = np.arange(0, (1. + ts_score_value),
step=ts_score_value)[:ts_bin_count]
cold_rating_values = hot_rating_values[::-1]
ts_percentile_array = stats.scoreatpercentile(
ts_array[np.isfinite(ts_array)], percentiles)
for bins_i in range(len(ts_percentile_array))[:-1]:
bool_array = ((ts_array > ts_percentile_array[bins_i]) &
(ts_array <= ts_percentile_array[bins_i + 1]))
cold_rating_array[bool_array] = cold_rating_values[bins_i]
hot_rating_array[bool_array] = hot_rating_values[bins_i]
# gdc.array_to_raster(cold_rating_array, raster_dict['cold_rating'])
# gdc.array_to_raster(hot_rating_array, raster_dict['hot_rating'])
# Cleanup
del ts_array, ts_percentile_array
del cold_rating_values, hot_rating_values
del ts_score_value, percentiles
# Process by block
logging.info('\nProcessing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(env.mask_cols,
env.mask_rows))
for b_i, b_j in gdc.block_gen(env.mask_rows, env.mask_cols, bs):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = gdc.array_to_block(env.mask_array, b_i, b_j,
bs).astype(np.bool)
# block_nodata_mask = ~block_data_mask
block_rows, block_cols = block_data_mask.shape
block_geo = gdc.array_offset_geo(env.mask_geo, b_j, b_i)
block_extent = gdc.geo_extent(block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
# logging.debug(' Block extent: {}'.format(block_extent))
# logging.debug(' Block geo: {}'.format(block_geo))
# Don't skip empty blocks since block rating needs to be written
# back to the array at the end of the block loop
block_region_mask = gdc.array_to_block(region_mask, b_i, b_j, bs)
if not np.any(block_region_mask):
logging.debug(' Empty block')
block_empty_flag = True
else:
block_empty_flag = False
# New style continuous pixel weighting
cold_rating_block = gdc.array_to_block(cold_rating_array, b_i, b_j, bs)
hot_rating_block = gdc.array_to_block(hot_rating_array, b_i, b_j, bs)
# Rating arrays already have region_mask set
# cold_rating_block = np.ones(block_region_mask.shape, dtype=np.float32)
# hot_rating_block = np.ones(block_region_mask.shape, dtype=np.float32)
# cold_rating_block[~block_region_mask] = np.nan
# hot_rating_block[~block_region_mask] = np.nan
# del block_region_mask
if ndvi_rating_flag and not block_empty_flag:
# NDVI based rating
ndvi_array = gdc.raster_to_array(ndvi_raster,
1,
mask_extent=block_extent,
return_nodata=False)
# Don't let NDVI be negative
ndvi_array.clip(0., 0.833, out=ndvi_array)
# ndvi_array.clip(0.001, 0.833, out=ndvi_array)
cold_rating_block *= ndvi_array
cold_rating_block *= 1.20
ndvi_mask = (ndvi_array > 0)
# DEADBEEF - Can this calculation be masked to only NDVI > 0?
ndvi_mask = ndvi_array > 0
hot_rating_block[ndvi_mask] *= stats.norm.pdf(
np.log(ndvi_array[ndvi_mask]), math.log(0.15), 0.5)
hot_rating_block[ndvi_mask] *= 1.25
del ndvi_mask
# hot_rating_block *= stats.norm.pdf(
# np.log(ndvi_array), math.log(0.15), 0.5)
# hot_rating_block *= 1.25
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del ndvi_array
if albedo_rating_flag and not block_empty_flag:
# Albdo based rating
albedo_array = gdc.raster_to_array(albedo_raster,
1,
mask_extent=block_extent,
return_nodata=False)
albedo_cold_pdf = stats.norm.pdf(albedo_array, 0.21, 0.03)
albedo_hot_pdf = stats.norm.pdf(albedo_array, 0.21, 0.06)
del albedo_array
cold_rating_block *= albedo_cold_pdf
cold_rating_block *= 0.07
hot_rating_block *= albedo_hot_pdf
hot_rating_block *= 0.15
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del albedo_cold_pdf, albedo_hot_pdf
if nlcd_rating_flag and not block_empty_flag:
# NLCD based weighting, this could be CDL instead?
nlcd_array = nlcd_rating(
gdc.raster_to_array(nlcd_raster,
1,
mask_extent=block_extent,
return_nodata=False))
cold_rating_block *= nlcd_array
hot_rating_block *= nlcd_array
del nlcd_array
if ke_rating_flag and not block_empty_flag:
# SWB Ke based rating
ke_array = gdc.raster_to_array(ke_raster,
1,
mask_extent=block_extent,
return_nodata=False)
# Don't let NDVI be negative
ke_array.clip(0., 1., out=ke_array)
# Assumption, lower Ke is better for selecting the hot pixel
# As the power (2) decreases and approaches 1,
# the relationship gets more linear
# cold_rating_block *= (1 - ke_array ** 2)
# hot_rating_block *= (1 - ke_array ** 1.5)
# Linear inverse
# cold_rating_block *= (1. - ke_array)
hot_rating_block *= (1. - ke_array)
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del ke_array
# Clearness
# clearness = 1.0
# cold_rating *= clearness
# hot_rating *= clearness
# Reset nan values
# cold_rating_block[~region_mask] = np.nan
# hot_rating_block[~region_mask] = np.nan
# Save rating values
cold_rating_array = gdc.block_to_array(cold_rating_block,
cold_rating_array, b_i, b_j, bs)
hot_rating_array = gdc.block_to_array(hot_rating_block,
hot_rating_array, b_i, b_j, bs)
# Save rating rasters
if save_dict['cold_rating']:
gdc.block_to_raster(cold_rating_block, raster_dict['cold_rating'],
b_i, b_j, bs)
if save_dict['hot_rating']:
gdc.block_to_raster(hot_rating_block, raster_dict['hot_rating'],
b_i, b_j, bs)
# Save rating values
cold_rating_array = gdc.block_to_array(cold_rating_block,
cold_rating_array, b_i, b_j, bs)
hot_rating_array = gdc.block_to_array(hot_rating_block,
hot_rating_array, b_i, b_j, bs)
del cold_rating_block, hot_rating_block
# Select pixels above target percentile
# Only build suggestion arrays if saving
logging.debug('Building suggested pixel rasters')
if save_dict['cold_sugg']:
cold_rating_score = float(
stats.scoreatpercentile(
cold_rating_array[np.isfinite(cold_rating_array)],
cold_rating_pct))
# cold_rating_array, cold_rating_nodata = gdc.raster_to_array(
# raster_dict['cold_rating'], 1, mask_extent=env.mask_extent)
# if cold_rating_score < float(min_cold_rating_score):
# logging.error(('ERROR: The cold_rating_score ({}) is less ' +
# 'than the min_cold_rating_score ({})').format(
# cold_rating_score, min_cold_rating_score))
# sys.exit()
cold_sugg_mask = cold_rating_array >= cold_rating_score
gdc.array_to_raster(cold_sugg_mask,
raster_dict['cold_sugg'],
stats_flag=stats_flag)
logging.debug(' Cold Percentile: {}'.format(cold_rating_pct))
logging.debug(' Cold Score: {:.6f}'.format(cold_rating_score))
logging.debug(' Cold Pixels: {}'.format(np.sum(cold_sugg_mask)))
del cold_sugg_mask, cold_rating_array
if save_dict['hot_sugg']:
hot_rating_score = float(
stats.scoreatpercentile(
hot_rating_array[np.isfinite(hot_rating_array)],
hot_rating_pct))
# hot_rating_array, hot_rating_nodata = gdc.raster_to_array(
# raster_dict['hot_rating'], 1, mask_extent=env.mask_extent)
# if hot_rating_score < float(min_hot_rating_score):
# logging.error(('ERROR: The hot_rating_array ({}) is less ' +
# 'than the min_hot_rating_score ({})').format(
# hot_rating_array, min_hot_rating_score))
# sys.exit()
hot_sugg_mask = hot_rating_array >= hot_rating_score
gdc.array_to_raster(hot_sugg_mask,
raster_dict['hot_sugg'],
stats_flag=stats_flag)
logging.debug(' Hot Percentile: {}'.format(hot_rating_pct))
logging.debug(' Hot Score: {:.6f}'.format(hot_rating_score))
logging.debug(' Hot Pixels: {}'.format(np.sum(hot_sugg_mask)))
del hot_sugg_mask, hot_rating_array
# Raster Statistics
if stats_flag:
logging.info('Calculating Statistics')
for name, save_flag in save_dict.items():
if save_flag:
gdc.raster_statistics(raster_dict[name])
# Raster Pyramids
if pyramids_flag:
logging.info('Building Pyramids')
for name, save_flag in save_dict.items():
if save_flag:
gdc.raster_pyramids(raster_dict[name])
def main(gis_ws, cdl_year='', block_size=16384, mask_flag=False,
overwrite_flag=False, pyramids_flag=False, stats_flag=False,
agland_nodata=0, agmask_nodata=0):
"""Mask CDL values for non-agricultural pixels
Use CDL derived agmask (in CDL workspace) to define agricultural pixels
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
cdl_year (str): Comma separated list and/or range of years
block_size (int): Maximum block size to use for raster processing
mask_flag (bool): If True, mask pixels outside extent shapefile
overwrite_flag (bool): If True, overwrite output rasters
pyramids_flag (bool): If True, build pyramids/overviews
for the output rasters
stats_flag (bool): If True, compute statistics for the output rasters
agland_nodata: Integer of the nodata value in the agland raster
agmask_nodata: Integer of the nodata value in the agmask raster
Returns:
None
"""
logging.info('\nExtracting Agriculatural CDL Values')
cdl_format = '{0}_30m_cdls.img'
cdl_ws = os.path.join(gis_ws, 'cdl')
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
# Ag landuses are 1, all others in state are 0, outside state is nodata
# Crop 61 is fallow/idle and was excluded from analysis
# Crops 176 is Grassland/Pasture in the new national CDL rasters
# Crops 181 was Pasture/Hay in the old state CDL rasters
# Crops 182 was Cultivated Crop in the old state CDL rasters
agmask_remap = [
[1, 60, 1], [61, 65, 0],
# [1, 61, 1], [62, 65, 0],
[66, 80, 1], [81, 92, 0],
# [93, 180, 0], [181, 182, 1], [183, 203, 0], [204, 254, 1]]
# [93, 180, 0], [176, 1], [183, 203, 0], [204, 254, 1]]
[93, 203, 0], [204, 254, 1]]
# Check input folders
if not os.path.isdir(gis_ws):
logging.error(('\nERROR: The GIS workspace {} ' +
'does not exist\n').format(gis_ws))
sys.exit()
elif not os.path.isdir(cdl_ws):
logging.error(('\nERROR: The CDL workspace {} ' +
'does not exist\n').format(cdl_ws))
sys.exit()
elif mask_flag and not os.path.isfile(zone_raster_path):
logging.error(
('\nERROR: The zone raster {} does not exist\n' +
' Try re-running "clip_cdl_raster.py"').format(zone_raster_path))
sys.exit()
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('CDL Workspace: {}'.format(cdl_ws))
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
# Process each CDL year separately
for cdl_year in list(util.parse_int_set(cdl_year)):
logging.info('\n{0}'.format(cdl_year))
cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year))
agmask_path = os.path.join(
cdl_ws, 'agmask_{}_30m_cdls.img'.format(cdl_year))
agland_path = os.path.join(
cdl_ws, 'agland_{}_30m_cdls.img'.format(cdl_year))
if not os.path.isfile(cdl_path):
logging.error(('\nERROR: The CDL raster {} ' +
'does not exist\n').format(cdl_path))
continue
# Get color table and spatial reference from CDL raster
logging.info('Reading CDL color table')
cdl_raster_ds = gdal.Open(cdl_path, 0)
cdl_geo = gdc.raster_ds_geo(cdl_raster_ds)
cdl_rows, cdl_cols = gdc.raster_ds_shape(cdl_raster_ds)
cdl_extent = gdc.geo_extent(cdl_geo, cdl_rows, cdl_cols)
cdl_proj = gdc.raster_ds_proj(cdl_raster_ds)
# DEADBEEF - Why is this hardcoded?
cdl_cellsize = 30
# cdl_cellsize = gdc.raster_ds_cellsize(cdl_raster_ds)[0]
cdl_band = cdl_raster_ds.GetRasterBand(1)
cdl_rat = cdl_band.GetDefaultRAT()
cdl_classname_dict = dict()
for row_i in range(cdl_rat.GetRowCount()):
cdl_classname_dict[row_i] = cdl_rat.GetValueAsString(
row_i, cdl_rat.GetColOfUsage(2))
cdl_raster_ds = None
del cdl_raster_ds, cdl_band, cdl_rat
# Copy the input raster to hold the ag data
logging.debug('{}'.format(agland_path))
if os.path.isfile(agland_path) and overwrite_flag:
subprocess.call(['gdalmanage', 'delete', agland_path])
if not os.path.isfile(agland_path):
logging.info('Copying CDL raster')
logging.debug('{}'.format(cdl_path))
subprocess.call(
['gdal_translate', '-of', 'HFA', '-co', 'COMPRESSED=YES',
cdl_path, agland_path])
# '-a_nodata', agland_nodata
# Set the nodata value after copying
agland_ds = gdal.Open(agland_path, 1)
agland_band = agland_ds.GetRasterBand(1)
agland_band.SetNoDataValue(agland_nodata)
agland_ds = None
# Get the colormap from the input CDL raster
logging.debug('Re-building raster attribute tables')
agland_ds = gdal.Open(agland_path, 1)
agland_band = agland_ds.GetRasterBand(1)
agland_rat = agland_band.GetDefaultRAT()
agland_usage_list = [
agland_rat.GetUsageOfCol(col_i)
for col_i in range(agland_rat.GetColumnCount())]
# if 1 not in agland_usage_list:
# _rat.CreateColumn('Count', 1, 1)
if 2 not in agland_usage_list:
agland_rat.CreateColumn('Class_Name', 2, 2)
for row_i in range(agland_rat.GetRowCount()):
agland_rat.SetValueAsString(
row_i, agland_rat.GetColOfUsage(2),
cdl_classname_dict[row_i])
agland_band.SetDefaultRAT(agland_rat)
agland_ds = None
# Build an empty output raster to hold the ag mask
logging.info('\nBuilding empty ag mask raster')
logging.debug('{}'.format(agmask_path))
if os.path.isfile(agmask_path) and overwrite_flag:
subprocess.call(['gdalmanage', 'delete', agmask_path])
if not os.path.isfile(agmask_path):
gdc.build_empty_raster(
agmask_path, band_cnt=1, output_dtype=np.uint8,
output_nodata=agmask_nodata, output_proj=cdl_proj,
output_cs=cdl_cellsize, output_extent=cdl_extent)
# Set the nodata value after initializing
agmask_ds = gdal.Open(agmask_path, 1)
agmask_band = agmask_ds.GetRasterBand(1)
agmask_band.SetNoDataValue(agmask_nodata)
agmask_ds = None
# Set non-ag areas to nodata value
logging.info('\nProcessing by block')
logging.debug(' Input cols/rows: {0}/{1}'.format(cdl_cols, cdl_rows))
for b_i, b_j in gdc.block_gen(cdl_rows, cdl_cols, block_size):
logging.info(' Block y: {0:5d} x: {1:5d}'.format(b_i, b_j))
# Read in data for block
cdl_array = gdc.raster_to_block(
cdl_path, b_i, b_j, block_size,
fill_value=0, return_nodata=False)
cdl_mask = np.zeros(cdl_array.shape, dtype=np.bool)
remap_mask = np.zeros(cdl_array.shape, dtype=np.bool)
# Mask CDL values outside extent shapefile
if mask_flag and os.path.isfile(zone_raster_path):
zone_array = gdc.raster_to_block(
zone_raster_path, b_i, b_j, block_size)
cdl_array[zone_array == 0] = 0
# Reclassify to 1 for ag and 0 for non-ag
for [start, end, value] in agmask_remap:
if value == 0:
continue
logging.debug([start, end, value])
remap_mask |= (cdl_array >= start) & (cdl_array <= end)
cdl_mask[remap_mask] = True
del remap_mask
# Set non-ag areas in agmask to nodata
cdl_mask[~cdl_mask] = agmask_nodata
# Set non-ag areas in aglands to nodata
cdl_array[~cdl_mask] = agland_nodata
gdc.block_to_raster(cdl_array, agland_path, b_i, b_j, block_size)
gdc.block_to_raster(cdl_mask.astype(np.uint8), agmask_path,
b_i, b_j, block_size)
del cdl_array, cdl_mask
if stats_flag:
logging.info('Computing statistics')
if os.path.isfile(agland_path):
logging.debug('{}'.format(agland_path))
subprocess.call(
['gdalinfo', '-stats', '-nomd', '-noct', '-norat',
agland_path])
if os.path.isfile(agmask_path):
logging.debug('{}'.format(agmask_path))
subprocess.call(
['gdalinfo', '-stats', '-nomd', '-noct', '-norat',
agmask_path])
if pyramids_flag:
logging.info('Building pyramids')
if os.path.isfile(agland_path):
logging.debug('{}'.format(agland_path))
subprocess.call(
['gdaladdo', '-ro', agland_path] + levels.split())
if os.path.isfile(agmask_path):
logging.debug('{}'.format(agmask_path))
subprocess.call(
['gdaladdo', '-ro', agmask_path] + levels.split())
def main(image_ws, ini_path, blocksize=2048, smooth_flag=False,
stats_flag=False, overwrite_flag=False):
"""Prep a Landsat scene for METRIC
Args:
image_ws (str): the landsat scene folder that will be prepped
ini_path (str): file path of the input parameters file
blocksize (int): defines the size of blocks to process
smooth_flag (bool): If True, dilate/erode image to remove
fringe/edge pixels
stats_flag (bool): if True, compute raster statistics
overwrite_flag (bool): If True, overwrite existing files
Returns:
True is successful
"""
# Open config file
config = python_common.open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
calc_refl_toa_flag = python_common.read_param(
'calc_refl_toa_flag', True, config, 'INPUTS')
calc_refl_toa_qa_flag = python_common.read_param(
'calc_refl_toa_qa_flag', True, config, 'INPUTS')
# calc_refl_sur_ledaps_flag = python_common.read_param(
# 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS')
# calc_refl_sur_qa_flag = python_common.read_param(
# 'calc_refl_sur_qa_flag', False, config, 'INPUTS')
calc_ts_bt_flag = python_common.read_param(
'calc_ts_bt_flag', True, config, 'INPUTS')
# Use QA band to set common area
# Fmask cloud, shadow, & snow pixels will be removed from common area
calc_fmask_common_flag = python_common.read_param(
'calc_fmask_common_flag', True, config, 'INPUTS')
fmask_buffer_flag = python_common.read_param(
'fmask_buffer_flag', False, config, 'INPUTS')
fmask_erode_flag = python_common.read_param(
'fmask_erode_flag', False, config, 'INPUTS')
if fmask_erode_flag:
fmask_erode_cells = int(python_common.read_param(
'fmask_erode_cells', 10, config, 'INPUTS'))
if fmask_erode_cells == 0 and fmask_erode_flag:
fmask_erode_flag = False
# Number of cells to buffer Fmask clouds
# For now use the same buffer radius and apply to
if fmask_buffer_flag:
fmask_buffer_cells = int(python_common.read_param(
'fmask_buffer_cells', 25, config, 'INPUTS'))
if fmask_buffer_cells == 0 and fmask_buffer_flag:
fmask_buffer_flag = False
# Include hand made cloud masks
cloud_mask_flag = python_common.read_param(
'cloud_mask_flag', False, config, 'INPUTS')
cloud_mask_ws = ""
if cloud_mask_flag:
cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws')
# Extract separate Fmask rasters
calc_fmask_flag = python_common.read_param(
'calc_fmask_flag', True, config, 'INPUTS')
calc_fmask_cloud_flag = python_common.read_param(
'calc_fmask_cloud_flag', True, config, 'INPUTS')
calc_fmask_snow_flag = python_common.read_param(
'calc_fmask_snow_flag', True, config, 'INPUTS')
calc_fmask_water_flag = python_common.read_param(
'calc_fmask_water_flag', True, config, 'INPUTS')
# Keep Landsat DN, LEDAPS, and Fmask rasters
keep_dn_flag = python_common.read_param(
'keep_dn_flag', True, config, 'INPUTS')
# keep_sr_flag = python_common.read_param(
# 'keep_sr_flag', True, config, 'INPUTS')
# For this to work I would need to pass in the metric input file
# calc_elev_flag = python_common.read_param(
# 'calc_elev_flag', False, config, 'INPUTS')
# calc_landuse_flag = python_common.read_param(
# 'calc_landuse_flag', False, config, 'INPUTS')
# calc_acca_cloud_flag = python_common.read_param(
# 'calc_acca_cloud_flag', True, config, 'INPUTS')
# calc_acca_snow_flag = python_common.read_param(
# 'calc_acca_snow_flag', True, config, 'INPUTS')
# calc_ledaps_dem_land_flag = python_common.read_param(
# 'calc_ledaps_dem_land_flag', False, config, 'INPUTS')
# calc_ledaps_veg_flag = python_common.read_param(
# 'calc_ledaps_veg_flag', False, config, 'INPUTS')
# calc_ledaps_snow_flag = python_common.read_param(
# 'calc_ledaps_snow_flag', False, config, 'INPUTS')
# calc_ledaps_land_flag = python_common.read_param(
# 'calc_ledaps_land_flag', False, config, 'INPUTS')
# calc_ledaps_cloud_flag = python_common.read_param(
# 'calc_ledaps_cloud_flag', False, config, 'INPUTS')
# Interpolate/clip/project hourly rasters for each Landsat scene
# calc_metric_flag = python_common.read_param(
# 'calc_metric_flag', False, config, 'INPUTS')
calc_metric_ea_flag = python_common.read_param(
'calc_metric_ea_flag', False, config, 'INPUTS')
calc_metric_wind_flag = python_common.read_param(
'calc_metric_wind_flag', False, config, 'INPUTS')
calc_metric_etr_flag = python_common.read_param(
'calc_metric_etr_flag', False, config, 'INPUTS')
calc_metric_tair_flag = python_common.read_param(
'calc_metric_tair_flag', False, config, 'INPUTS')
# Interpolate/clip/project AWC and daily ETr/PPT rasters
# to compute SWB Ke for each Landsat scene
calc_swb_ke_flag = python_common.read_param(
'calc_swb_ke_flag', False, config, 'INPUTS')
if cloud_mask_flag:
spinup_days = python_common.read_param(
'swb_spinup_days', 30, config, 'INPUTS')
min_spinup_days = python_common.read_param(
'swb_min_spinup_days', 5, config, 'INPUTS')
# Round ea raster to N digits to save space
rounding_digits = python_common.read_param(
'rounding_digits', 3, config, 'INPUTS')
env = gdc.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore', divide='ignore')
gdal.UseExceptions()
# Input file paths
dn_image_dict = et_common.landsat_band_image_dict(
image.orig_data_ws, image.image_re)
# # Open METRIC config file
# if config_file:
# logging.info(
# log_f.format('METRIC INI File:', os.path.basename(config_file)))
# config = configparser.ConfigParser()
# try:
# config.read(config_file)
# except:
# logging.error('\nERROR: Config file could not be read, ' +
# 'is not an input file, or does not exist\n' +
# 'ERROR: config_file = {}\n').format(config_file)
# sys.exit()
# # Overwrite
# overwrite_flag = read_param('overwrite_flag', True, config)
#
# # Elevation and landuse parameters/flags from METRIC input file
# calc_elev_flag = read_param('save_dem_raster_flag', True, config)
# calc_landuse_flag = read_param(
# 'save_landuse_raster_flag', True, config)
# if calc_elev_flag:
# elev_pr_path = config.get('INPUTS','dem_raster')
# if calc_landuse_flag:
# landuse_pr_path = config.get('INPUTS', 'landuse_raster')
# else:
# overwrite_flag = False
# calc_elev_flag = False
# calc_landuse_flag = False
#
# Elev raster must exist
# if calc_elev_flag and not os.path.isfile(elev_pr_path):
# logging.error('\nERROR: Elevation raster {} does not exist\n'.format(
# elev_pr_path))
# return False
# Landuse raster must exist
# if calc_landuse_flag and not os.path.isfile(landuse_pr_path):
# logging.error('\nERROR: Landuse raster {} does not exist\n'.format(
# landuse_pr_path))
# return False
# Removing ancillary files before checking for inputs
if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')):
shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask'))
for item in os.listdir(image.orig_data_ws):
if (image.type == 'Landsat7' and
(item.endswith('_B8.TIF') or
item.endswith('_B6_VCID_2.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (image.type == 'Landsat8' and
(item.endswith('_B1.TIF') or
item.endswith('_B8.TIF') or
item.endswith('_B9.TIF') or
item.endswith('_B11.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (item.endswith('_VER.jpg') or
item.endswith('_VER.txt') or
item.endswith('_GCP.txt') or
item == 'README.GTF'):
os.remove(os.path.join(image.orig_data_ws, item))
# Check correction level (image must be L1T to process)
if image.correction != 'L1TP':
logging.debug(' Image is not L1TP corrected, skipping')
return False
# calc_fmask_common_flag = False
# calc_refl_toa_flag = False
# calc_ts_bt_flag = False
# calc_metric_ea_flag = False
# calc_metric_wind_flag = False
# calc_metric_etr_flag = False
# overwrite_flag = False
# QA band must exist
if (calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys()):
logging.warning(
('\nQA band does not exist but calc_fmask_common_flag=True' +
'\n Setting calc_fmask_common_flag=False\n {}').format(
os.path.basename(image.qa_input_raster)))
calc_fmask_common_flag = False
if cloud_mask_flag and not os.path.isdir(cloud_mask_ws):
logging.warning(
('\ncloud_mask_ws is not a directory but cloud_mask_flag=True.' +
'\n Setting cloud_mask_flag=False\n {}').format(cloud_mask_ws))
cloud_mask_flag = False
# Check for Landsat TOA images
if (calc_refl_toa_flag and
(set(list(image.band_toa_dict.keys()) + [image.thermal_band, image.qa_band]) !=
set(dn_image_dict.keys()))):
logging.warning(
'\nMissing Landsat images but calc_refl_toa_flag=True' +
'\n Setting calc_refl_toa_flag=False')
calc_refl_toa_flag = False
# Check for Landsat brightness temperature image
if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys():
logging.warning(
'\nThermal band image does not exist but calc_ts_bt_flag=True' +
'\n Setting calc_ts_bt_flag=False')
calc_ts_bt_flag = False
# DEADBEEF - Should the function return False if Ts doesn't exist?
# return False
# Check for METRIC hourly/daily input folders
if calc_metric_ea_flag:
metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder')
if not os.path.isdir(metric_ea_input_ws):
logging.warning(
('\nHourly Ea folder does not exist but calc_metric_ea_flag=True' +
'\n Setting calc_metric_ea_flag=False\n {}').format(
metric_ea_input_ws))
calc_metric_ea_flag = False
if calc_metric_wind_flag:
metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder')
if not os.path.isdir(metric_wind_input_ws):
logging.warning(
('\nHourly wind folder does not exist but calc_metric_wind_flag=True' +
'\n Setting calc_metric_wind_flag=False\n {}').format(
metric_wind_input_ws))
calc_metric_wind_flag = False
if calc_metric_etr_flag:
metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder')
if not os.path.isdir(metric_etr_input_ws):
logging.warning(
('\nHourly ETr folder does not exist but calc_metric_etr_flag=True' +
'\n Setting calc_metric_etr_flag=False\n {}').format(
metric_etr_input_ws))
calc_metric_etr_flag = False
if calc_metric_tair_flag:
metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder')
if not os.path.isdir(metric_tair_input_ws):
logging.warning(
('\nHourly Tair folder does not exist but calc_metric_tair_flag=True' +
'\n Setting calc_metric_tair_flag=False\n {}').format(
metric_tair_input_ws))
calc_metric_tair_flag = False
if (calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag):
metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re'))
metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re'))
if calc_swb_ke_flag:
awc_input_path = config.get('INPUTS', 'awc_input_path')
etr_input_ws = config.get('INPUTS', 'etr_input_folder')
ppt_input_ws = config.get('INPUTS', 'ppt_input_folder')
etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re'))
ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re'))
if not os.path.isfile(awc_input_path):
logging.warning(
('\nAWC raster does not exist but calc_swb_ke_flag=True' +
'\n Setting calc_swb_ke_flag=False\n {}').format(
awc_input_path))
calc_swb_ke_flag = False
if not os.path.isdir(etr_input_ws):
logging.warning(
('\nDaily ETr folder does not exist but calc_swb_ke_flag=True' +
'\n Setting calc_swb_ke_flag=False\n {}').format(
etr_input_ws))
calc_swb_ke_flag = False
if not os.path.isdir(ppt_input_ws):
logging.warning(
('\nDaily PPT folder does not exist but calc_swb_ke_flag=True' +
'\n Setting calc_swb_ke_flag=False\n {}').format(
ppt_input_ws))
calc_swb_ke_flag = False
# Build folders for support rasters
if ((calc_fmask_common_flag or calc_refl_toa_flag or
# calc_refl_sur_ledaps_flag or
calc_ts_bt_flag or
calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag or
calc_swb_ke_flag) and
not os.path.isdir(image.support_ws)):
os.makedirs(image.support_ws)
if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws):
os.makedirs(image.refl_toa_ws)
# if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws):
# os.makedirs(image.refl_sur_ws)
# Apply overwrite flag
if overwrite_flag:
overwrite_list = [
image.fmask_cloud_raster, image.fmask_snow_raster,
image.fmask_water_raster
# image.elev_raster, image.landuse_raster
# image.common_area_raster
]
for overwrite_path in overwrite_list:
try:
python_common.remove_file(image.fmask_cloud_raster)
except:
pass
# Use QA band to build common area rasters
logging.info('\nCommon Area Raster')
qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0)
common_geo = gdc.raster_ds_geo(qa_ds)
common_extent = gdc.raster_ds_extent(qa_ds)
common_proj = gdc.raster_ds_proj(qa_ds)
common_osr = gdc.raster_ds_osr(qa_ds)
# Initialize common_area as all non-fill QA values
qa_array = gdc.raster_ds_to_array(qa_ds, return_nodata=False)
common_array = qa_array != 1
common_rows, common_cols = common_array.shape
del qa_ds
# First try applying user defined cloud masks
cloud_mask_path = os.path.join(
cloud_mask_ws, image.folder_id + '_mask.shp')
if cloud_mask_flag and os.path.isfile(cloud_mask_path):
logging.info(' Applying cloud mask shapefile')
feature_path = os.path.join(
cloud_mask_ws, (image.folder_id + '_mask.shp'))
logging.info(' {}'.format(feature_path))
cloud_mask_memory_ds = gdc.polygon_to_raster_ds(
feature_path, nodata_value=0, burn_value=1,
output_osr=common_osr, output_cs=30,
output_extent=common_extent)
cloud_array = gdc.raster_ds_to_array(
cloud_mask_memory_ds, return_nodata=False)
# DEADBEEF - If user sets a cloud mask,
# it is probably better than Fmask
# Eventually change "if" calc_fmask_common_flag: to "elif"
common_array[cloud_array == 1] = 0
del cloud_mask_memory_ds, cloud_array
if calc_fmask_common_flag:
fmask_array = et_numpy.bqa_fmask_func(qa_array)
fmask_mask = (fmask_array >= 2) & (fmask_array <= 4)
if fmask_erode_flag:
logging.info(
(' Eroding and dilating Fmask clouds, shadows, and snow ' +
'{} cells\n to remove errantly masked pixels.').format(
fmask_erode_cells))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_erode_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_erode_cells,
structure=ndimage.generate_binary_structure(2, 2))
if fmask_buffer_flag:
logging.info(
(' Buffering Fmask clouds, shadows, and snow ' +
'{} cells').format(fmask_buffer_cells))
# Only buffer clouds, shadow, and snow (not water or nodata)
if fmask_mask is None:
fmask_mask = (fmask_array >= 2) & (fmask_array <= 4)
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_buffer_cells,
structure=ndimage.generate_binary_structure(2, 2))
# Reset common_array for buffered cells
common_array[fmask_mask] = 0
del fmask_array, fmask_mask
if common_array is not None:
# Erode and dilate to remove fringe on edge
# Default is to not smooth, but user can force smoothing
if smooth_flag:
common_array = smooth_func(common_array)
# Check that there are some cloud free pixels
if not np.any(common_array):
logging.error(' ERROR: There are no cloud/snow free pixels')
return False
# Always overwrite common area raster
# if not os.path.isfile(image.common_area_raster):
gdc.array_to_raster(
common_array, image.common_area_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Print common geo/extent
logging.debug(' Common geo: {}'.format(common_geo))
logging.debug(' Common extent: {}'.format(common_extent))
# Extract Fmask components as separate rasters
if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or
calc_fmask_water_flag):
logging.info('\nFmask')
fmask_array = et_numpy.bqa_fmask_func(qa_array)
# Save Fmask data as separate rasters
if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)):
gdc.array_to_raster(
fmask_array.astype(np.uint8), image.fmask_output_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
if (calc_fmask_cloud_flag and
not os.path.isfile(image.fmask_cloud_raster)):
fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4)
gdc.array_to_raster(
fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_cloud_array
if (calc_fmask_snow_flag and
not os.path.isfile(image.fmask_snow_raster)):
fmask_snow_array = (fmask_array == 3)
gdc.array_to_raster(
fmask_snow_array.astype(np.uint8), image.fmask_snow_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_snow_array
if (calc_fmask_water_flag and
not os.path.isfile(image.fmask_water_raster)):
fmask_water_array = (fmask_array == 1)
gdc.array_to_raster(
fmask_water_array.astype(np.uint8), image.fmask_water_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_water_array
del fmask_array
# # Calculate elevation
# if calc_elev_flag and not os.path.isfile(elev_path):
# logging.info('Elevation')
# elev_array, elev_nodata = gdc.raster_to_array(
# elev_pr_path, 1, common_extent)
# gdc.array_to_raster(
# elev_array, elev_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del elev_array, elev_nodata, elev_path
#
# # Calculate landuse
# if calc_landuse_flag and not os.path.isfile(landuse_raster):
# logging.info('Landuse')
# landuse_array, landuse_nodata = gdc.raster_to_array(
# landuse_pr_path, 1, common_extent)
# gdc.array_to_raster(
# landuse_array, landuse_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del landuse_array, landuse_nodata, landuse_raster
# Calculate toa reflectance
# f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32)
if calc_refl_toa_flag:
logging.info('Top-of-Atmosphere Reflectance')
if os.path.isfile(image.refl_toa_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(
image.refl_toa_raster))
python_common.remove_file(image.refl_toa_raster)
if not os.path.isfile(image.refl_toa_raster):
# First build empty composite raster
gdc.build_empty_raster(
image.refl_toa_raster, image.band_toa_cnt, np.float32, None,
env.snap_proj, env.cellsize, common_extent)
# cos_theta_solar_flt = et_common.cos_theta_solar_func(
# image.sun_elevation)
# Process by block
logging.info('Processing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(
common_cols, common_rows))
for b_i, b_j in gdc.block_gen(common_rows, common_cols, blocksize):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = gdc.array_to_block(
common_array, b_i, b_j, blocksize).astype(np.bool)
block_rows, block_cols = block_data_mask.shape
block_geo = gdc.array_offset_geo(common_geo, b_j, b_i)
block_extent = gdc.geo_extent(
block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
logging.debug(' Block extent: {}'.format(block_extent))
logging.debug(' Block geo: {}'.format(block_geo))
# Process each TOA band
# for band, band_i in sorted(image.band_toa_dict.items()):
for band, dn_image in sorted(dn_image_dict.items()):
if band not in image.band_toa_dict.keys():
continue
# thermal_band_flag = (band == image.thermal_band)
# Set 0 as nodata value
gdc.raster_path_set_nodata(dn_image, 0)
# Calculate TOA reflectance
dn_array, dn_nodata = gdc.raster_to_array(
dn_image, 1, block_extent)
dn_array = dn_array.astype(np.float64)
# dn_array = dn_array.astype(np.float32)
dn_array[dn_array == 0] = np.nan
#
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
refl_toa_array = et_numpy.l457_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.dr, image.esun_dict[band],
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band])
elif image.type in ['Landsat8']:
refl_toa_array = et_numpy.l8_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.refl_mult_dict[band],
image.refl_add_dict[band])
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band])
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band],
# image.refl_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array,
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band],
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band],
# image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# refl_toa_array = et_numpy.refl_toa_band_func(
# dn_array, cos_theta_solar_flt,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag)
gdc.block_to_raster(
refl_toa_array.astype(np.float32),
image.refl_toa_raster,
b_i, b_j, band=image.band_toa_dict[band])
# gdc.array_to_comp_raster(
# refl_toa_array.astype(np.float32),
# image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
del refl_toa_array, dn_array
if stats_flag:
gdc.raster_statistics(image.refl_toa_raster)
# # Process each TOA band
# # for band, band_i in sorted(image.band_toa_dict.items()):
# for band, dn_image in sorted(dn_image_dict.items()):
# thermal_band_flag = (band == image.thermal_band)
# # Set 0 as nodata value
# gdc.raster_path_set_nodata(dn_image, 0)
# # Calculate TOA reflectance
# dn_array, dn_nodata = gdc.raster_to_array(
# dn_image, 1, common_extent)
# dn_array = dn_array.astype(np.float64)
# # dn_array = dn_array.astype(np.float32)
# dn_array[dn_array == 0] = np.nan
# #
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band])
# # image.rad_mult_dict[band], image.rad_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array, image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# # image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band], image.refl_add_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# # refl_toa_array = et_numpy.refl_toa_band_func(
# # dn_array, cos_theta_solar_flt,
# # image.dr, image.esun_dict[band],
# # image.lmin_dict[band], image.lmax_dict[band],
# # image.qcalmin_dict[band], image.qcalmax_dict[band],
# # thermal_band_flag)
# gdc.array_to_comp_raster(
# refl_toa_array.astype(np.float32), image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
# del refl_toa_array, dn_array
# Calculate brightness temperature
if calc_ts_bt_flag:
logging.info('Brightness Temperature')
if os.path.isfile(image.ts_bt_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(image.ts_bt_raster))
python_common.remove_file(image.ts_bt_raster)
if not os.path.isfile(image.ts_bt_raster):
band = image.thermal_band
thermal_dn_path = dn_image_dict[band]
gdc.raster_path_set_nodata(thermal_dn_path, 0)
thermal_dn_array, thermal_dn_nodata = gdc.raster_to_array(
thermal_dn_path, 1, common_extent, return_nodata=True)
thermal_dn_mask = thermal_dn_array != thermal_dn_nodata
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
ts_bt_array = et_numpy.l457_ts_bt_band_func(
thermal_dn_array,
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band],
# image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
elif image.type in ['Landsat8']:
ts_bt_array = et_numpy.l8_ts_bt_band_func(
thermal_dn_array,
image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
# thermal_rad_array = et_numpy.refl_toa_band_func(
# thermal_dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag=True)
# ts_bt_array = et_numpy.ts_bt_func(
# thermal_rad_array, image.k1_dict[image.thermal_band],
# image.k2_dict[image.thermal_band])
ts_bt_array[~thermal_dn_mask] = np.nan
gdc.array_to_raster(
ts_bt_array, image.ts_bt_raster,
output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array,
stats_flag=stats_flag)
# del thermal_dn_array, thermal_rad_array
del thermal_dn_path, thermal_dn_array, ts_bt_array
# Interpolate/project/clip METRIC hourly/daily rasters
if (calc_metric_ea_flag or
calc_metric_wind_flag or
calc_metric_etr_flag):
logging.info('METRIC hourly/daily rasters')
# Get bracketing hours from image acquisition time
image_prev_dt = image.acq_datetime.replace(
minute=0, second=0, microsecond=0)
image_next_dt = image_prev_dt + timedelta(seconds=3600)
# Get NLDAS properties from one of the images
input_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(input_ws, file_name)
for file_name in os.listdir(input_ws)
for match in [metric_hourly_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
except IndexError:
logging.error(' No hourly file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d %H00')))
return False
try:
input_ds = gdal.Open(input_path)
input_osr = gdc.raster_ds_osr(input_ds)
# input_proj = gdc.osr_proj(input_osr)
input_extent = gdc.raster_ds_extent(input_ds)
input_cs = gdc.raster_ds_cellsize(input_ds, x_only=True)
# input_geo = input_extent.geo(input_cs)
input_x, input_y = input_extent.origin()
input_ds = None
except:
logging.error(' Could not get default input image properties')
logging.error(' {}'.format(input_path))
return False
# Project Landsat scene extent to NLDAS GCS
common_gcs_osr = common_osr.CloneGeogCS()
common_gcs_extent = gdc.project_extent(
common_extent, common_osr, common_gcs_osr,
cellsize=env.cellsize)
common_gcs_extent.buffer_extent(0.1)
common_gcs_extent.adjust_to_snap(
'EXPAND', input_x, input_y, input_cs)
# common_gcs_geo = common_gcs_extent.geo(input_cs)
def metric_weather_func(output_raster, input_ws, input_re,
prev_dt, next_dt,
resample_method=gdal.GRA_NearestNeighbour,
rounding_flag=False):
"""Interpolate/project/clip METRIC hourly rasters"""
logging.debug(' Output: {}'.format(output_raster))
if os.path.isfile(output_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
python_common.remove_file(output_raster)
else:
logging.debug(' Skipping, file already exists ' +
'and overwrite is False')
return False
prev_ws = os.path.join(input_ws, str(prev_dt.year))
next_ws = os.path.join(input_ws, str(next_dt.year))
# Technically previous and next could come from different days
# or even years, although this won't happen in the U.S.
try:
prev_path = [
os.path.join(prev_ws, input_name)
for input_name in os.listdir(prev_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(prev_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input prev: {}'.format(prev_path))
except IndexError:
logging.error(' No previous hourly file')
logging.error(' {}'.format(prev_dt))
return False
try:
next_path = [
os.path.join(next_ws, input_name)
for input_name in os.listdir(next_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(next_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input next: {}'.format(next_path))
except IndexError:
logging.error(' No next hourly file')
logging.error(' {}'.format(next_dt))
return False
# Band numbers are 1's based
prev_band = int(prev_dt.strftime('%H')) + 1
next_band = int(next_dt.strftime('%H')) + 1
logging.debug(' Input prev band: {}'.format(prev_band))
logging.debug(' Input next band: {}'.format(next_band))
# Read arrays
prev_array = gdc.raster_to_array(
prev_path, band=prev_band, mask_extent=common_gcs_extent,
return_nodata=False)
next_array = gdc.raster_to_array(
next_path, band=next_band, mask_extent=common_gcs_extent,
return_nodata=False)
if not np.any(prev_array) or not np.any(next_array):
logging.warning('\nWARNING: Input NLDAS array is all nodata\n')
return None
output_array = hourly_interpolate_func(
prev_array, next_array,
prev_dt, next_dt, image.acq_datetime)
output_array = gdc.project_array(
output_array, resample_method,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent, output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
if rounding_flag:
output_array = np.around(output_array, rounding_digits)
# Force output to 32-bit float
gdc.array_to_raster(
output_array.astype(np.float32), output_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
return True
# Ea - Project to Landsat scene after clipping
if calc_metric_ea_flag:
logging.info(' Hourly vapor pressure (Ea)')
metric_weather_func(
image.metric_ea_raster, metric_ea_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_Bilinear, rounding_flag=True)
# Wind - Project to Landsat scene after clipping
if calc_metric_wind_flag:
logging.info(' Hourly windspeed')
metric_weather_func(
image.metric_wind_raster, metric_wind_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Hourly reference ET (ETr)')
metric_weather_func(
image.metric_etr_raster, metric_etr_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr 24hr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Daily reference ET (ETr)')
logging.debug(' Output: {}'.format(
image.metric_etr_24hr_raster))
if (os.path.isfile(image.metric_etr_24hr_raster) and
overwrite_flag):
logging.debug(' Overwriting output')
os.remove(image.metric_etr_24hr_raster)
if not os.path.isfile(image.metric_etr_24hr_raster):
etr_prev_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(etr_prev_ws, file_name)
for file_name in os.listdir(etr_prev_ws)
for match in [metric_daily_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
logging.debug(' Input: {}'.format(input_path))
except IndexError:
logging.error(' No daily file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d')))
return False
output_array = gdc.raster_to_array(
input_path, mask_extent=common_gcs_extent,
return_nodata=False)
output_array = gdc.project_array(
output_array, gdal.GRA_NearestNeighbour,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent,
output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
# output_array = np.around(output_array, rounding_digits)
gdc.array_to_raster(
output_array, image.metric_etr_24hr_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
del input_path
# Tair - Project to Landsat scene after clipping
if calc_metric_tair_flag:
logging.info(' Hourly air temperature (Tair)')
metric_weather_func(
image.metric_tair_raster, metric_tair_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# Cleanup
del image_prev_dt, image_next_dt
# Soil Water Balance
if calc_swb_ke_flag:
logging.info('Daily soil water balance')
# Check if output file already exists
logging.debug(' Ke: {}'.format(image.ke_raster))
if os.path.isfile(image.ke_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
python_common.remove_file(image.ke_raster)
else:
logging.debug(' Skipping, file already ' +
'exists and overwrite is False')
return False
ke_array = et_common.raster_swb_func(
image.acq_datetime, common_osr, env.cellsize, common_extent,
awc_input_path, etr_input_ws, etr_input_re,
ppt_input_ws, ppt_input_re,
spinup_days=spinup_days, min_spinup_days=min_spinup_days)
# Apply common area mask
ke_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest 2 digits
np.around(ke_array, 2, out=ke_array)
# Force output to 32-bit float
gdc.array_to_raster(
ke_array.astype(np.float32), image.ke_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Remove Landsat TOA rasters
if not keep_dn_flag:
for landsat_item in python_common.build_file_list(
image.orig_data_ws, image.image_re):
os.remove(os.path.join(image.orig_data_ws, landsat_item))
return True
def get_random_point_in_raster(raster_path, groupsize, blocksize):
""""""
# Check that raster exists
if not os.path.isfile(raster_path):
logging.error(
'\nERROR: Pixel region ({}) does not exist\n'.format(raster_path))
return None
# Get full geo transform and shape from input raster
raster_ds = gdal.Open(raster_path)
raster_geo = gdc.raster_ds_geo(raster_ds)
raster_rows, raster_cols = gdc.raster_ds_shape(raster_ds)
raster_ds = None
xy_list = []
# Process blocks randomly
logging.info('\nProcessing by block')
logging.debug(' Raster cols/rows: {}/{}'.format(raster_cols, raster_rows))
for b_i, b_j in gdc.block_gen_random(raster_rows, raster_cols, blocksize):
logging.info(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_array = gdc.raster_to_block(raster_path,
b_i,
b_j,
blocksize,
return_nodata=False)
block_rows, block_cols = block_array.shape
block_geo = gdc.array_offset_geo(raster_geo, b_j, b_i)
block_extent = gdc.geo_extent(block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
logging.debug(' Block extent: {}'.format(block_extent))
logging.debug(' Block geo: {}'.format(block_geo))
# Check that region mask is not all nodata / 0
if not np.any(block_array):
logging.debug(' Empty block')
continue
# logging.warning(' Empty region mask, automated calibration ' +
# 'will not be able to calculate ETrF')
# return None, None
# Group cells if touching
label_array, label_cnt = ndimage.label(
block_array, structure=ndimage.generate_binary_structure(2, 2))
# block_array, structure=ndimage.generate_binary_structure(2, 1))
# label_mask = label_array > 0
del block_array
# For each group, calculate number of pixels and replace label value
# count_array = np.copy(label_array)
# blobs = ndimage.find_objects(label_array)
# for i, blob_slice in enumerate(blobs):
for i, blob_slice in enumerate(ndimage.find_objects(label_array)):
label_array[blob_slice] = np.where(
label_array[blob_slice] == (i + 1),
np.sum(label_array[blob_slice] == (i + 1)),
label_array[blob_slice])
# count_array[blob_slice] = np.sum(label_array[blob_slice]==(i+1))
groupsize_max = np.max(label_array)
logging.debug(' Max block groupsize: {}'.format(groupsize_max))
# Blocks should be p randomly
# If a target group is not found or looking for the largest
# read all blocks and track largest found group
# If a target group is found, don't process other blocks
if groupsize == -1 or groupsize_max < groupsize:
yi, xi = np.where(label_array >= groupsize_max)
x, y = gdc.array_offsets_xy(block_geo,
random.choice(list(zip(xi, yi))))
xy_list.append([groupsize_max, x, y])
elif groupsize_max >= groupsize:
yi, xi = np.where(label_array >= groupsize)
x, y = gdc.array_offsets_xy(block_geo,
random.choice(list(zip(xi, yi))))
xy_list = [[groupsize_max, x, y]]
break
# If there are multiple points, return the point associated with the
# largest group
if xy_list:
groupsize_max, x, y = sorted(xy_list, reverse=True)[0]
logging.debug(' Max groupsize: {}'.format(groupsize_max))
return x, y
else:
return None, None