def common_area_func(image_list):
""""""
common_nodata = 0
# Assume all Landsat images have same extent
common_shape = drigo.raster_path_shape(image_list[0])
common_array = np.ones(common_shape, dtype=np.bool)
for i, image in enumerate(image_list):
image_array, image_nodata = drigo.raster_to_array(image)
common_array &= (image_array != common_nodata)
del image_array, image_nodata
return common_array
def clip_project_raster_func(input_raster, resampling_type, input_osr,
input_cs, input_extent, ouput_osr, output_cs,
output_extent):
"""Clip and then project an input raster"""
# Read array from input raster using input extent
input_array = drigo.raster_to_array(input_raster,
1,
input_extent,
return_nodata=False)
# Project and clip array to block
output_array = drigo.project_array(input_array, resampling_type, input_osr,
input_cs, input_extent, ouput_osr,
output_cs, output_extent)
return output_array
def main(img_ws=os.getcwd(),
ancillary_ws=os.getcwd(),
output_ws=os.getcwd(),
etr_flag=False,
eto_flag=False,
start_date=None,
end_date=None,
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False,
use_cimis_eto_flag=False):
"""Compute daily ETr/ETo from CIMIS data
Parameters
----------
img_ws : str
Root folder of GRIDMET data.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
etr_flag : bool, optional
If True, compute alfalfa reference ET (ETr).
eto_flag : bool, optional
If True, compute grass reference ET (ETo).
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
use_cimis_eto_flag : bool, optional
If True, use CIMIS ETo raster if one of the component rasters is
missing and ETo/ETr cannot be computed (te default is False).
Returns
-------
None
"""
logging.info('\nComputing CIMIS ETo/ETr')
np.seterr(invalid='ignore')
# Use CIMIS ETo raster directly instead of computing from components
# Currently this will only be applied if one of the inputs is missing
use_cimis_eto_flag = True
# Compute ETr and/or ETo
if not etr_flag and not eto_flag:
logging.info(' ETo/ETr flag(s) not set, defaulting to ETr')
etr_flag = True
# If a date is not set, process 2017
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2017, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2017, 12, 31)
logging.info(' End date: {}'.format(end_dt))
etr_folder = 'etr'
eto_folder = 'eto'
etr_fmt = 'etr_{}_daily_cimis.img'
eto_fmt = 'eto_{}_daily_cimis.img'
# DEM for air pressure calculation
mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img')
dem_raster = os.path.join(ancillary_ws, 'cimis_elev.img')
lat_raster = os.path.join(ancillary_ws, 'cimis_lat.img')
# lon_raster = os.path.join(ancillary_ws, 'cimis_lon.img')
# Interpolate zero windspeed pixels
# interpolate_zero_u2_flag = False
# Interpolate edge and coastal cells
# interpolate_edge_flag = False
# Resample type
# 0 = GRA_NearestNeighbour, Nearest neighbour (select on one input pixel)
# 1 = GRA_Bilinear,Bilinear (2x2 kernel)
# 2 = GRA_Cubic, Cubic Convolution Approximation (4x4 kernel)
# 3 = GRA_CubicSpline, Cubic B-Spline Approximation (4x4 kernel)
# 4 = GRA_Lanczos, Lanczos windowed sinc interpolation (6x6 kernel)
# 5 = GRA_Average, Average (computes the average of all non-NODATA contributing pixels)
# 6 = GRA_Mode, Mode (selects the value which appears most often of all the sampled points)
resample_type = gdal.GRA_CubicSpline
# Wind speed is measured at 2m
zw = 2
# Output workspaces
etr_ws = os.path.join(output_ws, etr_folder)
eto_ws = os.path.join(output_ws, eto_folder)
if etr_flag and not os.path.isdir(etr_ws):
os.makedirs(etr_ws)
if eto_flag and not os.path.isdir(eto_ws):
os.makedirs(eto_ws)
# Check ETr/ETo functions
test_flag = False
# Check that the daily_refet_func produces the correct values
if test_flag:
doy_test = 245
elev_test = 1050.0
lat_test = 39.9396 * math.pi / 180
tmin_test = 11.07
tmax_test = 34.69
rs_test = 22.38
u2_test = 1.94
zw_test = 2.5
tdew_test = -3.22
ea_test = refet.calcs.saturation_vapor_pressure_func(tdew_test)
pair_test = 101.3 * np.power((285 - 0.0065 * elev_test) / 285, 5.26)
q_test = 0.622 * ea_test / (pair_test - (0.378 * ea_test))
refet_obj = refet.Daily(tmin=tmin_test,
tmax=tmax_test,
q=q_test,
rs=rs_test,
u2=u2_test,
zw=zw_test,
elev=elev_test,
doy=doy_test,
lat=lat_test)
etr = float(refet_obj.etr())
eto = float(refet_obj.eto())
print('ETr: 8.89', etr)
print('ETo: 6.16', eto)
sys.exit()
# Get CIMIS grid properties from mask
cimis_mask_ds = gdal.Open(mask_raster)
cimis_osr = drigo.raster_ds_osr(cimis_mask_ds)
cimis_proj = drigo.osr_proj(cimis_osr)
cimis_cs = drigo.raster_ds_cellsize(cimis_mask_ds, x_only=True)
cimis_extent = drigo.raster_ds_extent(cimis_mask_ds)
cimis_full_geo = cimis_extent.geo(cimis_cs)
cimis_x, cimis_y = cimis_extent.origin()
cimis_mask_ds = None
logging.debug(' Projection: {}'.format(cimis_proj))
logging.debug(' Cellsize: {}'.format(cimis_cs))
logging.debug(' Geo: {}'.format(cimis_full_geo))
logging.debug(' Extent: {}'.format(cimis_extent))
# Manually set CIMIS grid properties
# cimis_extent = drigo.Extent((-400000, -650000, 600000, 454000))
# cimis_cs = 2000
# cimis_geo = drigo.extent_geo(cimis_extent, cellsize)
# cimis_epsg = 3310 # NAD_1983_California_Teale_Albers
# cimis_x, cimis_y = (0,0)
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
cimis_extent = drigo.Extent(output_extent)
cimis_extent.adjust_to_snap('EXPAND', cimis_x, cimis_y, cimis_cs)
cimis_geo = cimis_extent.geo(cimis_cs)
logging.debug(' Geo: {}'.format(cimis_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if extent_path.lower().endswith('.shp'):
cimis_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
cimis_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
cimis_extent = drigo.project_extent(cimis_extent, extent_osr,
cimis_osr, extent_cs)
cimis_extent.adjust_to_snap('EXPAND', cimis_x, cimis_y, cimis_cs)
cimis_geo = cimis_extent.geo(cimis_cs)
logging.debug(' Geo: {}'.format(cimis_geo))
logging.debug(' Extent: {}'.format(cimis_extent))
else:
cimis_geo = cimis_full_geo
# Latitude
lat_array = drigo.raster_to_array(lat_raster,
mask_extent=cimis_extent,
return_nodata=False)
lat_array = lat_array.astype(np.float32)
lat_array *= math.pi / 180
# Elevation data
elev_array = drigo.raster_to_array(dem_raster,
mask_extent=cimis_extent,
return_nodata=False)
elev_array = elev_array.astype(np.float32)
# Process each year in the input workspace
logging.info("")
for year_str in sorted(os.listdir(img_ws)):
logging.debug('{}'.format(year_str))
if not re.match('^\d{4}$', year_str):
logging.debug(' Not a 4 digit year folder, skipping')
continue
year_ws = os.path.join(img_ws, year_str)
year_int = int(year_str)
# year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
logging.info('{}'.format(year_str))
# Output paths
etr_raster = os.path.join(etr_ws, etr_fmt.format(year_str))
eto_raster = os.path.join(eto_ws, eto_fmt.format(year_str))
if etr_flag and (overwrite_flag or not os.path.isfile(etr_raster)):
logging.debug(' {}'.format(etr_raster))
drigo.build_empty_raster(etr_raster,
band_cnt=366,
output_dtype=np.float32,
output_proj=cimis_proj,
output_cs=cimis_cs,
output_extent=cimis_extent,
output_fill_flag=True)
if eto_flag and (overwrite_flag or not os.path.isfile(eto_raster)):
logging.debug(' {}'.format(eto_raster))
drigo.build_empty_raster(eto_raster,
band_cnt=366,
output_dtype=np.float32,
output_proj=cimis_proj,
output_cs=cimis_cs,
output_extent=cimis_extent,
output_fill_flag=True)
# Process each date in the year
for date_str in sorted(os.listdir(year_ws)):
logging.debug('{}'.format(date_str))
try:
date_dt = dt.datetime.strptime(date_str, '%Y_%m_%d')
except ValueError:
logging.debug(
' Invalid folder date format (YYYY_MM_DD), skipping')
continue
if start_dt is not None and date_dt < start_dt:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' After end date, skipping')
continue
logging.info(date_str)
date_ws = os.path.join(year_ws, date_str)
doy = int(date_dt.strftime('%j'))
# Set file paths
tmax_path = os.path.join(date_ws, 'Tx.img')
tmin_path = os.path.join(date_ws, 'Tn.img')
tdew_path = os.path.join(date_ws, 'Tdew.img')
rso_path = os.path.join(date_ws, 'Rso.img')
rs_path = os.path.join(date_ws, 'Rs.img')
u2_path = os.path.join(date_ws, 'U2.img')
eto_path = os.path.join(date_ws, 'ETo.img')
# k_path = os.path.join(date_ws, 'K.img')
# rnl_path = os.path.join(date_ws, 'Rnl.img')
input_list = [
tmin_path, tmax_path, tdew_path, u2_path, rs_path, rso_path
]
# If any input raster is missing, skip the day
# Unless ETo is present (and use_cimis_eto_flag is True)
day_skip_flag = False
for t_path in input_list:
if not os.path.isfile(t_path):
logging.info(' {} is missing'.format(t_path))
day_skip_flag = True
if (day_skip_flag and use_cimis_eto_flag
and os.path.isfile(eto_path)):
logging.info(' Using CIMIS ETo directly')
eto_array = drigo.raster_to_array(eto_path,
1,
cimis_extent,
return_nodata=False)
eto_array = eto_array.astype(np.float32)
if not np.any(eto_array):
logging.info(' {} is empty or missing'.format(eto_path))
logging.info(' Skipping date')
continue
# ETr
if etr_flag:
drigo.array_to_comp_raster(1.2 * eto_array,
etr_raster,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# 1.2 * eto_array, etr_raster,
# output_geo=cimis_geo, output_proj=cimis_proj,
# stats_flag=stats_flag)
# ETo
if eto_flag:
drigo.array_to_comp_raster(eto_array,
eto_raster,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# eto_array, eto_raster,
# output_geo=cimis_geo, output_proj=cimis_proj,
# stats_flag=stats_flag)
del eto_array
continue
elif not day_skip_flag:
# Read in rasters
tmin_array = drigo.raster_to_array(tmin_path,
1,
cimis_extent,
return_nodata=False)
tmax_array = drigo.raster_to_array(tmax_path,
1,
cimis_extent,
return_nodata=False)
tdew_array = drigo.raster_to_array(tdew_path,
1,
cimis_extent,
return_nodata=False)
# rso_array = drigo.raster_to_array(
# rso_path, 1, cimis_extent, return_nodata=False)
rs_array = drigo.raster_to_array(rs_path,
1,
cimis_extent,
return_nodata=False)
u2_array = drigo.raster_to_array(u2_path,
1,
cimis_extent,
return_nodata=False)
# k_array = drigo.raster_to_array(
# k_path, 1, cimis_extent, return_nodata=False)
# rnl_array = drigo.raster_to_array(
# rnl_path, 1, cimis_extent, return_nodata=False)
# Check that all input arrays have data
for t_name, t_array in [[tmin_path, tmin_array],
[tmax_path, tmax_array],
[tdew_path, tdew_array],
[u2_path, u2_array],
[rs_path, rs_array]]:
if not np.any(t_array):
logging.warning(
' {} is empty or missing'.format(t_name))
day_skip_flag = True
if day_skip_flag:
logging.warning(' Skipping date')
continue
# # DEADBEEF - Some arrays have a 500m cellsize
# # i.e. 2011-07-25, 2010-01-01 -> 2010-07-27
# tmin_array = rescale_array_func(tmin_array, elev_array, 'tmin')
# tmax_array = rescale_array_func(tmax_array, elev_array, 'tmax')
# tdew_array = rescale_array_func(tdew_array, elev_array, 'tdew')
# rso_array = rescale_array_func(rso_array, elev_array, 'rso')
# rs_array = rescale_array_func(rs_array, elev_array, 'rs')
# u2_array = rescale_array_func(u2_array, elev_array, 'u2')
# # k_array = rescale_array_func(k_array, elev_array, 'k')
# # rnl_array = rescale_array_func(rnl_array, elev_array, 'rnl')
# Compute Ea from Tdew
ea_array = refet.calcs.saturation_vapor_pressure_func(
tdew_array)
# # Calculate q from tdew by first calculating ea from tdew
# ea_array = refet.calcs.saturation_vapor_pressure_func(tdew_array)
# pair_array = refet.calcs.air_pressure_func(elev_array)
# q_array = 0.622 * ea_array / (pair_array - (0.378 * ea_array))
# del es_array, pair_array, tdew_array
# # Calculate rhmin/rhmax from tdew
# ea_tmax = refet._calcs.saturation_vapor_pressure_func(tmax_array)
# ea_tmin = refet._calcs.saturation_vapor_pressure_func(tmin_array)
# rhmin = ea_tdew * 2 / (ea_tmax + ea_tmin);
# rhmax = ea_tdew * 2 / (ea_tmax + ea_tmin);
# del ea_tmax, ea_tmin
refet_obj = refet.Daily(tmin=tmin_array,
tmax=tmax_array,
ea=ea_array,
rs=rs_array,
uz=u2_array,
zw=zw,
elev=elev_array,
lat=lat_array,
doy=doy,
method='asce')
# rso_type='ARRAY', rso=rso_array
# ETr
if etr_flag:
drigo.array_to_comp_raster(refet_obj.etr().astype(
np.float32),
etr_raster,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# refet_obj.etr().astype(np.float32), etr_raster,
# output_geo=cimis_geo, output_proj=cimis_proj,
# stats_flag=stats_flag)
# ETo
if eto_flag:
drigo.array_to_comp_raster(refet_obj.eto().astype(
np.float32),
eto_raster,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# refet_obj.eto().astype(np.float32), eto_raster,
# output_geo=cimis_geo, output_proj=cimis_proj,
# stats_flag=stats_flag)
# Cleanup
del tmin_array, tmax_array, u2_array, rs_array, ea_array
# del rnl, rs, rso
else:
logging.info(' Skipping date')
continue
if stats_flag and etr_flag:
drigo.raster_statistics(etr_raster)
if stats_flag and eto_flag:
drigo.raster_statistics(eto_raster)
logging.debug('\nScript Complete')
def main(grb_ws,
ancillary_ws,
output_ws,
etr_flag=False,
eto_flag=False,
scene_list_path=None,
start_dt=None,
end_dt=None,
times_str='',
extent_path=None,
output_extent=None,
daily_flag=True,
stats_flag=True,
overwrite_flag=False):
"""Compute hourly ETr/ETo from NLDAS data
Parameters
----------
grb_ws : str
Folder of NLDAS GRB files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
etr_flag : bool, optional
If True, compute alfalfa reference ET (ETr).
eto_flag : bool, optional
If True, compute grass reference ET (ETo).
scene_list_path : str, optional
Landsat scene keep list file path.
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
times : str, optional
Comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8").
Parsed with python_common.parse_int_set().
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
daily_flag : bool, optional
If True, save daily ETr/ETo sum raster (the default is True).
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nComputing NLDAS hourly ETr/ETo')
np.seterr(invalid='ignore')
# Compute ETr and/or ETo
if not etr_flag and not eto_flag:
logging.info(' ETo/ETr flag(s) not set, defaulting to ETr')
etr_flag = True
# Only process a specific hours
if not times_str:
time_list = range(0, 24, 1)
else:
time_list = list(_utils.parse_int_set(times_str))
time_list = ['{:02d}00'.format(t) for t in time_list]
etr_folder = 'etr'
eto_folder = 'eto'
hour_fmt = '{}_{:04d}{:02d}{:02d}_hourly_nldas.img'
# hour_fmt = '{}_{:04d}{:02d}{:02d}_{4:04d}_nldas.img'
day_fmt = '{}_{:04d}{:02d}{:02d}_nldas.img'
# input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb'
input_re = re.compile('NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' +
'(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$')
# # Landsat Collection 1 Product ID
# landsat_re = re.compile(
# '^(?:LT04|LT05|LE07|LC08)_\w{4}_\d{3}\d{3}_(?P<DATE>\d{8})_'
# '\w{8}_\w{2}_\w{2}')
# Landsat Custom Scene ID
landsat_re = re.compile('^(?:LT04|LT05|LE07|LC08)_\d{6}_(?P<DATE>\d{8})')
# Assume NLDAS is NAD83
# input_epsg = 'EPSG:4269'
# Ancillary raster paths
mask_path = os.path.join(ancillary_ws, 'nldas_mask.img')
elev_path = os.path.join(ancillary_ws, 'nldas_elev.img')
lat_path = os.path.join(ancillary_ws, 'nldas_lat.img')
lon_path = os.path.join(ancillary_ws, 'nldas_lon.img')
# Process Landsat scene list and start/end input parameters
if not scene_list_path and (not start_dt or not end_dt):
logging.error(
'\nERROR: A Landsat scene list or start/end dates must be set, '
'exiting\n')
return False
if scene_list_path is not None and os.path.isfile(scene_list_path):
# Build a date list from the Landsat scene keep list file
logging.info('\nReading dates from scene keep list file')
logging.info(' {}'.format(scene_list_path))
with open(scene_list_path) as input_f:
keep_list = input_f.readlines()
date_list = sorted([
dt.datetime.strptime(m.group('DATE'),
'%Y%m%d').strftime('%Y-%m-%d')
for image_id in keep_list for m in [landsat_re.match(image_id)]
if m
])
logging.debug(' {}'.format(', '.join(date_list)))
else:
date_list = []
if start_dt and end_dt:
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
else:
start_dt = dt.datetime.strptime(date_list[0], '%Y-%m-%d')
end_dt = dt.datetime.strptime(date_list[-1], '%Y-%m-%d')
# This allows GDAL to throw Python Exceptions
# gdal.UseExceptions()
# mem_driver = gdal.GetDriverByName('MEM')
# Get the NLDAS spatial reference from the mask raster
nldas_ds = gdal.Open(mask_path)
nldas_osr = drigo.raster_ds_osr(nldas_ds)
nldas_proj = drigo.osr_proj(nldas_osr)
nldas_cs = drigo.raster_ds_cellsize(nldas_ds, x_only=True)
nldas_extent = drigo.raster_ds_extent(nldas_ds)
nldas_geo = nldas_extent.geo(nldas_cs)
nldas_x, nldas_y = nldas_extent.origin()
nldas_ds = None
logging.debug(' Projection: {}'.format(nldas_proj))
logging.debug(' Cellsize: {}'.format(nldas_cs))
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
nldas_extent = drigo.Extent(output_extent)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if not os.path.isfile(extent_path):
logging.error('\nThe extent object does not exist, exiting\n'
' {}'.format(extent_path))
return False
elif extent_path.lower().endswith('.shp'):
nldas_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
nldas_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
nldas_extent = drigo.project_extent(nldas_extent, extent_osr,
nldas_osr, extent_cs)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
logging.debug('')
# Read the NLDAS mask array if present
if mask_path and os.path.isfile(mask_path):
mask_array, mask_nodata = drigo.raster_to_array(
mask_path,
mask_extent=nldas_extent,
fill_value=0,
return_nodata=True)
mask_array = mask_array != mask_nodata
else:
mask_array = None
# Read ancillary arrays (or subsets?)
elev_array = drigo.raster_to_array(elev_path,
mask_extent=nldas_extent,
return_nodata=False)
# pair_array = et_common.air_pressure_func(elev_array)
lat_array = drigo.raster_to_array(lat_path,
mask_extent=nldas_extent,
return_nodata=False)
lon_array = drigo.raster_to_array(lon_path,
mask_extent=nldas_extent,
return_nodata=False)
# Hourly RefET functions expects lat/lon in radians
lat_array *= (math.pi / 180)
lon_array *= (math.pi / 180)
# Build output folder
etr_ws = os.path.join(output_ws, etr_folder)
eto_ws = os.path.join(output_ws, eto_folder)
if etr_flag and not os.path.isdir(etr_ws):
os.makedirs(etr_ws)
if eto_flag and not os.path.isdir(eto_ws):
os.makedirs(eto_ws)
# DEADBEEF - Instead of processing all available files, the following
# code will process files for target dates
# for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)):
# logging.info(input_dt.date())
# Iterate all available files and check dates if necessary
# Each sub folder in the main folder has all imagery for 1 day
# (in UTC time)
# The path for each subfolder is the /YYYY/DOY
errors = defaultdict(list)
for root, folders, files in os.walk(grb_ws):
root_split = os.path.normpath(root).split(os.sep)
# If the year/doy is outside the range, skip
if (re.match('\d{4}', root_split[-2])
and re.match('\d{3}', root_split[-1])):
root_dt = dt.datetime.strptime(
'{}_{}'.format(root_split[-2], root_split[-1]), '%Y_%j')
logging.info('{}'.format(root_dt.date()))
if ((start_dt is not None and root_dt < start_dt)
or (end_dt is not None and root_dt > end_dt)):
continue
elif date_list and root_dt.date().isoformat() not in date_list:
continue
# If the year is outside the range, don't search subfolders
elif re.match('\d{4}', root_split[-1]):
root_year = int(root_split[-1])
logging.info('Year: {}'.format(root_year))
if ((start_dt is not None and root_year < start_dt.year)
or (end_dt is not None and root_year > end_dt.year)):
folders[:] = []
else:
folders[:] = sorted(folders)
continue
else:
continue
logging.debug(' {}'.format(root))
# Start off assuming every file needs to be processed
day_skip_flag = False
# Build output folders if necessary
etr_year_ws = os.path.join(etr_ws, str(root_dt.year))
eto_year_ws = os.path.join(eto_ws, str(root_dt.year))
if etr_flag and not os.path.isdir(etr_year_ws):
os.makedirs(etr_year_ws)
if eto_flag and not os.path.isdir(eto_year_ws):
os.makedirs(eto_year_ws)
# Build daily total paths
etr_day_path = os.path.join(
etr_year_ws,
day_fmt.format('etr', root_dt.year, root_dt.month, root_dt.day))
eto_day_path = os.path.join(
eto_year_ws,
day_fmt.format('eto', root_dt.year, root_dt.month, root_dt.day))
etr_hour_path = os.path.join(
etr_year_ws,
hour_fmt.format('etr', root_dt.year, root_dt.month, root_dt.day))
eto_hour_path = os.path.join(
eto_year_ws,
hour_fmt.format('eto', root_dt.year, root_dt.month, root_dt.day))
# logging.debug(' {}'.format(etr_hour_path))
# If daily ETr/ETo files are present, day can be skipped
if not overwrite_flag and daily_flag:
if etr_flag and not os.path.isfile(etr_day_path):
pass
elif eto_flag and not os.path.isfile(eto_day_path):
pass
else:
day_skip_flag = True
# If the hour and daily files don't need to be made, skip the day
if not overwrite_flag:
if etr_flag and not os.path.isfile(etr_hour_path):
pass
elif eto_flag and not os.path.isfile(eto_hour_path):
pass
elif day_skip_flag:
logging.debug(' File(s) already exist, skipping')
continue
# Create a single raster for each day with 24 bands
# Each time step will be stored in a separate band
if etr_flag:
logging.debug(' {}'.format(etr_day_path))
drigo.build_empty_raster(etr_hour_path,
band_cnt=24,
output_dtype=np.float32,
output_proj=nldas_proj,
output_cs=nldas_cs,
output_extent=nldas_extent,
output_fill_flag=True)
if eto_flag:
logging.debug(' {}'.format(eto_day_path))
drigo.build_empty_raster(eto_hour_path,
band_cnt=24,
output_dtype=np.float32,
output_proj=nldas_proj,
output_cs=nldas_cs,
output_extent=nldas_extent,
output_fill_flag=True)
# Sum all ETr/ETo images in each folder to generate a UTC day total
etr_day_array = 0
eto_day_array = 0
# Process each hour file
for input_name in sorted(files):
logging.info(' {}'.format(input_name))
input_match = input_re.match(input_name)
if input_match is None:
logging.debug(' Regular expression didn\'t match, skipping')
continue
input_dt = dt.datetime(int(input_match.group('YEAR')),
int(input_match.group('MONTH')),
int(input_match.group('DAY')))
input_doy = int(input_dt.strftime('%j'))
time_str = input_match.group('TIME')
band_num = int(time_str[:2]) + 1
# if start_dt is not None and input_dt < start_dt:
# continue
# elif end_dt is not None and input_dt > end_dt:
# continue
# elif date_list and input_dt.date().isoformat() not in date_list:
# continue
if not daily_flag and time_str not in time_list:
logging.debug(' Time not in list and not daily, skipping')
continue
input_path = os.path.join(root, input_name)
logging.debug(' Time: {} {}'.format(input_dt.date(), time_str))
logging.debug(' Band: {}'.format(band_num))
# Determine band numbering/naming
input_band_dict = grib_band_names(input_path)
# Read input bands
input_ds = gdal.Open(input_path)
# Temperature should be in C for et_common.refet_hourly_func()
if 'Temperature [K]' in input_band_dict.keys():
temp_band_units = 'K'
temp_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['Temperature [K]'],
mask_extent=nldas_extent,
return_nodata=False)
elif 'Temperature [C]' in input_band_dict.keys():
temp_band_units = 'C'
temp_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['Temperature [C]'],
mask_extent=nldas_extent,
return_nodata=False)
else:
logging.error('Unknown Temperature units, skipping')
logging.error(' {}'.format(input_band_dict.keys()))
continue
# DEADBEEF - Having issue with T appearing to be C but labeled as K
# Try to determine temperature units from values
temp_mean = float(np.nanmean(temp_array))
temp_units_dict = {20: 'C', 293: 'K'}
temp_array_units = temp_units_dict[min(
temp_units_dict, key=lambda x: abs(x - temp_mean))]
if temp_array_units == 'K' and temp_band_units == 'K':
logging.debug(' Converting temperature from K to C')
temp_array -= 273.15
elif temp_array_units == 'C' and temp_band_units == 'C':
pass
elif temp_array_units == 'C' and temp_band_units == 'K':
logging.debug((
' Temperature units are K in the GRB band name, ' +
'but values appear to be C\n Mean temperature: {:.2f}\n'
+ ' Values will NOT be adjusted').format(temp_mean))
elif temp_array_units == 'K' and temp_band_units == 'C':
logging.debug((
' Temperature units are C in the GRB band name, ' +
'but values appear to be K\n Mean temperature: {:.2f}\n'
+
' Values will be adjusted from K to C').format(temp_mean))
temp_array -= 273.15
try:
sph_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['Specific humidity [kg/kg]'],
mask_extent=nldas_extent,
return_nodata=False)
rs_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict[
'Downward shortwave radiation flux [W/m^2]'],
mask_extent=nldas_extent,
return_nodata=False)
wind_u_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['u-component of wind [m/s]'],
mask_extent=nldas_extent,
return_nodata=False)
wind_v_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['v-component of wind [m/s]'],
mask_extent=nldas_extent,
return_nodata=False)
input_ds = None
except KeyError as e:
errors[input_path].append(e)
logging.error(' KeyError: {} Skipping: {}'.format(
e, input_ds.GetDescription()))
continue
rs_array *= 0.0036 # W m-2 to MJ m-2 hr-1
wind_array = np.sqrt(wind_u_array**2 + wind_v_array**2)
del wind_u_array, wind_v_array
# Compute vapor pressure from specific humidity
pair_array = refet.calcs._air_pressure(elev=elev_array)
ea_array = refet.calcs._actual_vapor_pressure(q=sph_array,
pair=pair_array)
refet_obj = refet.Hourly(tmean=temp_array,
ea=ea_array,
rs=rs_array,
uz=wind_array,
zw=10,
elev=elev_array,
lat=lat_array,
lon=lon_array,
doy=input_doy,
time=int(time_str) / 100,
method='asce')
# ETr
if etr_flag:
etr_array = refet_obj.etr()
if daily_flag:
etr_day_array += etr_array
if time_str in time_list:
drigo.array_to_comp_raster(etr_array.astype(np.float32),
etr_hour_path,
band=band_num,
stats_flag=False)
del etr_array
# ETo
if eto_flag:
eto_array = refet_obj.eto()
if eto_flag and daily_flag:
eto_day_array += eto_array
if eto_flag and time_str in time_list:
drigo.array_to_comp_raster(eto_array.astype(np.float32),
eto_hour_path,
band=band_num,
stats_flag=False)
del eto_array
del temp_array, sph_array, rs_array, wind_array
del pair_array, ea_array
if stats_flag and etr_flag:
drigo.raster_statistics(etr_hour_path)
if stats_flag and eto_flag:
drigo.raster_statistics(eto_hour_path)
# Save the projected ETr/ETo as 32-bit floats
if not day_skip_flag and daily_flag:
if etr_flag:
try:
drigo.array_to_raster(etr_day_array.astype(np.float32),
etr_day_path,
output_geo=nldas_geo,
output_proj=nldas_proj,
stats_flag=stats_flag)
except AttributeError:
pass
if eto_flag:
try:
drigo.array_to_raster(eto_day_array.astype(np.float32),
eto_day_path,
output_geo=nldas_geo,
output_proj=nldas_proj,
stats_flag=stats_flag)
except AttributeError:
pass
del etr_day_array, eto_day_array
if len(errors) > 0:
logging.info('\nThe following errors were encountered:')
for key, value in errors.items():
logging.error(' Filepath: {}, error: {}'.format(key, value))
logging.debug('\nScript Complete')
def main(grb_ws, ancillary_ws, output_ws, variables=['pr'],
scene_list_path=None, start_dt=None, end_dt=None, times_str='',
extent_path=None, output_extent=None,
stats_flag=True, overwrite_flag=False):
"""Extract NLDAS target variable(s)
Parameters
----------
grb_ws : str
Folder of NLDAS GRB files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
variable : list
NLDAS variables to download (the default is ['pr']).
Choices: 'ppt', 'srad', 'sph', 'tair', tmmn', 'tmmx', 'vs'.
keep_list_path : str, optional
Landsat scene keep list file path.
start_dt : datetime, optional
Start date.
end_dt : datetime, optional
End date.
times : str
Comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8").
Parsed with python_common.parse_int_set().
extent_path : str
File path defining the output extent.
output_extent : list
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtract NLDAS target variable(s)')
# input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb'
input_re = re.compile(
'NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' +
'(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$')
output_fmt = '{}_{:04d}{:02d}{:02d}_hourly_nldas.img'
# output_fmt = '{}_{:04d}{:02d}{:02d}_{:04d}_nldas.img'
# # Landsat Collection 1 Product ID
# landsat_re = re.compile(
# '^(?:LT04|LT05|LE07|LC08)_\w{4}_\d{3}\d{3}_(?P<DATE>\d{8})_'
# '\w{8}_\w{2}_\w{2}')
# Landsat Custom Scene ID
landsat_re = re.compile(
'^(?:LT04|LT05|LE07|LC08)_\d{6}_(?P<DATE>\d{8})')
# Only process a specific hours
if not times_str:
time_list = range(0, 24, 1)
else:
time_list = list(_utils.parse_int_set(times_str))
time_list = ['{:02d}00'.format(t) for t in time_list]
# Assume NLDAS is NAD83
# input_epsg = 'EPSG:4269'
# NLDAS rasters to extract
data_full_list = ['pr', 'srad', 'sph', 'tair', 'tmmn', 'tmmx', 'vs']
if not variables:
logging.error('\nERROR: variables parameter is empty\n')
sys.exit()
elif type(variables) is not list:
# DEADBEEF - I could try converting comma separated strings to lists?
logging.warning('\nERROR: variables parameter must be a list\n')
sys.exit()
elif not set(variables).issubset(set(data_full_list)):
logging.error('\nERROR: variables parameter is invalid\n {}'.format(
variables))
sys.exit()
# Ancillary raster paths
mask_path = os.path.join(ancillary_ws, 'nldas_mask.img')
# Process Landsat scene list and start/end input parameters
if not scene_list_path and (not start_dt or not end_dt):
logging.error(
'\nERROR: A Landsat scene list or start/end dates must be set, '
'exiting\n')
return False
if scene_list_path is not None and os.path.isfile(scene_list_path):
# Build a date list from the Landsat scene keep list file
logging.info('\nReading dates from scene keep list file')
logging.info(' {}'.format(scene_list_path))
with open(scene_list_path) as input_f:
keep_list = input_f.readlines()
date_list = sorted([
dt.datetime.strptime(m.group('DATE'), '%Y%m%d').strftime('%Y-%m-%d')
for image_id in keep_list
for m in [landsat_re.match(image_id)] if m])
logging.debug(' {}'.format(', '.join(date_list)))
else:
date_list = []
if start_dt and end_dt:
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
else:
start_dt = dt.datetime.strptime(date_list[0], '%Y-%m-%d')
end_dt = dt.datetime.strptime(date_list[-1], '%Y-%m-%d')
# This allows GDAL to throw Python Exceptions
# gdal.UseExceptions()
# mem_driver = gdal.GetDriverByName('MEM')
# Get the NLDAS spatial reference from the mask raster
nldas_ds = gdal.Open(mask_path)
nldas_osr = drigo.raster_ds_osr(nldas_ds)
nldas_proj = drigo.osr_proj(nldas_osr)
nldas_cs = drigo.raster_ds_cellsize(nldas_ds, x_only=True)
nldas_extent = drigo.raster_ds_extent(nldas_ds)
nldas_geo = nldas_extent.geo(nldas_cs)
nldas_x, nldas_y = nldas_extent.origin()
nldas_ds = None
logging.debug(' Projection: {}'.format(nldas_proj))
logging.debug(' Cellsize: {}'.format(nldas_cs))
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
nldas_extent = drigo.Extent(output_extent)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if not os.path.isfile(extent_path):
logging.error(
'\nThe extent object does not exist, exiting\n'
' {}'.format(extent_path))
return False
elif extent_path.lower().endswith('.shp'):
nldas_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
nldas_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
nldas_extent = drigo.project_extent(
nldas_extent, extent_osr, nldas_osr, extent_cs)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
logging.debug('')
# Read the NLDAS mask array if present
if mask_path and os.path.isfile(mask_path):
mask_array, mask_nodata = drigo.raster_to_array(
mask_path, mask_extent=nldas_extent, fill_value=0,
return_nodata=True)
mask_array = mask_array != mask_nodata
else:
mask_array = None
# NLDAS band name dictionary
nldas_band_dict = dict()
nldas_band_dict['pr'] = 'Total precipitation [kg/m^2]'
nldas_band_dict['srad'] = 'Downward shortwave radiation flux [W/m^2]'
nldas_band_dict['sph'] = 'Specific humidity [kg/kg]'
nldas_band_dict['tair'] = 'Temperature [C]'
nldas_band_dict['tmmn'] = 'Temperature [C]'
nldas_band_dict['tmmx'] = 'Temperature [C]'
nldas_band_dict['vs'] = [
'u-component of wind [m/s]', 'v-component of wind [m/s]']
# NLDAS band name dictionary
# nldas_band_dict = dict()
# nldas_band_dict['pr'] = 'precipitation_amount'
# nldas_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air'
# nldas_band_dict['sph'] = 'specific_humidity'
# nldas_band_dict['tmmn'] = 'air_temperature'
# nldas_band_dict['tmmx'] = 'air_temperature'
# nldas_band_dict['vs'] = 'wind_speed'
# NLDAS band name dictionary (EarthEngine keys, GRID_ELEMENT values)
# nldas_band_dict = dict()
# nldas_band_dict['total_precipitation'] = 'Total precipitation [kg/m^2]'
# nldas_band_dict['shortwave_radiation'] = 'Downward shortwave radiation flux [W/m^2]'
# nldas_band_dict['specific_humidity'] = 'Specific humidity [kg/kg]'
# nldas_band_dict['pressure'] = 'Pressure [Pa]'
# nldas_band_dict['temperature'] = 'Temperature [C]'
# nldas_band_dict['wind_u'] = 'u-component of wind [m/s]'
# nldas_band_dict['wind_v'] = 'v-component of wind [m/s]'
# Process each variable
logging.info('\nReading NLDAS GRIBs')
for input_var in variables:
logging.info("Variable: {}".format(input_var))
# Build output folder
var_ws = os.path.join(output_ws, input_var)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Each sub folder in the main folde has all imagery for 1 day
# The path for each subfolder is the /YYYY/DOY
# This approach will process files for target dates
# for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)):
# logging.info(input_dt.date())
# Iterate all available files and check dates if necessary
for root, folders, files in os.walk(grb_ws):
root_split = os.path.normpath(root).split(os.sep)
# If the year/doy is outside the range, skip
if (re.match('\d{4}', root_split[-2]) and
re.match('\d{3}', root_split[-1])):
root_dt = dt.datetime.strptime('{}_{}'.format(
root_split[-2], root_split[-1]), '%Y_%j')
logging.info('{}-{:02d}-{:02d}'.format(
root_dt.year, root_dt.month, root_dt.day))
if ((start_dt is not None and root_dt < start_dt) or
(end_dt is not None and root_dt > end_dt)):
continue
elif date_list and root_dt.date().isoformat() not in date_list:
continue
# If the year is outside the range, don't search subfolders
elif re.match('\d{4}', root_split[-1]):
root_year = int(root_split[-1])
logging.info('Year: {}'.format(root_year))
if ((start_dt is not None and root_year < start_dt.year) or
(end_dt is not None and root_year > end_dt.year)):
folders[:] = []
else:
folders[:] = sorted(folders)
continue
else:
continue
# Create a single raster for each day with 24 bands
# Each time step will be stored in a separate band
output_name = output_fmt.format(
input_var, root_dt.year, root_dt.month, root_dt.day)
output_path = os.path.join(
var_ws, str(root_dt.year), output_name)
logging.debug(' {}'.format(output_path))
if os.path.isfile(output_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(output_path)
logging.debug(' {}'.format(root))
if not os.path.isdir(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
drigo.build_empty_raster(
output_path, band_cnt=24, output_dtype=np.float32,
output_proj=nldas_proj, output_cs=nldas_cs,
output_extent=nldas_extent, output_fill_flag=True)
# Iterate through hourly files
for input_name in sorted(files):
logging.info(' {}'.format(input_name))
input_path = os.path.join(root, input_name)
input_match = input_re.match(input_name)
if input_match is None:
logging.debug(
' Regular expression didn\'t match, skipping')
continue
input_dt = dt.datetime(
int(input_match.group('YEAR')),
int(input_match.group('MONTH')),
int(input_match.group('DAY')))
time_str = input_match.group('TIME')
band_num = int(time_str[:2]) + 1
# if start_dt is not None and input_dt < start_dt:
# continue
# elif end_dt is not None and input_dt > end_dt:
# continue
# elif date_list and input_dt.date().isoformat() not in date_list:
# continue
if time_str not in time_list:
logging.debug(' Time not in list, skipping')
continue
logging.debug(' Time: {} {}'.format(
input_dt.date(), time_str))
logging.debug(' Band: {}'.format(band_num))
# Determine band numbering/naming
input_band_dict = grib_band_names(input_path)
# Extract array and save
input_ds = gdal.Open(input_path)
# Convert Kelvin to Celsius (old NLDAS files were in K i think)
if input_var in ['tair', 'tmmx', 'tmmn']:
# Temperature should be in C for et_common.refet_hourly_func()
if 'Temperature [K]' in input_band_dict.keys():
temp_band_units = 'K'
output_array = drigo.raster_ds_to_array(
input_ds, band=input_band_dict['Temperature [K]'],
mask_extent=nldas_extent, return_nodata=False)
elif 'Temperature [C]' in input_band_dict.keys():
temp_band_units = 'C'
output_array = drigo.raster_ds_to_array(
input_ds, band=input_band_dict['Temperature [C]'],
mask_extent=nldas_extent, return_nodata=False)
else:
logging.error('Unknown Temperature units, skipping')
logging.error(' {}'.format(input_band_dict.keys()))
continue
# DEADBEEF - Having issue with T appearing to be C but labeled as K
# Try to determine temperature units from values
temp_mean = float(np.nanmean(output_array))
temp_units_dict = {20: 'C', 293: 'K'}
temp_array_units = temp_units_dict[
min(temp_units_dict, key=lambda x:abs(x - temp_mean))]
if temp_array_units == 'K' and temp_band_units == 'K':
logging.debug(' Converting temperature from K to C')
output_array -= 273.15
elif temp_array_units == 'C' and temp_band_units == 'C':
pass
elif temp_array_units == 'C' and temp_band_units == 'K':
logging.debug(
(' Temperature units are K in the GRB band name, ' +
'but values appear to be C\n Mean temperature: {:.2f}\n' +
' Values will NOT be adjusted').format(temp_mean))
elif temp_array_units == 'K' and temp_band_units == 'C':
logging.debug(
(' Temperature units are C in the GRB band name, ' +
'but values appear to be K\n Mean temperature: {:.2f}\n' +
' Values will be adjusted from K to C').format(temp_mean))
output_array -= 273.15
# Compute wind speed from vectors
elif input_var == 'vs':
wind_u_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['u-component of wind [m/s]'],
mask_extent=nldas_extent, return_nodata=False)
wind_v_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['v-component of wind [m/s]'],
mask_extent=nldas_extent, return_nodata=False)
output_array = np.sqrt(
wind_u_array ** 2 + wind_v_array ** 2)
# Read all other variables directly
else:
output_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict[nldas_band_dict[input_var]],
mask_extent=nldas_extent, return_nodata=False)
# Save the projected array as 32-bit floats
drigo.array_to_comp_raster(
output_array.astype(np.float32), output_path,
band=band_num)
# drigo.block_to_raster(
# ea_array.astype(np.float32), output_path, band=band)
# drigo.array_to_raster(
# output_array.astype(np.float32), output_path,
# output_geo=nldas_geo, output_proj=nldas_proj,
# stats_flag=stats_flag)
del output_array
input_ds = None
if stats_flag:
drigo.raster_statistics(output_path)
logging.debug('\nScript Complete')
def main(grb_ws,
ancillary_ws,
output_ws,
scene_list_path=None,
start_dt=None,
end_dt=None,
times_str='',
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Extract hourly NLDAS vapour pressure rasters
Parameters
----------
grb_ws : str
Folder of NLDAS GRB files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
scene_list_path : str, optional
Landsat scene keep list file path.
start_dt : datetime, optional
Start date.
end_dt : datetime, optional
End date.
times : str, optional
Comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8").
Parsed with python_common.parse_int_set().
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting NLDAS vapour pressure rasters')
# input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb'
input_re = re.compile('NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' +
'(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$')
# # Landsat Collection 1 Product ID
# landsat_re = re.compile(
# '^(?:LT04|LT05|LE07|LC08)_\w{4}_\d{3}\d{3}_(?P<DATE>\d{8})_'
# '\w{8}_\w{2}_\w{2}')
# Landsat Custom Scene ID
landsat_re = re.compile('^(?:LT04|LT05|LE07|LC08)_\d{6}_(?P<DATE>\d{8})')
output_folder = 'ea'
output_fmt = 'ea_{:04d}{:02d}{:02d}_hourly_nldas.img'
# output_fmt = 'ea_{:04d}{:02d}{:02d}_{:04d}_nldas.img'
# Only process a specific hours
if not times_str:
time_list = range(0, 24, 1)
else:
time_list = list(_utils.parse_int_set(times_str))
time_list = ['{:02d}00'.format(t) for t in time_list]
# Assume NLDAS is NAD83
# input_epsg = 'EPSG:4269'
# Ancillary raster paths
mask_path = os.path.join(ancillary_ws, 'nldas_mask.img')
elev_path = os.path.join(ancillary_ws, 'nldas_elev.img')
# Process Landsat scene list and start/end input parameters
if not scene_list_path and (not start_dt or not end_dt):
logging.error(
'\nERROR: A Landsat scene list or start/end dates must be set, '
'exiting\n')
return False
if scene_list_path is not None and os.path.isfile(scene_list_path):
# Build a date list from the Landsat scene keep list file
logging.info('\nReading dates from scene keep list file')
logging.info(' {}'.format(scene_list_path))
with open(scene_list_path) as input_f:
keep_list = input_f.readlines()
date_list = sorted([
dt.datetime.strptime(m.group('DATE'),
'%Y%m%d').strftime('%Y-%m-%d')
for image_id in keep_list for m in [landsat_re.match(image_id)]
if m
])
logging.debug(' {}'.format(', '.join(date_list)))
else:
date_list = []
if start_dt and end_dt:
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
else:
start_dt = dt.datetime.strptime(date_list[0], '%Y-%m-%d')
end_dt = dt.datetime.strptime(date_list[-1], '%Y-%m-%d')
# This allows GDAL to throw Python Exceptions
# gdal.UseExceptions()
# mem_driver = gdal.GetDriverByName('MEM')
# Get the NLDAS spatial reference from the mask raster
nldas_ds = gdal.Open(mask_path)
nldas_osr = drigo.raster_ds_osr(nldas_ds)
nldas_proj = drigo.osr_proj(nldas_osr)
nldas_cs = drigo.raster_ds_cellsize(nldas_ds, x_only=True)
nldas_extent = drigo.raster_ds_extent(nldas_ds)
nldas_geo = nldas_extent.geo(nldas_cs)
nldas_x, nldas_y = nldas_extent.origin()
nldas_ds = None
logging.debug(' Projection: {}'.format(nldas_proj))
logging.debug(' Cellsize: {}'.format(nldas_cs))
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
nldas_extent = drigo.Extent(output_extent)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if not os.path.isfile(extent_path):
logging.error('\nThe extent object does not exist, exiting\n'
' {}'.format(extent_path))
return False
elif extent_path.lower().endswith('.shp'):
nldas_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
nldas_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
nldas_extent = drigo.project_extent(nldas_extent, extent_osr,
nldas_osr, extent_cs)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
logging.debug('')
# Read the NLDAS mask array if present
if mask_path and os.path.isfile(mask_path):
mask_array, mask_nodata = drigo.raster_to_array(
mask_path,
mask_extent=nldas_extent,
fill_value=0,
return_nodata=True)
mask_array = mask_array != mask_nodata
else:
mask_array = None
# Read elevation arrays (or subsets?)
elev_array = drigo.raster_to_array(elev_path,
mask_extent=nldas_extent,
return_nodata=False)
pair_array = refet.calcs._air_pressure(elev_array)
# Build output folder
var_ws = os.path.join(output_ws, output_folder)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Each sub folder in the main folder has all imagery for 1 day
# The path for each subfolder is the /YYYY/DOY
# This approach will process files for target dates
# for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)):
# logging.info(input_dt.date())
# Iterate all available files and check dates if necessary
for root, folders, files in os.walk(grb_ws):
root_split = os.path.normpath(root).split(os.sep)
# If the year/doy is outside the range, skip
if (re.match('\d{4}', root_split[-2])
and re.match('\d{3}', root_split[-1])):
root_dt = dt.datetime.strptime(
'{}_{}'.format(root_split[-2], root_split[-1]), '%Y_%j')
logging.info('{}'.format(root_dt.date()))
if ((start_dt is not None and root_dt < start_dt)
or (end_dt is not None and root_dt > end_dt)):
continue
elif date_list and root_dt.date().isoformat() not in date_list:
continue
# If the year is outside the range, don't search subfolders
elif re.match('\d{4}', root_split[-1]):
root_year = int(root_split[-1])
logging.info('Year: {}'.format(root_year))
if ((start_dt is not None and root_year < start_dt.year)
or (end_dt is not None and root_year > end_dt.year)):
folders[:] = []
else:
folders[:] = sorted(folders)
continue
else:
continue
# Create a single raster for each day with 24 bands
# Each time step will be stored in a separate band
output_name = output_fmt.format(root_dt.year, root_dt.month,
root_dt.day)
output_path = os.path.join(var_ws, str(root_dt.year), output_name)
logging.debug(' {}'.format(output_path))
if os.path.isfile(output_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(output_path)
logging.debug(' {}'.format(root))
if not os.path.isdir(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
drigo.build_empty_raster(output_path,
band_cnt=24,
output_dtype=np.float32,
output_proj=nldas_proj,
output_cs=nldas_cs,
output_extent=nldas_extent,
output_fill_flag=True)
# Iterate through hourly files
for input_name in sorted(files):
logging.info(' {}'.format(input_name))
input_path = os.path.join(root, input_name)
input_match = input_re.match(input_name)
if input_match is None:
logging.debug(' Regular expression didn\'t match, skipping')
continue
input_dt = dt.datetime(int(input_match.group('YEAR')),
int(input_match.group('MONTH')),
int(input_match.group('DAY')))
input_doy = int(input_dt.strftime('%j'))
time_str = input_match.group('TIME')
band_num = int(time_str[:2]) + 1
# if start_dt is not None and input_dt < start_dt:
# continue
# elif end_dt is not None and input_dt > end_dt:
# continue
# elif date_list and input_dt.date().isoformat() not in date_list:
# continue
if time_str not in time_list:
logging.debug(' Time not in list, skipping')
continue
logging.debug(' Time: {} {}'.format(input_dt.date(), time_str))
logging.debug(' Band: {}'.format(band_num))
# Determine band numbering/naming
input_band_dict = grib_band_names(input_path)
# Compute vapour pressure from specific humidity
input_ds = gdal.Open(input_path)
sph_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['Specific humidity [kg/kg]'],
mask_extent=nldas_extent,
return_nodata=False)
ea_array = refet.calcs._actual_vapor_pressure(q=sph_array,
pair=pair_array)
# ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array)
# Save the projected array as 32-bit floats
drigo.array_to_comp_raster(ea_array.astype(np.float32),
output_path,
band=band_num)
# drigo.block_to_raster(
# ea_array.astype(np.float32), output_path, band=band)
# drigo.array_to_raster(
# ea_array.astype(np.float32), output_path,
# output_geo=nldas_geo, output_proj=nldas_proj,
# stats_flag=stats_flag)
del sph_array
input_ds = None
if stats_flag:
drigo.raster_statistics(output_path)
logging.debug('\nScript Complete')
def main(netcdf_ws=os.getcwd(),
ancillary_ws=os.getcwd(),
output_ws=os.getcwd(),
etr_flag=False,
eto_flag=False,
start_date=None,
end_date=None,
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Compute daily ETr/ETo from GRIDMET data
Parameters
----------
netcdf_ws : str
Folder of GRIDMET netcdf files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
etr_flag : str, optional
If True, compute alfalfa reference ET (ETr) (the default is False).
eto_flag : str, optional
If True, compute grass reference ET (ETo) (the default is False).
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nComputing GRIDMET ETo/ETr')
np.seterr(invalid='ignore')
# Compute ETr and/or ETo
if not etr_flag and not eto_flag:
logging.info(' ETo/ETr flag(s) not set, defaulting to ETr')
etr_flag = True
# If a date is not set, process 2017
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2017, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2017, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# Save GRIDMET lat, lon, and elevation arrays
elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img')
lat_raster = os.path.join(ancillary_ws, 'gridmet_lat.img')
# Wind speed is measured at 2m
zw = 10
etr_fmt = 'etr_{}_daily_gridmet.img'
eto_fmt = 'eto_{}_daily_gridmet.img'
# gridmet_re = re.compile('(?P<VAR>\w+)_(?P<YEAR>\d{4}).nc')
# GRIDMET band name dictionary
gridmet_band_dict = dict()
gridmet_band_dict['eto'] = 'potential_evapotranspiration'
gridmet_band_dict['etr'] = 'potential_evapotranspiration'
# gridmet_band_dict['pr'] = 'precipitation_amount'
# gridmet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air'
# gridmet_band_dict['sph'] = 'specific_humidity'
# gridmet_band_dict['tmmn'] = 'air_temperature'
# gridmet_band_dict['tmmx'] = 'air_temperature'
# gridmet_band_dict['vs'] = 'wind_speed'
# Get extent/geo from elevation raster
gridmet_ds = gdal.Open(elev_raster)
gridmet_osr = drigo.raster_ds_osr(gridmet_ds)
gridmet_proj = drigo.osr_proj(gridmet_osr)
gridmet_cs = drigo.raster_ds_cellsize(gridmet_ds, x_only=True)
gridmet_extent = drigo.raster_ds_extent(gridmet_ds)
gridmet_full_geo = gridmet_extent.geo(gridmet_cs)
gridmet_x, gridmet_y = gridmet_extent.origin()
gridmet_ds = None
logging.debug(' Projection: {}'.format(gridmet_proj))
logging.debug(' Cellsize: {}'.format(gridmet_cs))
logging.debug(' Geo: {}'.format(gridmet_full_geo))
logging.debug(' Extent: {}'.format(gridmet_extent))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
gridmet_extent = drigo.Extent(output_extent)
gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y,
gridmet_cs)
gridmet_geo = gridmet_extent.geo(gridmet_cs)
logging.debug(' Geo: {}'.format(gridmet_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if not os.path.isfile(extent_path):
logging.error('\nThe extent object not exist, exiting\n'
' {}'.format(extent_path))
return False
elif extent_path.lower().endswith('.shp'):
gridmet_extent = drigo.feature_path_extent(extent_path)
# DEADBEEF - Consider moving call into a try/except block
# logging.error(
# '\nThere was a problem reading the extent object'
# '\nThe file path may be invalid or the file may not exist '
# 'or be corrupt.\n{}'.format(extent_path))
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
gridmet_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
gridmet_extent = drigo.project_extent(gridmet_extent, extent_osr,
gridmet_osr, extent_cs)
gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y,
gridmet_cs)
gridmet_geo = gridmet_extent.geo(gridmet_cs)
logging.debug(' Geo: {}'.format(gridmet_geo))
logging.debug(' Extent: {}'.format(gridmet_extent))
else:
gridmet_geo = gridmet_full_geo
# Get indices for slicing/clipping input arrays
g_i, g_j = drigo.array_geo_offsets(gridmet_full_geo,
gridmet_geo,
cs=gridmet_cs)
g_rows, g_cols = gridmet_extent.shape(cs=gridmet_cs)
# Flip row indices since GRIDMET arrays are flipped up/down
# Hard coding GRIDMET row count for now
row_a, row_b = 585 - (g_j + g_rows), 585 - g_j,
col_a, col_b = g_i, g_i + g_cols
# Read the elevation and latitude arrays
elev_array = drigo.raster_to_array(elev_raster,
mask_extent=gridmet_extent,
return_nodata=False)
lat_array = drigo.raster_to_array(lat_raster,
mask_extent=gridmet_extent,
return_nodata=False)
lat_array *= math.pi / 180
# Check elevation and latitude arrays
if np.all(np.isnan(elev_array)):
logging.error('\nERROR: The elevation array is all nodata, exiting\n')
sys.exit()
elif np.all(np.isnan(lat_array)):
logging.error('\nERROR: The latitude array is all nodata, exiting\n')
sys.exit()
# Build output folder
etr_ws = os.path.join(output_ws, 'etr')
eto_ws = os.path.join(output_ws, 'eto')
if etr_flag and not os.path.isdir(etr_ws):
os.makedirs(etr_ws)
if eto_flag and not os.path.isdir(eto_ws):
os.makedirs(eto_ws)
# By default, try to process all possible years
if start_dt.year == end_dt.year:
year_list = [str(start_dt.year)]
year_list = sorted(map(str, range((start_dt.year), (end_dt.year + 1))))
# Process each year separately
for year_str in year_list:
logging.info("\nYear: {}".format(year_str))
year_int = int(year_str)
year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
# Build input file path
eto_path = os.path.join(netcdf_ws, 'eto_{}.nc'.format(year_str))
etr_path = os.path.join(netcdf_ws, 'etr_{}.nc'.format(year_str))
if eto_flag and not os.path.isfile(eto_path):
logging.debug(
' ETo NetCDF doesn\'t exist\n {}'.format(eto_path))
continue
if etr_flag and not os.path.isfile(etr_path):
logging.debug(
' ETr NetCDF doesn\'t exist\n {}'.format(etr_path))
continue
# Create a single raster for each year with 365 bands
# Each day will be stored in a separate band
etr_raster = os.path.join(etr_ws, etr_fmt.format(year_str))
eto_raster = os.path.join(eto_ws, eto_fmt.format(year_str))
if etr_flag and (overwrite_flag or not os.path.isfile(etr_raster)):
logging.debug(' {}'.format(etr_raster))
drigo.build_empty_raster(etr_raster,
band_cnt=366,
output_dtype=np.float32,
output_proj=gridmet_proj,
output_cs=gridmet_cs,
output_extent=gridmet_extent,
output_fill_flag=True)
if eto_flag and (overwrite_flag or not os.path.isfile(eto_raster)):
logging.debug(' {}'.format(eto_raster))
drigo.build_empty_raster(eto_raster,
band_cnt=366,
output_dtype=np.float32,
output_proj=gridmet_proj,
output_cs=gridmet_cs,
output_extent=gridmet_extent,
output_fill_flag=True)
# DEADBEEF - Need to find a way to test if both of these conditionals
# did not pass and pass logging debug message to user
# Read in the GRIDMET NetCDF file
# Immediately clip input arrays to save memory
# Transpose arrays back to row x col
logging.info(' Reading NetCDFs into memory')
if eto_flag:
logging.debug(" {}".format(eto_path))
eto_nc_f = netCDF4.Dataset(eto_path, 'r')
eto_nc = eto_nc_f.variables[
gridmet_band_dict['eto']][:, row_a:row_b, col_a:col_b].copy()
eto_nc = np.flip(eto_nc, 1)
eto_nc_f.close()
del eto_nc_f
if etr_flag:
logging.debug(" {}".format(etr_path))
etr_nc_f = netCDF4.Dataset(etr_path, 'r')
etr_nc = etr_nc_f.variables[
gridmet_band_dict['etr']][:, row_a:row_b, col_a:col_b].copy()
etr_nc = np.flip(etr_nc, 1)
etr_nc_f.close()
del etr_nc_f
# A numpy array is returned when slicing a masked array
# if there are no masked pixels
# This is a hack to force the numpy array back to a masked array
# For now assume all arrays need to be converted
if eto_flag and type(eto_nc) != np.ma.core.MaskedArray:
eto_nc = np.ma.core.MaskedArray(eto_nc,
np.zeros(eto_nc.shape, dtype=bool))
if etr_flag and type(etr_nc) != np.ma.core.MaskedArray:
etr_nc = np.ma.core.MaskedArray(etr_nc,
np.zeros(etr_nc.shape, dtype=bool))
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year_int, 1, 1),
dt.datetime(year_int + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
logging.debug(' {} - before start date, skipping'.format(
date_dt.date()))
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' {} - after end date, skipping'.format(
date_dt.date()))
continue
else:
logging.info(' {}'.format(date_dt.date()))
doy = int(date_dt.strftime('%j'))
doy_i = range(1, year_days + 1).index(doy)
if eto_flag:
# Arrays are being read as masked array with a fill value of -9999
# Convert to basic numpy array arrays with nan values
try:
eto_ma = eto_nc[doy_i, :, :]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
eto_array = eto_ma.data.astype(np.float32)
eto_nodata = float(eto_ma.fill_value)
eto_array[eto_array == eto_nodata] = np.nan
# Since inputs are netcdf, need to create GDAL raster
# datasets in order to use gdal_common functions
# Create an in memory dataset of the full ETo array
eto_ds = drigo.array_to_mem_ds(eto_array,
output_geo=gridmet_geo,
output_proj=gridmet_proj)
# Then extract the subset from the in memory dataset
eto_array = drigo.raster_ds_to_array(
eto_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
# Save
drigo.array_to_comp_raster(eto_array.astype(np.float32),
eto_raster,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# eto_array.astype(np.float32), eto_raster,
# output_geo=gridmet_geo, output_proj=gridmet_proj,
# stats_flag=stats_flag)
# Cleanup
del eto_ds, eto_array
if etr_flag:
try:
etr_ma = etr_nc[doy_i, :, :]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
etr_array = etr_ma.data.astype(np.float32)
etr_nodata = float(etr_ma.fill_value)
etr_array[etr_array == etr_nodata] = np.nan
etr_ds = drigo.array_to_mem_ds(etr_array,
output_geo=gridmet_geo,
output_proj=gridmet_proj)
etr_array = drigo.raster_ds_to_array(
etr_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
drigo.array_to_comp_raster(etr_array.astype(np.float32),
etr_raster,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# etr_array.astype(np.float32), etr_raster,
# output_geo=gridmet_geo, output_proj=gridmet_proj,
# stats_flag=stats_flag)
del etr_ds, etr_array
if stats_flag and eto_flag:
drigo.raster_statistics(eto_raster)
if stats_flag and etr_flag:
drigo.raster_statistics(etr_raster)
# DEADBEEF - Code for computing ETo/ETr from the component variables
# # Build input file path
# tmin_path = os.path.join(netcdf_ws, 'tmmn_{}.nc'.format(year_str))
# tmax_path = os.path.join(netcdf_ws, 'tmmx_{}.nc'.format(year_str))
# sph_path = os.path.join(netcdf_ws, 'sph_{}.nc'.format(year_str))
# rs_path = os.path.join(netcdf_ws, 'srad_{}.nc'.format(year_str))
# wind_path = os.path.join(netcdf_ws, 'vs_{}.nc'.format(year_str))
# # Check that all input files are present
# missing_flag = False
# for input_path in [tmin_path, tmax_path, sph_path,
# rs_path, wind_path]:
# if not os.path.isfile(input_path):
# logging.debug(' Input NetCDF doesn\'t exist\n {}'.format(
# input_path))
# missing_flag = True
# if missing_flag:
# logging.debug(' skipping')
# continue
#
# # Create a single raster for each year with 365 bands
# # Each day will be stored in a separate band
# etr_raster = os.path.join(etr_ws, etr_fmt.format(year_str))
# eto_raster = os.path.join(eto_ws, eto_fmt.format(year_str))
# if etr_flag and (overwrite_flag or not os.path.isfile(etr_raster)):
# logging.debug(' {}'.format(etr_raster))
# drigo.build_empty_raster(
# etr_raster, band_cnt=366, output_dtype=np.float32,
# output_proj=gridmet_proj, output_cs=gridmet_cs,
# output_extent=gridmet_extent, output_fill_flag=True)
# if eto_flag and (overwrite_flag or not os.path.isfile(eto_raster)):
# logging.debug(' {}'.format(eto_raster))
# drigo.build_empty_raster(
# eto_raster, band_cnt=366, output_dtype=np.float32,
# output_proj=gridmet_proj, output_cs=gridmet_cs,
# output_extent=gridmet_extent, output_fill_flag=True)
# # DEADBEEF - Need to find a way to test if both of these conditionals
# # did not pass and pass logging debug message to user
#
# # Read in the GRIDMET NetCDF file
# # Immediately clip input arrays to save memory
# # Transpose arrays back to row x col
# logging.info(' Reading NetCDFs into memory')
# logging.debug(" {}".format(tmin_path))
# tmin_nc_f = netCDF4.Dataset(tmin_path, 'r')
# tmin_nc = tmin_nc_f.variables[gridmet_band_dict['tmmn']][
# :, row_a: row_b, col_a: col_b].copy()
# tmin_nc = np.flip(tmin_nc, 1)
# tmin_nc_f.close()
# del tmin_nc_f
#
# logging.debug(" {}".format(tmax_path))
# tmax_nc_f = netCDF4.Dataset(tmax_path, 'r')
# tmax_nc = tmax_nc_f.variables[gridmet_band_dict['tmmx']][
# :, row_a: row_b, col_a: col_b].copy()
# tmax_nc = np.flip(tmax_nc, 1)
# tmax_nc_f.close()
# del tmax_nc_f
#
# logging.debug(" {}".format(sph_path))
# sph_nc_f = netCDF4.Dataset(sph_path, 'r')
# sph_nc = sph_nc_f.variables[gridmet_band_dict['sph']][
# :, row_a: row_b, col_a: col_b].copy()
# sph_nc = np.flip(sph_nc, 1)
# sph_nc_f.close()
# del sph_nc_f
#
# logging.debug(" {}".format(rs_path))
# rs_nc_f = netCDF4.Dataset(rs_path, 'r')
# rs_nc = rs_nc_f.variables[gridmet_band_dict['srad']][
# :, row_a: row_b, col_a: col_b].copy()
# rs_nc = np.flip(rs_nc, 1)
# rs_nc_f.close()
# del rs_nc_f
#
# logging.debug(" {}".format(wind_path))
# wind_nc_f = netCDF4.Dataset(wind_path, 'r')
# wind_nc = wind_nc_f.variables[gridmet_band_dict['vs']][
# :, row_a: row_b, col_a: col_b].copy()
# wind_nc = np.flip(wind_nc, 1)
# wind_nc_f.close()
# del wind_nc_f
#
# # A numpy array is returned when slicing a masked array
# # if there are no masked pixels
# # This is a hack to force the numpy array back to a masked array
# # For now assume all arrays need to be converted
# if type(tmax_nc) != np.ma.core.MaskedArray:
# tmax_nc = np.ma.core.MaskedArray(
# tmax_nc, np.zeros(tmax_nc.shape, dtype=bool))
# if type(sph_nc) != np.ma.core.MaskedArray:
# sph_nc = np.ma.core.MaskedArray(
# sph_nc, np.zeros(sph_nc.shape, dtype=bool))
# if type(rs_nc) != np.ma.core.MaskedArray:
# rs_nc = np.ma.core.MaskedArray(
# rs_nc, np.zeros(rs_nc.shape, dtype=bool))
# if type(wind_nc) != np.ma.core.MaskedArray:
# wind_nc = np.ma.core.MaskedArray(
# wind_nc, np.zeros(wind_nc.shape, dtype=bool))
#
# # Check all valid dates in the year
# year_dates = _utils.date_range(
# dt.datetime(year_int, 1, 1), dt.datetime(year_int + 1, 1, 1))
# for date_dt in year_dates:
# if start_dt is not None and date_dt < start_dt:
# logging.debug(' {} - before start date, skipping'.format(
# date_dt.date()))
# continue
# elif end_dt is not None and date_dt > end_dt:
# logging.debug(' {} - after end date, skipping'.format(
# date_dt.date()))
# continue
# else:
# logging.info(' {}'.format(date_dt.date()))
#
# doy = int(date_dt.strftime('%j'))
# doy_i = range(1, year_days + 1).index(doy)
#
# # Arrays are being read as masked array with a fill value of -9999
# # Convert to basic numpy array arrays with nan values
# try:
# tmin_ma = tmin_nc[doy_i, :, :]
# except IndexError:
# logging.info(' date not in netcdf, skipping')
# continue
# tmin_array = tmin_ma.data.astype(np.float32)
# tmin_nodata = float(tmin_ma.fill_value)
# tmin_array[tmin_array == tmin_nodata] = np.nan
#
# try:
# tmax_ma = tmax_nc[doy_i, :, :]
# except IndexError:
# logging.info(' date not in netcdf, skipping')
# continue
# tmax_array = tmax_ma.data.astype(np.float32)
# tmax_nodata = float(tmax_ma.fill_value)
# tmax_array[tmax_array == tmax_nodata] = np.nan
#
# try:
# sph_ma = sph_nc[doy_i, :, :]
# except IndexError:
# logging.info(' date not in netcdf, skipping')
# continue
# sph_array = sph_ma.data.astype(np.float32)
# sph_nodata = float(sph_ma.fill_value)
# sph_array[sph_array == sph_nodata] = np.nan
#
# try:
# rs_ma = rs_nc[doy_i, :, :]
# except IndexError:
# logging.info(' date not in netcdf, skipping')
# continue
# rs_array = rs_ma.data.astype(np.float32)
# rs_nodata = float(rs_ma.fill_value)
# rs_array[rs_array == rs_nodata] = np.nan
#
# try:
# wind_ma = wind_nc[doy_i, :, :]
# except IndexError:
# logging.info(' date not in netcdf, skipping')
# continue
# wind_array = wind_ma.data.astype(np.float32)
# wind_nodata = float(wind_ma.fill_value)
# wind_array[wind_array == wind_nodata] = np.nan
# del tmin_ma, tmax_ma, sph_ma, rs_ma, wind_ma
#
# # Since inputs are netcdf, need to create GDAL raster
# # datasets in order to use gdal_common functions
# # Create an in memory dataset of the full ETo array
# tmin_ds = drigo.array_to_mem_ds(
# tmin_array, output_geo=gridmet_geo,
# # tmin_array, output_geo=gridmet_full_geo,
# output_proj=gridmet_proj)
# tmax_ds = drigo.array_to_mem_ds(
# tmax_array, output_geo=gridmet_geo,
# # tmax_array, output_geo=gridmet_full_geo,
# output_proj=gridmet_proj)
# sph_ds = drigo.array_to_mem_ds(
# sph_array, output_geo=gridmet_geo,
# # sph_array, output_geo=gridmet_full_geo,
# output_proj=gridmet_proj)
# rs_ds = drigo.array_to_mem_ds(
# rs_array, output_geo=gridmet_geo,
# # rs_array, output_geo=gridmet_full_geo,
# output_proj=gridmet_proj)
# wind_ds = drigo.array_to_mem_ds(
# wind_array, output_geo=gridmet_geo,
# # wind_array, output_geo=gridmet_full_geo,
# output_proj=gridmet_proj)
#
# # Then extract the subset from the in memory dataset
# tmin_array = drigo.raster_ds_to_array(
# tmin_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
# tmax_array = drigo.raster_ds_to_array(
# tmax_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
# sph_array = drigo.raster_ds_to_array(
# sph_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
# rs_array = drigo.raster_ds_to_array(
# rs_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
# wind_array = drigo.raster_ds_to_array(
# wind_ds, 1, mask_extent=gridmet_extent, return_nodata=False)
# del tmin_ds, tmax_ds, sph_ds, rs_ds, wind_ds
#
# # Adjust units
# tmin_array -= 273.15
# tmax_array -= 273.15
# rs_array *= 0.0864
#
# # Compute vapor pressure from specific humidity
# pair_array = refet.calcs._air_pressure(elev=elev_array)
# ea_array = refet.calcs._actual_vapor_pressure(
# q=sph_array, pair=pair_array)
#
# # ETr/ETo
# refet_obj = refet.Daily(
# tmin=tmin_array, tmax=tmax_array, ea=ea_array, rs=rs_array,
# uz=wind_array, zw=zw, elev=elev_array, lat=lat_array, doy=doy,
# method='asce')
# if etr_flag:
# etr_array = refet_obj.etr()
# if eto_flag:
# eto_array = refet_obj.eto()
#
# # Cleanup
# del tmin_array, tmax_array, sph_array, rs_array, wind_array
# del pair_array, ea_array
#
# # Save the projected array as 32-bit floats
# if etr_flag:
# drigo.array_to_comp_raster(
# etr_array.astype(np.float32), etr_raster,
# band=doy, stats_flag=False)
# # drigo.array_to_raster(
# # etr_array.astype(np.float32), etr_raster,
# # output_geo=gridmet_geo, output_proj=gridmet_proj,
# # stats_flag=stats_flag)
# del etr_array
# if eto_flag:
# drigo.array_to_comp_raster(
# eto_array.astype(np.float32), eto_raster,
# band=doy, stats_flag=False)
# # drigo.array_to_raster(
# # eto_array.astype(np.float32), eto_raster,
# # output_geo=gridmet_geo, output_proj=gridmet_proj,
# # stats_flag=stats_flag)
# del eto_array
#
# del tmin_nc
# del tmax_nc
# del sph_nc
# del rs_nc
# del wind_nc
#
# if stats_flag and etr_flag:
# drigo.raster_statistics(etr_raster)
# if stats_flag and eto_flag:
# drigo.raster_statistics(eto_raster)
logging.debug('\nScript Complete')
def main(ancillary_ws, overwrite_flag=False):
"""Process CIMIS ancillary data
Parameters
----------
ancillary_ws : str
Folder of ancillary rasters.
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nProcess CIMIS ancillary data')
# Site URL
site_url = 'http://cimis.casil.ucdavis.edu/cimis/'
# DEM for air pressure calculation
# http://topotools.cr.usgs.gov/gmted_viewer/gmted2010_global_grids.php
elev_full_url = 'http://edcintl.cr.usgs.gov/downloads/sciweb1/shared/topo/downloads/GMTED/Grid_ZipFiles/mn30_grd.zip'
elev_full_zip = os.path.join(ancillary_ws, 'mn30_grd.zip')
elev_full_raster = os.path.join(ancillary_ws, 'mn30_grd')
# Get CIMIS grid properties from 2010/01/01 ETo raster
# Grid of the spatial cimis input rasters
# cimis_extent = drigo.Extent((-410000, -660000, 610000, 460000))
# cimis_cs = 2000
# cimis_geo = drigo.extent_geo(cimis_extent, cimis_cs)
# Spatial reference parameters
cimis_proj4 = (
'+proj=aea +lat_1=34 +lat_2=40.5 +lat_0=0 +lon_0=-120 +x_0=0 '
'+y_0=-4000000 +ellps=GRS80 +datum=NAD83 +units=m +no_defs')
cimis_osr = drigo.proj4_osr(cimis_proj4)
# cimis_epsg = 3310 # NAD_1983_California_Teale_Albers
# cimis_osr = drigo.epsg_osr(cimis_epsg)
# Comment this line out if building GeoTIFF instead of IMG
cimis_osr.MorphToESRI()
cimis_proj = cimis_osr.ExportToWkt()
# snap_xmin, snap_ymin = (0, 0)
# Build output workspace if it doesn't exist
if not os.path.isdir(ancillary_ws):
os.makedirs(ancillary_ws)
# File paths
mask_url = site_url + '/2010/01/01/ETo.asc.gz'
# mask_gz = os.path.join(ancillary_ws, 'cimis_mask.asc.gz')
mask_ascii = os.path.join(ancillary_ws, 'cimis_mask.asc')
mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img')
elev_raster = os.path.join(ancillary_ws, 'cimis_elev.img')
lat_raster = os.path.join(ancillary_ws, 'cimis_lat.img')
lon_raster = os.path.join(ancillary_ws, 'cimis_lon.img')
# Download an ETo ASCII raster to generate the mask raster
if overwrite_flag or not os.path.isfile(mask_raster):
logging.info('\nCIMIS mask')
logging.debug(' Downloading')
logging.debug(" {}".format(mask_url))
logging.debug(" {}".format(mask_ascii))
_utils.url_download(mask_url, mask_ascii)
# DEADBEEF - The files do not appeared to be compressed even though
# they have a .asc.gz file extension on the server.
# logging.debug(" {}".format(mask_gz))
# _utils.url_download(mask_url, mask_gz)
#
# they are named .asc.gz
# # Uncompress '.gz' file to a new file
# logging.debug(' Uncompressing')
# logging.debug(' {}'.format(mask_ascii))
# try:
# input_f = gzip.open(mask_gz, 'rb')
# output_f = open(mask_ascii, 'wb')
# output_f.write(input_f.read())
# output_f.close()
# input_f.close()
# del input_f, output_f
# except:
# logging.error(" ERROR EXTRACTING FILE")
# os.remove(mask_gz)
# # Set spatial reference of the ASCII files
# if build_prj_flag:
# prj_file = open(mask_asc.replace('.asc','.prj'), 'w')
# prj_file.write(output_proj)
# prj_file.close()
# Convert the ASCII raster to a IMG raster
logging.debug(' Computing mask')
logging.debug(' {}'.format(mask_raster))
mask_array = drigo.raster_to_array(mask_ascii, return_nodata=False)
cimis_geo = drigo.raster_path_geo(mask_ascii)
# cimis_extent = drigo.raster_path_extent(mask_ascii)
logging.debug(' {}'.format(cimis_geo))
mask_array = np.isfinite(mask_array).astype(np.uint8)
drigo.array_to_raster(mask_array,
mask_raster,
output_geo=cimis_geo,
output_proj=cimis_proj,
output_nodata=0)
# drigo.ascii_to_raster(
# mask_ascii, mask_raster, np.float32, cimis_proj)
os.remove(mask_ascii)
# Compute latitude/longitude rasters
if ((overwrite_flag or not os.path.isfile(lat_raster)
or not os.path.isfile(lat_raster)) and os.path.isfile(mask_raster)):
logging.info('\nCIMIS latitude/longitude')
logging.debug(' {}'.format(lat_raster))
lat_array, lon_array = drigo.raster_lat_lon_func(mask_raster)
drigo.array_to_raster(lat_array,
lat_raster,
output_geo=cimis_geo,
output_proj=cimis_proj)
logging.debug(' {}'.format(lon_raster))
drigo.array_to_raster(lon_array,
lon_raster,
output_geo=cimis_geo,
output_proj=cimis_proj)
# Compute DEM raster
if overwrite_flag or not os.path.isfile(elev_raster):
logging.info('\nCIMIS DEM')
logging.debug(' Downloading GMTED2010 DEM')
logging.debug(" {}".format(elev_full_url))
logging.debug(" {}".format(elev_full_zip))
if overwrite_flag or not os.path.isfile(elev_full_zip):
_utils.url_download(elev_full_url, elev_full_zip)
# Uncompress '.gz' file to a new file
logging.debug(' Uncompressing')
logging.debug(' {}'.format(elev_full_raster))
if overwrite_flag or not os.path.isfile(elev_full_raster):
try:
with zipfile.ZipFile(elev_full_zip, "r") as z:
z.extractall(ancillary_ws)
except:
logging.error(" ERROR EXTRACTING FILE")
os.remove(elev_full_zip)
# Get the extent and cellsize from the mask
logging.debug(' Projecting to CIMIS grid')
cimis_cs = drigo.raster_path_cellsize(mask_raster)[0]
cimis_extent = drigo.raster_path_extent(mask_raster)
logging.debug(' Extent: {}'.format(cimis_extent))
logging.debug(' Cellsize: {}'.format(cimis_cs))
logging.info(' {}'.format(mask_ascii))
if overwrite_flag and os.path.isfile(elev_raster):
subprocess.call(['gdalmanage', 'delete', elev_raster])
if not os.path.isfile(elev_raster):
subprocess.call([
'gdalwarp', '-r', 'average', '-t_srs', cimis_proj4, '-te',
str(cimis_extent.xmin),
str(cimis_extent.ymin),
str(cimis_extent.xmax),
str(cimis_extent.ymax), '-tr',
str(cimis_cs),
str(cimis_cs), '-of', 'HFA', '-co', 'COMPRESSED=TRUE',
elev_full_raster, elev_raster
],
cwd=ancillary_ws)
logging.debug('\nScript Complete')
def main(cimis_ws=os.getcwd(),
gridmet_ws=None,
ancillary_ws=os.getcwd(),
etr_flag=False,
eto_flag=False,
start_date=None,
end_date=None,
stats_flag=True,
overwrite_flag=False):
"""Fill missing CIMIS days with projected data from GRIDMET
Parameters
----------
cimis_ws : str
Root folder path of CIMIS data.
gridmet_ws : str
Root folder path of GRIDMET data.
ancillary_ws : str
Folder of ancillary rasters.
etr_flag : bool, optional
If True, compute alfalfa reference ET (ETr).
eto_flag : bool, optional
If True, compute grass reference ET (ETo).
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
Notes
-----
Currently missing (CGM 2014-08-15)
2010-11-16 -> 2010-11-23
"""
logging.info('\nFilling CIMIS with GRIDMET')
cimis_re = re.compile(
'(?P<VAR>etr)_(?P<YYYY>\d{4})_daily_(?P<GRID>\w+).img$')
# gridmet_re = re.compile(
# '(?P<VAR>ppt)_(?P<YYY>\d{4})_daily_(?P<GRID>\w+).img$')
gridmet_fmt = 'etr_{}_daily_gridmet.img'
# Compute ETr and/or ETo
if not etr_flag and not eto_flag:
logging.info(' ETo/ETr flag(s) not set, defaulting to ETr')
etr_flag = True
logging.debug(' CIMIS: {}'.format(cimis_ws))
logging.debug(' GRIDMET: {}'.format(gridmet_ws))
# If a date is not set, process 2017
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2017, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2017, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# Get GRIDMET spatial reference and cellsize from elevation raster
# gridmet_elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img')
# Get CIMIS spatial reference and cellsize from mask raster
cimis_mask_raster = os.path.join(ancillary_ws, 'cimis_mask.img')
# Resample type
# 0 = GRA_NearestNeighbour, Nearest neighbour (select on one input pixel)
# 1 = GRA_Bilinear,Bilinear (2x2 kernel)
# 2 = GRA_Cubic, Cubic Convolution Approximation (4x4 kernel)
# 3 = GRA_CubicSpline, Cubic B-Spline Approximation (4x4 kernel)
# 4 = GRA_Lanczos, Lanczos windowed sinc interpolation (6x6 kernel)
# 5 = GRA_Average, Average (computes the average of all non-NODATA contributing pixels)
# 6 = GRA_Mode, Mode (selects the value which appears most often of all the sampled points)
resample_type = gdal.GRA_Bilinear
# ETo/ETr workspaces
cimis_eto_ws = os.path.join(cimis_ws, 'eto')
cimis_etr_ws = os.path.join(cimis_ws, 'etr')
gridmet_eto_ws = os.path.join(gridmet_ws, 'eto')
gridmet_etr_ws = os.path.join(gridmet_ws, 'etr')
# This allows GDAL to throw Python Exceptions
# gdal.UseExceptions()
# mem_driver = gdal.GetDriverByName('MEM')
# Get CIMIS grid properties from mask
logging.info('\nCIMIS Properties')
cimis_mask_ds = gdal.Open(cimis_mask_raster)
cimis_osr = drigo.raster_ds_osr(cimis_mask_ds)
cimis_proj = drigo.osr_proj(cimis_osr)
cimis_cs = drigo.raster_ds_cellsize(cimis_mask_ds, x_only=True)
cimis_extent = drigo.raster_ds_extent(cimis_mask_ds)
cimis_geo = cimis_extent.geo(cimis_cs)
cimis_mask_ds = None
logging.debug(' Projection: {}'.format(cimis_proj))
logging.debug(' Cellsize: {}'.format(cimis_cs))
logging.debug(' Geo: {}'.format(cimis_geo))
logging.debug(' Extent: {}'.format(cimis_extent))
# Read the CIMIS mask array if present
cimis_mask, cimis_mask_nodata = drigo.raster_to_array(cimis_mask_raster)
cimis_mask = cimis_mask != cimis_mask_nodata
# # Get extent/geo from elevation raster
# logging.info('\nGRIDMET Properties')
# gridmet_ds = gdal.Open(gridmet_elev_raster)
# gridmet_osr = drigo.raster_ds_osr(gridmet_ds)
# gridmet_proj = drigo.osr_proj(gridmet_osr)
# gridmet_cs = drigo.raster_ds_cellsize(gridmet_ds, x_only=True)
# gridmet_full_extent = drigo.raster_ds_extent(gridmet_ds)
# gridmet_full_geo = gridmet_full_extent.geo(gridmet_cs)
# gridmet_x, gridmet_y = gridmet_full_extent.origin()
# gridmet_ds = None
# logging.debug(' Projection: {}'.format(gridmet_proj))
# logging.debug(' Cellsize: {}'.format(gridmet_cs))
# logging.debug(' Geo: {}'.format(gridmet_full_geo))
# logging.debug(' Extent: {}'.format(gridmet_full_extent))
# # Project CIMIS extent to the GRIDMET spatial reference
# logging.info('\nGet CIMIS extent in GRIDMET spat. ref.')
# gridmet_sub_extent = drigo.project_extent(
# cimis_extent, cimis_osr, gridmet_osr, cimis_cs)
# gridmet_sub_extent.buffer_extent(4 * gridmet_cs)
# gridmet_sub_extent.adjust_to_snap(
# 'EXPAND', gridmet_x, gridmet_y, gridmet_cs)
# gridmet_sub_geo = gridmet_sub_extent.geo(gridmet_cs)
# logging.debug(' Geo: {}'.format(gridmet_sub_geo))
# logging.debug(' Extent: {}'.format(gridmet_sub_extent))
# Process Missing ETo
if eto_flag:
logging.info('\nETo')
for cimis_name in sorted(os.listdir(cimis_eto_ws)):
logging.debug("\n{}".format(cimis_name))
cimis_match = cimis_re.match(cimis_name)
if not cimis_match:
logging.debug(' Regular expression didn\'t match, skipping')
continue
year = int(cimis_match.group('YYYY'))
logging.info(" {}".format(str(year)))
if start_dt is not None and year < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year > end_dt.year:
logging.debug(' After end date, skipping')
continue
cimis_path = os.path.join(cimis_eto_ws, cimis_name)
gridmet_path = os.path.join(gridmet_eto_ws,
gridmet_fmt.format(str(year)))
if not os.path.isfile(gridmet_path):
logging.debug(' GRIDMET raster does not exist, skipping')
continue
if not os.path.isfile(cimis_path):
logging.error(' CIMIS raster does not exist, skipping')
continue
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year, 1, 1),
dt.datetime(year + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
continue
elif end_dt is not None and date_dt > end_dt:
continue
doy = int(date_dt.strftime('%j'))
# Look for arrays that don't have data
eto_array = drigo.raster_to_array(cimis_path,
band=doy,
return_nodata=False)
if np.any(np.isfinite(eto_array)):
logging.debug(' {} - no missing data, skipping'.format(
date_dt.strftime('%Y-%m-%d')))
continue
else:
logging.info(' {}'.format(date_dt.strftime('%Y-%m-%d')))
# # This is much faster but doesn't apply the CIMIS mask
# # Create an in memory dataset of the full ETo array
# eto_full_rows, eto_full_cols = eto_full_array[:,:,doy_i].shape
# eto_full_type, eto_full_nodata = numpy_to_gdal_type(np.float32)
# eto_full_ds = mem_driver.Create(
# '', eto_full_cols, eto_full_rows, 1, eto_full_type)
# eto_full_ds.SetProjection(gridmet_proj)
# eto_full_ds.SetGeoTransform(gridmet_full_geo)
# eto_full_band = eto_full_ds.GetRasterBand(1)
# # eto_full_band.Fill(eto_full_nodata)
# eto_full_band.SetNoDataValue(eto_full_nodata)
# eto_full_band.WriteArray(eto_full_array[:,:,doy_i], 0, 0)
#
# # Extract the subset
# eto_sub_array, eto_sub_nodata = drigo.raster_ds_to_array(
# eto_full_ds, 1, gridmet_sub_extent)
# eto_sub_rows, eto_sub_cols = eto_sub_array.shape
# eto_full_ds = None
#
# # Create projected raster
# eto_sub_ds = mem_driver.Create(
# '', eto_sub_cols, eto_sub_rows, 1, eto_full_type)
# eto_sub_ds.SetProjection(gridmet_proj)
# eto_sub_ds.SetGeoTransform(gridmet_sub_geo)
# eto_sub_band = eto_sub_ds.GetRasterBand(1)
# eto_sub_band.Fill(eto_sub_nodata)
# eto_sub_band.SetNoDataValue(eto_sub_nodata)
# eto_sub_band.WriteArray(eto_sub_array, 0, 0)
# eto_sub_ds.FlushCache()
#
# # Project input DEM to CIMIS spat. ref.
# drigo.project_raster_ds(
# eto_sub_ds, gridmet_path, resample_type,
# env.snap_proj, env.cellsize, cimis_extent)
# eto_sub_ds = None
# Extract the subset
gridmet_ds = gdal.Open(gridmet_path)
gridmet_extent = drigo.raster_ds_extent(gridmet_ds)
gridmet_cs = drigo.raster_ds_cellsize(gridmet_ds, x_only=True)
gridmet_osr = drigo.raster_ds_osr(gridmet_ds)
eto_full_array = drigo.raster_ds_to_array(gridmet_ds,
band=doy,
return_nodata=False)
gridmet_ds = None
# Get the projected subset of the full ETo array
# This is slower than projecting the subset above
eto_sub_array = drigo.project_array(eto_full_array,
resample_type, gridmet_osr,
gridmet_cs, gridmet_extent,
cimis_osr, cimis_cs,
cimis_extent)
# Save the projected array
drigo.array_to_comp_raster(eto_sub_array,
cimis_path,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# eto_sub_array, output_path, output_geo=cimis_geo,
# output_proj=cimis_proj, stats_flag=False)
# drigo.array_to_raster(
# eto_sub_array, output_path,
# output_geo=cimis_geo, output_proj=cimis_proj,
# mask_array=cimis_mask, stats_flag=False)
del eto_sub_array, eto_full_array
if stats_flag:
drigo.raster_statistics(cimis_path)
# Process Missing ETr
if etr_flag:
logging.info('\nETr')
for cimis_name in sorted(os.listdir(cimis_etr_ws)):
cimis_match = cimis_re.match(cimis_name)
if not cimis_match:
continue
year = int(cimis_match.group('YYYY'))
if start_dt is not None and year < start_dt.year:
continue
elif end_dt is not None and year > end_dt.year:
continue
logging.info("{}".format(str(year)))
cimis_path = os.path.join(cimis_etr_ws, cimis_name)
gridmet_path = os.path.join(gridmet_etr_ws,
gridmet_fmt.format(str(year)))
if not os.path.isfile(gridmet_path):
continue
if not os.path.isfile(cimis_path):
logging.error(' CIMIS raster does not exist')
continue
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year, 1, 1),
dt.datetime(year + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
continue
elif end_dt is not None and date_dt > end_dt:
continue
doy = int(date_dt.strftime('%j'))
# Look for arrays that don't have data
etr_array = drigo.raster_to_array(cimis_path,
band=doy,
return_nodata=False)
if np.any(np.isfinite(etr_array)):
logging.debug(' {} - skipping'.format(
date_dt.strftime('%Y-%m-%d')))
continue
else:
logging.info(' {}'.format(date_dt.strftime('%Y-%m-%d')))
# Extract the subset
gridmet_ds = gdal.Open(gridmet_path)
gridmet_extent = drigo.raster_ds_extent(gridmet_ds)
gridmet_cs = drigo.raster_ds_cellsize(gridmet_ds, x_only=True)
gridmet_osr = drigo.raster_ds_osr(gridmet_ds)
etr_full_array = drigo.raster_ds_to_array(gridmet_ds,
band=doy,
return_nodata=False)
gridmet_ds = None
# Get the projected subset of the full ETr array
# This is slower than projecting the subset
etr_sub_array = drigo.project_array(etr_full_array,
resample_type, gridmet_osr,
gridmet_cs, gridmet_extent,
cimis_osr, cimis_cs,
cimis_extent)
# # Save the projected array
drigo.array_to_comp_raster(etr_sub_array,
cimis_path,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# etr_sub_array, output_path,
# output_geo=cimis_geo, output_proj=cimis_proj,
# mask_array=cimis_mask, stats_flag=False)
del etr_sub_array, etr_full_array
if stats_flag:
drigo.raster_statistics(cimis_path)
logging.debug('\nScript Complete')
def load_year_array_func(input_ws,
input_re,
date_list,
mask_osr,
mask_cs,
mask_extent,
name='ETr',
return_geo_array=True):
"""Load
Parameters
----------
input_ws : str
input_re
date_list : list
output_osr
output_cs : float
output_extent
name : str
return_geo_array : bool
If True, return array geo-spatial properties (the default is True).
Returns
-------
ndarray
"""
logging.info('\n{}'.format(name))
logging.debug(' {} workspace: {}'.format(name, input_ws))
year_str_list = sorted(
list(set([date.strftime('%Y') for date in date_list])))
if not os.path.isdir(input_ws):
logging.error('\nERROR: The {} folder does not exist:\n {}'.format(
name, input_ws))
sys.exit()
input_dict = {
input_match.group('YYYY'): os.path.join(input_ws, input_name)
for input_name in os.listdir(os.path.join(input_ws))
for input_match in [input_re.match(input_name)]
if (input_match and input_match.group('YYYY')
and input_match.group('YYYY') in year_str_list)
}
if not input_dict:
logging.error((' No {0} files found in {1} for {2}\n'
' The {0} year folder may be empty or the regular '
'expression is invalid\n Exiting').format(
name, input_ws, ', '.join(year_str_list)))
sys.exit()
# Assume all rasters have same projection, cellsize, and snap
for date_obj in date_list:
try:
input_path = input_dict[date_obj.strftime('%Y')]
break
except KeyError:
logging.debug(' {} raster for date {} does not exist'.format(
name, date_obj.strftime('%Y%m%d')))
sys.exit()
input_ds = gdal.Open(input_path, 0)
input_osr = drigo.raster_ds_osr(input_ds)
# input_proj = drigo.osr_proj(input_osr)
input_cs = drigo.raster_ds_cellsize(input_ds, x_only=True)
input_x, input_y = drigo.raster_ds_origin(input_ds)
input_ds = None
# Get mask extent in the original spat. ref.
output_extent = drigo.project_extent(mask_extent, mask_osr, input_osr,
mask_cs)
output_extent.adjust_to_snap('EXPAND', input_x, input_y, input_cs)
output_rows, output_cols = output_extent.shape(cs=input_cs)
# Initialize the common array
output_array = np.full((len(date_list), output_rows, output_cols), np.nan,
np.float32)
# Read in the raster for each date
for date_i, date_obj in enumerate(date_list):
try:
input_path = input_dict[date_obj.strftime('%Y')]
except KeyError:
logging.debug(' {} - {} raster does not exist'.format(
date_obj.strftime('%Y%m%d'), name))
continue
output_array[date_i, :, :] = drigo.raster_to_array(
input_path,
band=int(date_obj.strftime('%j')),
mask_extent=output_extent,
return_nodata=False,
)
if return_geo_array:
return output_array, input_osr, input_cs, output_extent
else:
return output_array
def pixel_rating(image_ws,
ini_path,
bs=None,
stats_flag=False,
overwrite_flag=None):
"""Calculate pixel rating
Parameters
----------
image_ws : str
Image folder path.
ini_path : str
Pixel regions config file path.
bs : int, optional
Processing block size (the default is None). If set, this blocksize
parameter will be used instead of the value in the INI file.
stats_flag : bool, optional
if True, compute raster statistics (the default is False).
ovewrite_flag : bool, optional
If True, overwrite existing files (the default is None).
Returns
-------
None
"""
logging.info('Generating suggested hot/cold pixel regions')
log_fmt = ' {:<18s} {}'
env = drigo.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore')
# # Check that image_ws is valid
# image_id_re = re.compile(
# '^(LT04|LT05|LE07|LC08)_(?:\w{4})_(\d{3})(\d{3})_'
# '(\d{4})(\d{2})(\d{2})_(?:\d{8})_(?:\d{2})_(?:\w{2})$')
# if not os.path.isdir(image_ws) or not image_id_re.match(image_id):
# logging.error('\nERROR: Image folder is invalid or does not exist\n')
# return False
# Folder Paths
region_ws = os.path.join(image_ws, 'PIXEL_REGIONS')
# Open config file
config = dripy.open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
# Arrays are processed by block
if bs is None:
bs = dripy.read_param('block_size', 1024, config)
logging.info(log_fmt.format('Block Size:', bs))
# Raster pyramids/statistics
pyramids_flag = dripy.read_param('pyramids_flag', False, config)
if pyramids_flag:
gdal.SetConfigOption('HFA_USE_RRD', 'YES')
if stats_flag is None:
stats_flag = dripy.read_param('statistics_flag', False, config)
# Overwrite
if overwrite_flag is None:
overwrite_flag = dripy.read_param('overwrite_flag', True, config)
# Check that common_area raster exists
if not os.path.isfile(image.common_area_raster):
logging.error(
'\nERROR: A common area raster was not found.' +
'\nERROR: Please rerun prep tool to build these files.\n' +
' {}\n'.format(image.common_area_raster))
sys.exit()
# Use common_area to set mask parameters
common_ds = gdal.Open(image.common_area_raster)
# env.mask_proj = raster_ds_proj(common_ds)
env.mask_geo = drigo.raster_ds_geo(common_ds)
env.mask_rows, env.mask_cols = drigo.raster_ds_shape(common_ds)
env.mask_extent = drigo.geo_extent(env.mask_geo, env.mask_rows,
env.mask_cols)
env.mask_array = drigo.raster_ds_to_array(common_ds)[0]
env.mask_path = image.common_area_raster
env.snap_osr = drigo.raster_path_osr(image.common_area_raster)
env.snap_proj = env.snap_osr.ExportToWkt()
env.cellsize = drigo.raster_path_cellsize(image.common_area_raster)[0]
common_ds = None
logging.debug(' {:<18s} {}'.format('Mask Extent:', env.mask_extent))
# Read Pixel Regions config file
# Currently there is no code to support applying an NLCD mask
apply_nlcd_mask = False
# apply_nlcd_mask = dripy.read_param('apply_nlcd_mask', False, config)
apply_cdl_ag_mask = dripy.read_param('apply_cdl_ag_mask', False, config)
apply_field_mask = dripy.read_param('apply_field_mask', False, config)
apply_ndwi_mask = dripy.read_param('apply_ndwi_mask', True, config)
apply_ndvi_mask = dripy.read_param('apply_ndvi_mask', True, config)
# Currently the code to apply a study area mask is commented out
# apply_study_area_mask = dripy.read_param(
# 'apply_study_area_mask', False, config)
albedo_rating_flag = dripy.read_param('albedo_rating_flag', True, config)
nlcd_rating_flag = dripy.read_param('nlcd_rating_flag', True, config)
ndvi_rating_flag = dripy.read_param('ndvi_rating_flag', True, config)
ts_rating_flag = dripy.read_param('ts_rating_flag', True, config)
ke_rating_flag = dripy.read_param('ke_rating_flag', False, config)
# if apply_study_area_mask:
# study_area_path = config.get('INPUTS', 'study_area_path')
if apply_nlcd_mask or nlcd_rating_flag:
nlcd_raster = config.get('INPUTS', 'landuse_raster')
if apply_cdl_ag_mask:
cdl_ag_raster = config.get('INPUTS', 'cdl_ag_raster')
cdl_buffer_cells = dripy.read_param('cdl_buffer_cells', 0, config)
cdl_ag_eroded_name = dripy.read_param('cdl_ag_eroded_name',
'cdl_ag_eroded_{}.img', config)
if apply_field_mask:
field_raster = config.get('INPUTS', 'fields_raster')
cold_rating_pct = dripy.read_param('cold_percentile', 99, config)
hot_rating_pct = dripy.read_param('hot_percentile', 99, config)
# min_cold_rating_score = dripy.read_param('min_cold_rating_score', 0.3, config)
# min_hot_rating_score = dripy.read_param('min_hot_rating_score', 0.3, config)
ts_bin_count = int(dripy.read_param('ts_bin_count', 10, config))
if 100 % ts_bin_count != 0:
logging.warning(
'WARNING: ts_bins_count of {} is not a divisor ' +
'of 100. Using default ts_bins_count = 4'.format(ts_bin_count))
ts_bin_count = 10
bin_size = 1. / (ts_bin_count - 1)
hot_rating_values = np.arange(0., 1. + bin_size, step=bin_size)
cold_rating_values = hot_rating_values[::-1]
# Input raster paths
r_fmt = '.img'
if 'Landsat' in image.type:
albedo_raster = image.albedo_sur_raster
ndvi_raster = image.ndvi_toa_raster
ndwi_raster = image.ndwi_toa_raster
ts_raster = image.ts_raster
ke_raster = image.ke_raster
# Check config file input paths
# if apply_study_area_mask and not os.path.isfile(study_area_path):
# logging.error(
# ('\nERROR: The study area shapefile {} does ' +
# 'not exist\n').format(study_area_path))
# sys.exit()
if ((apply_nlcd_mask or nlcd_rating_flag)
and not os.path.isfile(nlcd_raster)):
logging.error(('\nERROR: The NLCD raster {} does ' +
'not exist\n').format(nlcd_raster))
sys.exit()
if apply_cdl_ag_mask and not os.path.isfile(cdl_ag_raster):
logging.error(('\nERROR: The CDL Ag raster {} does ' +
'not exist\n').format(cdl_ag_raster))
sys.exit()
if apply_field_mask and not os.path.isfile(field_raster):
logging.error(('\nERROR: The field raster {} does ' +
'not exist\n').format(field_raster))
sys.exit()
if (not (isinstance(cold_rating_pct,
(int, float)) and (0 <= cold_rating_pct <= 100))):
logging.error(
'\nERROR: cold_percentile must be a value between 0 and 100\n')
sys.exit()
if (not (isinstance(hot_rating_pct,
(int, float)) and (0 <= hot_rating_pct <= 100))):
logging.error(
'\nERROR: hot_percentile must be a value between 0 and 100\n')
sys.exit()
# Set raster names
raster_dict = dict()
# Output Rasters
raster_dict['region_mask'] = os.path.join(region_ws, 'region_mask' + r_fmt)
raster_dict['cold_rating'] = os.path.join(region_ws,
'cold_pixel_rating' + r_fmt)
raster_dict['hot_rating'] = os.path.join(region_ws,
'hot_pixel_rating' + r_fmt)
raster_dict['cold_sugg'] = os.path.join(region_ws,
'cold_pixel_suggestion' + r_fmt)
raster_dict['hot_sugg'] = os.path.join(region_ws,
'hot_pixel_suggestion' + r_fmt)
# Read pixel region raster flags
save_dict = dict()
save_dict['region_mask'] = dripy.read_param('save_region_mask_flag', False,
config)
save_dict['cold_rating'] = dripy.read_param('save_rating_rasters_flag',
False, config)
save_dict['hot_rating'] = dripy.read_param('save_rating_rasters_flag',
False, config)
save_dict['cold_sugg'] = dripy.read_param('save_suggestion_rasters_flag',
True, config)
save_dict['hot_sugg'] = dripy.read_param('save_suggestion_rasters_flag',
True, config)
# Output folder
if not os.path.isdir(region_ws):
os.mkdir(region_ws)
# Remove existing files if necessary
region_ws_file_list = [
os.path.join(region_ws, item) for item in os.listdir(region_ws)
]
if overwrite_flag and region_ws_file_list:
for raster_path in raster_dict.values():
if raster_path in region_ws_file_list:
dripy.remove_file(raster_path)
# Check scene specific input paths
if apply_ndwi_mask and not os.path.isfile(ndwi_raster):
logging.error(
'ERROR: NDWI raster does not exist\n {}'.format(ndwi_raster))
sys.exit()
elif apply_ndvi_mask and not os.path.isfile(ndvi_raster):
logging.error(
'ERROR: NDVI raster does not exist\n {}'.format(ndvi_raster))
sys.exit()
elif ke_rating_flag and not os.path.isfile(ke_raster):
logging.error(
('ERROR: The Ke raster does not exist\n {}').format(ke_raster))
sys.exit()
# Remove existing and build new empty rasters if necessary
# If processing by block, rating rasters must be built
logging.debug('\nBuilding empty rasters')
for name, save_flag in sorted(save_dict.items()):
if save_flag and 'rating' in name:
drigo.build_empty_raster(raster_dict[name], 1, np.float32)
elif save_flag:
drigo.build_empty_raster(raster_dict[name],
1,
np.uint8,
output_nodata=0)
if apply_cdl_ag_mask:
logging.info('Building CDL ag mask')
cdl_array = drigo.raster_to_array(cdl_ag_raster,
mask_extent=env.mask_extent,
return_nodata=False)
if cdl_buffer_cells > 0:
logging.info(' Eroding CDL by {} cells'.format(cdl_buffer_cells))
structure_array = np.ones((cdl_buffer_cells, cdl_buffer_cells),
dtype=np.int)
# Deadbeef - This could blow up in memory on bigger rasters
cdl_array = ndimage.binary_erosion(
cdl_array, structure_array).astype(structure_array.dtype)
cdl_ag_eroded_raster = os.path.join(
image.support_ws, cdl_ag_eroded_name.format(cdl_buffer_cells))
drigo.array_to_raster(cdl_array,
cdl_ag_eroded_raster,
output_geo=env.mask_geo,
output_proj=env.snap_proj,
mask_array=env.mask_array,
output_nodata=0,
stats_flag=False)
cdl_array = None
del cdl_array
# Build region mask
logging.debug('Building region mask')
region_mask = np.copy(env.mask_array).astype(np.bool)
if apply_field_mask:
field_mask, field_nodata = drigo.raster_to_array(
field_raster, mask_extent=env.mask_extent, return_nodata=True)
region_mask &= field_mask != field_nodata
del field_mask, field_nodata
if apply_ndwi_mask:
ndwi_array = drigo.raster_to_array(ndwi_raster,
1,
mask_extent=env.mask_extent,
return_nodata=False)
region_mask &= ndwi_array > 0.0
del ndwi_array
if apply_ndvi_mask:
ndvi_array = drigo.raster_to_array(ndvi_raster,
1,
mask_extent=env.mask_extent,
return_nodata=False)
region_mask &= ndvi_array > 0.12
del ndvi_array
if apply_cdl_ag_mask:
cdl_array, cdl_nodata = drigo.raster_to_array(
cdl_ag_eroded_raster,
mask_extent=env.mask_extent,
return_nodata=True)
region_mask &= cdl_array != cdl_nodata
del cdl_array, cdl_nodata
if save_dict['region_mask']:
drigo.array_to_raster(region_mask,
raster_dict['region_mask'],
stats_flag=False)
# Initialize rating arrays
# This needs to be done before the ts_rating if block
cold_rating_array = np.ones(env.mask_array.shape, dtype=np.float32)
hot_rating_array = np.ones(env.mask_array.shape, dtype=np.float32)
cold_rating_array[~region_mask] = np.nan
hot_rating_array[~region_mask] = np.nan
# Temperature pixel rating - grab the max and min value for the entire
# Ts image in a memory safe way by using gdal_common blocks
# The following is a percentile based approach
if ts_rating_flag:
logging.debug('Computing Ts percentile rating')
ts_array = drigo.raster_to_array(ts_raster,
mask_extent=env.mask_extent,
return_nodata=False)
ts_array[~region_mask] = np.nan
percentiles = list(
range(0, (100 + ts_bin_count), int(100 / ts_bin_count)))
ts_score_value = 1. / (ts_bin_count - 1)
hot_rating_values = np.arange(0, (1. + ts_score_value),
step=ts_score_value)[:ts_bin_count]
cold_rating_values = hot_rating_values[::-1]
ts_percentile_array = np.percentile(ts_array[np.isfinite(ts_array)],
percentiles)
# Deprecated - per SciPy help this function will become obsolete
# ts_percentile_array = stats.scoreatpercentile(
# ts_array[np.isfinite(ts_array)], percentiles)
for bins_i in range(len(ts_percentile_array))[:-1]:
bool_array = ((ts_array > ts_percentile_array[bins_i]) &
(ts_array <= ts_percentile_array[bins_i + 1]))
cold_rating_array[bool_array] = cold_rating_values[bins_i]
hot_rating_array[bool_array] = hot_rating_values[bins_i]
# drigo.array_to_raster(cold_rating_array, raster_dict['cold_rating'])
# drigo.array_to_raster(hot_rating_array, raster_dict['hot_rating'])
# Cleanup
del ts_array, ts_percentile_array
del cold_rating_values, hot_rating_values
del ts_score_value, percentiles
# Process by block
logging.info('\nProcessing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(env.mask_cols,
env.mask_rows))
for b_i, b_j in drigo.block_gen(env.mask_rows, env.mask_cols, bs):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = drigo.array_to_block(env.mask_array, b_i, b_j,
bs).astype(np.bool)
# block_nodata_mask = ~block_data_mask
block_rows, block_cols = block_data_mask.shape
block_geo = drigo.array_offset_geo(env.mask_geo, b_j, b_i)
block_extent = drigo.geo_extent(block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
# logging.debug(' Block extent: {}'.format(block_extent))
# logging.debug(' Block geo: {}'.format(block_geo))
# Don't skip empty blocks since block rating needs to be written
# back to the array at the end of the block loop
block_region_mask = drigo.array_to_block(region_mask, b_i, b_j, bs)
if not np.any(block_region_mask):
logging.debug(' Empty block')
block_empty_flag = True
else:
block_empty_flag = False
# New style continuous pixel weighting
cold_rating_block = drigo.array_to_block(cold_rating_array, b_i, b_j,
bs)
hot_rating_block = drigo.array_to_block(hot_rating_array, b_i, b_j, bs)
# Rating arrays already have region_mask set
# cold_rating_block = np.ones(block_region_mask.shape, dtype=np.float32)
# hot_rating_block = np.ones(block_region_mask.shape, dtype=np.float32)
# cold_rating_block[~block_region_mask] = np.nan
# hot_rating_block[~block_region_mask] = np.nan
# del block_region_mask
if ndvi_rating_flag and not block_empty_flag:
# NDVI based rating
ndvi_array = drigo.raster_to_array(ndvi_raster,
1,
mask_extent=block_extent,
return_nodata=False)
# Don't let NDVI be negative
ndvi_array.clip(0., 0.833, out=ndvi_array)
# ndvi_array.clip(0.001, 0.833, out=ndvi_array)
cold_rating_block *= ndvi_array
cold_rating_block *= 1.20
ndvi_mask = (ndvi_array > 0)
# DEADBEEF - Can this calculation be masked to only NDVI > 0?
ndvi_mask = ndvi_array > 0
hot_rating_block[ndvi_mask] *= stats.norm.pdf(
np.log(ndvi_array[ndvi_mask]), math.log(0.15), 0.5)
hot_rating_block[ndvi_mask] *= 1.25
del ndvi_mask
# hot_rating_block *= stats.norm.pdf(
# np.log(ndvi_array), math.log(0.15), 0.5)
# hot_rating_block *= 1.25
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del ndvi_array
if albedo_rating_flag and not block_empty_flag:
# Albdo based rating
albedo_array = drigo.raster_to_array(albedo_raster,
1,
mask_extent=block_extent,
return_nodata=False)
albedo_cold_pdf = stats.norm.pdf(albedo_array, 0.21, 0.03)
albedo_hot_pdf = stats.norm.pdf(albedo_array, 0.21, 0.06)
del albedo_array
cold_rating_block *= albedo_cold_pdf
cold_rating_block *= 0.07
hot_rating_block *= albedo_hot_pdf
hot_rating_block *= 0.15
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del albedo_cold_pdf, albedo_hot_pdf
if nlcd_rating_flag and not block_empty_flag:
# NLCD based weighting, this could be CDL instead?
nlcd_array = nlcd_rating(
drigo.raster_to_array(nlcd_raster,
1,
mask_extent=block_extent,
return_nodata=False))
cold_rating_block *= nlcd_array
hot_rating_block *= nlcd_array
del nlcd_array
if ke_rating_flag and not block_empty_flag:
# SWB Ke based rating
ke_array = drigo.raster_to_array(ke_raster,
1,
mask_extent=block_extent,
return_nodata=False)
# Don't let Ke be negative
ke_array.clip(0., 1., out=ke_array)
# Assumption, lower Ke is better for selecting the hot pixel
# As the power (2) decreases and approaches 1,
# the relationship gets more linear
# cold_rating_block *= (1 - ke_array ** 2)
# hot_rating_block *= (1 - ke_array ** 1.5)
# Linear inverse
# cold_rating_block *= (1. - ke_array)
hot_rating_block *= (1. - ke_array)
# cold_rating_block.clip(0., 1., out=cold_rating_block)
# hot_rating_block.clip(0., 1., out=hot_rating_block)
del ke_array
# Clearness
# clearness = 1.0
# cold_rating *= clearness
# hot_rating *= clearness
# Reset nan values
# cold_rating_block[~region_mask] = np.nan
# hot_rating_block[~region_mask] = np.nan
# Save rating values
cold_rating_array = drigo.block_to_array(cold_rating_block,
cold_rating_array, b_i, b_j,
bs)
hot_rating_array = drigo.block_to_array(hot_rating_block,
hot_rating_array, b_i, b_j, bs)
# Save rating rasters
if save_dict['cold_rating']:
drigo.block_to_raster(cold_rating_block,
raster_dict['cold_rating'], b_i, b_j, bs)
if save_dict['hot_rating']:
drigo.block_to_raster(hot_rating_block, raster_dict['hot_rating'],
b_i, b_j, bs)
# Save rating values
cold_rating_array = drigo.block_to_array(cold_rating_block,
cold_rating_array, b_i, b_j,
bs)
hot_rating_array = drigo.block_to_array(hot_rating_block,
hot_rating_array, b_i, b_j, bs)
del cold_rating_block, hot_rating_block
# Select pixels above target percentile
# Only build suggestion arrays if saving
logging.debug('Building suggested pixel rasters')
if save_dict['cold_sugg']:
cold_rating_score = float(
np.percentile(cold_rating_array[np.isfinite(cold_rating_array)],
cold_rating_pct))
# Deprecated - per SciPy help this function will become obsolete
# cold_rating_score = float(stats.scoreatpercentile(
# cold_rating_array[np.isfinite(cold_rating_array)],
# cold_rating_pct))
# cold_rating_array, cold_rating_nodata = drigo.raster_to_array(
# raster_dict['cold_rating'], 1, mask_extent=env.mask_extent)
# if cold_rating_score < float(min_cold_rating_score):
# logging.error(('ERROR: The cold_rating_score ({}) is less ' +
# 'than the min_cold_rating_score ({})').format(
# cold_rating_score, min_cold_rating_score))
# sys.exit()
cold_sugg_mask = cold_rating_array >= cold_rating_score
drigo.array_to_raster(cold_sugg_mask,
raster_dict['cold_sugg'],
stats_flag=stats_flag)
logging.debug(' Cold Percentile: {}'.format(cold_rating_pct))
logging.debug(' Cold Score: {:.6f}'.format(cold_rating_score))
logging.debug(' Cold Pixels: {}'.format(np.sum(cold_sugg_mask)))
del cold_sugg_mask, cold_rating_array
if save_dict['hot_sugg']:
hot_rating_score = float(
np.percentile(hot_rating_array[np.isfinite(hot_rating_array)],
hot_rating_pct))
# Deprecated - per SciPy help this function will become obsolete
# hot_rating_score = float(stats.scoreatpercentile(
# hot_rating_array[np.isfinite(hot_rating_array)],
# hot_rating_pct))
# hot_rating_array, hot_rating_nodata = drigo.raster_to_array(
# raster_dict['hot_rating'], 1, mask_extent=env.mask_extent)
# if hot_rating_score < float(min_hot_rating_score):
# logging.error(('ERROR: The hot_rating_array ({}) is less ' +
# 'than the min_hot_rating_score ({})').format(
# hot_rating_array, min_hot_rating_score))
# sys.exit()
hot_sugg_mask = hot_rating_array >= hot_rating_score
drigo.array_to_raster(hot_sugg_mask,
raster_dict['hot_sugg'],
stats_flag=stats_flag)
logging.debug(' Hot Percentile: {}'.format(hot_rating_pct))
logging.debug(' Hot Score: {:.6f}'.format(hot_rating_score))
logging.debug(' Hot Pixels: {}'.format(np.sum(hot_sugg_mask)))
del hot_sugg_mask, hot_rating_array
# Raster Statistics
if stats_flag:
logging.info('Calculating Statistics')
for name, save_flag in save_dict.items():
if save_flag:
drigo.raster_statistics(raster_dict[name])
# Raster Pyramids
if pyramids_flag:
logging.info('Building Pyramids')
for name, save_flag in save_dict.items():
if save_flag:
drigo.raster_pyramids(raster_dict[name])
def main(start_dt,
end_dt,
netcdf_ws,
ancillary_ws,
output_ws,
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Extract DAYMET temperature
Parameters
----------
start_dt : datetime
Start date.
end_dt : datetime
End date.
netcdf_ws : str
Folder of DAYMET netcdf files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
extent_path : str, optional
File path defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting DAYMET vapor pressure')
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
# Get DAYMET spatial reference from an ancillary raster
mask_raster = os.path.join(ancillary_ws, 'daymet_mask.img')
elev_raster = os.path.join(ancillary_ws, 'daymet_elev.img')
daymet_re = re.compile('daymet_v3_(?P<VAR>\w+)_(?P<YEAR>\d{4})_na.nc4$')
# DAYMET band name dictionary
# daymet_band_dict = dict()
# daymet_band_dict['prcp'] = 'precipitation_amount'
# daymet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air'
# daymet_band_dict['sph'] = 'specific_humidity'
# daymet_band_dict['tmin'] = 'air_temperature'
# daymet_band_dict['tmax'] = 'air_temperature'
# Get extent/geo from mask raster
daymet_ds = gdal.Open(mask_raster)
daymet_osr = drigo.raster_ds_osr(daymet_ds)
daymet_proj = drigo.osr_proj(daymet_osr)
daymet_cs = drigo.raster_ds_cellsize(daymet_ds, x_only=True)
daymet_extent = drigo.raster_ds_extent(daymet_ds)
daymet_geo = daymet_extent.geo(daymet_cs)
daymet_x, daymet_y = daymet_extent.origin()
daymet_ds = None
logging.debug(' Projection: {}'.format(daymet_proj))
logging.debug(' Cellsize: {}'.format(daymet_cs))
logging.debug(' Geo: {}'.format(daymet_geo))
logging.debug(' Extent: {}'.format(daymet_extent))
logging.debug(' Origin: {} {}'.format(daymet_x, daymet_y))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
# Assume input extent is in decimal degrees
output_extent = drigo.project_extent(drigo.Extent(output_extent),
drigo.epsg_osr(4326), daymet_osr,
0.001)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if extent_path.lower().endswith('.shp'):
output_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
output_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
output_extent = drigo.project_extent(output_extent, extent_osr,
daymet_osr, extent_cs)
output_extent = drigo.intersect_extents([daymet_extent, output_extent])
output_extent.adjust_to_snap('EXPAND', daymet_x, daymet_y, daymet_cs)
output_geo = output_extent.geo(daymet_cs)
logging.debug(' Geo: {}'.format(output_geo))
logging.debug(' Extent: {}'.format(output_extent))
else:
output_extent = daymet_extent.copy()
output_geo = daymet_geo[:]
# output_shape = output_extent.shape(cs=daymet_cs)
xi, yi = drigo.array_geo_offsets(daymet_geo, output_geo, daymet_cs)
output_rows, output_cols = output_extent.shape(daymet_cs)
logging.debug(' Shape: {} {}'.format(output_rows, output_cols))
logging.debug(' Offsets: {} {} (x y)'.format(xi, yi))
# Read the elevation array
elev_array = drigo.raster_to_array(elev_raster,
mask_extent=output_extent,
return_nodata=False)
pair_array = refet.calcs._air_pressure_func(elev_array)
del elev_array
# Process each variable
input_var = 'vp'
output_var = 'ea'
logging.info("\nVariable: {}".format(input_var))
# Build output folder
var_ws = os.path.join(output_ws, output_var)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Process each file in the input workspace
for input_name in sorted(os.listdir(netcdf_ws)):
logging.debug("{}".format(input_name))
input_match = daymet_re.match(input_name)
if not input_match:
logging.debug(' Regular expression didn\'t match, skipping')
continue
elif input_match.group('VAR') != input_var:
logging.debug(' Variable didn\'t match, skipping')
continue
year_str = input_match.group('YEAR')
logging.info(" Year: {}".format(year_str))
year_int = int(year_str)
year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
# Build input file path
input_raster = os.path.join(netcdf_ws, input_name)
# if not os.path.isfile(input_raster):
# logging.debug(
# ' Input raster doesn\'t exist, skipping {}'.format(
# input_raster))
# continue
# Build output folder
output_year_ws = os.path.join(var_ws, year_str)
if not os.path.isdir(output_year_ws):
os.makedirs(output_year_ws)
# Read in the DAYMET NetCDF file
input_nc_f = netCDF4.Dataset(input_raster, 'r')
# logging.debug(input_nc_f.variables)
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year_int, 1, 1),
dt.datetime(year_int + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
logging.debug(' {} - before start date, skipping'.format(
date_dt.date()))
continue
elif end_dt is not None and date_dt > end_dt:
logging.debug(' {} - after end date, skipping'.format(
date_dt.date()))
continue
else:
logging.info(' {}'.format(date_dt.date()))
output_path = os.path.join(
output_year_ws,
'{}_{}_daymet.img'.format(output_var,
date_dt.strftime('%Y%m%d')))
if os.path.isfile(output_path):
logging.debug(' {}'.format(output_path))
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(output_path)
doy = int(date_dt.strftime('%j'))
doy_i = range(1, year_days + 1).index(doy)
# Arrays are being read as masked array with a fill value of -9999
# Convert to basic numpy array arrays with nan values
try:
input_ma = input_nc_f.variables[input_var][doy_i,
yi:yi + output_rows,
xi:xi + output_cols]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
input_nodata = float(input_ma.fill_value)
sph_array = input_ma.data.astype(np.float32)
sph_array[sph_array == input_nodata] = np.nan
# Compute ea [kPa] from specific humidity [kg/kg]
ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array)
# Save the array as 32-bit floats
drigo.array_to_raster(ea_array.astype(np.float32),
output_path,
output_geo=output_geo,
output_proj=daymet_proj,
stats_flag=stats_flag)
del input_ma, ea_array, sph_array
input_nc_f.close()
del input_nc_f
logging.debug('\nScript Complete')
def main(start_dt,
end_dt,
netcdf_ws,
ancillary_ws,
output_ws,
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Extract GRIDMET temperature
Parameters
----------
start_dt : datetime
Start date.
end_dt : datetime
End date.
netcdf_ws : str
Folder of GRIDMET netcdf files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
extent_path : str, optional
File path a raster defining the output extent.
output_extent : list, optional
Decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting GRIDMET vapor pressure')
logging.debug(' Start date: {}'.format(start_dt))
logging.debug(' End date: {}'.format(end_dt))
# Save GRIDMET lat, lon, and elevation arrays
elev_raster = os.path.join(ancillary_ws, 'gridmet_elev.img')
output_fmt = '{}_{}_daily_gridmet.img'
gridmet_re = re.compile('(?P<VAR>\w+)_(?P<YEAR>\d{4}).nc$')
# GRIDMET band name dictionary
gridmet_band_dict = dict()
gridmet_band_dict['pr'] = 'precipitation_amount'
gridmet_band_dict['srad'] = 'surface_downwelling_shortwave_flux_in_air'
gridmet_band_dict['sph'] = 'specific_humidity'
gridmet_band_dict['tmmn'] = 'air_temperature'
gridmet_band_dict['tmmx'] = 'air_temperature'
gridmet_band_dict['vs'] = 'wind_speed'
# Get extent/geo from elevation raster
gridmet_ds = gdal.Open(elev_raster)
gridmet_osr = drigo.raster_ds_osr(gridmet_ds)
gridmet_proj = drigo.osr_proj(gridmet_osr)
gridmet_cs = drigo.raster_ds_cellsize(gridmet_ds, x_only=True)
gridmet_extent = drigo.raster_ds_extent(gridmet_ds)
gridmet_full_geo = gridmet_extent.geo(gridmet_cs)
gridmet_x, gridmet_y = gridmet_extent.origin()
gridmet_ds = None
logging.debug(' Projection: {}'.format(gridmet_proj))
logging.debug(' Cellsize: {}'.format(gridmet_cs))
logging.debug(' Geo: {}'.format(gridmet_full_geo))
logging.debug(' Extent: {}'.format(gridmet_extent))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
gridmet_extent = drigo.Extent(output_extent)
gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y,
gridmet_cs)
gridmet_geo = gridmet_extent.geo(gridmet_cs)
logging.debug(' Geo: {}'.format(gridmet_geo))
logging.debug(' Extent: {}'.format(gridmet_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if not os.path.isfile(extent_path):
logging.error('\nThe extent object does not exist, exiting\n'
' {}'.format(extent_path))
return False
elif extent_path.lower().endswith('.shp'):
gridmet_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
gridmet_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
gridmet_extent = drigo.project_extent(gridmet_extent, extent_osr,
gridmet_osr, extent_cs)
gridmet_extent.adjust_to_snap('EXPAND', gridmet_x, gridmet_y,
gridmet_cs)
gridmet_geo = gridmet_extent.geo(gridmet_cs)
logging.debug(' Geo: {}'.format(gridmet_geo))
logging.debug(' Extent: {}'.format(gridmet_extent))
else:
gridmet_geo = gridmet_full_geo
# Get indices for slicing/clipping input arrays
g_i, g_j = drigo.array_geo_offsets(gridmet_full_geo,
gridmet_geo,
cs=gridmet_cs)
g_rows, g_cols = gridmet_extent.shape(cs=gridmet_cs)
# Flip row indices since GRIDMET arrays are flipped up/down
# Hard coding GRIDMET row count for now
row_a, row_b = 585 - (g_j + g_rows), 585 - g_j,
col_a, col_b = g_i, g_i + g_cols
# Read the elevation array
elev_array = drigo.raster_to_array(elev_raster,
mask_extent=gridmet_extent,
return_nodata=False)
pair_array = refet.calcs._air_pressure(elev_array)
del elev_array
# Process each variable
input_var = 'sph'
output_var = 'ea'
logging.info("\nVariable: {}".format(input_var))
# Build output folder
var_ws = os.path.join(output_ws, output_var)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Process each file in the input workspace
for input_name in sorted(os.listdir(netcdf_ws)):
input_match = gridmet_re.match(input_name)
if not input_match:
logging.debug("{}".format(input_name))
logging.debug(' Regular expression didn\'t match, skipping')
continue
elif input_match.group('VAR') != input_var:
logging.debug("{}".format(input_name))
logging.debug(' Variable didn\'t match, skipping')
continue
else:
logging.info("{}".format(input_name))
year_str = input_match.group('YEAR')
logging.info(" {}".format(year_str))
year_int = int(year_str)
year_days = int(dt.datetime(year_int, 12, 31).strftime('%j'))
if start_dt is not None and year_int < start_dt.year:
logging.debug(' Before start date, skipping')
continue
elif end_dt is not None and year_int > end_dt.year:
logging.debug(' After end date, skipping')
continue
# Build input file path
input_raster = os.path.join(netcdf_ws, input_name)
# if not os.path.isfile(input_raster):
# logging.debug(
# ' Input NetCDF doesn\'t exist, skipping {}'.format(
# input_raster))
# continue
# Create a single raster for each year with 365 bands
# Each day will be stored in a separate band
output_path = os.path.join(var_ws,
output_fmt.format(output_var, year_str))
logging.debug(' {}'.format(output_path))
if os.path.isfile(output_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(output_path)
drigo.build_empty_raster(output_path,
band_cnt=366,
output_dtype=np.float32,
output_proj=gridmet_proj,
output_cs=gridmet_cs,
output_extent=gridmet_extent,
output_fill_flag=True)
# Read in the GRIDMET NetCDF file
# Immediately clip input array to save memory
input_nc_f = netCDF4.Dataset(input_raster, 'r')
input_nc = input_nc_f.variables[
gridmet_band_dict[input_var]][:, row_a:row_b, col_a:col_b].copy()
input_nc = np.flip(input_nc, 1)
input_nc_f.close()
del input_nc_f
# A numpy array is returned when slicing a masked array
# if there are no masked pixels
# This is a hack to force the numpy array back to a masked array
if type(input_nc) != np.ma.core.MaskedArray:
input_nc = np.ma.core.MaskedArray(
input_nc, np.zeros(input_nc.shape, dtype=bool))
# Check all valid dates in the year
year_dates = _utils.date_range(dt.datetime(year_int, 1, 1),
dt.datetime(year_int + 1, 1, 1))
for date_dt in year_dates:
if start_dt is not None and date_dt < start_dt:
# logging.debug(' before start date, skipping')
continue
elif end_dt is not None and date_dt > end_dt:
# logging.debug(' after end date, skipping')
continue
logging.info(' {}'.format(date_dt.strftime('%Y_%m_%d')))
doy = int(date_dt.strftime('%j'))
doy_i = range(1, year_days + 1).index(doy)
# Arrays are being read as masked array with a fill value of -9999
# Convert to basic numpy array arrays with nan values
try:
input_full_ma = input_nc[doy_i, :, :]
except IndexError:
logging.info(' date not in netcdf, skipping')
continue
input_full_array = input_full_ma.data.astype(np.float32)
input_full_nodata = float(input_full_ma.fill_value)
input_full_array[input_full_array == input_full_nodata] = np.nan
# Since inputs are netcdf, need to create GDAL raster
# datasets in order to use gdal_common functions
# Create an in memory dataset of the full ETo array
input_full_ds = drigo.array_to_mem_ds(input_full_array,
output_geo=gridmet_geo,
output_proj=gridmet_proj)
# Then extract the subset from the in memory dataset
sph_array = drigo.raster_ds_to_array(input_full_ds,
1,
mask_extent=gridmet_extent,
return_nodata=False)
# Compute ea [kPa] from specific humidity [kg/kg]
ea_array = (sph_array * pair_array) / (0.622 + 0.378 * sph_array)
# Save the projected array as 32-bit floats
drigo.array_to_comp_raster(ea_array.astype(np.float32),
output_path,
band=doy,
stats_flag=False)
# drigo.array_to_raster(
# ea_array.astype(np.float32), output_path,
# output_geo=gridmet_geo, output_proj=gridmet_proj,
# stats_flag=False)
del sph_array, ea_array
if stats_flag:
drigo.raster_statistics(output_path)
logging.debug('\nScript Complete')
def main(image_ws, ini_path, bs=2048, stats_flag=False, overwrite_flag=False):
"""Prep a Landsat scene for METRIC
Parameters
----------
image_ws : str
Landsat scene folder that will be prepped.
ini_path : str
File path of the input parameters file.
bs : int, optional
Processing block size (the default is 2048).
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
True is successful
"""
# Open config file
config = dripy.open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
calc_refl_toa_flag = dripy.read_param(
'calc_refl_toa_flag', True, config, 'INPUTS')
calc_refl_toa_qa_flag = dripy.read_param(
'calc_refl_toa_qa_flag', True, config, 'INPUTS')
# calc_refl_sur_ledaps_flag = dripy.read_param(
# 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS')
# calc_refl_sur_qa_flag = dripy.read_param(
# 'calc_refl_sur_qa_flag', False, config, 'INPUTS')
calc_ts_bt_flag = dripy.read_param(
'calc_ts_bt_flag', True, config, 'INPUTS')
# Use QA band to set common area
# Fmask cloud, shadow, & snow pixels will be removed from common area
calc_fmask_common_flag = dripy.read_param(
'calc_fmask_common_flag', True, config, 'INPUTS')
fmask_smooth_flag = dripy.read_param(
'fmask_smooth_flag', False, config, 'INPUTS')
fmask_buffer_flag = dripy.read_param(
'fmask_buffer_flag', False, config, 'INPUTS')
fmask_erode_flag = dripy.read_param(
'fmask_erode_flag', False, config, 'INPUTS')
if fmask_smooth_flag:
fmask_smooth_cells = int(dripy.read_param(
'fmask_smooth_cells', 1, config, 'INPUTS'))
if fmask_smooth_cells == 0 and fmask_smooth_flag:
fmask_smooth_flag = False
if fmask_erode_flag:
fmask_erode_cells = int(dripy.read_param(
'fmask_erode_cells', 1, config, 'INPUTS'))
if fmask_erode_cells == 0 and fmask_erode_flag:
fmask_erode_flag = False
if fmask_buffer_flag:
fmask_buffer_cells = int(dripy.read_param(
'fmask_buffer_cells', 1, config, 'INPUTS'))
if fmask_buffer_cells == 0 and fmask_buffer_flag:
fmask_buffer_flag = False
# Remove edge (fringe) cells
edge_smooth_flag = dripy.read_param(
'edge_smooth_flag', True, config, 'INPUTS')
# Include hand made cloud masks
cloud_mask_flag = dripy.read_param(
'cloud_mask_flag', False, config, 'INPUTS')
cloud_mask_ws = ""
if cloud_mask_flag:
cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws')
# Extract separate Fmask rasters
calc_fmask_flag = dripy.read_param(
'calc_fmask_flag', True, config, 'INPUTS')
calc_fmask_cloud_flag = dripy.read_param(
'calc_fmask_cloud_flag', True, config, 'INPUTS')
calc_fmask_snow_flag = dripy.read_param(
'calc_fmask_snow_flag', True, config, 'INPUTS')
calc_fmask_water_flag = dripy.read_param(
'calc_fmask_water_flag', True, config, 'INPUTS')
# Keep Landsat DN, LEDAPS, and Fmask rasters
keep_dn_flag = dripy.read_param(
'keep_dn_flag', True, config, 'INPUTS')
# keep_sr_flag = dripy.read_param(
# 'keep_sr_flag', True, config, 'INPUTS')
# For this to work I would need to pass in the metric input file
# calc_elev_flag = dripy.read_param(
# 'calc_elev_flag', False, config, 'INPUTS')
# calc_landuse_flag = dripy.read_param(
# 'calc_landuse_flag', False, config, 'INPUTS')
# calc_acca_cloud_flag = dripy.read_param(
# 'calc_acca_cloud_flag', True, config, 'INPUTS')
# calc_acca_snow_flag = dripy.read_param(
# 'calc_acca_snow_flag', True, config, 'INPUTS')
# calc_ledaps_dem_land_flag = dripy.read_param(
# 'calc_ledaps_dem_land_flag', False, config, 'INPUTS')
# calc_ledaps_veg_flag = dripy.read_param(
# 'calc_ledaps_veg_flag', False, config, 'INPUTS')
# calc_ledaps_snow_flag = dripy.read_param(
# 'calc_ledaps_snow_flag', False, config, 'INPUTS')
# calc_ledaps_land_flag = dripy.read_param(
# 'calc_ledaps_land_flag', False, config, 'INPUTS')
# calc_ledaps_cloud_flag = dripy.read_param(
# 'calc_ledaps_cloud_flag', False, config, 'INPUTS')
# Interpolate/clip/project hourly rasters for each Landsat scene
# calc_metric_flag = dripy.read_param(
# 'calc_metric_flag', False, config, 'INPUTS')
calc_metric_ea_flag = dripy.read_param(
'calc_metric_ea_flag', False, config, 'INPUTS')
calc_metric_wind_flag = dripy.read_param(
'calc_metric_wind_flag', False, config, 'INPUTS')
calc_metric_etr_flag = dripy.read_param(
'calc_metric_etr_flag', False, config, 'INPUTS')
calc_metric_tair_flag = dripy.read_param(
'calc_metric_tair_flag', False, config, 'INPUTS')
# Interpolate/clip/project AWC and daily ETr/PPT rasters
# to compute SWB Ke for each Landsat scene
calc_swb_ke_flag = dripy.read_param(
'calc_swb_ke_flag', False, config, 'INPUTS')
if cloud_mask_flag:
spinup_days = dripy.read_param(
'swb_spinup_days', 30, config, 'INPUTS')
min_spinup_days = dripy.read_param(
'swb_min_spinup_days', 5, config, 'INPUTS')
# Round ea raster to N digits to save space
rounding_digits = dripy.read_param(
'rounding_digits', 3, config, 'INPUTS')
env = drigo.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore', divide='ignore')
gdal.UseExceptions()
# Input file paths
dn_image_dict = et_common.landsat_band_image_dict(
image.orig_data_ws, image.image_name_re)
# # Open METRIC config file
# if config_file:
# logging.info(
# log_f.format('METRIC INI File:', os.path.basename(config_file)))
# config = configparser.ConfigParser()
# try:
# config.read(config_file)
# except:
# logging.error('\nERROR: Config file could not be read, ' +
# 'is not an input file, or does not exist\n' +
# 'ERROR: config_file = {}\n').format(config_file)
# sys.exit()
# # Overwrite
# overwrite_flag = dripy.read_param('overwrite_flag', True, config)
#
# # Elevation and landuse parameters/flags from METRIC input file
# calc_elev_flag = dripy.read_param('save_dem_raster_flag', True, config)
# calc_landuse_flag = dripy.read_param(
# 'save_landuse_raster_flag', True, config)
# if calc_elev_flag:
# elev_pr_path = config.get('INPUTS','dem_raster')
# if calc_landuse_flag:
# landuse_pr_path = config.get('INPUTS', 'landuse_raster')
# else:
# overwrite_flag = False
# calc_elev_flag = False
# calc_landuse_flag = False
#
# Elev raster must exist
# if calc_elev_flag and not os.path.isfile(elev_pr_path):
# logging.error('\nERROR: Elevation raster {} does not exist\n'.format(
# elev_pr_path))
# return False
# Landuse raster must exist
# if calc_landuse_flag and not os.path.isfile(landuse_pr_path):
# logging.error('\nERROR: Landuse raster {} does not exist\n'.format(
# landuse_pr_path))
# return False
# Removing ancillary files before checking for inputs
if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')):
shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask'))
for item in os.listdir(image.orig_data_ws):
if (image.type == 'Landsat7' and
(item.endswith('_B8.TIF') or
item.endswith('_B6_VCID_2.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (image.type == 'Landsat8' and
(item.endswith('_B1.TIF') or
item.endswith('_B8.TIF') or
item.endswith('_B9.TIF') or
item.endswith('_B11.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (item.endswith('_VER.jpg') or
item.endswith('_VER.txt') or
item.endswith('_GCP.txt') or
item == 'README.GTF'):
os.remove(os.path.join(image.orig_data_ws, item))
# Check correction level (image must be L1T to process)
if image.correction != 'L1TP':
logging.debug(' Image is not L1TP corrected, skipping')
return False
# calc_fmask_common_flag = False
# calc_refl_toa_flag = False
# calc_ts_bt_flag = False
# calc_metric_ea_flag = False
# calc_metric_wind_flag = False
# calc_metric_etr_flag = False
# overwrite_flag = False
# QA band must exist
if calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys():
logging.warning(
'\nQA band does not exist but calc_fmask_common_flag=True'
'\n Setting calc_fmask_common_flag=False\n {}'.format(
image.qa_band))
calc_fmask_common_flag = False
if cloud_mask_flag and not os.path.isdir(cloud_mask_ws):
logging.warning(
'\ncloud_mask_ws is not a directory but cloud_mask_flag=True.'
'\n Setting cloud_mask_flag=False\n {}'.format(cloud_mask_ws))
cloud_mask_flag = False
# Check for Landsat TOA images
if (calc_refl_toa_flag and
(set(list(image.band_toa_dict.keys()) +
[image.thermal_band, image.qa_band]) !=
set(dn_image_dict.keys()))):
logging.warning(
'\nMissing Landsat images but calc_refl_toa_flag=True'
'\n Setting calc_refl_toa_flag=False')
calc_refl_toa_flag = False
# Check for Landsat brightness temperature image
if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys():
logging.warning(
'\nThermal band image does not exist but calc_ts_bt_flag=True'
'\n Setting calc_ts_bt_flag=False')
calc_ts_bt_flag = False
# DEADBEEF - Should the function return False if Ts doesn't exist?
# return False
# Check for METRIC hourly/daily input folders
if calc_metric_ea_flag:
metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder')
if not os.path.isdir(metric_ea_input_ws):
logging.warning(
'\nHourly Ea folder does not exist but calc_metric_ea_flag=True'
'\n Setting calc_metric_ea_flag=False\n {}'.format(
metric_ea_input_ws))
calc_metric_ea_flag = False
if calc_metric_wind_flag:
metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder')
if not os.path.isdir(metric_wind_input_ws):
logging.warning(
'\nHourly wind folder does not exist but calc_metric_wind_flag=True'
'\n Setting calc_metric_wind_flag=False\n {}'.format(
metric_wind_input_ws))
calc_metric_wind_flag = False
if calc_metric_etr_flag:
metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder')
if not os.path.isdir(metric_etr_input_ws):
logging.warning(
'\nHourly ETr folder does not exist but calc_metric_etr_flag=True'
'\n Setting calc_metric_etr_flag=False\n {}'.format(
metric_etr_input_ws))
calc_metric_etr_flag = False
if calc_metric_tair_flag:
metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder')
if not os.path.isdir(metric_tair_input_ws):
logging.warning(
'\nHourly Tair folder does not exist but calc_metric_tair_flag=True'
'\n Setting calc_metric_tair_flag=False\n {}'.format(
metric_tair_input_ws))
calc_metric_tair_flag = False
if (calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag):
metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re'))
metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re'))
if calc_swb_ke_flag:
awc_input_path = config.get('INPUTS', 'awc_input_path')
etr_input_ws = config.get('INPUTS', 'etr_input_folder')
ppt_input_ws = config.get('INPUTS', 'ppt_input_folder')
etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re'))
ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re'))
if not os.path.isfile(awc_input_path):
logging.warning(
'\nAWC raster does not exist but calc_swb_ke_flag=True'
'\n Setting calc_swb_ke_flag=False\n {}'.format(
awc_input_path))
calc_swb_ke_flag = False
if not os.path.isdir(etr_input_ws):
logging.warning(
'\nDaily ETr folder does not exist but calc_swb_ke_flag=True'
'\n Setting calc_swb_ke_flag=False\n {}'.format(
etr_input_ws))
calc_swb_ke_flag = False
if not os.path.isdir(ppt_input_ws):
logging.warning(
'\nDaily PPT folder does not exist but calc_swb_ke_flag=True'
'\n Setting calc_swb_ke_flag=False\n {}'.format(
ppt_input_ws))
calc_swb_ke_flag = False
# Build folders for support rasters
if ((calc_fmask_common_flag or calc_refl_toa_flag or
# calc_refl_sur_ledaps_flag or
calc_ts_bt_flag or
calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag or
calc_swb_ke_flag) and
not os.path.isdir(image.support_ws)):
os.makedirs(image.support_ws)
if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws):
os.makedirs(image.refl_toa_ws)
# if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws):
# os.makedirs(image.refl_sur_ws)
# DEADBEEF - This is being further down just for the Fmask images
# # Apply overwrite flag
# if overwrite_flag:
# overwrite_list = [
# image.fmask_cloud_raster, image.fmask_snow_raster,
# image.fmask_water_raster
# # image.elev_raster, image.landuse_raster
# # image.common_area_raster
# ]
# for overwrite_path in overwrite_list:
# try:
# dripy.remove_file(image.fmask_cloud_raster)
# except:
# pass
# Use QA band to build common area rasters
logging.info('\nCommon Area Raster')
qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0)
common_geo = drigo.raster_ds_geo(qa_ds)
common_extent = drigo.raster_ds_extent(qa_ds)
common_proj = drigo.raster_ds_proj(qa_ds)
common_osr = drigo.raster_ds_osr(qa_ds)
# Initialize common_area as all non-fill QA values
qa_array = drigo.raster_ds_to_array(qa_ds, return_nodata=False)
common_array = qa_array != 1
common_rows, common_cols = common_array.shape
del qa_ds
# Erode and dilate to remove fringe on edge
# Default is to not smooth, but user can force smoothing
# This needs to be applied before Fmask
if edge_smooth_flag and image.prefix in ['LT05', 'LE07']:
struct = ndimage.generate_binary_structure(2, 2).astype(np.uint8)
if image.prefix == 'LT05':
cells = 8
elif image.prefix == 'LE07':
cells = 2
else:
cells = 0
common_array = ndimage.binary_dilation(
ndimage.binary_erosion(common_array, struct, cells),
struct, cells)
# Try applying user defined cloud masks to common_area
cloud_mask_path = os.path.join(
cloud_mask_ws, image.folder_id + '_mask.shp')
if cloud_mask_flag and os.path.isfile(cloud_mask_path):
logging.info(' Applying cloud mask shapefile')
feature_path = os.path.join(
cloud_mask_ws, (image.folder_id + '_mask.shp'))
logging.info(' {}'.format(feature_path))
cloud_mask_memory_ds = drigo.polygon_to_raster_ds(
feature_path, nodata_value=0, burn_value=1,
output_osr=common_osr, output_cs=30,
output_extent=common_extent)
cloud_array = drigo.raster_ds_to_array(
cloud_mask_memory_ds, return_nodata=False)
# DEADBEEF - If user sets a cloud mask,
# it is probably better than Fmask
# Eventually change "if" calc_fmask_common_flag: to "elif"
common_array[cloud_array == 1] = 0
del cloud_mask_memory_ds, cloud_array
# Remove Fmask cloud, shadow, and snow pixels from common_area
if calc_fmask_common_flag:
logging.info(' Applying Fmask to common area')
fmask_array = et_numpy.bqa_fmask_func(qa_array)
fmask_mask = (fmask_array >= 2) & (fmask_array <= 4)
if fmask_smooth_flag:
logging.debug(
' Smoothing (dilate/erode/erode/dilate) Fmask clouds, shadows,'
' and snow pixels by {} cells'.format(fmask_smooth_cells))
# ArcGIS smoothing procedure
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_smooth_cells,
structure=ndimage.generate_binary_structure(2, 2))
if fmask_erode_flag:
logging.debug(
' Eroding Fmask clouds, shadows, and snow pixels by '
'{} cells'.format(fmask_erode_cells))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_erode_cells,
structure=ndimage.generate_binary_structure(2, 2))
if fmask_buffer_flag:
logging.debug(
' Dilating (buffering) Fmask clouds, shadows, and snow pixels '
'by {} cells'.format(fmask_buffer_cells))
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_buffer_cells,
structure=ndimage.generate_binary_structure(2, 2))
# Reset common_array for buffered cells
common_array[fmask_mask] = 0
del fmask_array, fmask_mask
# Check that there are some cloud free pixels
if not np.any(common_array):
logging.error(
' ERROR: There are no cloud/snow free pixels, returning False')
return False
# Always overwrite common area raster
# if not os.path.isfile(image.common_area_raster):
drigo.array_to_raster(
common_array, image.common_area_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Print common geo/extent
logging.debug(' Common geo: {}'.format(common_geo))
logging.debug(' Common extent: {}'.format(common_extent))
# Extract Fmask components as separate rasters
if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or
calc_fmask_water_flag):
logging.info('\nFmask')
fmask_array = et_numpy.bqa_fmask_func(qa_array)
# Remove existing Fmask rasters
if (calc_fmask_flag and overwrite_flag and
os.path.isfile(image.fmask_output_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_output_raster))
dripy.remove_file(image.fmask_output_raster)
if (calc_fmask_cloud_flag and overwrite_flag and
os.path.isfile(image.fmask_cloud_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_cloud_raster))
dripy.remove_file(image.fmask_cloud_raster)
if (calc_fmask_snow_flag and overwrite_flag and
os.path.isfile(image.fmask_snow_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_snow_raster))
dripy.remove_file(image.fmask_snow_raster)
if (calc_fmask_water_flag and overwrite_flag and
os.path.isfile(image.fmask_water_raster)):
logging.debug(' Overwriting: {}'.format(
image.fmask_water_raster))
dripy.remove_file(image.fmask_water_raster)
# Save Fmask data as separate rasters
if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)):
logging.debug(' Saving Fmask raster')
drigo.array_to_raster(
fmask_array.astype(np.uint8), image.fmask_output_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
if (calc_fmask_cloud_flag and
not os.path.isfile(image.fmask_cloud_raster)):
logging.debug(' Saving Fmask cloud raster')
fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4)
drigo.array_to_raster(
fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_cloud_array
if (calc_fmask_snow_flag and
not os.path.isfile(image.fmask_snow_raster)):
logging.debug(' Saving Fmask snow raster')
fmask_snow_array = (fmask_array == 3)
drigo.array_to_raster(
fmask_snow_array.astype(np.uint8), image.fmask_snow_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_snow_array
if (calc_fmask_water_flag and
not os.path.isfile(image.fmask_water_raster)):
logging.debug(' Saving Fmask water raster')
fmask_water_array = (fmask_array == 1)
drigo.array_to_raster(
fmask_water_array.astype(np.uint8), image.fmask_water_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_water_array
del fmask_array
# # Calculate elevation
# if calc_elev_flag and not os.path.isfile(elev_path):
# logging.info('Elevation')
# elev_array, elev_nodata = drigo.raster_to_array(
# elev_pr_path, 1, common_extent)
# drigo.array_to_raster(
# elev_array, elev_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del elev_array, elev_nodata, elev_path
#
# # Calculate landuse
# if calc_landuse_flag and not os.path.isfile(landuse_raster):
# logging.info('Landuse')
# landuse_array, landuse_nodata = drigo.raster_to_array(
# landuse_pr_path, 1, common_extent)
# drigo.array_to_raster(
# landuse_array, landuse_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del landuse_array, landuse_nodata, landuse_raster
# Calculate toa reflectance
# f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32)
if calc_refl_toa_flag:
logging.info('Top-of-Atmosphere Reflectance')
if os.path.isfile(image.refl_toa_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(
image.refl_toa_raster))
dripy.remove_file(image.refl_toa_raster)
if not os.path.isfile(image.refl_toa_raster):
# First build empty composite raster
drigo.build_empty_raster(
image.refl_toa_raster, image.band_toa_cnt, np.float32, None,
env.snap_proj, env.cellsize, common_extent)
# cos_theta_solar_flt = et_common.cos_theta_solar_func(
# image.sun_elevation)
# Process by block
logging.info('Processing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(
common_cols, common_rows))
for b_i, b_j in drigo.block_gen(common_rows, common_cols, bs):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = drigo.array_to_block(
common_array, b_i, b_j, bs).astype(np.bool)
block_rows, block_cols = block_data_mask.shape
block_geo = drigo.array_offset_geo(common_geo, b_j, b_i)
block_extent = drigo.geo_extent(
block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
logging.debug(' Block extent: {}'.format(block_extent))
logging.debug(' Block geo: {}'.format(block_geo))
# Process each TOA band
# for band, band_i in sorted(image.band_toa_dict.items()):
for band, dn_image in sorted(dn_image_dict.items()):
if band not in image.band_toa_dict.keys():
continue
# thermal_band_flag = (band == image.thermal_band)
# Set 0 as nodata value
drigo.raster_path_set_nodata(dn_image, 0)
# Calculate TOA reflectance
dn_array, dn_nodata = drigo.raster_to_array(
dn_image, 1, block_extent)
dn_array = dn_array.astype(np.float64)
# dn_array = dn_array.astype(np.float32)
dn_array[dn_array == 0] = np.nan
#
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
refl_toa_array = et_numpy.l457_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.dr, image.esun_dict[band],
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band])
elif image.type in ['Landsat8']:
refl_toa_array = et_numpy.l8_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.refl_mult_dict[band],
image.refl_add_dict[band])
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band])
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band],
# image.refl_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array,
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band],
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band],
# image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# refl_toa_array = et_numpy.refl_toa_band_func(
# dn_array, cos_theta_solar_flt,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag)
drigo.block_to_raster(
refl_toa_array.astype(np.float32),
image.refl_toa_raster,
b_i, b_j, band=image.band_toa_dict[band])
# drigo.array_to_comp_raster(
# refl_toa_array.astype(np.float32),
# image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
del refl_toa_array, dn_array
if stats_flag:
drigo.raster_statistics(image.refl_toa_raster)
# # Process each TOA band
# # for band, band_i in sorted(image.band_toa_dict.items()):
# for band, dn_image in sorted(dn_image_dict.items()):
# thermal_band_flag = (band == image.thermal_band)
# # Set 0 as nodata value
# drigo.raster_path_set_nodata(dn_image, 0)
# # Calculate TOA reflectance
# dn_array, dn_nodata = drigo.raster_to_array(
# dn_image, 1, common_extent)
# dn_array = dn_array.astype(np.float64)
# # dn_array = dn_array.astype(np.float32)
# dn_array[dn_array == 0] = np.nan
# #
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band])
# # image.rad_mult_dict[band], image.rad_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array, image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# # image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band], image.refl_add_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# # refl_toa_array = et_numpy.refl_toa_band_func(
# # dn_array, cos_theta_solar_flt,
# # image.dr, image.esun_dict[band],
# # image.lmin_dict[band], image.lmax_dict[band],
# # image.qcalmin_dict[band], image.qcalmax_dict[band],
# # thermal_band_flag)
# drigo.array_to_comp_raster(
# refl_toa_array.astype(np.float32), image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
# del refl_toa_array, dn_array
# Calculate brightness temperature
if calc_ts_bt_flag:
logging.info('Brightness Temperature')
if os.path.isfile(image.ts_bt_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(image.ts_bt_raster))
dripy.remove_file(image.ts_bt_raster)
if not os.path.isfile(image.ts_bt_raster):
band = image.thermal_band
thermal_dn_path = dn_image_dict[band]
drigo.raster_path_set_nodata(thermal_dn_path, 0)
thermal_dn_array, thermal_dn_nodata = drigo.raster_to_array(
thermal_dn_path, 1, common_extent, return_nodata=True)
thermal_dn_mask = thermal_dn_array != thermal_dn_nodata
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
ts_bt_array = et_numpy.l457_ts_bt_band_func(
thermal_dn_array,
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band],
# image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
elif image.type in ['Landsat8']:
ts_bt_array = et_numpy.l8_ts_bt_band_func(
thermal_dn_array,
image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
# thermal_rad_array = et_numpy.refl_toa_band_func(
# thermal_dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag=True)
# ts_bt_array = et_numpy.ts_bt_func(
# thermal_rad_array, image.k1_dict[image.thermal_band],
# image.k2_dict[image.thermal_band])
ts_bt_array[~thermal_dn_mask] = np.nan
drigo.array_to_raster(
ts_bt_array, image.ts_bt_raster,
output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array,
stats_flag=stats_flag)
# del thermal_dn_array, thermal_rad_array
del thermal_dn_path, thermal_dn_array, ts_bt_array
# Interpolate/project/clip METRIC hourly/daily rasters
if (calc_metric_ea_flag or
calc_metric_wind_flag or
calc_metric_etr_flag):
logging.info('METRIC hourly/daily rasters')
# Get bracketing hours from image acquisition time
image_prev_dt = image.acq_datetime.replace(
minute=0, second=0, microsecond=0)
image_next_dt = image_prev_dt + timedelta(seconds=3600)
# Get NLDAS properties from one of the images
input_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(input_ws, file_name)
for file_name in os.listdir(input_ws)
for match in [metric_hourly_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
except IndexError:
logging.error(' No hourly file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d %H00')))
return False
try:
input_ds = gdal.Open(input_path)
input_osr = drigo.raster_ds_osr(input_ds)
# input_proj = drigo.osr_proj(input_osr)
input_extent = drigo.raster_ds_extent(input_ds)
input_cs = drigo.raster_ds_cellsize(input_ds, x_only=True)
# input_geo = input_extent.geo(input_cs)
input_x, input_y = input_extent.origin()
input_ds = None
except:
logging.error(' Could not get default input image properties')
logging.error(' {}'.format(input_path))
return False
# Project Landsat scene extent to NLDAS GCS
common_gcs_osr = common_osr.CloneGeogCS()
common_gcs_extent = drigo.project_extent(
common_extent, common_osr, common_gcs_osr,
cellsize=env.cellsize)
common_gcs_extent.buffer_extent(0.1)
common_gcs_extent.adjust_to_snap(
'EXPAND', input_x, input_y, input_cs)
# common_gcs_geo = common_gcs_extent.geo(input_cs)
def metric_weather_func(output_raster, input_ws, input_re,
prev_dt, next_dt,
resample_method=gdal.GRA_NearestNeighbour,
rounding_flag=False):
"""Interpolate/project/clip METRIC hourly rasters"""
logging.debug(' Output: {}'.format(output_raster))
if os.path.isfile(output_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
dripy.remove_file(output_raster)
else:
logging.debug(' Skipping, file already exists ' +
'and overwrite is False')
return False
prev_ws = os.path.join(input_ws, str(prev_dt.year))
next_ws = os.path.join(input_ws, str(next_dt.year))
# Technically previous and next could come from different days
# or even years, although this won't happen in the U.S.
try:
prev_path = [
os.path.join(prev_ws, input_name)
for input_name in os.listdir(prev_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(prev_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input prev: {}'.format(prev_path))
except IndexError:
logging.error(' No previous hourly file')
logging.error(' {}'.format(prev_dt))
return False
try:
next_path = [
os.path.join(next_ws, input_name)
for input_name in os.listdir(next_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(next_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input next: {}'.format(next_path))
except IndexError:
logging.error(' No next hourly file')
logging.error(' {}'.format(next_dt))
return False
# Band numbers are 1's based
prev_band = int(prev_dt.strftime('%H')) + 1
next_band = int(next_dt.strftime('%H')) + 1
logging.debug(' Input prev band: {}'.format(prev_band))
logging.debug(' Input next band: {}'.format(next_band))
# Read arrays
prev_array = drigo.raster_to_array(
prev_path, band=prev_band, mask_extent=common_gcs_extent,
return_nodata=False)
next_array = drigo.raster_to_array(
next_path, band=next_band, mask_extent=common_gcs_extent,
return_nodata=False)
if not np.any(prev_array) or not np.any(next_array):
logging.warning('\nWARNING: Input NLDAS array is all nodata\n')
return None
output_array = hourly_interpolate_func(
prev_array, next_array,
prev_dt, next_dt, image.acq_datetime)
output_array = drigo.project_array(
output_array, resample_method,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent, output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
if rounding_flag:
output_array = np.around(output_array, rounding_digits)
# Force output to 32-bit float
drigo.array_to_raster(
output_array.astype(np.float32), output_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
return True
# Ea - Project to Landsat scene after clipping
if calc_metric_ea_flag:
logging.info(' Hourly vapor pressure (Ea)')
metric_weather_func(
image.metric_ea_raster, metric_ea_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_Bilinear, rounding_flag=True)
# Wind - Project to Landsat scene after clipping
if calc_metric_wind_flag:
logging.info(' Hourly windspeed')
metric_weather_func(
image.metric_wind_raster, metric_wind_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Hourly reference ET (ETr)')
metric_weather_func(
image.metric_etr_raster, metric_etr_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr 24hr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Daily reference ET (ETr)')
logging.debug(' Output: {}'.format(
image.metric_etr_24hr_raster))
if (os.path.isfile(image.metric_etr_24hr_raster) and
overwrite_flag):
logging.debug(' Overwriting output')
os.remove(image.metric_etr_24hr_raster)
if not os.path.isfile(image.metric_etr_24hr_raster):
etr_prev_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(etr_prev_ws, file_name)
for file_name in os.listdir(etr_prev_ws)
for match in [metric_daily_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
logging.debug(' Input: {}'.format(input_path))
except IndexError:
logging.error(' No daily file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d')))
return False
output_array = drigo.raster_to_array(
input_path, mask_extent=common_gcs_extent,
return_nodata=False)
output_array = drigo.project_array(
output_array, gdal.GRA_NearestNeighbour,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent,
output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
# output_array = np.around(output_array, rounding_digits)
drigo.array_to_raster(
output_array, image.metric_etr_24hr_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
del input_path
# Tair - Project to Landsat scene after clipping
if calc_metric_tair_flag:
logging.info(' Hourly air temperature (Tair)')
metric_weather_func(
image.metric_tair_raster, metric_tair_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# Cleanup
del image_prev_dt, image_next_dt
# Soil Water Balance
if calc_swb_ke_flag:
logging.info('Daily soil water balance')
# Check if output file already exists
logging.debug(' Ke: {}'.format(image.ke_raster))
if os.path.isfile(image.ke_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
dripy.remove_file(image.ke_raster)
else:
logging.debug(' Skipping, file already '
'exists and overwrite is False')
return False
ke_array = et_common.raster_swb_func(
image.acq_datetime, common_osr, env.cellsize, common_extent,
awc_input_path, etr_input_ws, etr_input_re,
ppt_input_ws, ppt_input_re,
spinup_days=spinup_days, min_spinup_days=min_spinup_days)
# Apply common area mask
ke_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest 2 digits
np.around(ke_array, 2, out=ke_array)
# Force output to 32-bit float
drigo.array_to_raster(
ke_array.astype(np.float32), image.ke_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Remove Landsat TOA rasters
if not keep_dn_flag:
for landsat_item in dripy.build_file_list(
image.orig_data_ws, image.image_name_re):
os.remove(os.path.join(image.orig_data_ws, landsat_item))
return True
def monte_carlo(image_ws,
metric_ini_path,
mc_ini_path,
mc_iter=None,
cold_tgt_pct=None,
hot_tgt_pct=None,
groupsize=64,
blocksize=4096,
multipoint_flag=False,
shapefile_flag=False,
stats_flag=False,
overwrite_flag=False,
debug_flag=False,
no_etrf_final_plots=None,
no_etrf_temp_plots=None):
"""METRIC Monte Carlo
Parameters
----------
image_ws : str
The workspace (path) of the landsat scene folder.
metric_ini_path : str
The METRIC config file (path).
mc_ini_path : str
The Monte Carlo config file (path).
mc_iter : int, optional
Iteration number for Monte Carlo processing.
cold_tgt_pct : float, optional
Target percentage of pixels with ETrF > than cold Kc.
hot_tgt_pct : float, optional
Target percentage of pixels with ETrF < than hot Kc.
groupsize : int, optional
Script will try to place calibration point randomly
into a labeled group of clustered values with at least n pixels.
-1 = In the largest group
0 = Anywhere in the image (not currently implemented)
1 >= In any group with a pixel count greater or equal to n
blocksize : int, optional
Processing block size (the default is 4096).
shapefile_flag : bool, optional
If True, save calibration points to shapefile (the default is False).
multipoint_flag : bool, optional
If True, save cal. points to multipoint shapefile (the default is False).
stats_flag : bool, optional
If True, compute raster statistics (the default is False).
ovewrite_flag : bool, optional
If True, overwrite existing files (the default is False).
debug_flag : bool, optional
If True, enable debug level logging (the default is False).
no_final_plots : bool, optional
If True, don't save final ETrF histograms (the default is None).
This will override the flag in the INI file.
no_temp_plots : bool
If True, don't save temp ETrF histogram (the default is None).
This will override the flag in the INI file.
Returns
-------
None
"""
logging.info('METRIC Automated Calibration')
# Open config file
config = open_ini(mc_ini_path)
# Get input parameters
logging.debug(' Reading Input File')
etrf_training_path = config.get('INPUTS', 'etrf_training_path')
# Adjust Kc cold target value based on day of year
# etrf_doy_adj_path = read_param(
# 'etrf_doy_adj_path', None, config, 'INPUTS')
# Intentionally set default to None, to trigger error in eval call
kc_cold_doy_dict = read_param('kc_cold_doy_dict', None, config, 'INPUTS')
kc_hot_doy_dict = read_param('kc_hot_doy_dict', None, config, 'INPUTS')
# If the "no_" flags were set True, honor them and set the flag False
# If the "no_" flags were not set by the user, use the INI flag values
# If not set in the INI, default to False (don't save any plots)
if no_etrf_temp_plots:
save_etrf_temp_plots = False
else:
save_etrf_temp_plots = read_param('save_etrf_temp_plots', False,
config, 'INPUTS')
if no_etrf_final_plots:
save_etrf_final_plots = False
else:
save_etrf_final_plots = read_param('save_etrf_final_plots', False,
config, 'INPUTS')
save_ndvi_plots = read_param('save_ndvi_plots', False, config, 'INPUTS')
max_cal_iter = read_param('max_cal_iterations', 5, config, 'INPUTS')
max_point_iter = read_param('max_point_iterations', 10, config, 'INPUTS')
ts_diff_threshold = read_param('ts_diff_threshold', 4, config, 'INPUTS')
etr_ws = config.get('INPUTS', 'etr_ws')
ppt_ws = config.get('INPUTS', 'ppt_ws')
etr_re = re.compile(config.get('INPUTS', 'etr_re'))
ppt_re = re.compile(config.get('INPUTS', 'ppt_re'))
awc_path = config.get('INPUTS', 'awc_path')
spinup_days = read_param('swb_spinup_days', 5, config, 'INPUTS')
min_spinup_days = read_param('swb_min_spinup_days', 30, config, 'INPUTS')
log_fmt = ' {:<18s} {}'
break_line = '\n{}'.format('#' * 80)
env = drigo.env
image = et_image.Image(image_ws, env)
logging.info(log_fmt.format('Image:', image.folder_id))
# Check inputs
for file_path in [awc_path]:
if not os.path.isfile(file_path):
logging.error('\nERROR: File {} does not exist'.format(file_path))
sys.exit()
for folder in [etr_ws, ppt_ws]:
if not os.path.isdir(folder):
logging.error('\nERROR: Folder {} does not exist'.format(folder))
sys.exit()
# if (etrf_doy_adj_path and not
# os.path.isfile(etrf_doy_adj_path)):
# logging.error(
# '\nERROR: File {} does not exist.'.format(
# etrf_doy_adj_path))
# sys.exit()
# Use iteration number to file iteration string
if mc_iter is None:
mc_str = ''
mc_fmt = '.img'
elif int(mc_iter) < 0:
logging.error('\nERROR: Iteration number must be a positive integer')
return False
else:
mc_str = 'MC{:02d}_'.format(int(mc_iter))
mc_fmt = '_{:02d}.img'.format(int(mc_iter))
logging.info(' {:<18s} {}'.format('Iteration:', mc_iter))
# Folder names
etrf_ws = os.path.join(image_ws, 'ETRF')
# indices_ws = image.indices_ws
region_ws = os.path.join(image_ws, 'PIXEL_REGIONS')
pixels_ws = os.path.join(image_ws, 'PIXELS')
plots_ws = os.path.join(image_ws, 'PLOTS')
if shapefile_flag and not os.path.isdir(pixels_ws):
os.mkdir(pixels_ws)
if not os.path.isdir(plots_ws):
os.mkdir(plots_ws)
# File names
r_fmt = '.img'
etrf_path = os.path.join(etrf_ws, 'et_rf' + mc_fmt)
region_path = os.path.join(region_ws, 'region_mask' + r_fmt)
# Initialize calibration parameters dictionary
logging.info(break_line)
logging.info('Calibration Parameters')
cal_dict = dict()
logging.debug(' Reading target cold/hot Kc from INI')
# Using eval is potentially a really bad way of reading this in
try:
kc_cold_doy_dict = eval('{' + kc_cold_doy_dict + '}')
except:
kc_cold_doy_dict = {1: 1.05, 366: 1.05}
logging.info(
' ERROR: kc_cold_doy_dict was not parsed, using default values')
try:
kc_hot_doy_dict = eval('{' + kc_hot_doy_dict + '}')
except:
kc_hot_doy_dict = {1: 0.1, 366: 0.1}
logging.info(
' ERROR: kc_hot_doy_dict was not parsed, using default values')
logging.debug(' Kc cold dict: {}'.format(kc_cold_doy_dict))
logging.debug(' Kc hot dict: {}\n'.format(kc_hot_doy_dict))
# doy_cold, kc_cold = zip(*sorted(kc_cold_doy_dict.items()))
cal_dict['cold_tgt_kc'] = np.interp(
image.acq_doy,
*zip(*sorted(kc_cold_doy_dict.items())),
left=1.05,
right=1.05)
# doy_hot, kc_hot = zip(*sorted(kc_hot_doy_dict.items()))
cal_dict['hot_tgt_kc'] = np.interp(image.acq_doy,
*zip(*sorted(kc_hot_doy_dict.items())),
left=0.1,
right=0.1)
# if etrf_doy_adj_path:
# doy_adj_df = pd.read_csv(etrf_doy_adj_path)
# doy_adj = float(
# doy_adj_df[doy_adj_df['DOY'] == image.acq_doy]['ETRF_ADJ'])
# cal_dict['cold_tgt_kc'] = cal_dict['cold_tgt_kc'] + doy_adj
# Get hot/cold etrf fraction sizes
if cold_tgt_pct is None or hot_tgt_pct is None:
logging.info('ETrF Tail Size Percentages')
logging.info(' Reading target tail size from file')
cold_tgt_pct, hot_tgt_pct = auto_calibration.etrf_fractions(
etrf_training_path)
if cold_tgt_pct is None or hot_tgt_pct is None:
logging.error('\nERROR: Tail sizes were not mannually set or '
'read from the the file\n')
return False
cal_dict['cold_tgt_pct'] = cold_tgt_pct
cal_dict['hot_tgt_pct'] = hot_tgt_pct
logging.info(pixel_str_fmt.format('', 'Cold Pixel', 'Hot Pixel'))
logging.info(
pixel_flt_fmt.format('Target kc:', cal_dict['cold_tgt_kc'],
cal_dict['hot_tgt_kc']))
logging.info(
pixel_pct_fmt.format('Tail Size:', cal_dict['cold_tgt_pct'],
cal_dict['hot_tgt_pct']))
# # Create calibration database
# # Set overwrite false to use existing database if it exists
# cal_ws = os.path.join(image_ws, cal_folder)
# if not os.path.isdir(cal_ws):
# os.mkdir(cal_ws)
# cal_path = os.path.join(cal_ws, cal_name)
# logging.info('{:<20s} {}\{}'.format(
# 'Calibration DB:', cal_folder, cal_name))
# calibration_database.create_calibration_database(
# image_ws, cal_path, overwrite_db_flag)
# del cal_ws
# Remove previous calibrations from database
# logging.info(break_line)
# calibration_database.remove_calibration_points(
# image_ws, cal_path, cal_initials, mc_iter)
# Get ETrF and region mask (from pixel rating)
# Assume they have identical extents
try:
region_mask = drigo.raster_to_array(region_path, return_nodata=False)
region_mask = region_mask.astype(np.bool)
except:
logging.error(
'\nERROR: Pixel regions mask does not exist or could not be read.\n'
' Please try re-running the METRIC Pixel Rating tool.')
logging.debug(' {} '.format(region_path))
return False
# Remove previous plots
logging.info(break_line)
auto_calibration.remove_histograms(plots_ws, mc_iter)
# Generate the NDVI histogram
if save_ndvi_plots:
logging.info(break_line)
logging.info('NDVI Histograms')
if os.path.isfile(image.ndvi_toa_raster):
ndvi_array = drigo.raster_to_array(image.ndvi_toa_raster,
return_nodata=False)
else:
logging.error(
'\nERROR: NDVI raster does not exist. METRIC Model 1 may not '
'have run successfully.')
logging.debug(' {} '.format(image.ndvi_toa_raster))
# Only process ag. ETrF pixels
ndvi_array[~region_mask] = np.nan
ndvi_sub_array = ndvi_array[region_mask]
if np.any(ndvi_sub_array):
auto_calibration.save_ndvi_histograms(ndvi_sub_array, plots_ws,
mc_iter)
else:
logging.error(
'\nERROR: Empty NDVI array, histogram was not built\n')
# Place points in suggested region allowing for a number of iterations
# dependent on whether or not Ts meets certain criteria
logging.info(break_line)
pixel_point_iters = 0
while pixel_point_iters <= max_point_iter:
if pixel_point_iters == max_point_iter:
logging.error('\nERROR: Suitable hot and cold pixels could not be '
'determined. The scene will not calibrate.\n')
return False
cold_xy, hot_xy = pixel_points.pixel_points(
image_ws,
groupsize=groupsize,
blocksize=blocksize,
mc_iter=mc_iter,
shapefile_flag=shapefile_flag,
multipoint_flag=multipoint_flag,
overwrite_flag=overwrite_flag,
pixel_point_iters=pixel_point_iters)
if any(x is None for x in cold_xy) or any(x is None for x in hot_xy):
logging.error(('\nPixel points coordinates are invalid. '
'The scene will not calibrate.'
'\n Cold: {}\n Hot: {}').format(cold_xy, hot_xy))
return False
cold_ts = drigo.raster_value_at_xy(image.ts_raster, cold_xy)
hot_ts = drigo.raster_value_at_xy(image.ts_raster, hot_xy)
if cold_ts > hot_ts:
logging.info(
'\nThe cold pixel is hotter than the hot pixel. Placing '
'the points again.\n')
logging.info(break_line)
pixel_point_iters += 1
elif abs(hot_ts - cold_ts) < ts_diff_threshold:
logging.info((
'\nThere is less than a {} degree difference in Ts hot and cold. '
'Placing the points again.\n').format(ts_diff_threshold))
logging.info(break_line)
pixel_point_iters += 1
# raise et_common.TemperatureError
else:
break
# Adjust Kc hot for soil water balance
logging.info(break_line)
cal_dict = auto_calibration.hot_kc_swb_adjust(cal_dict, hot_xy,
env.snap_osr, image.acq_date,
awc_path, etr_ws, etr_re,
ppt_ws, ppt_re, spinup_days,
min_spinup_days)
# Adjust Kc cold based on NDVI
# cal_dict['tgt_c_kc'] = auto_calibration.kc_ndvi_adjust(
# cal_dict['tgt_c_kc'], cold_xy, ndvi_path, 'Cold')
# Check that Kc hot (Ke) is not too high?
if cal_dict['hot_tgt_kc'] >= 1.0:
logging.error('\nERROR: Target Kc hot is too high for automated '
'calibration\n ETrF will not be computed')
return False
elif (cal_dict['cold_tgt_kc'] - cal_dict['hot_tgt_kc']) <= 0.05:
logging.error('\nERROR: Target Kc hot and Kc cold are too close for '
'automated calibration\n ETrF will not be computed')
return False
# Initialize Kc values at targets
cal_dict['kc_cold'] = cal_dict['cold_tgt_kc']
cal_dict['kc_hot'] = cal_dict['hot_tgt_kc']
# Iterate until max calibrations is reached or error is small
cal_flag = False
cal_iter = 1
while not cal_flag:
logging.info(break_line)
logging.info('Calibration Iteration: {}'.format(cal_iter))
# Run METRIC Model2 for initial ETrF map
logging.info(break_line)
metric_model2.metric_model2(image_ws,
metric_ini_path,
mc_iter=mc_iter,
kc_cold=cal_dict['kc_cold'],
kc_hot=cal_dict['kc_hot'],
cold_xy=cold_xy,
hot_xy=hot_xy,
overwrite_flag=overwrite_flag)
# Read in ETrF array
if os.path.isfile(etrf_path):
etrf_array = drigo.raster_to_array(etrf_path, return_nodata=False)
else:
logging.warning(
('WARNING: ETrF raster does not exist. METRIC Model 2 '
'may not have run successfully.\n {}').format(etrf_path))
break
etrf_geo = drigo.raster_path_geo(etrf_path)
# Only process ag. ETrF pixels
etrf_array[~region_mask] = np.nan
etrf_sub_array = etrf_array[np.isfinite(etrf_array)]
if not np.any(etrf_sub_array):
logging.error(
'\nERROR: Empty ETrF array, scene cannot be calibrated\n')
break
# Calculate calibration parameters
logging.debug(break_line)
cal_dict = auto_calibration.calibration_params(cal_dict,
etrf_sub_array)
# Plot intermediates calibration histograms
if save_etrf_temp_plots:
logging.debug(break_line)
auto_calibration.save_etrf_histograms(etrf_sub_array, plots_ws,
cal_dict, mc_iter, cal_iter)
# Check calibration
logging.debug(break_line)
cal_flag = auto_calibration.check_calibration(cal_dict)
# Don't re-calibrate if initial calibration was suitable
if cal_flag:
break
# Limit calibration attempts
# cal_iter index is 1 based for Monte Carlo
# cal_iter is 0 for stand alone mode
elif cal_iter >= max_cal_iter:
logging.info(break_line)
logging.info(
('All {} iteration attempts were made, '
'the scene will not calibrate.').format(max_cal_iter))
if os.path.isfile(etrf_path):
os.remove(etrf_path)
return False
# break
# Adjust Kc value of calibration points (instead of moving them)
cal_dict = auto_calibration.kc_calibration_adjust(
cal_dict, etrf_sub_array)
# # Select new calibration points based on ETrF distribution
# logging.info(break_line)
# cold_xy, hot_xy = auto_calibration.build_pixel_points(
# etrf_array, etrf_geo, cal_dict,
# shapefile_flag=shapefile_flag, pixels_ws=pixels_ws)
del etrf_array, etrf_geo
# Increment calibration iteration counter
cal_iter += 1
# Only save 'final' results if the scene was calibrated
if cal_flag and save_etrf_final_plots:
# Plot final ETrF distribution
# logging.info(break_line)
auto_calibration.save_etrf_histograms(etrf_sub_array, plots_ws,
cal_dict, mc_iter, None)
# Save final calibration points to database
# logging.info(break_line)
# calibration_database.save_calibration_points(
# image_ws, cal_path, cal_dict, mc_iter, 0)
return True
def metric_weather_func(output_raster, input_ws, input_re,
prev_dt, next_dt,
resample_method=gdal.GRA_NearestNeighbour,
rounding_flag=False):
"""Interpolate/project/clip METRIC hourly rasters"""
logging.debug(' Output: {}'.format(output_raster))
if os.path.isfile(output_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
dripy.remove_file(output_raster)
else:
logging.debug(' Skipping, file already exists ' +
'and overwrite is False')
return False
prev_ws = os.path.join(input_ws, str(prev_dt.year))
next_ws = os.path.join(input_ws, str(next_dt.year))
# Technically previous and next could come from different days
# or even years, although this won't happen in the U.S.
try:
prev_path = [
os.path.join(prev_ws, input_name)
for input_name in os.listdir(prev_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(prev_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input prev: {}'.format(prev_path))
except IndexError:
logging.error(' No previous hourly file')
logging.error(' {}'.format(prev_dt))
return False
try:
next_path = [
os.path.join(next_ws, input_name)
for input_name in os.listdir(next_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(next_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input next: {}'.format(next_path))
except IndexError:
logging.error(' No next hourly file')
logging.error(' {}'.format(next_dt))
return False
# Band numbers are 1's based
prev_band = int(prev_dt.strftime('%H')) + 1
next_band = int(next_dt.strftime('%H')) + 1
logging.debug(' Input prev band: {}'.format(prev_band))
logging.debug(' Input next band: {}'.format(next_band))
# Read arrays
prev_array = drigo.raster_to_array(
prev_path, band=prev_band, mask_extent=common_gcs_extent,
return_nodata=False)
next_array = drigo.raster_to_array(
next_path, band=next_band, mask_extent=common_gcs_extent,
return_nodata=False)
if not np.any(prev_array) or not np.any(next_array):
logging.warning('\nWARNING: Input NLDAS array is all nodata\n')
return None
output_array = hourly_interpolate_func(
prev_array, next_array,
prev_dt, next_dt, image.acq_datetime)
output_array = drigo.project_array(
output_array, resample_method,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent, output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
if rounding_flag:
output_array = np.around(output_array, rounding_digits)
# Force output to 32-bit float
drigo.array_to_raster(
output_array.astype(np.float32), output_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
return True
def main(grb_ws=os.getcwd(),
ancillary_ws=os.getcwd(),
output_ws=os.getcwd(),
keep_list_path=None,
start_date=None,
end_date=None,
times_str='',
extent_path=None,
output_extent=None,
stats_flag=True,
overwrite_flag=False):
"""Extract hourly NLDAS wind rasters
Parameters
----------
grb_ws : str
Folder of NLDAS GRB files.
ancillary_ws : str
Folder of ancillary rasters.
output_ws : str
Folder of output rasters.
keep_list_path : str, optional
Landsat scene keep list file path.
start_date : str, optional
ISO format date (YYYY-MM-DD).
end_date : str, optional
ISO format date (YYYY-MM-DD).
times : str, optional
Comma separated values and/or ranges of UTC hours (i.e. "1, 2, 5-8").
Parsed with python_common.parse_int_set().
extent_path : str, optional
File path defining the output extent.
output_extent : ?, optional
List decimal degrees values defining output extent.
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
None
"""
logging.info('\nExtracting NLDAS wind rasters')
# input_fmt = 'NLDAS_FORA0125_H.A{:04d}{:02d}{:02d}.{}.002.grb'
input_re = re.compile('NLDAS_FORA0125_H.A(?P<YEAR>\d{4})(?P<MONTH>\d{2})' +
'(?P<DAY>\d{2}).(?P<TIME>\d{4}).002.grb$')
output_folder = 'wind'
output_fmt = 'wind_{:04d}{:02d}{:02d}_hourly_nldas.img'
# output_fmt = 'wind_{:04d}{:02d}{:02d}_{:04d}_nldas.img'
# If a date is not set, process 2017
try:
start_dt = dt.datetime.strptime(start_date, '%Y-%m-%d')
logging.debug(' Start date: {}'.format(start_dt))
except:
start_dt = dt.datetime(2017, 1, 1)
logging.info(' Start date: {}'.format(start_dt))
try:
end_dt = dt.datetime.strptime(end_date, '%Y-%m-%d')
logging.debug(' End date: {}'.format(end_dt))
except:
end_dt = dt.datetime(2017, 12, 31)
logging.info(' End date: {}'.format(end_dt))
# Only process a specific hours
if not times_str:
time_list = range(0, 24, 1)
else:
time_list = list(_utils.parse_int_set(times_str))
time_list = ['{:02d}00'.format(t) for t in time_list]
# Assume NLDAS is NAD83
# input_epsg = 'EPSG:4269'
# Ancillary raster paths
mask_path = os.path.join(ancillary_ws, 'nldas_mask.img')
# Build a date list from the Landsat scene keep list file
date_list = []
if keep_list_path is not None and os.path.isfile(keep_list_path):
logging.info('\nReading dates from scene keep list file')
logging.info(' {}'.format(keep_list_path))
landsat_re = re.compile(
'^(?:LT04|LT05|LE07|LC08)_(?:\d{3})(?:\d{3})_' +
'(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})')
with open(keep_list_path) as input_f:
keep_list = input_f.readlines()
keep_list = [
image_id.strip() for image_id in keep_list
if landsat_re.match(image_id.strip())
]
date_list = [
dt.datetime.strptime(image_id[12:20],
'%Y%m%d').strftime('%Y-%m-%d')
for image_id in keep_list
]
logging.debug(' {}'.format(', '.join(date_list)))
# DEADBEE
# # Build a date list from landsat_ws scene folders or tar.gz files
# date_list = []
# if landsat_ws is not None and os.path.isdir(landsat_ws):
# logging.info('\nReading dates from Landsat IDs')
# logging.info(' {}'.format(landsat_ws))
# landsat_re = re.compile(
# '^(?:LT04|LT05|LE07|LC08)_(?:\d{3})(?:\d{3})_' +
# '(?P<year>\d{4})(?P<month>\d{2})(?P<day>\d{2})')
# for root, dirs, files in os.walk(landsat_ws, topdown=True):
# # If root matches, don't explore subfolders
# try:
# landsat_match = landsat_re.match(os.path.basename(root))
# date_list.append(dt.datetime.strptime(
# '_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat())
# dirs[:] = []
# except:
# pass
#
# for file in files:
# try:
# landsat_match = landsat_re.match(file)
# date_list.append(dt.datetime.strptime(
# '_'.join(landsat_match.groups()), '%Y_%m_%d').date().isoformat())
# except:
# pass
# date_list = sorted(list(set(date_list)))
# This allows GDAL to throw Python Exceptions
# gdal.UseExceptions()
# mem_driver = gdal.GetDriverByName('MEM')
# Get the NLDAS spatial reference from the mask raster
nldas_ds = gdal.Open(mask_path)
nldas_osr = drigo.raster_ds_osr(nldas_ds)
nldas_proj = drigo.osr_proj(nldas_osr)
nldas_cs = drigo.raster_ds_cellsize(nldas_ds, x_only=True)
nldas_extent = drigo.raster_ds_extent(nldas_ds)
nldas_geo = nldas_extent.geo(nldas_cs)
nldas_x, nldas_y = nldas_extent.origin()
nldas_ds = None
logging.debug(' Projection: {}'.format(nldas_proj))
logging.debug(' Cellsize: {}'.format(nldas_cs))
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
# Subset data to a smaller extent
if output_extent is not None:
logging.info('\nComputing subset extent & geo')
logging.debug(' Extent: {}'.format(output_extent))
nldas_extent = drigo.Extent(output_extent)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(output_extent))
elif extent_path is not None:
logging.info('\nComputing subset extent & geo')
if extent_path.lower().endswith('.shp'):
nldas_extent = drigo.feature_path_extent(extent_path)
extent_osr = drigo.feature_path_osr(extent_path)
extent_cs = None
else:
nldas_extent = drigo.raster_path_extent(extent_path)
extent_osr = drigo.raster_path_osr(extent_path)
extent_cs = drigo.raster_path_cellsize(extent_path, x_only=True)
nldas_extent = drigo.project_extent(nldas_extent, extent_osr,
nldas_osr, extent_cs)
nldas_extent.adjust_to_snap('EXPAND', nldas_x, nldas_y, nldas_cs)
nldas_geo = nldas_extent.geo(nldas_cs)
logging.debug(' Geo: {}'.format(nldas_geo))
logging.debug(' Extent: {}'.format(nldas_extent))
logging.debug('')
# Read the NLDAS mask array if present
if mask_path and os.path.isfile(mask_path):
mask_array, mask_nodata = drigo.raster_to_array(
mask_path,
mask_extent=nldas_extent,
fill_value=0,
return_nodata=True)
mask_array = mask_array != mask_nodata
else:
mask_array = None
# Build output folder
var_ws = os.path.join(output_ws, output_folder)
if not os.path.isdir(var_ws):
os.makedirs(var_ws)
# Each sub folder in the main folde has all imagery for 1 day
# The path for each subfolder is the /YYYY/DOY
# This approach will process files for target dates
# for input_dt in date_range(start_dt, end_dt + dt.timedelta(1)):
# logging.info(input_dt.date())
# Iterate all available files and check dates if necessary
logging.info('\nReading NLDAS GRIBs')
for root, folders, files in os.walk(grb_ws, topdown=True):
root_split = os.path.normpath(root).split(os.sep)
# If the year/doy is outside the range, skip
if (re.match('\d{4}', root_split[-2])
and re.match('\d{3}', root_split[-1])):
root_dt = dt.datetime.strptime(
'{}_{}'.format(root_split[-2], root_split[-1]), '%Y_%j')
logging.info('{}-{:02d}-{:02d}'.format(root_dt.year, root_dt.month,
root_dt.day))
if ((start_dt is not None and root_dt < start_dt)
or (end_dt is not None and root_dt > end_dt)):
continue
elif date_list and root_dt.date().isoformat() not in date_list:
continue
# If the year is outside the range, don't search subfolders
elif re.match('\d{4}', root_split[-1]):
root_year = int(root_split[-1])
logging.info('Year: {}'.format(root_year))
if ((start_dt is not None and root_year < start_dt.year)
or (end_dt is not None and root_year > end_dt.year)):
folders[:] = []
else:
folders[:] = sorted(folders)
continue
else:
continue
# Create a single raster for each day with 24 bands
# Each time step will be stored in a separate band
output_name = output_fmt.format(root_dt.year, root_dt.month,
root_dt.day)
output_path = os.path.join(var_ws, str(root_dt.year), output_name)
logging.debug(' {}'.format(output_path))
if os.path.isfile(output_path):
if not overwrite_flag:
logging.debug(' File already exists, skipping')
continue
else:
logging.debug(' File already exists, removing existing')
os.remove(output_path)
logging.debug(' {}'.format(root))
if not os.path.isdir(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
drigo.build_empty_raster(output_path,
band_cnt=24,
output_dtype=np.float32,
output_proj=nldas_proj,
output_cs=nldas_cs,
output_extent=nldas_extent,
output_fill_flag=True)
# Iterate through hourly files
for input_name in sorted(files):
logging.info(' {}'.format(input_name))
input_path = os.path.join(root, input_name)
input_match = input_re.match(input_name)
if input_match is None:
logging.debug(' Regular expression didn\'t match, skipping')
continue
input_dt = dt.datetime(int(input_match.group('YEAR')),
int(input_match.group('MONTH')),
int(input_match.group('DAY')))
time_str = input_match.group('TIME')
band_num = int(time_str[:2]) + 1
# if start_dt is not None and input_dt < start_dt:
# continue
# elif end_dt is not None and input_dt > end_dt:
# continue
# elif date_list and input_dt.date().isoformat() not in date_list:
# continue
if time_str not in time_list:
logging.debug(' Time not in list, skipping')
continue
logging.debug(' Time: {} {}'.format(input_dt.date(), time_str))
logging.debug(' Band: {}'.format(band_num))
# Determine band numbering/naming
input_band_dict = grib_band_names(input_path)
# Compute magnitude of wind from components
input_ds = gdal.Open(input_path)
wind_u_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['u-component of wind [m/s]'],
mask_extent=nldas_extent,
return_nodata=False)
wind_v_array = drigo.raster_ds_to_array(
input_ds,
band=input_band_dict['v-component of wind [m/s]'],
mask_extent=nldas_extent,
return_nodata=False)
wind_array = np.sqrt(wind_u_array**2 + wind_v_array**2)
# Save the projected array as 32-bit floats
drigo.array_to_comp_raster(wind_array.astype(np.float32),
output_path,
band=band_num)
# drigo.block_to_raster(
# ea_array.astype(np.float32), output_path, band=band_num)
# drigo.array_to_raster(
# wind_array.astype(np.float32), output_path,
# output_geo=nldas_geo, output_proj=nldas_proj,
# stats_flag=stats_flag)
del wind_array, wind_u_array, wind_v_array
input_ds = None
if stats_flag:
drigo.raster_statistics(output_path)
logging.debug('\nScript Complete')
