def temp_adjust_parameters(config_path):
"""Calculate GSFLOW Temperature Adjustment Parameters
Parameters
----------
config_path : str
Project configuration file (.ini) path.
Returns
-------
None
"""
# Hardcoded HRU field formats for now
tmax_field_fmt = 'TMAX_{:02d}'
tmin_field_fmt = 'TMIN_{:02d}'
tmax_adj_field_fmt = 'TMX_ADJ_{:02d}'
tmin_adj_field_fmt = 'TMN_ADJ_{:02d}'
# Initialize hru_parameters class
hru = support.HRUParameters(config_path)
# Open input parameter config file
inputs_cfg = ConfigParser.ConfigParser()
try:
inputs_cfg.readfp(open(config_path))
except Exception as e:
logging.error('\nERROR: Config file could not be read, '
'is not an input file, or does not exist\n'
' config_file = {}\n'
' Exception: {}\n'.format(config_path, e))
sys.exit()
# Log DEBUG to file
log_file_name = 'temp_adjust_parameters_log.txt'
log_console = logging.FileHandler(filename=os.path.join(
hru.log_ws, log_file_name),
mode='w')
log_console.setLevel(logging.DEBUG)
log_console.setFormatter(logging.Formatter('%(message)s'))
logging.getLogger('').addHandler(log_console)
logging.info('\nGSFLOW Temperature Adjustment Parameters')
# Units
temp_obs_units = support.get_param('temp_obs_units', 'C',
inputs_cfg).upper()
temp_units_list = ['C', 'F', 'K']
# Compare against the upper case of the values in the list
# but don't modify the acceptable units list
if temp_obs_units not in temp_units_list:
logging.error('\nERROR: Invalid observed temperature units ({})\n '
'Valid units are: {}'.format(temp_obs_units,
', '.join(temp_units_list)))
sys.exit()
# Check input paths
if not arcpy.Exists(hru.polygon_path):
logging.error('\nERROR: Fishnet ({}) does not exist'.format(
hru.polygon_path))
sys.exit()
# Temperature calculation method
try:
temp_calc_method = inputs_cfg.get('INPUTS',
'temperature_calc_method').upper()
except:
temp_calc_method = '1STA'
logging.info(' Defaulting temperature_calc_method = {}'.format(
temp_calc_method))
temp_calc_options = ['ZONES', '1STA', 'LAPSE']
if temp_calc_method not in temp_calc_options:
logging.error(
'\nERROR: Invalid temperature calculation method ({})\n '
'Valid methods are: {}'.format(temp_calc_method,
', '.join(temp_calc_options)))
sys.exit()
if temp_calc_method == 'LAPSE':
logging.warning(
'\nWARNING: If temperature calculation set to LAPSE,'
'\n it is not necessary to run the temp_adjust_parameters.py'
'\n Exiting')
return False
if temp_calc_method == 'ZONES':
temp_zone_orig_path = inputs_cfg.get('INPUTS', 'temp_zone_path')
try:
temp_zone_id_field = inputs_cfg.get('INPUTS', 'temp_zone_id_field')
except:
logging.error(
'\nERROR: temp_zone_id_field must be set in INI to apply '
'zone specific temperature adjustments\n')
sys.exit()
try:
temp_hru_id_field = inputs_cfg.get('INPUTS', 'temp_hru_id_field')
except:
temp_hru_id_field = None
logging.warning(
' temp_hru_id_field was not set in the INI file\n'
' Temperature adjustments will not be changed to match station '
'values')
# Field name for TSTA hard coded, but could be changed to be read from
# config file like temp_zone
hru_tsta_field = 'HRU_TSTA'
try:
tmax_obs_field_fmt = inputs_cfg.get('INPUTS',
'tmax_obs_field_format')
except:
tmax_obs_field_fmt = 'TMAX_{:02d}'
logging.info(' Defaulting tmax_obs_field_format = {}'.format(
tmax_obs_field_fmt))
try:
tmin_obs_field_fmt = inputs_cfg.get('INPUTS',
'temp_obs_field_format')
except:
tmin_obs_field_fmt = 'TMIN_{:02d}'
logging.info(' Defaulting tmin_obs_field_format = {}'.format(
tmin_obs_field_fmt))
if not arcpy.Exists(temp_zone_orig_path):
logging.error(
'\nERROR: Temperature Zone ({}) does not exist'.format(
temp_zone_orig_path))
sys.exit()
# temp_zone_path must be a polygon shapefile
if arcpy.Describe(temp_zone_orig_path).datasetType != 'FeatureClass':
logging.error(
'\nERROR: temp_zone_path must be a polygon shapefile')
sys.exit()
# Check temp_zone_id_field
if temp_zone_id_field.upper() in ['FID', 'OID']:
temp_zone_id_field = arcpy.Describe(
temp_zone_orig_path).OIDFieldName
logging.warning('\n NOTE: Using {} to set {}\n'.format(
temp_zone_id_field, hru.temp_zone_id_field))
elif not arcpy.ListFields(temp_zone_orig_path, temp_zone_id_field):
logging.error(
'\nERROR: temp_zone_id_field field {} does not exist\n'.format(
temp_zone_id_field))
sys.exit()
# Need to check that field is an int type
# Should we only check active cells (HRU_TYPE > 0)?
elif not [
f.type for f in arcpy.Describe(temp_zone_orig_path).fields
if (f.name == temp_zone_id_field
and f.type in ['SmallInteger', 'Integer'])
]:
logging.error(
'\nERROR: temp_zone_id_field field {} must be an integer type\n'
.format(temp_zone_id_field))
sys.exit()
# Need to check that field values are all positive
# Should we only check active cells (HRU_TYPE > 0)?
elif min([
row[0] for row in arcpy.da.SearchCursor(
temp_zone_orig_path, [temp_zone_id_field])
]) <= 0:
logging.error(
'\nERROR: temp_zone_id_field values cannot be negative\n')
sys.exit()
# Check hru_tsta_field
if not arcpy.ListFields(temp_zone_orig_path, hru_tsta_field):
logging.error(
'\nERROR: hru_tsta_field field {} does not exist\n'.format(
hru_tsta_field))
sys.exit()
# Need to check that field is an int type
# Only check active cells (HRU_TYPE >0)?!
elif not [
f.type for f in arcpy.Describe(temp_zone_orig_path).fields
if (f.name == hru_tsta_field
and f.type in ['SmallInteger', 'Integer'])
]:
logging.error(
'\nERROR: hru_tsta_field field {} must be an integer type\n'.
format(hru_tsta_field))
sys.exit()
# Need to check that field values are all positive
# Only check active cells (HRU_TYPE >0)?!
elif min([
row[0] for row in arcpy.da.SearchCursor(
temp_zone_orig_path, [hru_tsta_field])
]) <= 0:
logging.error(
'\nERROR: hru_tsta_field values cannot be negative\n')
sys.exit()
# Check temp_hru_id_field
# temp_hru_id values are checked later
if temp_hru_id_field is not None:
if not arcpy.ListFields(temp_zone_orig_path, temp_hru_id_field):
logging.error(
'\nERROR: temp_hru_id_field field {} does not exist\n'.
format(temp_hru_id_field))
sys.exit()
# Need to check that field is an int type
elif not [
f.type for f in arcpy.Describe(temp_zone_orig_path).fields
if (f.name == temp_hru_id_field
and f.type in ['SmallInteger', 'Integer'])
]:
logging.error(
'\nERROR: temp_hru_id_field field {} must be an integer type\n'
.format(temp_hru_id_field))
sys.exit()
# Need to check that field values are not negative (0 is okay)
elif min([
row[0] for row in arcpy.da.SearchCursor(
temp_zone_orig_path, [temp_hru_id_field])
]) < 0:
logging.error(
'\nERROR: temp_hru_id_field values cannot be negative\n')
sys.exit()
elif temp_calc_method == '1STA':
# If a zone shapefile is not used, temperature must be set manually
tmax_obs_list = inputs_cfg.get('INPUTS', 'tmax_obs_list')
tmin_obs_list = inputs_cfg.get('INPUTS', 'tmin_obs_list')
# Check that values are floats
try:
tmax_obs_list = map(float, tmax_obs_list.split(','))
except ValueError:
logging.error('\nERROR: tmax_obs_list (mean monthly tmax) '
'values could not be parsed as floats\n')
sys.exit()
try:
tmin_obs_list = map(float, tmin_obs_list.split(','))
except ValueError:
logging.error('\nERROR: tmin_obs_list (mean monthly tmin) '
'values could not be parsed as floats\n')
sys.exit()
# Check that there are 12 values
if len(tmax_obs_list) != 12:
logging.error('\nERROR: There must be exactly 12 mean monthly '
'observed tmax values based to tmax_obs_list\n')
sys.exit()
logging.info(
' Observed mean monthly tmax ({}):\n {}\n'
' (Script will assume these are listed in month order, '
'i.e. Jan, Feb, ...)'.format(temp_obs_units,
', '.join(map(str, tmax_obs_list))))
if len(tmin_obs_list) != 12:
logging.error('\nERROR: There must be exactly 12 mean monthly '
'observed tmin values based to tmin_obs_list\n')
sys.exit()
logging.info(
' Observed mean monthly tmin ({}):\n {}\n'
' (Script will assume these are listed in month order, '
'i.e. Jan, Feb, ...)'.format(temp_obs_units,
', '.join(map(str, tmin_obs_list))))
# Check if all the values are 0
if tmax_obs_list == ([0.0] * 12):
logging.error(
'\nERROR: The observed tmax values are all 0.\n'
' To compute tmax adjustments, please set the tmax_obs_list '
'parameter in the INI with\n observed mean monthly tmax '
'values (i.e. from a weather station)')
sys.exit()
if tmin_obs_list == ([0.0] * 12):
logging.error(
'\nERROR: The observed tmin values are all 0.\n'
' To compute tmin adjustments, please set the tmin_obs_list '
'parameter in the INI with\n observed mean monthly tmin '
'values (i.e. from a weather station)')
sys.exit()
# Get the temperature HRU ID
try:
temp_hru_id = inputs_cfg.getint('INPUTS', 'temp_hru_id')
except:
temp_hru_id = 0
# Check that the temp_hru_id is a valid cell hru_id
# If temp_hru_id is 0, temperature adjustments will not be adjusted
if temp_hru_id > 0:
# Check that HRU_ID is valid
logging.info(' Temperature HRU_ID: {}'.format(temp_hru_id))
arcpy.MakeTableView_management(
hru.polygon_path, "layer",
"{} = {}".format(hru.id_field, temp_hru_id))
if (temp_hru_id != 0 and int(
arcpy.GetCount_management("layer").getOutput(0)) == 0):
logging.error(
'\nERROR: temp_hru_id {0} is not a valid cell hru_id'
'\nERROR: temp adjustments will NOT be forced to 1'
' at cell {0}\n'.format(temp_hru_id))
temp_hru_id = 0
arcpy.Delete_management("layer")
else:
logging.info(
' Temperatures adjustments will not be adjusted to match '
'station values\n (temp_hru_id = 0)')
# Could add a second check that HRU_TSTA has values >0
# Build output folders if necessary
temp_adj_temp_ws = os.path.join(hru.param_ws, 'temp_adjust')
if not os.path.isdir(temp_adj_temp_ws):
os.mkdir(temp_adj_temp_ws)
temp_zone_path = os.path.join(temp_adj_temp_ws, 'temp_zone.shp')
# temp_zone_clip_path = os.path.join(temp_adj_temp_ws, 'temp_zone_clip.shp')
# Set ArcGIS environment variables
arcpy.CheckOutExtension('Spatial')
env.overwriteOutput = True
# env.pyramid = 'PYRAMIDS -1'
env.pyramid = 'PYRAMIDS 0'
env.workspace = hru.param_ws
env.scratchWorkspace = hru.scratch_ws
# Set month list based on flags
month_list = range(1, 13)
tmax_field_list = [tmax_field_fmt.format(m) for m in month_list]
tmin_field_list = [tmin_field_fmt.format(m) for m in month_list]
tmax_adj_field_list = [tmax_adj_field_fmt.format(m) for m in month_list]
tmin_adj_field_list = [tmin_adj_field_fmt.format(m) for m in month_list]
# Check fields
logging.info('\nAdding temperature adjust fields if necessary')
# Temperature zone fields
support.add_field_func(hru.polygon_path, hru.temp_zone_id_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.hru_tsta_field, 'SHORT')
# Temperature adjustment fields
for tmax_adj_field in tmax_adj_field_list:
support.add_field_func(hru.polygon_path, tmax_adj_field, 'DOUBLE')
for tmin_adj_field in tmin_adj_field_list:
support.add_field_func(hru.polygon_path, tmin_adj_field, 'DOUBLE')
# Calculate temperature zone ID
if temp_calc_method == 'ZONES':
logging.info('\nCalculating cell HRU Temperature Zone ID')
temp_zone_desc = arcpy.Describe(temp_zone_orig_path)
temp_zone_sr = temp_zone_desc.spatialReference
logging.debug(' Zones: {}'.format(temp_zone_orig_path))
logging.debug(' Projection: {}'.format(temp_zone_sr.name))
logging.debug(' GCS: {}'.format(temp_zone_sr.GCS.name))
# Reset temp_ZONE_ID
logging.info(' Resetting {} to 0'.format(hru.temp_zone_id_field))
arcpy.CalculateField_management(hru.polygon_path,
hru.temp_zone_id_field, 0, 'PYTHON')
# If temp_zone spat_ref doesn't match hru_param spat_ref
# Project temp_zone to hru_param spat ref
# Otherwise, read temp_zone directly
if hru.sr.name != temp_zone_sr.name:
logging.info(' Projecting temperature zones...')
# Set preferred transforms
transform_str = support.transform_func(hru.sr, temp_zone_sr)
logging.debug(' Transform: {}'.format(transform_str))
# Project temp_zone shapefile
arcpy.Project_management(temp_zone_orig_path, temp_zone_path,
hru.sr, transform_str, temp_zone_sr)
del transform_str
else:
arcpy.Copy_management(temp_zone_orig_path, temp_zone_path)
# # Remove all unnecessary fields
# for field in arcpy.ListFields(temp_zone_path):
# skip_field_list = temp_obs_field_list + [temp_zone_id_field, 'Shape']
# if field.name not in skip_field_list:
# try:
# arcpy.DeleteField_management(temp_zone_path, field.name)
# except:
# pass
# Set temperature zone ID
logging.info(' Setting {}'.format(hru.temp_zone_id_field))
support.zone_by_centroid_func(temp_zone_path, hru.temp_zone_id_field,
temp_zone_id_field, hru.polygon_path,
hru.point_path, hru)
# support.zone_by_area_func(
# temp_zone_layer, hru.temp_zone_id_field, temp_zone_id_field,
# hru.polygon_path, hru, hru_area_field, None, 50)
# Set HRU_TSTA
logging.info(' Setting {}'.format(hru.hru_tsta_field))
support.zone_by_centroid_func(temp_zone_path, hru.hru_tsta_field,
hru_tsta_field, hru.polygon_path,
hru.point_path, hru)
del temp_zone_desc, temp_zone_sr
elif temp_calc_method == '1STA':
# Set all cells to zone 1
arcpy.CalculateField_management(hru.polygon_path,
hru.temp_zone_id_field, 1, 'PYTHON')
# Calculate adjustments
logging.info('\nCalculating mean monthly temperature adjustments')
if temp_calc_method == 'ZONES':
# Read mean monthly values for each zone
tmax_obs_dict = dict()
tmin_obs_dict = dict()
tmax_obs_field_list = [
tmax_obs_field_fmt.format(m) for m in month_list
]
tmin_obs_field_list = [
tmin_obs_field_fmt.format(m) for m in month_list
]
tmax_fields = [temp_zone_id_field] + tmax_obs_field_list
tmin_fields = [temp_zone_id_field] + tmin_obs_field_list
logging.debug(' Tmax Obs. Fields: {}'.format(', '.join(tmax_fields)))
logging.debug(' Tmin Obs. Fields: {}'.format(', '.join(tmax_fields)))
with arcpy.da.SearchCursor(temp_zone_path, tmax_fields) as s_cursor:
for row in s_cursor:
tmax_obs_dict[int(row[0])] = map(float, row[1:13])
with arcpy.da.SearchCursor(temp_zone_path, tmin_fields) as s_cursor:
for row in s_cursor:
tmin_obs_dict[int(row[0])] = map(float, row[1:13])
# Convert values to Celsius if necessary to match PRISM
if temp_obs_units == 'F':
tmax_obs_dict = {
z: [(t - 32) * (5.0 / 9) for t in t_list]
for z, t_list in tmax_obs_dict.items()
}
tmin_obs_dict = {
z: [(t - 32) * (5.0 / 9) for t in t_list]
for z, t_list in tmin_obs_dict.items()
}
elif temp_obs_units == 'K':
tmax_obs_dict = {
z: [(t - 273.15) for t in t_list]
for z, t_list in tmax_obs_dict.items()
}
tmin_obs_dict = {
z: [(t - 273.15) for t in t_list]
for z, t_list in tmin_obs_dict.items()
}
tmax_zone_list = sorted(tmax_obs_dict.keys())
tmin_zone_list = sorted(tmin_obs_dict.keys())
logging.debug(' Tmax Zones: {}'.format(tmax_zone_list))
logging.debug(' Tmin Zones: {}'.format(tmin_zone_list))
# Print the observed temperature values
logging.debug(' Observed Tmax')
for zone, tmax_obs in tmax_obs_dict.items():
logging.debug(' {}: {}'.format(
zone, ', '.join(['{:.2f}'.format(x) for x in tmax_obs])))
logging.debug(' Observed Tmin')
for zone, tmin_obs in tmin_obs_dict.items():
logging.debug(' {}: {}'.format(
zone, ', '.join(['{:.2f}'.format(x) for x in tmin_obs])))
# Default all zones to an adjustment of 0
tmax_adj_dict = {z: [0] * 12 for z in tmax_zone_list}
tmin_adj_dict = {z: [0] * 12 for z in tmin_zone_list}
# Get list of HRU_IDs for each zone
fields = [hru.temp_zone_id_field, hru.id_field]
zone_hru_id_dict = defaultdict(list)
with arcpy.da.SearchCursor(hru.polygon_path, fields) as s_cursor:
for row in s_cursor:
zone_hru_id_dict[int(row[0])].append(int(row[1]))
# Check that TEMP_HRU_IDs are in the correct zone
# Default all temperature zone HRU IDs to 0
temp_hru_id_dict = {z: 0 for z in tmax_zone_list}
if temp_hru_id_field is not None:
fields = [temp_zone_id_field, temp_hru_id_field]
logging.debug(
' Temp Zone ID field: {}'.format(temp_zone_id_field))
logging.debug(' Temp HRU ID field: {}'.format(temp_hru_id_field))
with arcpy.da.SearchCursor(temp_zone_path, fields) as s_cursor:
for row in s_cursor:
temp_zone = int(row[0])
hru_id = int(row[1])
if hru_id == 0 or hru_id in zone_hru_id_dict[temp_zone]:
temp_hru_id_dict[temp_zone] = hru_id
logging.debug(' {}: {}'.format(temp_zone, hru_id))
else:
logging.error(
'\nERROR: HRU_ID {} is not in temperature ZONE {}'.
format(hru_id, temp_hru_id_dict[temp_zone]))
sys.exit()
# Get gridded tmax values for each TEMP_HRU_ID
fields = [hru.temp_zone_id_field, hru.id_field] + tmax_field_list
with arcpy.da.SearchCursor(hru.polygon_path, fields) as s_cursor:
for row in s_cursor:
temp_zone = int(row[0])
hru_id = int(row[1])
if hru_id == 0:
pass
elif hru_id in temp_hru_id_dict.values():
tmax_gridded_list = map(float, row[2:14])
tmax_obs_list = tmax_obs_dict[temp_zone]
tmax_adj_list = [
float(o) - t
for o, t in zip(tmax_obs_list, tmax_gridded_list)
]
tmax_adj_dict[temp_zone] = tmax_adj_list
# Get gridded tmin values for each TEMP_HRU_ID
fields = [hru.temp_zone_id_field, hru.id_field] + tmin_field_list
with arcpy.da.SearchCursor(hru.polygon_path, fields) as s_cursor:
for row in s_cursor:
temp_zone = int(row[0])
hru_id = int(row[1])
if hru_id == 0:
pass
elif hru_id in temp_hru_id_dict.values():
tmin_gridded_list = map(float, row[2:14])
tmin_obs_list = tmin_obs_dict[temp_zone]
tmin_adj_list = [
float(o) - t
for o, t in zip(tmin_obs_list, tmin_gridded_list)
]
tmin_adj_dict[temp_zone] = tmin_adj_list
del temp_hru_id_dict, zone_hru_id_dict, fields
logging.debug(' Tmax Adjustment Factors:')
for k, v in tmax_adj_dict.items():
logging.debug(' {}: {}'.format(
k, ', '.join(['{:.3f}'.format(x) for x in v])))
logging.debug(' Tmin Adjustment Factors:')
for k, v in tmin_adj_dict.items():
logging.debug(' {}: {}'.format(
k, ', '.join(['{:.3f}'.format(x) for x in v])))
logging.debug('\nWriting adjustment values to hru_params')
fields = [hru.temp_zone_id_field]
fields.extend(tmax_field_list + tmax_adj_field_list)
fields.extend(tmin_field_list + tmin_adj_field_list)
with arcpy.da.UpdateCursor(hru.polygon_path, fields) as u_cursor:
for row in u_cursor:
zone = int(row[0])
for i, month in enumerate(month_list):
tmax_i = fields.index(tmax_field_fmt.format(month))
tmax_adj_i = fields.index(tmax_adj_field_fmt.format(month))
row[tmax_adj_i] = (row[tmax_i] - tmax_obs_dict[zone][i] +
tmax_adj_dict[zone][i])
tmin_i = fields.index(tmin_field_fmt.format(month))
tmin_adj_i = fields.index(tmin_adj_field_fmt.format(month))
row[tmin_adj_i] = (row[tmin_i] - tmin_obs_dict[zone][i] +
tmin_adj_dict[zone][i])
u_cursor.updateRow(row)
del row
elif temp_calc_method == '1STA':
# Get gridded temperature at temp_HRU_ID
tmax_fields = [hru.id_field] + tmax_field_list
tmin_fields = [hru.id_field] + tmin_field_list
logging.debug(' Tmax Fields: {}'.format(', '.join(tmax_field_list)))
logging.debug(' Tmin Fields: {}'.format(', '.join(tmin_field_list)))
# Convert values to Celsius if necessary to match PRISM
if temp_obs_units == 'F':
tmax_obs_list = [(t - 32) * (5.0 / 9) for t in tmax_obs_list]
tmin_obs_list = [(t - 32) * (5.0 / 9) for t in tmin_obs_list]
elif temp_obs_units == 'K':
tmax_obs_list = [t - 273.15 for t in tmax_obs_list]
tmin_obs_list = [t - 273.15 for t in tmin_obs_list]
if temp_obs_units != 'C':
logging.info('\nConverted Mean Monthly Tmax ({}):\n {}'.format(
temp_obs_units, ', '.join(map(str, tmax_obs_list))))
logging.info('Converted Mean Monthly Tmin ({}):\n {}'.format(
temp_obs_units, ', '.join(map(str, tmin_obs_list))))
# Scale all adjustments so gridded temperature will match observed
# temperature at target cell
if temp_hru_id != 0:
tmax_gridded_list = map(
float,
arcpy.da.SearchCursor(
hru.polygon_path, tmax_fields,
'"{}" = {}'.format(hru.id_field, temp_hru_id)).next()[1:])
logging.debug(' Gridded Tmax: {}'.format(', '.join(
['{:.2f}'.format(p) for p in tmax_gridded_list])))
tmin_gridded_list = map(
float,
arcpy.da.SearchCursor(
hru.polygon_path, tmin_fields,
'"{}" = {}'.format(hru.id_field, temp_hru_id)).next()[1:])
logging.debug(' Gridded Tmin: {}'.format(', '.join(
['{:.2f}'.format(p) for p in tmin_gridded_list])))
# Difference of MEASURED or OBSERVED TEMP to GRIDDED TEMP
tmax_adj_list = [
float(o) - t for o, t in zip(tmax_obs_list, tmax_gridded_list)
]
logging.info(' Obs./Gridded: {}'.format(', '.join(
['{:.3f}'.format(p) for p in tmax_adj_list])))
tmin_adj_list = [
float(o) - t for o, t in zip(tmin_obs_list, tmin_gridded_list)
]
logging.info(' Obs./Gridded: {}'.format(', '.join(
['{:.3f}'.format(p) for p in tmin_adj_list])))
else:
tmax_adj_list = [0 for p in tmax_obs_list]
tmin_adj_list = [0 for p in tmin_obs_list]
# Use single mean monthly tmax for all cells
# Assume tmax_obs_list is in month order
fields = tmax_field_list + tmax_adj_field_list
with arcpy.da.UpdateCursor(hru.polygon_path, fields) as u_cursor:
for row in u_cursor:
for i, month in enumerate(month_list):
tmax_i = fields.index(tmax_field_fmt.format(month))
tmax_adj_i = fields.index(tmax_adj_field_fmt.format(month))
row[tmax_adj_i] = (row[tmax_i] - tmax_obs_list[i] +
tmax_adj_list[i])
u_cursor.updateRow(row)
del row
# Use single mean monthly tmax for all cells
# Assume tmax_obs_list is in month order
fields = tmin_field_list + tmin_adj_field_list
with arcpy.da.UpdateCursor(hru.polygon_path, fields) as u_cursor:
for row in u_cursor:
for i, month in enumerate(month_list):
tmin_i = fields.index(tmin_field_fmt.format(month))
tmin_adj_i = fields.index(tmin_adj_field_fmt.format(month))
row[tmin_adj_i] = (row[tmin_i] - tmin_obs_list[i] +
tmin_adj_list[i])
u_cursor.updateRow(row)
del row
def hru_parameters(config_path):
"""Calculate GSFLOW HRU Parameters
Parameters
----------
config_path : str
Project configuration file (.ini) path.
Returns
-------
None
"""
# Initialize hru parameters class
hru = support.HRUParameters(config_path)
# Open input parameter config file
inputs_cfg = ConfigParser.ConfigParser()
try:
inputs_cfg.readfp(open(config_path))
except Exception as e:
logging.error(
'\nERROR: Config file could not be read, '
'is not an input file, or does not exist\n'
' config_file = {}\n'
' Exception: {}\n'.format(config_path, e))
sys.exit()
# Log DEBUG to file
log_file_name = 'hru_parameters_log.txt'
log_console = logging.FileHandler(
filename=os.path.join(hru.log_ws, log_file_name), mode='w')
log_console.setLevel(logging.DEBUG)
log_console.setFormatter(logging.Formatter('%(message)s'))
logging.getLogger('').addHandler(log_console)
logging.info('\nGSFLOW HRU Parameters')
# Read parameters from config file
study_area_orig_path = inputs_cfg.get('INPUTS', 'study_area_path')
try:
set_lake_flag = inputs_cfg.getboolean('INPUTS', 'set_lake_flag')
except ConfigParser.NoOptionError:
set_lake_flag = False
logging.info(
' Missing INI parameter, setting {} = {}'.format(
'set_lake_flag', set_lake_flag))
if set_lake_flag:
lake_orig_path = inputs_cfg.get('INPUTS', 'lake_path')
lake_zone_field = inputs_cfg.get('INPUTS', 'lake_zone_field')
lake_area_pct = inputs_cfg.getfloat('INPUTS', 'lake_area_pct')
# Model points
model_inputs_path = inputs_cfg.get('INPUTS', 'model_points_path')
try:
model_points_zone_field = inputs_cfg.get(
'INPUTS', 'model_points_zone_field')
except:
model_points_zone_field = 'FID'
logging.info(
' Missing INI parameter, setting {} = {}'.format(
'model_points_zone_field', model_points_zone_field))
try:
model_points_type_field = inputs_cfg.get(
'INPUTS', 'model_points_type_field')
except:
model_points_type_field = 'TYPE'
logging.info(
' Missing INI parameter, setting {} = {}'.format(
'model_points_type_field', model_points_type_field))
# Control flags
try:
calc_flow_acc_dem_flag = inputs_cfg.getboolean(
'INPUTS', 'calc_flow_acc_dem_flag')
except ConfigParser.NoOptionError:
calc_flow_acc_dem_flag = False
logging.info(
' Missing INI parameter, setting {} = {}'.format(
'calc_flow_acc_dem_flag', calc_flow_acc_dem_flag))
try:
calc_topo_index_flag = inputs_cfg.getboolean(
'INPUTS', 'calc_topo_index_flag')
except ConfigParser.NoOptionError:
calc_topo_index_flag = False
logging.info(
' Missing INI parameter, setting {} = {}'.format(
'calc_topo_index_flag', calc_topo_index_flag))
# try:
# set_ppt_zones_flag = inputs_cfg.getboolean(
# 'INPUTS', 'set_ppt_zones_flag')
# except ConfigParser.NoOptionError:
# set_ppt_zones_flag = False
# logging.info(
# ' Missing INI parameter, setting {} = {}'.format(
# 'set_ppt_zones_flag', set_ppt_zones_flag))
# Check input paths
if not arcpy.Exists(hru.polygon_path):
logging.error(
'\nERROR: Fishnet ({}) does not exist'.format(
hru.polygon_path))
sys.exit()
if set_lake_flag:
if not arcpy.Exists(lake_orig_path):
logging.error(
'\nERROR: Lake layer ({}) does not exist'.format(
lake_orig_path))
sys.exit()
# lake_path must be a polygon shapefile
if arcpy.Describe(lake_orig_path).datasetType != 'FeatureClass':
logging.error(
'\nERROR: lake_path must be a polygon shapefile')
sys.exit()
# Check lake_zone_field
if lake_zone_field.upper() in ['', 'FID', 'NONE']:
lake_zone_field = arcpy.Describe(lake_orig_path).OIDFieldName
logging.warning(
'\n NOTE: Using {} to set {}\n'.format(
lake_zone_field, hru.lake_id_field))
elif not arcpy.ListFields(lake_orig_path, lake_zone_field):
logging.error(
'\nERROR: lake_zone_field field {} does not exist\n'.format(
lake_zone_field))
sys.exit()
# Need to check that lake_zone_field is an int type
elif not [f.type for f in arcpy.Describe(lake_orig_path).fields
if (f.name == lake_zone_field and
f.type in ['SmallInteger', 'Integer'])]:
logging.error(
'\nERROR: lake_zone_field field {} must be an '
'integer type\n'.format(lake_zone_field))
sys.exit()
# Check model points
if not os.path.isfile(model_inputs_path):
logging.error(
'\nERROR: Model points shapefiles does not exist'
'\nERROR: {}'.format(model_inputs_path))
sys.exit()
# model_points_path must be a point shapefile
elif arcpy.Describe(model_inputs_path).datasetType != 'FeatureClass':
logging.error(
'\nERROR: model_points_path must be a point shapefile')
sys.exit()
# For now, study area has to be a polygon
if arcpy.Describe(study_area_orig_path).datasetType != 'FeatureClass':
logging.error(
'\nERROR: For now, study area must be a polygon shapefile')
sys.exit()
# Build output folder if necessary
hru_temp_ws = os.path.join(hru.param_ws, 'hru_temp')
if not os.path.isdir(hru_temp_ws):
os.mkdir(hru_temp_ws)
# Output paths
study_area_path = os.path.join(hru_temp_ws, 'study_area.shp')
lake_path = os.path.join(hru_temp_ws, 'lakes.shp')
lake_clip_path = os.path.join(hru_temp_ws, 'lake_clip.shp')
model_points_path = os.path.join(hru_temp_ws, 'model_points.shp')
# Set ArcGIS environment variables
arcpy.CheckOutExtension('Spatial')
env.overwriteOutput = True
env.pyramid = 'PYRAMIDS -1'
# env.pyramid = 'PYRAMIDS 0'
env.workspace = hru.param_ws
env.scratchWorkspace = hru.scratch_ws
# Create HRU points at polygon centroids
if not arcpy.Exists(hru.point_path):
logging.info('\n Building HRU point shapefile')
# FeatureToPoint will copy all fields in hru.polygon_path
# arcpy.FeatureToPoint_management(
# hru.polygon_path, hru.point_path)
# Build point_path directly
arcpy.CreateFeatureclass_management(
os.path.dirname(hru.point_path),
os.path.basename(hru.point_path), 'POINT')
arcpy.DefineProjection_management(hru.point_path, hru.sr)
arcpy.AddField_management(
hru.point_path, hru.fid_field, 'LONG')
hru_centroid_list = [
row for row in arcpy.da.SearchCursor(
hru.polygon_path, ['OID@', 'SHAPE@XY'])]
with arcpy.da.InsertCursor(
hru.point_path,
['OID@', 'SHAPE@XY', hru.fid_field]) as update_c:
for hru_centroid in hru_centroid_list:
update_c.insertRow(
[hru_centroid[0], hru_centroid[1], hru_centroid[0]])
del hru_centroid_list
# Check existing HRU points
else:
# Remove any extra fields
field_remove_list = [
f.name for f in arcpy.ListFields(hru.point_path)
if f.name not in ['FID', 'Shape', hru.fid_field]]
# Skip if there is only one field in the shapefile
if field_remove_list and len(field_remove_list) > 1:
logging.info('\n Removing HRU point fields')
for field in field_remove_list:
logging.debug(' {}'.format(field))
try:
arcpy.DeleteField_management(hru.point_path, field)
except Exception as e:
logging.debug(' Unhandled exception: {}'.format(e))
continue
# Save original FID
if len(arcpy.ListFields(hru.point_path, hru.fid_field)) == 0:
arcpy.AddField_management(
hru.point_path, hru.fid_field, 'LONG')
arcpy.CalculateField_management(
hru.point_path, hru.fid_field, '!FID!', 'PYTHON')
if len(arcpy.ListFields(hru.point_path, 'Id')) > 0:
arcpy.DeleteField_management(hru.point_path, 'Id')
del field_remove_list
# Add all output fields
logging.info('\nAdding fields if necessary')
logging.info(
' Note: You may see duplicate field names when writing to a network '
'drive')
# HRU/DEM Fields
support.add_field_func(hru.polygon_path, hru.fid_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.id_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.type_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.dem_mean_field, 'DOUBLE')
#support.add_field_func(hru.polygon_path, hru.dem_median_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_min_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_max_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_adj_field, 'DOUBLE')
if calc_flow_acc_dem_flag:
support.add_field_func(hru.polygon_path, hru.dem_flowacc_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_sum_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_count_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_sink_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.crt_elev_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.crt_fill_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_aspect_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.dem_slope_deg_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_slope_rad_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.dem_slope_pct_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.area_field, 'DOUBLE')
if calc_topo_index_flag:
support.add_field_func(hru.polygon_path, hru.topo_index_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.row_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.col_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.x_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.y_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.lat_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.lon_field, 'DOUBLE')
# Lake fields
support.add_field_func(hru.polygon_path, hru.lake_id_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.lake_area_field, 'DOUBLE')
# Stream fields
support.add_field_func(hru.polygon_path, hru.iseg_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.irunbound_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.flow_dir_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.krch_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.irch_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.jrch_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.iseg_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.reach_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.rchlen_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.maxreach_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.outseg_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.iupseg_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.subbasin_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.segbasin_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.outflow_field, 'LONG')
support.add_field_func(hru.polygon_path, hru.strm_top_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.strm_slope_field, 'DOUBLE')
# Sink field
support.add_field_func(hru.polygon_path, hru.hru_sink_field, 'LONG')
# Precipitation zone fields
support.add_field_func(hru.polygon_path, hru.ppt_zone_id_field, 'SHORT')
support.add_field_func(hru.polygon_path, hru.hru_psta_field, 'SHORT')
# Temperature zone fields
# if temp_calc_method == 'ZONES':
# support.add_field_func(hru.polygon_path, hru.temp_zone_id_field, 'SHORT')
# support.add_field_func(hru.polygon_path, hru.hru_tsta_field, 'SHORT')
# DEM based
support.add_field_func(hru.polygon_path, hru.jh_tmax_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.jh_tmin_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.jh_coef_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.snarea_thresh_field, 'DOUBLE')
# Aspect based
support.add_field_func(hru.polygon_path, hru.tmax_adj_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.tmin_adj_field, 'DOUBLE')
# Vegetation fields
support.add_field_func(hru.polygon_path, hru.cov_type_field, 'SHORT')
support.add_field_func(hru.polygon_path, hru.covden_sum_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.covden_win_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.rad_trncf_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.snow_intcp_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.srain_intcp_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.wrain_intcp_field, 'DOUBLE')
# Soil fields
support.add_field_func(hru.polygon_path, hru.awc_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.clay_pct_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.sand_pct_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.ksat_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.soil_type_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.soil_root_max_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.moist_init_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.moist_max_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.rechr_init_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.rechr_max_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.ssr2gw_rate_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.ssr2gw_k_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.slowcoef_lin_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.slowcoef_sq_field, 'DOUBLE')
# Impervious fields
support.add_field_func(hru.polygon_path, hru.imperv_pct_field, 'DOUBLE')
support.add_field_func(hru.polygon_path, hru.carea_max_field, 'DOUBLE')
# PRISM mean monthly fields
month_list = ['{:02d}'.format(m) for m in range(1, 13)]
# month_list.extend(['14'])
for prism_data_name in ['PPT', 'TMAX', 'TMIN']:
for month in month_list:
support.add_field_func(
hru.polygon_path,
'{}_{}'.format(prism_data_name, month), 'DOUBLE')
# PRISM mean monthly precipitation ratio fields
for month in month_list:
if month == '14':
continue
support.add_field_func(
hru.polygon_path, 'PPT_RT_{}'.format(month), 'DOUBLE')
# Temperature adjust fields are added in temp_adjust_parameters.py if needed
# for month in month_list:
# if month == '14':
# continue
# support.add_field_func(
# hru.polygon_path, 'TMX_ADJ_{}'.format(month), 'DOUBLE')
# for month in month_list:
# if month == '14':
# continue
# support.add_field_func(
# hru.polygon_path, 'TMN_ADJ_{}'.format(month), 'DOUBLE')
# Id field is added by default to new fishnets
if arcpy.ListFields(hru.polygon_path, 'Id'):
arcpy.DeleteField_management(hru.polygon_path, 'Id')
logging.info('\nCalculating parameters')
# Keep original FID for subsetting in zonal stats
logging.info(' Saving original HRU FID to {}'.format(
hru.fid_field))
arcpy.CalculateField_management(
hru.polygon_path, hru.fid_field, '!FID!', 'PYTHON')
# Cell X/Y
logging.info(' Calculating cell X/Y')
cell_xy_func(hru.polygon_path, hru.x_field, hru.y_field)
# Create unique ID, start at top left corner, work down rows
# Row/Col numbered from top left corner (1's based numbering)
logging.info(' Calculating cell ID/row/col')
cell_id_col_row_func(
hru.polygon_path, hru.id_field, hru.col_field, hru.row_field,
hru.extent, hru.cs)
# Cell Lat/Lon
logging.info(' Calculating cell lat/lon')
cell_lat_lon_func(
hru.polygon_path, hru.lat_field, hru.lon_field, hru.sr.GCS)
# Cell Area
logging.info(' Calculating cell area (acres)')
arcpy.CalculateField_management(
hru.polygon_path, hru.area_field, '!SHAPE.AREA@acres!', 'PYTHON')
# Reset HRU_TYPE
logging.info('\nResetting {} to 0'.format(hru.type_field))
arcpy.CalculateField_management(
hru.polygon_path, hru.type_field, 0, 'PYTHON')
# Reset LAKE_ID
if set_lake_flag:
logging.info('Resetting {} to 0'.format(hru.lake_id_field))
arcpy.CalculateField_management(
hru.polygon_path, hru.lake_id_field, 0, 'PYTHON')
if set_lake_flag:
logging.info('Resetting {} to 0'.format(hru.lake_area_field))
arcpy.CalculateField_management(
hru.polygon_path, hru.lake_area_field, 0, 'PYTHON')
# Calculate HRU Type
logging.info('\nCalculating cell HRU Type')
study_area_desc = arcpy.Describe(study_area_orig_path)
study_area_sr = study_area_desc.spatialReference
logging.debug(' Study area: {}'.format(study_area_orig_path))
logging.debug(' Study area spat. ref.: {}'.format(
study_area_sr.name))
logging.debug(' Study area GCS: {}'.format(
study_area_sr.GCS.name))
# If study area spat_ref doesn't match hru_param spat_ref
# Project study area to hru_param spat ref
# Otherwise, read study_area directly
if hru.sr.name != study_area_sr.name:
logging.info(' Projecting study area...')
# Set preferred transforms
transform_str = support.transform_func(hru.sr, study_area_sr)
logging.debug(' Transform: {}'.format(transform_str))
# Project study area shapefile
arcpy.Project_management(
study_area_orig_path, study_area_path, hru.sr,
transform_str, study_area_sr)
del transform_str
else:
arcpy.Copy_management(study_area_orig_path, study_area_path)
support.zone_by_centroid_func(
study_area_path, hru.type_field, 1,
hru.polygon_path, hru.point_path, hru)
# Calculate HRU Type for lakes (HRU_TYPE = 2)
if set_lake_flag:
logging.info('\nCalculating cell HRU Type & ID for lakes')
lake_layer = 'lake_layer'
lake_desc = arcpy.Describe(lake_orig_path)
lake_sr = lake_desc.spatialReference
logging.debug(' Lakes: {}'.format(lake_orig_path))
logging.debug(' Lakes spat. ref.: {}'.format(lake_sr.name))
logging.debug(' Lakes GCS: {}'.format(lake_sr.GCS.name))
# If lakes spat_ref doesn't match hru_param spat_ref
# Project lakes to hru_param spat ref
# Otherwise, read lakes directly
if hru.sr.name != lake_sr.name:
logging.info(' Projecting lakes...')
# Set preferred transforms
transform_str = support.transform_func(hru.sr, lake_sr)
logging.debug(' Transform: {}'.format(transform_str))
# Project lakes shapefile
arcpy.Project_management(
lake_orig_path, lake_path, hru.sr, transform_str, lake_sr)
arcpy.MakeFeatureLayer_management(lake_path, lake_layer)
del lake_path, transform_str
else:
arcpy.MakeFeatureLayer_management(
lake_orig_path, lake_layer)
# Clip lakes by study area after projecting lakes
logging.info(' Clipping lakes...')
arcpy.Clip_analysis(lake_layer, study_area_path, lake_clip_path)
# Remove all unnecesary fields
for field in arcpy.ListFields(lake_clip_path):
if field.name not in [lake_zone_field, 'Shape']:
try:
arcpy.DeleteField_management(lake_clip_path, field.name)
except Exception as e:
logging.debug(' Unhandled exception: {}'.format(e))
continue
# Set lake HRU_TYPE
logging.info(' Setting lake {}'.format(hru.type_field))
support.zone_by_area_func(
lake_clip_path, hru.type_field, 2,
hru.polygon_path, hru, hru.area_field,
hru.lake_area_field, lake_area_pct)
# Set lake ID
logging.info(' Setting {}'.format(hru.lake_id_field))
support.zone_by_area_func(
lake_clip_path, hru.lake_id_field, lake_zone_field,
hru.polygon_path, hru, hru.area_field,
hru.lake_area_field, lake_area_pct)
# Cleanup
del lake_layer, lake_desc, lake_sr
# Read in model points shapefile
logging.info('\nChecking model points shapefile')
model_points_desc = arcpy.Describe(model_inputs_path)
model_points_sr = model_points_desc.spatialReference
logging.debug(' Points: {}'.format(model_inputs_path))
logging.debug(' Points spat. ref.: {}'.format(model_points_sr.name))
logging.debug(' Points GCS: {}'.format(model_points_sr.GCS.name))
# If model points spat_ref doesn't match hru_param spat_ref
# Project model points to hru_param spat ref
# Otherwise, read model points directly
if hru.sr.name != model_points_sr.name:
logging.info(
' Model points projection does not match fishnet.\n'
' Projecting model points.\n')
# Set preferred transforms
transform_str = support.transform_func(hru.sr, model_points_sr)
logging.debug(' Transform: {}'.format(transform_str))
arcpy.Project_management(
model_inputs_path, model_points_path,
hru.sr, transform_str, model_points_sr)
else:
arcpy.Copy_management(model_inputs_path, model_points_path)
model_points_lyr = 'model_points_lyr'
arcpy.MakeFeatureLayer_management(model_points_path, model_points_lyr)
# Check model point types
logging.info(' Checking model point types')
model_point_types = [str(r[0]).upper() for r in arcpy.da.SearchCursor(
model_points_path, [model_points_type_field])]
if not set(model_point_types).issubset(set(['OUTLET', 'SUBBASIN', 'SWALE'])):
logging.error('\nERROR: Unsupported model point type(s) found, exiting')
logging.error('\n Model point types: {}\n'.format(model_point_types))
sys.exit()
elif not set(model_point_types).intersection(set(['OUTLET', 'SWALE'])):
logging.error(
'\nERROR: At least one model point must be an OUTLET or SWALE, '
'exiting\n')
sys.exit()
else:
logging.debug(' {}'.format(', '.join(model_point_types)))
if 'SWALE' in model_point_types:
arcpy.SelectLayerByAttribute_management(
model_points_lyr, 'NEW_SELECTION', '"TYPE" = \'SWALE\'')
logging.info(' Setting swale (sink) cells to {}=3'.format(
hru.type_field))
hru_polygon_lyr = 'hru_polygon_lyr'
arcpy.MakeFeatureLayer_management(hru.polygon_path, hru_polygon_lyr)
arcpy.SelectLayerByAttribute_management(hru_polygon_lyr, 'CLEAR_SELECTION')
arcpy.SelectLayerByLocation_management(
hru_polygon_lyr, 'INTERSECT', model_points_lyr)
arcpy.CalculateField_management(
hru_polygon_lyr, hru.type_field, 3, 'PYTHON')
arcpy.SelectLayerByAttribute_management(hru_polygon_lyr, 'CLEAR_SELECTION')
arcpy.SelectLayerByAttribute_management(model_points_lyr, 'CLEAR_SELECTION')
arcpy.Delete_management(hru_polygon_lyr)
del hru_polygon_lyr
arcpy.Delete_management(model_points_lyr)
del model_points_lyr
# Setting HRU_PSTA to default value of 1
if all([row[0] == 0 for row in arcpy.da.SearchCursor(
hru.polygon_path, [hru.hru_psta_field])]):
logging.info('Setting {} to 1'.format(
hru.hru_psta_field))
arcpy.CalculateField_management(
hru.polygon_path, hru.hru_psta_field, '1', 'PYTHON')
# Cleanup
del study_area_desc, study_area_sr
def ppt_ratio_parameters(config_path, overwrite_flag=False, debug_flag=False):
"""Calculate GSFLOW PPT Ratio Parameters
Args:
config_file (str): Project config file path
ovewrite_flag (bool): if True, overwrite existing files
debug_flag (bool): if True, enable debug level logging
Returns:
None
"""
# Hardcoded HRU field formats for now
ppt_field_format = 'PPT_{:02d}'
ratio_field_format = 'PPT_RT_{:02d}'
# Initialize hru_parameters class
hru = support.HRUParameters(config_path)
# Open input parameter config file
inputs_cfg = ConfigParser.ConfigParser()
try:
inputs_cfg.readfp(open(config_path))
except Exception as e:
logging.error('\nERROR: Config file could not be read, '
'is not an input file, or does not exist\n'
' config_file = {}\n'
' Exception: {}\n'.format(config_path, e))
sys.exit()
# Log DEBUG to file
log_file_name = 'ppt_ratio_parameters_log.txt'
log_console = logging.FileHandler(filename=os.path.join(
hru.log_ws, log_file_name),
mode='w')
log_console.setLevel(logging.DEBUG)
log_console.setFormatter(logging.Formatter('%(message)s'))
logging.getLogger('').addHandler(log_console)
logging.info('\nGSFLOW PPT Ratio Parameters')
# Units
ppt_obs_units = support.get_param('ppt_obs_units', 'mm',
inputs_cfg).lower()
ppt_units_list = ['mm', 'cm', 'm', 'in', 'ft']
# Compare against the upper case of the values in the list
# but don't modify the acceptable units list
if ppt_obs_units not in ppt_units_list:
logging.warning(
('WARNING: Invalid PPT obs. units ({})\n '
'Valid units are: {}').format(ppt_obs_units,
', '.join(ppt_units_list)))
# Convert units while reading obs values
if ppt_obs_units == 'mm':
units_factor = 1
elif ppt_obs_units == 'cm':
units_factor = 10
elif ppt_obs_units == 'm':
units_factor = 1000
elif ppt_obs_units == 'in':
units_factor = 25.4
elif ppt_obs_units == 'ft':
units_factor = 304.8
else:
units_factor = 1
# Check input paths
if not arcpy.Exists(hru.polygon_path):
logging.error('\nERROR: Fishnet ({}) does not exist'.format(
hru.polygon_path))
sys.exit()
# PPT Zones
set_ppt_zones_flag = inputs_cfg.getboolean('INPUTS', 'set_ppt_zones_flag')
if set_ppt_zones_flag:
ppt_zone_orig_path = inputs_cfg.get('INPUTS', 'ppt_zone_path')
try:
ppt_zone_field = inputs_cfg.get('INPUTS', 'ppt_zone_field')
except:
logging.error(
'\nERROR: ppt_zone_field must be set in INI to apply '
'zone specific ppt ratios\n')
sys.exit()
try:
ppt_hru_id_field = inputs_cfg.get('INPUTS', 'ppt_hru_id_field')
except:
ppt_hru_id_field = None
logging.warning(
' ppt_hru_id_field was not set in the INI file\n'
' PPT ratios will not be adjusted to match station values'.
format(ppt_zone_field, hru.ppt_zone_id_field))
# Field name for PSTA hard coded, but could be changed to be read from
# config file like ppt_zone
hru_psta_field = 'HRU_PSTA'
try:
ppt_obs_field_format = inputs_cfg.get('INPUTS',
'ppt_obs_field_format')
except:
ppt_obs_field_format = 'PPT_{:02d}'
logging.info(' Defaulting ppt_obs_field_format = {}'.format(
ppt_obs_field_format))
if not arcpy.Exists(ppt_zone_orig_path):
logging.error('\nERROR: PPT Zone ({}) does not exist'.format(
ppt_zone_orig_path))
sys.exit()
# ppt_zone_path must be a polygon shapefile
if arcpy.Describe(ppt_zone_orig_path).datasetType != 'FeatureClass':
logging.error('\nERROR: ppt_zone_path must be a polygon shapefile')
sys.exit()
# Check ppt_zone_field
if ppt_zone_field.upper() in ['FID', 'OID']:
ppt_zone_field = arcpy.Describe(ppt_zone_orig_path).OIDFieldName
logging.warning('\n NOTE: Using {} to set {}\n'.format(
ppt_zone_field, hru.ppt_zone_id_field))
elif not arcpy.ListFields(ppt_zone_orig_path, ppt_zone_field):
logging.error(
'\nERROR: ppt_zone_field field {} does not exist\n'.format(
ppt_zone_field))
sys.exit()
# Need to check that field is an int type
# Only check active cells (HRU_TYPE >0)?!
elif not [
f.type for f in arcpy.Describe(ppt_zone_orig_path).fields
if (f.name == ppt_zone_field
and f.type in ['SmallInteger', 'Integer'])
]:
logging.error(
'\nERROR: ppt_zone_field field {} must be an integer type\n'.
format(ppt_zone_field))
sys.exit()
# Need to check that field values are all positive
# Only check active cells (HRU_TYPE >0)?!
elif min([
row[0] for row in arcpy.da.SearchCursor(
ppt_zone_orig_path, [ppt_zone_field])
]) <= 0:
logging.error(
'\nERROR: ppt_zone_field values must be positive\n'.format(
ppt_zone_field))
sys.exit()
# Check hru_psta_field
if not arcpy.ListFields(ppt_zone_orig_path, hru_psta_field):
logging.error(
'\nERROR: hru_psta_field field {} does not exist\n'.format(
hru_psta_field))
sys.exit()
# Need to check that field is an int type
# Only check active cells (HRU_TYPE >0)?!
elif not [
f.type for f in arcpy.Describe(ppt_zone_orig_path).fields
if (f.name == hru_psta_field
and f.type in ['SmallInteger', 'Integer'])
]:
logging.error(
'\nERROR: hru_psta_field field {} must be an integer type\n'.
format(hru_psta_field))
sys.exit()
# Need to check that field values are all positive
# Only check active cells (HRU_TYPE >0)?!
elif min([
row[0] for row in arcpy.da.SearchCursor(
ppt_zone_orig_path, [hru_psta_field])
]) <= 0:
logging.error(
'\nERROR: hru_psta_field values must be positive\n'.format(
hru_psta_field))
sys.exit()
# Check ppt_hru_id_field
# ppt_hru_id values are checked later
if ppt_hru_id_field is not None:
if not arcpy.ListFields(ppt_zone_orig_path, ppt_hru_id_field):
logging.error(
'\nERROR: ppt_hru_id_field field {} does not exist\n'.
format(ppt_hru_id_field))
sys.exit()
# Need to check that field is an int type
elif not [
f.type for f in arcpy.Describe(ppt_zone_orig_path).fields
if (f.name == ppt_hru_id_field
and f.type in ['SmallInteger', 'Integer'])
]:
logging.error(
'\nERROR: ppt_hru_id_field field {} must be an integer type\n'
.format(ppt_hru_id_field))
sys.exit()
# Need to check that field values are not negative (0 is okay)
elif min([
row[0] for row in arcpy.da.SearchCursor(
ppt_zone_orig_path, [ppt_hru_id_field])
]) < 0:
logging.error(
'\nERROR: ppt_hru_id_field values cannot be negative\n'.
format(ppt_hru_id_field))
sys.exit()
else:
# If a zone shapefile is not used, PPT must be set manually
ppt_obs_list = inputs_cfg.get('INPUTS', 'ppt_obs_list')
# Check that values are floats
try:
ppt_obs_list = map(float, ppt_obs_list.split(','))
except ValueError:
logging.error('\nERROR: ppt_obs_list (mean monthly precipitation) '
'values could not be parsed as floats\n')
sys.exit()
# Check that there are 12 values
if len(ppt_obs_list) != 12:
logging.error(
'\nERROR: There must be exactly 12 mean monthly '
'observed precipitation values based to ppt_obs_list\n')
sys.exit()
logging.info((' Observed Mean Monthly PPT ({}):\n {}\n (Script '
'will assume these are listed in month order, i.e. Jan, '
'Feb, ...)').format(ppt_obs_units, ppt_obs_list))
# Check if all the values are 0
if ppt_obs_list == ([0.0] * 12):
logging.error(
'\nERROR: The observed precipitation values are all 0.\n'
' To compute PPT ratios, please set the ppt_obs_list '
'parameter in the INI with\n observed mean monthly PPT '
'values (i.e. from a weather station)')
sys.exit()
# Adjust units (DEADBEEF - this might be better later on)
if units_factor != 1:
ppt_obs_list = [p * units_factor for p in ppt_obs_list]
logging.info('\n Converted Mean Monthly PPT ({}):\n {}'.format(
ppt_obs_units, ppt_obs_list))
# Get the PPT HRU ID
try:
ppt_hru_id = inputs_cfg.getint('INPUTS', 'ppt_hru_id')
except:
ppt_hru_id = 0
# Check that the ppt_hru_id is a valid cell hru_id
# If ppt_hru_id is 0, PPT ratios will not be adjusted
if ppt_hru_id > 0:
# Check that HRU_ID is valid
logging.info(' PPT HRU_ID: {}'.format(ppt_hru_id))
arcpy.MakeTableView_management(
hru.polygon_path, "layer",
"{} = {}".format(hru.id_field, ppt_hru_id))
if (ppt_hru_id != 0 and int(
arcpy.GetCount_management("layer").getOutput(0)) == 0):
logging.error(
('\nERROR: ppt_hru_id {} is not a valid cell hru_id'
'\nERROR: ppt_ratios will NOT be forced to 1'
' at cell {}\n').format(ppt_hru_id))
ppt_hru_id = 0
arcpy.Delete_management("layer")
else:
logging.info(
' PPT ratios will not be adjusted to match station values\n'
' (ppt_hru_id = 0)')
# Could add a second check that HRU_PSTA has values >0
# Build output folders if necesssary
ppt_ratio_temp_ws = os.path.join(hru.param_ws, 'ppt_ratio_temp')
if not os.path.isdir(ppt_ratio_temp_ws):
os.mkdir(ppt_ratio_temp_ws)
ppt_zone_path = os.path.join(ppt_ratio_temp_ws, 'ppt_zone.shp')
# ppt_zone_clip_path = os.path.join(ppt_ratio_temp_ws, 'ppt_zone_clip.shp')
# Set ArcGIS environment variables
arcpy.CheckOutExtension('Spatial')
env.overwriteOutput = True
# env.pyramid = 'PYRAMIDS -1'
env.pyramid = 'PYRAMIDS 0'
env.workspace = hru.param_ws
env.scratchWorkspace = hru.scratch_ws
# Set month list based on flags
month_list = range(1, 13)
ppt_field_list = [ppt_field_format.format(m) for m in month_list]
ratio_field_list = [ratio_field_format.format(m) for m in month_list]
# Check fields
logging.info('\nAdding PPT fields if necessary')
# PPT zone fields
support.add_field_func(hru.polygon_path, hru.ppt_zone_id_field, 'LONG')
# PPT ratio fields
for ppt_field in ppt_field_list:
support.add_field_func(hru.polygon_path, ppt_field, 'DOUBLE')
# Calculate PPT zone ID
if set_ppt_zones_flag:
logging.info('\nCalculating cell HRU PPT zone ID')
ppt_zone_desc = arcpy.Describe(ppt_zone_orig_path)
ppt_zone_sr = ppt_zone_desc.spatialReference
logging.debug(' PPT zones: {}'.format(ppt_zone_orig_path))
logging.debug(' PPT zones spat. ref.: {}'.format(ppt_zone_sr.name))
logging.debug(' PPT zones GCS: {}'.format(
ppt_zone_sr.GCS.name))
# Reset PPT_ZONE_ID
if set_ppt_zones_flag:
logging.info(' Resetting {} to 0'.format(hru.ppt_zone_id_field))
arcpy.CalculateField_management(hru.polygon_path,
hru.ppt_zone_id_field, 0, 'PYTHON')
# If ppt_zone spat_ref doesn't match hru_param spat_ref
# Project ppt_zone to hru_param spat ref
# Otherwise, read ppt_zone directly
if hru.sr.name != ppt_zone_sr.name:
logging.info(' Projecting PPT zones...')
# Set preferred transforms
transform_str = support.transform_func(hru.sr, ppt_zone_sr)
logging.debug(' Transform: {}'.format(transform_str))
# Project ppt_zone shapefile
arcpy.Project_management(ppt_zone_orig_path, ppt_zone_path, hru.sr,
transform_str, ppt_zone_sr)
del transform_str
else:
arcpy.Copy_management(ppt_zone_orig_path, ppt_zone_path)
# # Remove all unnecesary fields
# for field in arcpy.ListFields(ppt_zone_path):
# skip_field_list = ppt_obs_field_list + [ppt_zone_field, 'Shape']
# if field.name not in skip_field_list:
# try:
# arcpy.DeleteField_management(ppt_zone_path, field.name)
# except:
# pass
# Set ppt zone ID
logging.info(' Setting {}'.format(hru.ppt_zone_id_field))
support.zone_by_centroid_func(ppt_zone_path, hru.ppt_zone_id_field,
ppt_zone_field, hru.polygon_path,
hru.point_path, hru)
# support.zone_by_area_func(
# ppt_zone_layer, hru.ppt_zone_id_field, ppt_zone_field,
# hru.polygon_path, hru, hru_area_field, None, 50)
# Set HRU_PSTA
logging.info(' Setting {}'.format(hru.hru_psta_field))
support.zone_by_centroid_func(ppt_zone_path, hru.hru_psta_field,
hru_psta_field, hru.polygon_path,
hru.point_path, hru)
# Cleanup
del ppt_zone_desc, ppt_zone_sr
else:
# Set all cells to PPT zone 1
arcpy.CalculateField_management(hru.polygon_path,
hru.ppt_zone_id_field, 1, 'PYTHON')
# Calculate PPT ratios
logging.info('\nCalculating mean monthly PPT ratios')
if set_ppt_zones_flag:
# Read mean monthly PPT values for each zone
ppt_obs_dict = dict()
ppt_obs_field_list = [
ppt_obs_field_format.format(m) for m in month_list
]
fields = [ppt_zone_field] + ppt_obs_field_list
logging.debug(' Obs. Fields: {}'.format(', '.join(fields)))
with arcpy.da.SearchCursor(ppt_zone_path, fields) as s_cursor:
for row in s_cursor:
# Convert units while reading obs values
ppt_obs_dict[int(row[0])] = map(
lambda x: float(x) * units_factor, row[1:13])
# ppt_obs_dict[row[0]] = map(float, row[1:13])
ppt_zone_list = sorted(ppt_obs_dict.keys())
logging.debug(' PPT Zones: {}'.format(ppt_zone_list))
# Print the observed PPT values
logging.debug(' Observed PPT')
for zone, ppt_obs in ppt_obs_dict.items():
logging.debug(' {}: {}'.format(
zone, ', '.join(['{:.2f}'.format(x) for x in ppt_obs])))
# Default all zones to a PPT ratio of 1
ppt_ratio_dict = {z: [1] * 12 for z in ppt_zone_list}
# ppt_ratio_dict[0] = [1] * 12
# ppt_ratio_dict[0] = 1
# Get list of HRU_IDs for each PPT Zone
fields = [hru.ppt_zone_id_field, hru.id_field]
zone_hru_id_dict = defaultdict(list)
with arcpy.da.SearchCursor(hru.polygon_path, fields) as s_cursor:
for row in s_cursor:
zone_hru_id_dict[int(row[0])].append(int(row[1]))
# Check that PPT_HRU_IDs are in the correct zone
# Default all PPT Zone HRU IDs to 0
ppt_hru_id_dict = {z: 0 for z in ppt_zone_list}
if ppt_hru_id_field is not None:
fields = [ppt_zone_field, ppt_hru_id_field]
logging.debug(' PPT Zone ID field: {}'.format(ppt_zone_field))
logging.debug(' PPT HRU ID field: {}'.format(ppt_hru_id_field))
with arcpy.da.SearchCursor(ppt_zone_path, fields) as s_cursor:
for row in s_cursor:
ppt_zone = int(row[0])
hru_id = int(row[1])
if hru_id == 0 or hru_id in zone_hru_id_dict[ppt_zone]:
ppt_hru_id_dict[ppt_zone] = hru_id
logging.debug(' {}: {}'.format(ppt_zone, hru_id))
else:
logging.error(
'\nERROR: HRU_ID {} is not in PPT ZONE {}'.format(
hru_id, ppt_hru_id_dict[ppt_zone]))
sys.exit()
# Get gridded PPT values for each PPT_HRU_ID
fields = [hru.ppt_zone_id_field, hru.id_field] + ppt_field_list
# ppt_ratio_dict = dict()
with arcpy.da.SearchCursor(hru.polygon_path, fields) as s_cursor:
for row in s_cursor:
ppt_zone = int(row[0])
hru_id = int(row[1])
if hru_id == 0:
pass
elif hru_id in ppt_hru_id_dict.values():
ppt_gridded_list = map(float, row[2:14])
ppt_obs_list = ppt_obs_dict[ppt_zone]
ppt_ratio_list = [
float(o) / p if p > 0 else 0
for o, p in zip(ppt_obs_list, ppt_gridded_list)
]
ppt_ratio_dict[ppt_zone] = ppt_ratio_list
del ppt_hru_id_dict, zone_hru_id_dict, fields
logging.debug(' PPT Ratio Adjustment Factors:')
for k, v in ppt_ratio_dict.items():
logging.debug(' {}: {}'.format(
k, ', '.join(['{:.3f}'.format(x) for x in v])))
# DEADBEEF - ZONE_VALUE is calculated in zone_by_centroid_func
# There is probably a cleaner way of linking these two
fields = [hru.ppt_zone_id_field] + ppt_field_list + ratio_field_list
with arcpy.da.UpdateCursor(hru.polygon_path, fields) as u_cursor:
for row in u_cursor:
ppt_zone = int(row[0])
for i, month in enumerate(month_list):
ppt_i = fields.index(ppt_field_format.format(month))
ratio_i = fields.index(ratio_field_format.format(month))
if ppt_zone in ppt_zone_list:
ppt_obs = ppt_obs_dict[ppt_zone][i]
else:
ppt_obs = 0
if ppt_obs > 0:
row[ratio_i] = (ppt_ratio_dict[ppt_zone][i] *
row[ppt_i] / ppt_obs)
else:
row[ratio_i] = 0
u_cursor.updateRow(row)
del row
else:
# Get gridded precip at PPT_HRU_ID
fields = [hru.id_field] + ppt_field_list
logging.debug(' Fields: {}'.format(', '.join(fields)))
# Scale all ratios so gridded PPT will match observed PPT at target cell
if ppt_hru_id != 0:
ppt_gridded_list = map(
float,
arcpy.da.SearchCursor(
hru.polygon_path, fields,
'"{}" = {}'.format(hru.id_field, ppt_hru_id)).next()[1:])
logging.info(' Gridded PPT: {}'.format(', '.join(
['{:.2f}'.format(p) for p in ppt_gridded_list])))
# Ratio of MEASURED or OBSERVED PPT to GRIDDED PPT
# This will be multiplied by GRIDDED/OBSERVED below
ppt_ratio_list = [
float(o) / p if p > 0 else 0
for o, p in zip(ppt_obs_list, ppt_gridded_list)
]
logging.info(' Obs./Gridded: {}'.format(', '.join(
['{:.3f}'.format(p) for p in ppt_ratio_list])))
else:
ppt_ratio_list = [1 for p in ppt_obs_list]
# Use single mean monthly PPT for all cells
# Assume ppt_obs_list is in month order
fields = ppt_field_list + ratio_field_list
with arcpy.da.UpdateCursor(hru.polygon_path, fields) as u_cursor:
for row in u_cursor:
for i, month in enumerate(month_list):
ppt_i = fields.index(ppt_field_format.format(month))
ratio_i = fields.index(ratio_field_format.format(month))
if ppt_obs_list[i] > 0:
row[ratio_i] = (ppt_ratio_list[i] * row[ppt_i] /
ppt_obs_list[i])
else:
row[ratio_i] = 0
u_cursor.updateRow(row)
del row