def F(x, y):
"""This is the function returned by composite_quantity_setting_function
It can be passed to set_quantity
"""
isSet = numpy.zeros(len(x)) # 0/1 - record if each point has been set
quantityVal = x * 0 + numpy.nan # Function return value
# Record points which evaluated to nan on their first preference
# dataset.
was_ever_nan = (x * 0).astype(int)
lpf = len(poly_fun_pairs)
if (lpf <= 0):
raise Exception('Must have at least 1 fun-poly-pair')
# Make an array of 'transformed' spatial coordinates, for checking
# polygon inclusion
xll = domain.geo_reference.xllcorner
yll = domain.geo_reference.yllcorner
xy_array_trans = numpy.vstack([x + xll, y + yll]).transpose()
# Check that none of the pi polygons [except perhaps the last] is 'All'
for i in range(lpf - 1):
if (poly_fun_pairs[i][0] == 'All'):
# This is only ok if all the othe poly_fun_pairs are None
remaining_poly_fun_pairs_are_None = \
[poly_fun_pairs[j][0] is None for j in range(i+1,lpf)]
if (not all(remaining_poly_fun_pairs_are_None)):
raise Exception('Can only have the last polygon = All')
# Main Loop
# Apply the fi inside the pi
for i in range(lpf):
fi = poly_fun_pairs[i][1] # The function
pi = poly_fun_pairs[i][0] # The polygon
# Quick exit
if (pi is None):
continue
###################################################################
# Get indices fInds of points in polygon pi which are not already
# set
###################################################################
if (pi == 'All'):
# Get all unset points
fInside = (1 - isSet)
fInds = (fInside == 1).nonzero()[0]
else:
if (pi == 'Extent'):
# Here fi MUST be a gdal-compatible raster
if (not (type(fi) == str)):
msg = ' pi = "Extent" can only be used when fi is a' +\
' raster file name'
raise Exception(msg)
if (not os.path.exists(fi)):
msg = 'fi ' + str(fi) + ' is supposed to be a ' +\
' raster filename, but it could not be found'
raise Exception(msg)
# Then we get the extent from the raster itself
pi_path = su.getRasterExtent(fi, asPolygon=True)
if verbose:
print 'Extracting extent from raster: ', fi
print 'Extent: ', pi_path
elif ((type(pi) == str) and os.path.isfile(pi)):
# pi is a file
pi_path = su.read_polygon(pi)
else:
# pi is the actual polygon data
pi_path = pi
# Get the insides of unset points inside pi_path
notSet = (isSet == 0.).nonzero()[0]
fInds = inside_polygon(xy_array_trans[notSet, :], pi_path)
fInds = notSet[fInds]
if len(fInds) == 0:
# No points found, move on
continue
###################################################################
# Evaluate fi at the points inside pi
###################################################################
# We use various tricks to infer whether fi is a function,
# a constant, a file (raster or csv), or an array
if (hasattr(fi, '__call__')):
# fi is a function
quantityVal[fInds] = fi(x[fInds], y[fInds])
elif isinstance(fi, (int, long, float)):
# fi is a numerical constant
quantityVal[fInds] = fi * 1.0
elif (type(fi) is str and os.path.exists(fi)):
# fi is a file which is assumed to be
# a gdal-compatible raster OR an x,y,z elevation file
if os.path.splitext(fi)[1] in ['.txt', '.csv']:
fi_array = su.read_csv_optional_header(fi)
# Check the results
if fi_array.shape[1] is not 3:
print 'Treated input file ' + fi +\
' as xyz array with an optional header'
msg = 'Array should have 3 columns -- x,y,value'
raise Exception(msg)
newfi = make_nearestNeighbour_quantity_function(
fi_array,
domain,
k_nearest_neighbours=default_k_nearest_neighbours)
quantityVal[fInds] = newfi(x[fInds], y[fInds])
else:
# Treating input file as a raster
newfi = quantityRasterFun(
domain, fi, interpolation=default_raster_interpolation)
quantityVal[fInds] = newfi(x[fInds], y[fInds])
elif (type(fi) is numpy.ndarray):
if fi.shape[1] is not 3:
msg = 'Array should have 3 columns -- x,y,value'
raise Exception(msg)
newfi = make_nearestNeighbour_quantity_function(
fi,
domain,
k_nearest_neighbours=default_k_nearest_neighbours)
quantityVal[fInds] = newfi(x[fInds], y[fInds])
else:
print 'Error with function from'
print fi
msg = 'Cannot make function from type ' + str(type(fi))
raise Exception, msg
###################################################################
# Check for nan values
###################################################################
#nan_flag = (quantityVal[fInds] != quantityVal[fInds])
nan_flag = 1 * numpy.isnan(quantityVal[fInds])
nan_inds = nan_flag.nonzero()[0]
was_ever_nan[fInds[nan_inds]] = 1
if len(nan_inds) > 0:
if nan_treatment == 'exception':
msg = 'nan values generated by the poly_fun_pair at '\
'index ' + str(i) + ' '\
'in composite_quantity_setting_function. ' + \
'To allow these values to be set by later ' + \
'poly_fun pairs, pass the argument ' + \
'nan_treatment="fall_through" ' + \
'to composite_quantity_setting_function'
raise Exception(msg)
elif nan_treatment == 'fall_through':
msg = 'WARNING: nan values generated by the ' + \
'poly_fun_pair at index ' + str(i) + ' '\
'in composite_quantity_setting_function. ' + \
'They will be passed to later poly_fun_pairs'
if verbose: print msg
not_nan_inds = (1 - nan_flag).nonzero()[0]
if len(not_nan_inds) > 0:
fInds = fInds[not_nan_inds]
else:
# All values are nan
msg = '( Actually all the values were nan - ' + \
'Are you sure they should be? Possible error?)'
if verbose: print msg
continue
else:
msg = 'Found nan values in ' + \
'composite_quantity_setting_function but ' + \
'nan_treatment is not a recognized value'
raise Exception(msg)
# Record that the points have been set
isSet[fInds] = 1
# Enforce clip_range
if clip_range is not None:
lower_bound = clip_range[i][0]
upper_bound = clip_range[i][1]
quantityVal[fInds] = numpy.maximum(quantityVal[fInds],
lower_bound)
quantityVal[fInds] = numpy.minimum(quantityVal[fInds],
upper_bound)
# End of loop
# Find points which were nan on their first preference dataset + are
# inside nan_interpolation_region_polygon. Then reinterpolate their
# values from the other x,y, quantityVal points.
if (nan_interpolation_region_polygon is not None) &\
(was_ever_nan.sum() > 0):
if nan_interpolation_region_polygon == 'All':
points_to_reinterpolate = was_ever_nan.nonzero()[0]
else:
# nan_interpolation_region_polygon contains information on 1 or
# more polygons
# Inside those polygons, we need to re-interpolate points which
# first evaluted to na
possible_points_to_reint = was_ever_nan.nonzero()[0]
points_to_reinterpolate = numpy.array([]).astype(int)
for i in range(len(nan_interpolation_region_polygon)):
nan_pi = nan_interpolation_region_polygon[i]
# Ensure nan_pi = list of x,y points making a polygon
if (type(nan_pi) == str):
nan_pi = su.read_polygon(nan_pi)
points_in_nan_pi = inside_polygon(
xy_array_trans[possible_points_to_reint, :], nan_pi)
if len(points_in_nan_pi) > 0:
points_to_reinterpolate = numpy.hstack([
points_to_reinterpolate,
possible_points_to_reint[points_in_nan_pi]
])
if verbose:
print 'Re-interpolating ', len(points_to_reinterpolate),\
' points which were nan under their',\
' first-preference and are inside the',\
' nan_interpolation_region_polygon'
if len(points_to_reinterpolate) > 0:
msg = 'WARNING: nan interpolation is being applied. This ',\
'should be done in serial prior to distributing the ',\
'domain, as there is no parallel communication ',\
'implemented yet [so parallel results might depend on ',\
'the number of processes]'
if verbose:
print msg
# Find the interpolation points = points not needing reinterpolation
ip = x * 0 + 1
ip[points_to_reinterpolate] = 0
number_of_ip = ip.sum()
ip = ip.nonzero()[0]
# Check that none of the ip points has an nan value
nan_ip = (quantityVal[ip] != quantityVal[ip]).nonzero()[0]
if len(nan_ip) > 0:
print 'There are ', len(nan_ip), ' points outside the ',\
'nan_interpolation_region_polygon have nan values.'
print 'The user should ensure this does not happen.'
print 'The points have the following coordinates:'
print xy_array_trans[ip[nan_ip], :]
msg = "There are nan points outside of " +\
"nan_interpolation_region_polygon, even after all " +\
"fall-through's"
raise Exception(msg)
if (number_of_ip < default_k_nearest_neighbours):
raise Exception('Too few non-nan points to interpolate from')
# Make function for re-interpolation. Note this requires
# x,y,z in georeferenced coordinates, whereas x,y are ANUGA
# coordinates
reinterp_F = make_nearestNeighbour_quantity_function(
numpy.vstack([
xy_array_trans[ip, 0], xy_array_trans[ip, 1],
quantityVal[ip]
]).transpose(),
domain,
k_nearest_neighbours=default_k_nearest_neighbours)
# re-interpolate
quantityVal[points_to_reinterpolate] = reinterp_F(
x[points_to_reinterpolate], y[points_to_reinterpolate])
isSet[points_to_reinterpolate] = 1
# Check there are no remaining nan values
if (min(isSet) != 1):
print 'Some points remain as nan, which is not allowed'
unset_inds = (isSet != 1).nonzero()[0]
lui = min(5, len(unset_inds))
print 'There are ', len(unset_inds), ' such points'
print 'Here are a few:'
for i in range(lui):
print x[unset_inds[i]] + xll, y[unset_inds[i]] + yll
raise Exception('It seems the input data needs to be fixed')
return quantityVal
def F(x,y):
"""This is the function returned by composite_quantity_setting_function
It can be passed to set_quantity
"""
isSet = numpy.zeros(len(x)) # 0/1 - record if each point has been set
quantityVal = x*0 + numpy.nan # Function return value
# Record points which evaluated to nan on their first preference
# dataset.
was_ever_nan = (x*0).astype(int)
lpf = len(poly_fun_pairs)
if(lpf <= 0):
raise Exception('Must have at least 1 fun-poly-pair')
# Make an array of 'transformed' spatial coordinates, for checking
# polygon inclusion
xll = domain.geo_reference.xllcorner
yll = domain.geo_reference.yllcorner
xy_array_trans = numpy.vstack([x+xll,y+yll]).transpose()
# Check that none of the pi polygons [except perhaps the last] is 'All'
for i in range(lpf-1):
if(poly_fun_pairs[i][0]=='All'):
# This is only ok if all the othe poly_fun_pairs are None
remaining_poly_fun_pairs_are_None = \
[poly_fun_pairs[j][0] is None for j in range(i+1,lpf)]
if(not all(remaining_poly_fun_pairs_are_None)):
raise Exception('Can only have the last polygon = All')
# Main Loop
# Apply the fi inside the pi
for i in range(lpf):
fi = poly_fun_pairs[i][1] # The function
pi = poly_fun_pairs[i][0] # The polygon
# Quick exit
if(pi is None):
continue
###################################################################
# Get indices fInds of points in polygon pi which are not already
# set
###################################################################
if(pi == 'All'):
# Get all unset points
fInside = (1-isSet)
fInds = (fInside==1).nonzero()[0]
else:
if(pi == 'Extent'):
# Here fi MUST be a gdal-compatible raster
if(not (type(fi) == str)):
msg = ' pi = "Extent" can only be used when fi is a' +\
' raster file name'
raise Exception(msg)
if(not os.path.exists(fi)):
msg = 'fi ' + str(fi) + ' is supposed to be a ' +\
' raster filename, but it could not be found'
raise Exception(msg)
# Then we get the extent from the raster itself
pi_path = su.getRasterExtent(fi,asPolygon=True)
if verbose:
print 'Extracting extent from raster: ', fi
print 'Extent: ', pi_path
elif( (type(pi) == str) and os.path.isfile(pi) ):
# pi is a file
pi_path = su.read_polygon(pi)
else:
# pi is the actual polygon data
pi_path = pi
# Get the insides of unset points inside pi_path
notSet = (isSet==0.).nonzero()[0]
fInds = inside_polygon(xy_array_trans[notSet,:], pi_path)
fInds = notSet[fInds]
if len(fInds) == 0:
# No points found, move on
continue
###################################################################
# Evaluate fi at the points inside pi
###################################################################
# We use various tricks to infer whether fi is a function,
# a constant, a file (raster or csv), or an array
if(hasattr(fi,'__call__')):
# fi is a function
quantityVal[fInds] = fi(x[fInds], y[fInds])
elif isinstance(fi, (int, long, float)):
# fi is a numerical constant
quantityVal[fInds] = fi*1.0
elif ( type(fi) is str and os.path.exists(fi)):
# fi is a file which is assumed to be
# a gdal-compatible raster OR an x,y,z elevation file
if os.path.splitext(fi)[1] in ['.txt', '.csv']:
fi_array = su.read_csv_optional_header(fi)
# Check the results
if fi_array.shape[1] is not 3:
print 'Treated input file ' + fi +\
' as xyz array with an optional header'
msg = 'Array should have 3 columns -- x,y,value'
raise Exception(msg)
newfi = make_nearestNeighbour_quantity_function(
fi_array, domain,
k_nearest_neighbours = default_k_nearest_neighbours)
quantityVal[fInds] = newfi(x[fInds], y[fInds])
else:
# Treating input file as a raster
newfi = quantityRasterFun(domain, fi,
interpolation = default_raster_interpolation)
quantityVal[fInds] = newfi(x[fInds], y[fInds])
elif(type(fi) is numpy.ndarray):
if fi.shape[1] is not 3:
msg = 'Array should have 3 columns -- x,y,value'
raise Exception(msg)
newfi = make_nearestNeighbour_quantity_function(fi, domain,
k_nearest_neighbours = default_k_nearest_neighbours)
quantityVal[fInds] = newfi(x[fInds], y[fInds])
else:
print 'Error with function from'
print fi
msg='Cannot make function from type ' + str(type(fi))
raise Exception, msg
###################################################################
# Check for nan values
###################################################################
#nan_flag = (quantityVal[fInds] != quantityVal[fInds])
nan_flag = 1*numpy.isnan(quantityVal[fInds])
nan_inds = nan_flag.nonzero()[0]
was_ever_nan[fInds[nan_inds]] = 1
if len(nan_inds)>0:
if nan_treatment == 'exception':
msg = 'nan values generated by the poly_fun_pair at '\
'index ' + str(i) + ' '\
'in composite_quantity_setting_function. ' + \
'To allow these values to be set by later ' + \
'poly_fun pairs, pass the argument ' + \
'nan_treatment="fall_through" ' + \
'to composite_quantity_setting_function'
raise Exception(msg)
elif nan_treatment == 'fall_through':
msg = 'WARNING: nan values generated by the ' + \
'poly_fun_pair at index ' + str(i) + ' '\
'in composite_quantity_setting_function. ' + \
'They will be passed to later poly_fun_pairs'
if verbose: print msg
not_nan_inds = (1-nan_flag).nonzero()[0]
if len(not_nan_inds)>0:
fInds = fInds[not_nan_inds]
else:
# All values are nan
msg = '( Actually all the values were nan - ' + \
'Are you sure they should be? Possible error?)'
if verbose: print msg
continue
else:
msg = 'Found nan values in ' + \
'composite_quantity_setting_function but ' + \
'nan_treatment is not a recognized value'
raise Exception(msg)
# Record that the points have been set
isSet[fInds] = 1
# Enforce clip_range
if clip_range is not None:
lower_bound = clip_range[i][0]
upper_bound = clip_range[i][1]
quantityVal[fInds] = numpy.maximum(
quantityVal[fInds], lower_bound)
quantityVal[fInds] = numpy.minimum(
quantityVal[fInds], upper_bound)
# End of loop
# Find points which were nan on their first preference dataset + are
# inside nan_interpolation_region_polygon. Then reinterpolate their
# values from the other x,y, quantityVal points.
if (nan_interpolation_region_polygon is not None) &\
(was_ever_nan.sum() > 0):
if nan_interpolation_region_polygon == 'All':
points_to_reinterpolate = was_ever_nan.nonzero()[0]
else:
# nan_interpolation_region_polygon contains information on 1 or
# more polygons
# Inside those polygons, we need to re-interpolate points which
# first evaluted to na
possible_points_to_reint = was_ever_nan.nonzero()[0]
points_to_reinterpolate = numpy.array([]).astype(int)
for i in range(len(nan_interpolation_region_polygon)):
nan_pi = nan_interpolation_region_polygon[i]
# Ensure nan_pi = list of x,y points making a polygon
if(type(nan_pi) == str):
nan_pi = su.read_polygon(nan_pi)
points_in_nan_pi = inside_polygon(
xy_array_trans[possible_points_to_reint,:],
nan_pi)
if len(points_in_nan_pi)>0:
points_to_reinterpolate = numpy.hstack(
[points_to_reinterpolate,
possible_points_to_reint[points_in_nan_pi]])
if verbose:
print 'Re-interpolating ', len(points_to_reinterpolate),\
' points which were nan under their',\
' first-preference and are inside the',\
' nan_interpolation_region_polygon'
if len(points_to_reinterpolate) > 0:
msg = 'WARNING: nan interpolation is being applied. This ',\
'should be done in serial prior to distributing the ',\
'domain, as there is no parallel communication ',\
'implemented yet [so parallel results might depend on ',\
'the number of processes]'
if verbose:
print msg
# Find the interpolation points = points not needing reinterpolation
ip = x*0 + 1
ip[points_to_reinterpolate] = 0
number_of_ip = ip.sum()
ip = ip.nonzero()[0]
# Check that none of the ip points has an nan value
nan_ip = (quantityVal[ip] != quantityVal[ip]).nonzero()[0]
if len(nan_ip) > 0:
print 'There are ', len(nan_ip), ' points outside the ',\
'nan_interpolation_region_polygon have nan values.'
print 'The user should ensure this does not happen.'
print 'The points have the following coordinates:'
print xy_array_trans[ip[nan_ip],:]
msg = "There are nan points outside of " +\
"nan_interpolation_region_polygon, even after all " +\
"fall-through's"
raise Exception(msg)
if(number_of_ip < default_k_nearest_neighbours):
raise Exception('Too few non-nan points to interpolate from')
# Make function for re-interpolation. Note this requires
# x,y,z in georeferenced coordinates, whereas x,y are ANUGA
# coordinates
reinterp_F = make_nearestNeighbour_quantity_function(
numpy.vstack([xy_array_trans[ip,0], xy_array_trans[ip,1],
quantityVal[ip]]).transpose(),
domain,
k_nearest_neighbours = default_k_nearest_neighbours)
# re-interpolate
quantityVal[points_to_reinterpolate] = reinterp_F(
x[points_to_reinterpolate], y[points_to_reinterpolate])
isSet[points_to_reinterpolate] = 1
# Check there are no remaining nan values
if( min(isSet) != 1):
print 'Some points remain as nan, which is not allowed'
unset_inds = (isSet != 1).nonzero()[0]
lui = min(5, len(unset_inds))
print 'There are ', len(unset_inds), ' such points'
print 'Here are a few:'
for i in range(lui):
print x[unset_inds[i]] + xll, y[unset_inds[i]] + yll
raise Exception('It seems the input data needs to be fixed')
return quantityVal