def setup_pumping_stations(domain, project): """ Extract pumping station data from project class and apply the internal boundary operator """ pumping_station_data = project.pumping_station_data for i in range(len(pumping_station_data)): # Extract pumping station parameters ps = pumping_station_data[i] label = ps[0] pump_capacity = ps[1] pump_rate_of_increase = ps[2] pump_rate_of_decrease = ps[3] hw_to_start_pumping = ps[4] hw_to_stop_pumping = ps[5] # Pumping station basin polygon + elevation are used elsewhere exchange_line_0 = su.read_polygon(ps[8]) exchange_line_1 = su.read_polygon(ps[9]) exchange_lines = [exchange_line_0, exchange_line_1] smoothing_timescale = ps[10] print 'Need to implement elevation data adjustments' # Function which computes Q pump_behaviour = pumping_station_function( domain=domain, pump_capacity=pump_capacity, hw_to_start_pumping=hw_to_start_pumping, hw_to_stop_pumping=hw_to_stop_pumping, initial_pump_rate=0., pump_rate_of_increase=pump_rate_of_increase, pump_rate_of_decrease=pump_rate_of_decrease ) # Add operator as side-effect of this operation pumping_station = Internal_boundary_operator( domain, pump_behaviour, exchange_lines=exchange_lines, enquiry_gap=0., apron = 0.0, smoothing_timescale=smoothing_timescale, compute_discharge_implicitly=False, logging=True, label=label, verbose=True) return
def setup_pumping_stations(domain, project): """ Extract pumping station data from project class and apply the internal boundary operator """ pumping_station_data = project.pumping_station_data for i in range(len(pumping_station_data)): # Extract pumping station parameters ps = pumping_station_data[i] label = ps[0] pump_capacity = ps[1] pump_rate_of_increase = ps[2] pump_rate_of_decrease = ps[3] hw_to_start_pumping = ps[4] hw_to_stop_pumping = ps[5] # Pumping station basin polygon + elevation are used elsewhere exchange_line_0 = su.read_polygon(ps[8]) exchange_line_1 = su.read_polygon(ps[9]) exchange_lines = [exchange_line_0, exchange_line_1] smoothing_timescale = ps[10] print 'Need to implement elevation data adjustments' # Function which computes Q pump_behaviour = pumping_station_function( domain=domain, pump_capacity=pump_capacity, hw_to_start_pumping=hw_to_start_pumping, hw_to_stop_pumping=hw_to_stop_pumping, initial_pump_rate=0., pump_rate_of_increase=pump_rate_of_increase, pump_rate_of_decrease=pump_rate_of_decrease) # Add operator as side-effect of this operation pumping_station = Internal_boundary_operator( domain, pump_behaviour, exchange_lines=exchange_lines, enquiry_gap=0., apron=0.0, smoothing_timescale=smoothing_timescale, compute_discharge_implicitly=False, logging=True, label=label, verbose=True) return
def setup_bridges(domain, project): """ Extract bridge data from project class and apply the internal boundary operator Note that the bridge deck information is applied when setting the elevation """ bridge_data = project.bridge_data for i in range(len(bridge_data)): # Extract bridge parameters bd = bridge_data[i] label = bd[0] # bd[1] and bd[2] are used elsewhere to set deck elevation exchange_line_0 = su.read_polygon(bd[3]) exchange_line_1 = su.read_polygon(bd[4]) exchange_lines = [exchange_line_0, exchange_line_1] enquiry_gap = bd[5] #apron = bd[6] #assert apron==0.0, 'Apron must be zero until parallel apron issues fixed' internal_boundary_curve_file = bd[6] vertical_datum_offset = bd[7] smoothing_timescale = bd[8] # Function which computes Q rating_curve = hecras_internal_boundary_function( internal_boundary_curves_file=internal_boundary_curve_file, allow_sign_reversal=True, vertical_datum_offset=vertical_datum_offset) # Add operator as side-effect of this operation bridge = Internal_boundary_operator( domain, rating_curve, exchange_lines=exchange_lines, enquiry_gap=enquiry_gap, apron = 0.0, zero_outflow_momentum=False, smoothing_timescale=smoothing_timescale, logging=True, label=label, verbose=True) return
def setup_inlets(domain, project): """ Add inlets to domain """ inlet_data = project.inlet_data for i in range(len(inlet_data)): name = inlet_data[i][0] line_file = inlet_data[i][1] timeseries_file = inlet_data[i][2] start_time = inlet_data[i][3] # Add inlet timeseries = numpy.genfromtxt(timeseries_file, delimiter=',', skip_header=1) # Adjust start time timeseries[:, 0] = timeseries[:, 0] - start_time # Make discharge function qfun = scipy.interpolate.interp1d(timeseries[:, 0], timeseries[:, 1], kind='linear') # Make cross-section line line = su.read_polygon(line_file) anuga.Inlet_operator(domain, line, qfun, label='Inlet: ' + str(name)) return
def setup_inlets(domain, project): """ Add inlets to domain """ inlet_data = project.inlet_data for i in range(len(inlet_data)): name = inlet_data[i][0] line_file = inlet_data[i][1] timeseries_file = inlet_data[i][2] start_time = inlet_data[i][3] # Add inlet timeseries = numpy.genfromtxt( timeseries_file, delimiter=',', skip_header=1) # Adjust start time timeseries[:, 0] = timeseries[:, 0] - start_time # Make discharge function qfun = scipy.interpolate.interp1d( timeseries[:, 0], timeseries[:, 1], kind='linear') # Make cross-section line line = su.read_polygon(line_file) anuga.Inlet_operator(domain, line, qfun, label='Inlet: ' + str(name)) return
def setup_bridges(domain, project): """ Extract bridge data from project class and apply the internal boundary operator Note that the bridge deck information is applied when setting the elevation """ bridge_data = project.bridge_data for i in range(len(bridge_data)): # Extract bridge parameters bd = bridge_data[i] label = bd[0] # bd[1] and bd[2] are used elsewhere to set deck elevation exchange_line_0 = su.read_polygon(bd[3]) exchange_line_1 = su.read_polygon(bd[4]) exchange_lines = [exchange_line_0, exchange_line_1] enquiry_gap = bd[5] #apron = bd[6] #assert apron==0.0, 'Apron must be zero until parallel apron issues fixed' internal_boundary_curve_file = bd[6] vertical_datum_offset = bd[7] smoothing_timescale = bd[8] # Function which computes Q rating_curve = hecras_internal_boundary_function( internal_boundary_curves_file=internal_boundary_curve_file, allow_sign_reversal=True, vertical_datum_offset=vertical_datum_offset) # Add operator as side-effect of this operation bridge = Internal_boundary_operator( domain, rating_curve, exchange_lines=exchange_lines, enquiry_gap=enquiry_gap, apron=0.0, zero_outflow_momentum=False, smoothing_timescale=smoothing_timescale, logging=True, label=label, verbose=True) return
def setup_rainfall(domain, project): """ Function to add rainfall operators to the domain """ rain_data = project.rain_data for i in range(len(rain_data)): timeseries_file = rain_data[i][0] start_time = rain_data[i][1] interpolation_type = rain_data[i][2] # Get polygon defining rainfall extent if ((len(rain_data[i]) >= 4) and (rain_data[i][3] != 'All')): polygon = su.read_polygon(rain_data[i][3]) else: polygon = None if len(rain_data[i]) >= 5: multiplier = rain_data[i][4] * 1.0 else: multiplier = 1.0 rain_timeseries = scipy.genfromtxt(timeseries_file, delimiter=',', skip_header=1) # Adjust starttime rain_timeseries[:, 0] = rain_timeseries[:, 0] - start_time # Convert units to m/s (from mm/hr) rain_timeseries[:, 1] = rain_timeseries[:, 1] / (3600. * 1000.) * multiplier # Sanity check assert rain_timeseries[:, 1].min() >= 0., 'Negative rainfall input' # Make interpolation function and add to ANUGA as operator if rain_timeseries[:, 1].max() >= 0.: myrain = scipy.interpolate.interp1d(rain_timeseries[:, 0], rain_timeseries[:, 1], kind=interpolation_type) anuga.operators.rate_operators.Rate_operator(domain, rate=myrain, polygon=polygon, label=timeseries_file) return
def setup_rainfall(domain, project): """ Function to add rainfall operators to the domain """ rain_data = project.rain_data for i in range(len(rain_data)): timeseries_file = rain_data[i][0] start_time = rain_data[i][1] interpolation_type = rain_data[i][2] # Get polygon defining rainfall extent if ((len(rain_data[i]) >= 4) and (rain_data[i][3] != 'All')): polygon = su.read_polygon(rain_data[i][3]) else: polygon = None if len(rain_data[i]) >=5: multiplier = rain_data[i][4]*1.0 else: multiplier = 1.0 rain_timeseries = scipy.genfromtxt( timeseries_file, delimiter=',', skip_header=1) # Adjust starttime rain_timeseries[:, 0] = rain_timeseries[:, 0] - start_time # Convert units to m/s (from mm/hr) rain_timeseries[:, 1] = rain_timeseries[:, 1] / (3600. * 1000.) * multiplier # Sanity check assert rain_timeseries[:, 1].min() >= 0., 'Negative rainfall input' # Make interpolation function and add to ANUGA as operator if rain_timeseries[:, 1].max() >= 0.: myrain = scipy.interpolate.interp1d( rain_timeseries[:, 0], rain_timeseries[:, 1], kind=interpolation_type) anuga.operators.rate_operators.Rate_operator( domain, rate=myrain, polygon=polygon, label=timeseries_file) return
def get_initial_condition_data(data_source, worksheet, flag, print_info): """Convenience function to extract the initial condition data (and initial_condition_additions) from the xls worksheet The needs have become more elaborate over time, e.g. to support combining 2 line files into a polygon Given a character string referring to a quantity which has initial conditions in the xls worksheet (e.g. 'Elevation'), extract the associated data from the 'data_source' (an AnugaXls object) on worksheet 'worksheet' This assumes a particular format in the excel sheet """ # Read the polygon / value pairs quantity_data = data_source.get_paired_list(worksheet, flag, [1, 1], post_process=string_or_float) # If the polygon is a wildcard, assume it matches 2 lines, read them in, # and join them to make a polygon. This is a convenient shorthand # for when we have lkl for i in range(len(quantity_data)): polygon_files = glob.glob(quantity_data[i][0]) # Check it only matches 0 or 1 or 2 files msg = 'Polygon:' + str(i) + ' : ' + quantity_data[i][0] + \ ' for ' + flag + ' on worksheet' + \ worksheet + ' matches > 2 files. We can join at most 2 lines' + \ 'to make a polygon' assert len(polygon_files) <= 2, msg if len(polygon_files) == 0: # Check it is valid msg = 'Polygon:' + str(i) + ' : ' + quantity_data[i][0] + \ ' for ' + flag + ' on worksheet' + \ worksheet + ' matches no files, and is not All or None ' + \ 'or Extent (for a raster)' assert ((quantity_data[i][0] == 'All') | (quantity_data[i][0] is None) | (quantity_data[i][0] == 'Extent')), msg elif len(polygon_files) == 2: # If it matches 2, try to combine to 1. # This is often required to use pairs of breaklines as polygons # Do this by: # 1) Setting up the 2 lines as though they were in a # breakline object # 2) Using su.polygon_from_matching_breakLines print_info.append('Initial ' + flag) print_info.append('Combining these files to a polygon: ') print_info.append(str(polygon_files)) print_info.append('') l0 = su.read_polygon(polygon_files[0]) l1 = su.read_polygon(polygon_files[1]) fake_breakline = {polygon_files[0]: l0, polygon_files[1]: l1} fake_match = quantity_data[i][0].split('*')[0] out_poly = su.polygon_from_matching_breaklines( fake_match, fake_breakline) quantity_data[i][0] = out_poly # Get the clip_range for each polygon / function pair quantity_clip_range = data_source.get_fixed_size_subtable_by_columns( worksheet, flag, dimensions=[2, len(quantity_data)], offset=[3, 1], post_process=string_or_float) new_quantity_clip_range = reformat_clip_range(quantity_clip_range) # Get sub-grid size for spatial averaging, if applicable spatial_average = data_source.get_var(worksheet, flag, [5, 1], post_process=string_or_float) if type(spatial_average) == str: spatial_average = None return quantity_data, new_quantity_clip_range, spatial_average
def F(x, y): """This is the function returned by composite_quantity_setting_function It can be passed to set_quantity """ isSet = numpy.zeros(len(x)) # 0/1 - record if each point has been set quantityVal = x * 0 + numpy.nan # Function return value # Record points which evaluated to nan on their first preference # dataset. was_ever_nan = (x * 0).astype(int) lpf = len(poly_fun_pairs) if (lpf <= 0): raise Exception('Must have at least 1 fun-poly-pair') # Make an array of 'transformed' spatial coordinates, for checking # polygon inclusion xll = domain.geo_reference.xllcorner yll = domain.geo_reference.yllcorner xy_array_trans = numpy.vstack([x + xll, y + yll]).transpose() # Check that none of the pi polygons [except perhaps the last] is 'All' for i in range(lpf - 1): if (poly_fun_pairs[i][0] == 'All'): # This is only ok if all the othe poly_fun_pairs are None remaining_poly_fun_pairs_are_None = \ [poly_fun_pairs[j][0] is None for j in range(i+1,lpf)] if (not all(remaining_poly_fun_pairs_are_None)): raise Exception('Can only have the last polygon = All') # Main Loop # Apply the fi inside the pi for i in range(lpf): fi = poly_fun_pairs[i][1] # The function pi = poly_fun_pairs[i][0] # The polygon # Quick exit if (pi is None): continue ################################################################### # Get indices fInds of points in polygon pi which are not already # set ################################################################### if (pi == 'All'): # Get all unset points fInside = (1 - isSet) fInds = (fInside == 1).nonzero()[0] else: if (pi == 'Extent'): # Here fi MUST be a gdal-compatible raster if (not (type(fi) == str)): msg = ' pi = "Extent" can only be used when fi is a' +\ ' raster file name' raise Exception(msg) if (not os.path.exists(fi)): msg = 'fi ' + str(fi) + ' is supposed to be a ' +\ ' raster filename, but it could not be found' raise Exception(msg) # Then we get the extent from the raster itself pi_path = su.getRasterExtent(fi, asPolygon=True) if verbose: print 'Extracting extent from raster: ', fi print 'Extent: ', pi_path elif ((type(pi) == str) and os.path.isfile(pi)): # pi is a file pi_path = su.read_polygon(pi) else: # pi is the actual polygon data pi_path = pi # Get the insides of unset points inside pi_path notSet = (isSet == 0.).nonzero()[0] fInds = inside_polygon(xy_array_trans[notSet, :], pi_path) fInds = notSet[fInds] if len(fInds) == 0: # No points found, move on continue ################################################################### # Evaluate fi at the points inside pi ################################################################### # We use various tricks to infer whether fi is a function, # a constant, a file (raster or csv), or an array if (hasattr(fi, '__call__')): # fi is a function quantityVal[fInds] = fi(x[fInds], y[fInds]) elif isinstance(fi, (int, long, float)): # fi is a numerical constant quantityVal[fInds] = fi * 1.0 elif (type(fi) is str and os.path.exists(fi)): # fi is a file which is assumed to be # a gdal-compatible raster OR an x,y,z elevation file if os.path.splitext(fi)[1] in ['.txt', '.csv']: fi_array = su.read_csv_optional_header(fi) # Check the results if fi_array.shape[1] is not 3: print 'Treated input file ' + fi +\ ' as xyz array with an optional header' msg = 'Array should have 3 columns -- x,y,value' raise Exception(msg) newfi = make_nearestNeighbour_quantity_function( fi_array, domain, k_nearest_neighbours=default_k_nearest_neighbours) quantityVal[fInds] = newfi(x[fInds], y[fInds]) else: # Treating input file as a raster newfi = quantityRasterFun( domain, fi, interpolation=default_raster_interpolation) quantityVal[fInds] = newfi(x[fInds], y[fInds]) elif (type(fi) is numpy.ndarray): if fi.shape[1] is not 3: msg = 'Array should have 3 columns -- x,y,value' raise Exception(msg) newfi = make_nearestNeighbour_quantity_function( fi, domain, k_nearest_neighbours=default_k_nearest_neighbours) quantityVal[fInds] = newfi(x[fInds], y[fInds]) else: print 'Error with function from' print fi msg = 'Cannot make function from type ' + str(type(fi)) raise Exception, msg ################################################################### # Check for nan values ################################################################### #nan_flag = (quantityVal[fInds] != quantityVal[fInds]) nan_flag = 1 * numpy.isnan(quantityVal[fInds]) nan_inds = nan_flag.nonzero()[0] was_ever_nan[fInds[nan_inds]] = 1 if len(nan_inds) > 0: if nan_treatment == 'exception': msg = 'nan values generated by the poly_fun_pair at '\ 'index ' + str(i) + ' '\ 'in composite_quantity_setting_function. ' + \ 'To allow these values to be set by later ' + \ 'poly_fun pairs, pass the argument ' + \ 'nan_treatment="fall_through" ' + \ 'to composite_quantity_setting_function' raise Exception(msg) elif nan_treatment == 'fall_through': msg = 'WARNING: nan values generated by the ' + \ 'poly_fun_pair at index ' + str(i) + ' '\ 'in composite_quantity_setting_function. ' + \ 'They will be passed to later poly_fun_pairs' if verbose: print msg not_nan_inds = (1 - nan_flag).nonzero()[0] if len(not_nan_inds) > 0: fInds = fInds[not_nan_inds] else: # All values are nan msg = '( Actually all the values were nan - ' + \ 'Are you sure they should be? Possible error?)' if verbose: print msg continue else: msg = 'Found nan values in ' + \ 'composite_quantity_setting_function but ' + \ 'nan_treatment is not a recognized value' raise Exception(msg) # Record that the points have been set isSet[fInds] = 1 # Enforce clip_range if clip_range is not None: lower_bound = clip_range[i][0] upper_bound = clip_range[i][1] quantityVal[fInds] = numpy.maximum(quantityVal[fInds], lower_bound) quantityVal[fInds] = numpy.minimum(quantityVal[fInds], upper_bound) # End of loop # Find points which were nan on their first preference dataset + are # inside nan_interpolation_region_polygon. Then reinterpolate their # values from the other x,y, quantityVal points. if (nan_interpolation_region_polygon is not None) &\ (was_ever_nan.sum() > 0): if nan_interpolation_region_polygon == 'All': points_to_reinterpolate = was_ever_nan.nonzero()[0] else: # nan_interpolation_region_polygon contains information on 1 or # more polygons # Inside those polygons, we need to re-interpolate points which # first evaluted to na possible_points_to_reint = was_ever_nan.nonzero()[0] points_to_reinterpolate = numpy.array([]).astype(int) for i in range(len(nan_interpolation_region_polygon)): nan_pi = nan_interpolation_region_polygon[i] # Ensure nan_pi = list of x,y points making a polygon if (type(nan_pi) == str): nan_pi = su.read_polygon(nan_pi) points_in_nan_pi = inside_polygon( xy_array_trans[possible_points_to_reint, :], nan_pi) if len(points_in_nan_pi) > 0: points_to_reinterpolate = numpy.hstack([ points_to_reinterpolate, possible_points_to_reint[points_in_nan_pi] ]) if verbose: print 'Re-interpolating ', len(points_to_reinterpolate),\ ' points which were nan under their',\ ' first-preference and are inside the',\ ' nan_interpolation_region_polygon' if len(points_to_reinterpolate) > 0: msg = 'WARNING: nan interpolation is being applied. This ',\ 'should be done in serial prior to distributing the ',\ 'domain, as there is no parallel communication ',\ 'implemented yet [so parallel results might depend on ',\ 'the number of processes]' if verbose: print msg # Find the interpolation points = points not needing reinterpolation ip = x * 0 + 1 ip[points_to_reinterpolate] = 0 number_of_ip = ip.sum() ip = ip.nonzero()[0] # Check that none of the ip points has an nan value nan_ip = (quantityVal[ip] != quantityVal[ip]).nonzero()[0] if len(nan_ip) > 0: print 'There are ', len(nan_ip), ' points outside the ',\ 'nan_interpolation_region_polygon have nan values.' print 'The user should ensure this does not happen.' print 'The points have the following coordinates:' print xy_array_trans[ip[nan_ip], :] msg = "There are nan points outside of " +\ "nan_interpolation_region_polygon, even after all " +\ "fall-through's" raise Exception(msg) if (number_of_ip < default_k_nearest_neighbours): raise Exception('Too few non-nan points to interpolate from') # Make function for re-interpolation. Note this requires # x,y,z in georeferenced coordinates, whereas x,y are ANUGA # coordinates reinterp_F = make_nearestNeighbour_quantity_function( numpy.vstack([ xy_array_trans[ip, 0], xy_array_trans[ip, 1], quantityVal[ip] ]).transpose(), domain, k_nearest_neighbours=default_k_nearest_neighbours) # re-interpolate quantityVal[points_to_reinterpolate] = reinterp_F( x[points_to_reinterpolate], y[points_to_reinterpolate]) isSet[points_to_reinterpolate] = 1 # Check there are no remaining nan values if (min(isSet) != 1): print 'Some points remain as nan, which is not allowed' unset_inds = (isSet != 1).nonzero()[0] lui = min(5, len(unset_inds)) print 'There are ', len(unset_inds), ' such points' print 'Here are a few:' for i in range(lui): print x[unset_inds[i]] + xll, y[unset_inds[i]] + yll raise Exception('It seems the input data needs to be fixed') return quantityVal
def F(x,y): """This is the function returned by composite_quantity_setting_function It can be passed to set_quantity """ isSet = numpy.zeros(len(x)) # 0/1 - record if each point has been set quantityVal = x*0 + numpy.nan # Function return value # Record points which evaluated to nan on their first preference # dataset. was_ever_nan = (x*0).astype(int) lpf = len(poly_fun_pairs) if(lpf <= 0): raise Exception('Must have at least 1 fun-poly-pair') # Make an array of 'transformed' spatial coordinates, for checking # polygon inclusion xll = domain.geo_reference.xllcorner yll = domain.geo_reference.yllcorner xy_array_trans = numpy.vstack([x+xll,y+yll]).transpose() # Check that none of the pi polygons [except perhaps the last] is 'All' for i in range(lpf-1): if(poly_fun_pairs[i][0]=='All'): # This is only ok if all the othe poly_fun_pairs are None remaining_poly_fun_pairs_are_None = \ [poly_fun_pairs[j][0] is None for j in range(i+1,lpf)] if(not all(remaining_poly_fun_pairs_are_None)): raise Exception('Can only have the last polygon = All') # Main Loop # Apply the fi inside the pi for i in range(lpf): fi = poly_fun_pairs[i][1] # The function pi = poly_fun_pairs[i][0] # The polygon # Quick exit if(pi is None): continue ################################################################### # Get indices fInds of points in polygon pi which are not already # set ################################################################### if(pi == 'All'): # Get all unset points fInside = (1-isSet) fInds = (fInside==1).nonzero()[0] else: if(pi == 'Extent'): # Here fi MUST be a gdal-compatible raster if(not (type(fi) == str)): msg = ' pi = "Extent" can only be used when fi is a' +\ ' raster file name' raise Exception(msg) if(not os.path.exists(fi)): msg = 'fi ' + str(fi) + ' is supposed to be a ' +\ ' raster filename, but it could not be found' raise Exception(msg) # Then we get the extent from the raster itself pi_path = su.getRasterExtent(fi,asPolygon=True) if verbose: print 'Extracting extent from raster: ', fi print 'Extent: ', pi_path elif( (type(pi) == str) and os.path.isfile(pi) ): # pi is a file pi_path = su.read_polygon(pi) else: # pi is the actual polygon data pi_path = pi # Get the insides of unset points inside pi_path notSet = (isSet==0.).nonzero()[0] fInds = inside_polygon(xy_array_trans[notSet,:], pi_path) fInds = notSet[fInds] if len(fInds) == 0: # No points found, move on continue ################################################################### # Evaluate fi at the points inside pi ################################################################### # We use various tricks to infer whether fi is a function, # a constant, a file (raster or csv), or an array if(hasattr(fi,'__call__')): # fi is a function quantityVal[fInds] = fi(x[fInds], y[fInds]) elif isinstance(fi, (int, long, float)): # fi is a numerical constant quantityVal[fInds] = fi*1.0 elif ( type(fi) is str and os.path.exists(fi)): # fi is a file which is assumed to be # a gdal-compatible raster OR an x,y,z elevation file if os.path.splitext(fi)[1] in ['.txt', '.csv']: fi_array = su.read_csv_optional_header(fi) # Check the results if fi_array.shape[1] is not 3: print 'Treated input file ' + fi +\ ' as xyz array with an optional header' msg = 'Array should have 3 columns -- x,y,value' raise Exception(msg) newfi = make_nearestNeighbour_quantity_function( fi_array, domain, k_nearest_neighbours = default_k_nearest_neighbours) quantityVal[fInds] = newfi(x[fInds], y[fInds]) else: # Treating input file as a raster newfi = quantityRasterFun(domain, fi, interpolation = default_raster_interpolation) quantityVal[fInds] = newfi(x[fInds], y[fInds]) elif(type(fi) is numpy.ndarray): if fi.shape[1] is not 3: msg = 'Array should have 3 columns -- x,y,value' raise Exception(msg) newfi = make_nearestNeighbour_quantity_function(fi, domain, k_nearest_neighbours = default_k_nearest_neighbours) quantityVal[fInds] = newfi(x[fInds], y[fInds]) else: print 'Error with function from' print fi msg='Cannot make function from type ' + str(type(fi)) raise Exception, msg ################################################################### # Check for nan values ################################################################### #nan_flag = (quantityVal[fInds] != quantityVal[fInds]) nan_flag = 1*numpy.isnan(quantityVal[fInds]) nan_inds = nan_flag.nonzero()[0] was_ever_nan[fInds[nan_inds]] = 1 if len(nan_inds)>0: if nan_treatment == 'exception': msg = 'nan values generated by the poly_fun_pair at '\ 'index ' + str(i) + ' '\ 'in composite_quantity_setting_function. ' + \ 'To allow these values to be set by later ' + \ 'poly_fun pairs, pass the argument ' + \ 'nan_treatment="fall_through" ' + \ 'to composite_quantity_setting_function' raise Exception(msg) elif nan_treatment == 'fall_through': msg = 'WARNING: nan values generated by the ' + \ 'poly_fun_pair at index ' + str(i) + ' '\ 'in composite_quantity_setting_function. ' + \ 'They will be passed to later poly_fun_pairs' if verbose: print msg not_nan_inds = (1-nan_flag).nonzero()[0] if len(not_nan_inds)>0: fInds = fInds[not_nan_inds] else: # All values are nan msg = '( Actually all the values were nan - ' + \ 'Are you sure they should be? Possible error?)' if verbose: print msg continue else: msg = 'Found nan values in ' + \ 'composite_quantity_setting_function but ' + \ 'nan_treatment is not a recognized value' raise Exception(msg) # Record that the points have been set isSet[fInds] = 1 # Enforce clip_range if clip_range is not None: lower_bound = clip_range[i][0] upper_bound = clip_range[i][1] quantityVal[fInds] = numpy.maximum( quantityVal[fInds], lower_bound) quantityVal[fInds] = numpy.minimum( quantityVal[fInds], upper_bound) # End of loop # Find points which were nan on their first preference dataset + are # inside nan_interpolation_region_polygon. Then reinterpolate their # values from the other x,y, quantityVal points. if (nan_interpolation_region_polygon is not None) &\ (was_ever_nan.sum() > 0): if nan_interpolation_region_polygon == 'All': points_to_reinterpolate = was_ever_nan.nonzero()[0] else: # nan_interpolation_region_polygon contains information on 1 or # more polygons # Inside those polygons, we need to re-interpolate points which # first evaluted to na possible_points_to_reint = was_ever_nan.nonzero()[0] points_to_reinterpolate = numpy.array([]).astype(int) for i in range(len(nan_interpolation_region_polygon)): nan_pi = nan_interpolation_region_polygon[i] # Ensure nan_pi = list of x,y points making a polygon if(type(nan_pi) == str): nan_pi = su.read_polygon(nan_pi) points_in_nan_pi = inside_polygon( xy_array_trans[possible_points_to_reint,:], nan_pi) if len(points_in_nan_pi)>0: points_to_reinterpolate = numpy.hstack( [points_to_reinterpolate, possible_points_to_reint[points_in_nan_pi]]) if verbose: print 'Re-interpolating ', len(points_to_reinterpolate),\ ' points which were nan under their',\ ' first-preference and are inside the',\ ' nan_interpolation_region_polygon' if len(points_to_reinterpolate) > 0: msg = 'WARNING: nan interpolation is being applied. This ',\ 'should be done in serial prior to distributing the ',\ 'domain, as there is no parallel communication ',\ 'implemented yet [so parallel results might depend on ',\ 'the number of processes]' if verbose: print msg # Find the interpolation points = points not needing reinterpolation ip = x*0 + 1 ip[points_to_reinterpolate] = 0 number_of_ip = ip.sum() ip = ip.nonzero()[0] # Check that none of the ip points has an nan value nan_ip = (quantityVal[ip] != quantityVal[ip]).nonzero()[0] if len(nan_ip) > 0: print 'There are ', len(nan_ip), ' points outside the ',\ 'nan_interpolation_region_polygon have nan values.' print 'The user should ensure this does not happen.' print 'The points have the following coordinates:' print xy_array_trans[ip[nan_ip],:] msg = "There are nan points outside of " +\ "nan_interpolation_region_polygon, even after all " +\ "fall-through's" raise Exception(msg) if(number_of_ip < default_k_nearest_neighbours): raise Exception('Too few non-nan points to interpolate from') # Make function for re-interpolation. Note this requires # x,y,z in georeferenced coordinates, whereas x,y are ANUGA # coordinates reinterp_F = make_nearestNeighbour_quantity_function( numpy.vstack([xy_array_trans[ip,0], xy_array_trans[ip,1], quantityVal[ip]]).transpose(), domain, k_nearest_neighbours = default_k_nearest_neighbours) # re-interpolate quantityVal[points_to_reinterpolate] = reinterp_F( x[points_to_reinterpolate], y[points_to_reinterpolate]) isSet[points_to_reinterpolate] = 1 # Check there are no remaining nan values if( min(isSet) != 1): print 'Some points remain as nan, which is not allowed' unset_inds = (isSet != 1).nonzero()[0] lui = min(5, len(unset_inds)) print 'There are ', len(unset_inds), ' such points' print 'Here are a few:' for i in range(lui): print x[unset_inds[i]] + xll, y[unset_inds[i]] + yll raise Exception('It seems the input data needs to be fixed') return quantityVal
def start_sim(run_id, Runs, scenario_name, Scenario, session, **kwargs): yieldstep = kwargs['yieldstep'] finaltime = kwargs['finaltime'] logger = logging.getLogger(run_id) max_triangle_area = kwargs['max_triangle_area'] logger.info('Starting hydrata_project') if run_id == 'local_run': base_dir = os.getcwd() else: base_dir = os.getcwd() + '/base_dir/%s/' % run_id outname = run_id meshname = base_dir + 'outputs/' + run_id + '.msh' def get_filename(data_type, file_type): files = os.listdir('%sinputs/%s' % (base_dir, data_type)) filename = '%sinputs/%s/%s' % ( base_dir, data_type, [f for f in files if f[-4:] == file_type][0]) return filename boundary_data_filename = get_filename('boundary_data', '.shp') elevation_data_filename = get_filename('elevation_data', '.tif') try: structures_filename = get_filename('structures', '.shp') except OSError as e: structures_filename = None try: rain_data_filename = get_filename('rain_data', '.shp') except OSError as e: rain_data_filename = None try: inflow_data_filename = get_filename('inflow_data', '.shp') except OSError as e: inflow_data_filename = None try: friction_data_filename = get_filename('friction_data', '.shp') except OSError as e: friction_data_filename = None logger.info('boundary_data_filename: %s' % boundary_data_filename) logger.info('structures_filename: %s' % structures_filename) logger.info('rain_data_filename: %s' % rain_data_filename) logger.info('inflow_data_filename: %s' % inflow_data_filename) logger.info('friction_data_filename: %s' % friction_data_filename) logger.info('elevation_data_filename: %s' % elevation_data_filename) # create a list of project files vector_filenames = [ boundary_data_filename, structures_filename, rain_data_filename, inflow_data_filename, friction_data_filename ] # set the projection system for ANUGA calculations from the geotiff elevation data elevation_data_gdal = gdal.Open(elevation_data_filename) project_spatial_ref = osr.SpatialReference() project_spatial_ref.ImportFromWkt(elevation_data_gdal.GetProjectionRef()) project_spatial_ref_epsg_code = int( project_spatial_ref.GetAttrValue("AUTHORITY", 1)) # check the spatial reference system of the project files matches that of the calculation for filename in vector_filenames: if filename: prj_text = open(filename[:-4] + '.prj').read() srs = osr.SpatialReference() srs.ImportFromESRI([prj_text]) srs.AutoIdentifyEPSG() logger.info('filename is: %s' % filename) logger.info('EPSG is: %s' % srs.GetAuthorityCode(None)) if str(srs.GetAuthorityCode(None)) != str( project_spatial_ref_epsg_code): logger.warning('warning spatial refs are not maching: %s, %s' % (srs.GetAuthorityCode(None), project_spatial_ref_epsg_code)) logger.info('Setting up structures...') if structures_filename: structures = [] logger.info('processing structures from :%s' % structures_filename) ogr_shapefile = ogr.Open(structures_filename) ogr_layer = ogr_shapefile.GetLayer(0) ogr_layer_feature = ogr_layer.GetNextFeature() while ogr_layer_feature: structure = json.loads(ogr_layer_feature.GetGeometryRef(). ExportToJson())['coordinates'][0] structures.append(structure) ogr_layer_feature = None ogr_layer_feature = ogr_layer.GetNextFeature() logger.info('structures: %s' % structures) else: logger.warning('warning: no structures found.') structures = None logger.info('Setting up friction...') frictions = [] if friction_data_filename: logger.info('processing frictions from :%s' % friction_data_filename) ogr_shapefile = ogr.Open(friction_data_filename) ogr_layer = ogr_shapefile.GetLayer(0) ogr_layer_feature = ogr_layer.GetNextFeature() while ogr_layer_feature: friction_poly = json.loads(ogr_layer_feature.GetGeometryRef(). ExportToJson())['coordinates'][0] friction_value = float(ogr_layer_feature.GetField('mannings')) friction_couple = [friction_poly, friction_value] frictions.append(friction_couple) ogr_layer_feature = None ogr_layer_feature = ogr_layer.GetNextFeature() frictions.append(['All', 0.04]) logger.info('frictions: %s' % frictions) else: frictions.append(['All', 0.04]) logger.info('warning: no frictions found.') logger.info('Setting up boundary conditions...') ogr_shapefile = ogr.Open(boundary_data_filename) ogr_layer = ogr_shapefile.GetLayer(0) ogr_layer_definition = ogr_layer.GetLayerDefn() logger.info('ogr_layer_definition.GetGeomType: %s' % ogr_layer_definition.GetGeomType()) boundary_tag_index = 0 bdy_tags = {} bdy = {} ogr_layer_feature = ogr_layer.GetNextFeature() while ogr_layer_feature: boundary_tag_key = ogr_layer_feature.GetField('bdy_tag_k') boundary_tag_value = ogr_layer_feature.GetField('bdy_tag_v') bdy_tags[boundary_tag_key] = [ boundary_tag_index * 2, boundary_tag_index * 2 + 1 ] bdy[boundary_tag_key] = boundary_tag_value geom = ogr_layer_feature.GetGeometryRef().GetPoints() ogr_layer_feature = None ogr_layer_feature = ogr_layer.GetNextFeature() boundary_tag_index = boundary_tag_index + 1 logger.info('bdy_tags: %s' % bdy_tags) logger.info('bdy: %s' % bdy) boundary_data = su.read_polygon(boundary_data_filename) create_mesh_from_regions(boundary_data, boundary_tags=bdy_tags, maximum_triangle_area=max_triangle_area, interior_regions=None, interior_holes=structures, filename=meshname, use_cache=False, verbose=True) domain = Domain(meshname, use_cache=False, verbose=True) domain.set_name(outname) domain.set_datadir(base_dir + '/outputs') logger.info(domain.statistics()) poly_fun_pairs = [['Extent', elevation_data_filename.encode("utf-8")]] topography_function = qs.composite_quantity_setting_function( poly_fun_pairs, domain, nan_treatment='exception', ) friction_function = qs.composite_quantity_setting_function( frictions, domain) domain.set_quantity('friction', friction_function, verbose=True) domain.set_quantity('stage', 0.0) domain.set_quantity('elevation', topography_function, verbose=True, alpha=0.99) domain.set_minimum_storable_height(0.005) logger.info('Applying rainfall...') if rain_data_filename: ogr_shapefile = ogr.Open(rain_data_filename) ogr_layer = ogr_shapefile.GetLayer(0) rainfall = 0 ogr_layer_feature = ogr_layer.GetNextFeature() while ogr_layer_feature: rainfall = float(ogr_layer_feature.GetField('rate_mm_hr')) polygon = su.read_polygon(rain_data_filename) logger.info("applying Polygonal_rate_operator with rate, polygon:") logger.info(rainfall) logger.info(polygon) Polygonal_rate_operator(domain, rate=rainfall, factor=1.0e-6, polygon=polygon, default_rate=0.0) ogr_layer_feature = None ogr_layer_feature = ogr_layer.GetNextFeature() logger.info('Applying surface inflows...') if inflow_data_filename: ogr_shapefile = ogr.Open(inflow_data_filename) ogr_layer = ogr_shapefile.GetLayer(0) ogr_layer_definition = ogr_layer.GetLayerDefn() ogr_layer_feature = ogr_layer.GetNextFeature() while ogr_layer_feature: in_fixed = float(ogr_layer_feature.GetField('in_fixed')) line = ogr_layer_feature.GetGeometryRef().GetPoints() logger.info("applying Inlet_operator with line, in_fixed:") logger.info(line) logger.info(in_fixed) Inlet_operator(domain, line, in_fixed, verbose=False) ogr_layer_feature = None ogr_layer_feature = ogr_layer.GetNextFeature() logger.info('Applying Boundary Conditions...') logger.info('Available boundary tags: %s' % domain.get_boundary_tags()) Br = anuga.Reflective_boundary(domain) Bd = anuga.Dirichlet_boundary([0.0, 0.0, 0.0]) Bt = anuga.Transmissive_boundary(domain) for key, value in bdy.iteritems(): if value == 'Br': bdy[key] = Br elif value == 'Bd': bdy[key] = Bd elif value == 'Bt': bdy[key] = Bt else: logger.info( 'No matching boundary condition exists - please check your shapefile attributes in: %s' % boundary_data_filename) # set a default value for exterior & interior boundary if it is not already set try: bdy['exterior'] except KeyError: bdy['exterior'] = Br try: bdy['interior'] except KeyError: bdy['interior'] = Br logger.info('bdy: %s' % bdy) domain.set_boundary(bdy) domain = distribute(domain) logger.info('Beginning evolve phase...') for t in domain.evolve(yieldstep, finaltime): domain.write_time() print domain.timestepping_statistics() logger.info(domain.timestepping_statistics(track_speeds=True)) percentage_complete = round(domain.time / domain.finaltime, 3) * 100 logger.info('%s percent complete' % percentage_complete) if run_id != 'local_run': write_percentage_complete(run_id, Runs, scenario_name, Scenario, session, percentage_complete) domain.sww_merge(delete_old=True) barrier() finalize() sww_file = base_dir + '/outputs/' + run_id + '.sww' sww_file = sww_file.encode( 'utf-8', 'ignore') # sometimes run_id gets turned to a unicode object by celery util.Make_Geotif(swwFile=sww_file, output_quantities=['depth', 'velocity'], myTimeStep='max', CellSize=max_triangle_area, lower_left=None, upper_right=None, EPSG_CODE=project_spatial_ref_epsg_code, proj4string=None, velocity_extrapolation=True, min_allowed_height=1.0e-05, output_dir=(base_dir + '/outputs/'), bounding_polygon=boundary_data, internal_holes=structures, verbose=False, k_nearest_neighbours=3, creation_options=[]) logger.info("Done. Nice work.")
def elevation_setter(xc, yc): # Return scipy array of values out = xc * 0. # Get multiple elevation values in each triangle. # Process triangles in chunks to reduce function call overhead lx = len(xc) lx_div_cs = scipy.ceil(lx * 1. / (1. * chunk_size)).astype(int) # Crude check that xc/yc are the centroid values # erMess = ' Result of make_meanFun can ONLY be applied to a vector' +\ ' of ALL centroid coordinates\n' +\ ' (since mesh triangles are used to spatially average)' assert scipy.all(xc == domain.centroid_coordinates[:, 0]), erMess assert scipy.all(yc == domain.centroid_coordinates[:, 1]), erMess # Find triangles in which we want to average if polygons_for_averaging is not None: averaging_flag = 0*xc # Need georeferenced centroid coordinates to find which # are in the polygon xll = domain.geo_reference.xllcorner yll = domain.geo_reference.yllcorner centroid_coordinates_georef = scipy.vstack([xc + xll, yc + yll]).transpose() for j in range(len(polygons_for_averaging)): poly_j = polygons_for_averaging[j] # poly_j can either be a polygon, or a filename if type(poly_j) is str: poly_j = su.read_polygon(poly_j) points_in_poly_j = inside_polygon(centroid_coordinates_georef, poly_j) averaging_flag[points_in_poly_j] = 1 else: averaging_flag = 1 + 0*xc for i in range(lx_div_cs): # Evaluate in triangles lb:ub lb = i * chunk_size ub = min((i + 1) * chunk_size, lx) if verbose: print 'Averaging in triangles ', lb, '-', ub - 1 # Store x,y,triangleIndex px = scipy.array([]) py = scipy.array([]) p_indices = scipy.array([]) for j in range(lb, ub): # If we average this cell, then get a grid # of points in it. Otherwise just get the centroid # coordinates. if averaging_flag[j] == 1: mesh_tri = \ domain.mesh.vertex_coordinates[ range(3 * j, 3 * j + 3), :].tolist() pts = su.gridPointsInPolygon( mesh_tri, approx_grid_spacing=approx_grid_spacing) else: # Careful to keep this a 2D array pts = domain.centroid_coordinates[j,:, None].transpose() px = scipy.hstack([px, pts[:, 0]]) py = scipy.hstack([py, pts[:, 1]]) p_indices = scipy.hstack([p_indices, scipy.repeat(j, len(pts[:, 0]))]) # Get function values at all px,py if verbose: print ' Evaluating function at ', len(px), ' points' allTopo = q_function(px, py) # Set output values in lb:ub for j in range(lb, ub): out_indices = (p_indices == j).nonzero()[0] assert len(out_indices) > 0 if(averaging == 'mean'): out[j] = allTopo[out_indices].mean() elif(averaging == 'min'): out[j] = allTopo[out_indices].min() elif(averaging == 'max'): out[j] = allTopo[out_indices].max() else: raise Exception('Unknown value of averaging') return(out)
def process_project_data(self): """Process the input data ready for ANUGA """ # Print messages from the ProjectData.__init__ call # This allows us to log those messages without refactoring # (Consider refactoring though) if myid == 0: for p in self.print_info: print p print '' print '---------------------' print 'PROCESS_PROJECT_DATA' print '---------------------' print '' # Record the time and broadcast to other processers time_number = time.time() if numprocs > 1: for i in range(1, numprocs): send(time_number, i) else: time_number = receive(0) # We can either use interior regions, or breaklines if not self.interior_regions_data == []: assert self.pt_areas is None, \ 'Cannot define both ptAreas and non-empty interior regions' bounding_polygon_and_tags = \ read_boundary_tags_line_shapefile( self.bounding_polygon_and_tags_file, self.boundary_tags_attribute_name) self.bounding_polygon = bounding_polygon_and_tags[0] self.boundary_tags = bounding_polygon_and_tags[1] self.breaklines = su.readListOfBreakLines(self.breakline_files) (self.riverwalls, self.riverwall_par) = \ su.readListOfRiverWalls(self.riverwall_csv_files) if self.pt_areas is not None: self.region_point_areas = su.readRegionPtAreas( self.pt_areas, convert_length_to_area=self.region_resolutions_from_length) else: self.region_point_areas = None # Hack to override resolution # region_point_areas=\ # [ region_point_areas[i][0:2]+[150*150*0.5] for i in \ # range(len(region_point_areas))] # Redefine interior_regions to contain the polygon data + resolutions self.interior_regions = [[su.read_polygon(ir[0]), ir[1]] for ir in self.interior_regions_data] # Deal with intersections in the bounding polygon / breaklines / # riverwalls. At the moment we cannot add points to the bounding # polygon because the boundary tags are not adjusted -- so check that # the length of the bounding polygon doesn't change lbp = len(self.bounding_polygon) if type(self.break_line_intersect_point_movement_threshold) is not str: (self.bounding_polygon, self.breaklines, self.riverwalls) = \ su.add_intersections_to_domain_features( self.bounding_polygon, self.breaklines, self.riverwalls, point_movement_threshold=self.break_line_intersect_point_movement_threshold, verbose=True) msg = 'Bounding polygon had points added or dropped because of ' + \ 'intersections --' + \ 'This is not yet properly supported. Please add ' + \ ' the intersection points to the bounding polygon' assert lbp == len(self.bounding_polygon), msg # Here we make a unique ID based on the all the mesh geometry inputs # This tells us if we need to regenerate partitions, or use old ones mesh_dependency_information = [ self.bounding_polygon, self.interior_regions, self.riverwalls, self.breaklines, self.region_point_areas, self.default_res, self.boundary_tags ] if not self.use_existing_mesh_pickle: # Append the time to the mesh dependency so we don't reuse old # meshes mesh_dependency_information.append([time_number]) self.mesh_id_hash = hashlib.md5(json.dumps(mesh_dependency_information)).hexdigest() # Fix the output tif bounding polygon if self.output_tif_bounding_polygon is None: self.output_tif_bounding_polygon = self.bounding_polygon else: self.output_tif_bounding_polygon = \ su.read_polygon(self.output_tif_bounding_polygon) # Make proj4string from projection information # if isinstance(self.projection_information, int): # projection_information describes a UTM zone # e.g. '+units=m +ellps=WGS84 +zone=47 +south=False +proj=utm ' if self.projection_information < 0: self.proj4string = '+proj=utm +zone=' \ + str(abs(self.projection_information)) \ + ' +south +datum=WGS84 +units=m +no_defs' else: self.proj4string = '+proj=utm +zone=' \ + str(self.projection_information) \ + ' +datum=WGS84 +units=m +no_defs' elif isinstance(self.projection_information, str): self.proj4string = self.projection_information else: msg = 'Invalid projection information ' + \ ' -- must be a proj4string, or an integer' + \ ' defining a UTM zone [positive for northern hemisphere,' + \ ' negative for southern hemisphere]' raise Exception(msg) # Set up directories etc self.partition_basedir = 'PARTITIONS/' self.partition_dir = self.partition_basedir + 'Mesh_' +\ str(self.mesh_id_hash) self.meshname = self.output_dir + '/mesh.tsh'
def process_project_data(self): """Process the input data ready for ANUGA """ # Print messages from the ProjectData.__init__ call # This allows us to log those messages without refactoring # (Consider refactoring though) if myid == 0: for p in self.print_info: print p print '' print '---------------------' print 'PROCESS_PROJECT_DATA' print '---------------------' print '' # Record the time and broadcast to other processers time_number = time.time() if numprocs > 1: for i in range(1, numprocs): send(time_number, i) else: time_number = receive(0) # We can either use interior regions, or breaklines if not self.interior_regions_data == []: assert self.pt_areas is None, \ 'Cannot define both ptAreas and non-empty interior regions' bounding_polygon_and_tags = \ read_boundary_tags_line_shapefile( self.bounding_polygon_and_tags_file, self.boundary_tags_attribute_name) self.bounding_polygon = bounding_polygon_and_tags[0] self.boundary_tags = bounding_polygon_and_tags[1] self.breaklines = su.readListOfBreakLines(self.breakline_files) (self.riverwalls, self.riverwall_par) = \ su.readListOfRiverWalls(self.riverwall_csv_files) if self.pt_areas is not None: self.region_point_areas = su.readRegionPtAreas( self.pt_areas, convert_length_to_area=self.region_resolutions_from_length) else: self.region_point_areas = None # Hack to override resolution # region_point_areas=\ # [ region_point_areas[i][0:2]+[150*150*0.5] for i in \ # range(len(region_point_areas))] # Redefine interior_regions to contain the polygon data + resolutions self.interior_regions = [[su.read_polygon(ir[0]), ir[1]] for ir in self.interior_regions_data] # Deal with intersections in the bounding polygon / breaklines / # riverwalls. At the moment we cannot add points to the bounding # polygon because the boundary tags are not adjusted -- so check that # the length of the bounding polygon doesn't change lbp = len(self.bounding_polygon) if type(self.break_line_intersect_point_movement_threshold) is not str: (self.bounding_polygon, self.breaklines, self.riverwalls) = \ su.add_intersections_to_domain_features( self.bounding_polygon, self.breaklines, self.riverwalls, point_movement_threshold=self.break_line_intersect_point_movement_threshold, verbose=True) msg = 'Bounding polygon had points added or dropped because of ' + \ 'intersections --' + \ 'This is not yet properly supported. Please add ' + \ ' the intersection points to the bounding polygon' assert lbp == len(self.bounding_polygon), msg # Here we make a unique ID based on the all the mesh geometry inputs # This tells us if we need to regenerate partitions, or use old ones mesh_dependency_information = [ self.bounding_polygon, self.interior_regions, self.riverwalls, self.breaklines, self.region_point_areas, self.default_res, self.boundary_tags ] if not self.use_existing_mesh_pickle: # Append the time to the mesh dependency so we don't reuse old # meshes mesh_dependency_information.append([time_number]) self.mesh_id_hash = hashlib.md5( json.dumps(mesh_dependency_information)).hexdigest() # Fix the output tif bounding polygon if self.output_tif_bounding_polygon is None: self.output_tif_bounding_polygon = self.bounding_polygon else: self.output_tif_bounding_polygon = \ su.read_polygon(self.output_tif_bounding_polygon) # Make proj4string from projection information # if isinstance(self.projection_information, int): # projection_information describes a UTM zone # e.g. '+units=m +ellps=WGS84 +zone=47 +south=False +proj=utm ' if self.projection_information < 0: self.proj4string = '+proj=utm +zone=' \ + str(abs(self.projection_information)) \ + ' +south +datum=WGS84 +units=m +no_defs' else: self.proj4string = '+proj=utm +zone=' \ + str(self.projection_information) \ + ' +datum=WGS84 +units=m +no_defs' elif isinstance(self.projection_information, str): self.proj4string = self.projection_information else: msg = 'Invalid projection information ' + \ ' -- must be a proj4string, or an integer' + \ ' defining a UTM zone [positive for northern hemisphere,' + \ ' negative for southern hemisphere]' raise Exception(msg) # Set up directories etc self.partition_basedir = 'PARTITIONS/' self.partition_dir = self.partition_basedir + 'Mesh_' +\ str(self.mesh_id_hash) self.meshname = self.output_dir + '/mesh.tsh'
def elevation_setter(xc, yc): # Return scipy array of values out = xc * 0. # Get multiple elevation values in each triangle. # Process triangles in chunks to reduce function call overhead lx = len(xc) lx_div_cs = scipy.ceil(lx * 1. / (1. * chunk_size)).astype(int) # Crude check that xc/yc are the centroid values # erMess = ' Result of make_meanFun can ONLY be applied to a vector' +\ ' of ALL centroid coordinates\n' +\ ' (since mesh triangles are used to spatially average)' assert scipy.all(xc == domain.centroid_coordinates[:, 0]), erMess assert scipy.all(yc == domain.centroid_coordinates[:, 1]), erMess # Find triangles in which we want to average if polygons_for_averaging is not None: averaging_flag = 0 * xc # Need georeferenced centroid coordinates to find which # are in the polygon xll = domain.geo_reference.xllcorner yll = domain.geo_reference.yllcorner centroid_coordinates_georef = scipy.vstack([xc + xll, yc + yll]).transpose() for j in range(len(polygons_for_averaging)): poly_j = polygons_for_averaging[j] # poly_j can either be a polygon, or a filename if type(poly_j) is str: poly_j = su.read_polygon(poly_j) points_in_poly_j = inside_polygon(centroid_coordinates_georef, poly_j) averaging_flag[points_in_poly_j] = 1 else: averaging_flag = 1 + 0 * xc for i in range(lx_div_cs): # Evaluate in triangles lb:ub lb = i * chunk_size ub = min((i + 1) * chunk_size, lx) if verbose: print 'Averaging in triangles ', lb, '-', ub - 1 # Store x,y,triangleIndex px = scipy.array([]) py = scipy.array([]) p_indices = scipy.array([]) for j in range(lb, ub): # If we average this cell, then get a grid # of points in it. Otherwise just get the centroid # coordinates. if averaging_flag[j] == 1: mesh_tri = \ domain.mesh.vertex_coordinates[ range(3 * j, 3 * j + 3), :].tolist() pts = su.gridPointsInPolygon( mesh_tri, approx_grid_spacing=approx_grid_spacing) else: # Careful to keep this a 2D array pts = domain.centroid_coordinates[j, :, None].transpose() px = scipy.hstack([px, pts[:, 0]]) py = scipy.hstack([py, pts[:, 1]]) p_indices = scipy.hstack( [p_indices, scipy.repeat(j, len(pts[:, 0]))]) # Get function values at all px,py if verbose: print ' Evaluating function at ', len(px), ' points' allTopo = q_function(px, py) # Set output values in lb:ub for j in range(lb, ub): out_indices = (p_indices == j).nonzero()[0] assert len(out_indices) > 0 if (averaging == 'mean'): out[j] = allTopo[out_indices].mean() elif (averaging == 'min'): out[j] = allTopo[out_indices].min() elif (averaging == 'max'): out[j] = allTopo[out_indices].max() else: raise Exception('Unknown value of averaging') return (out)
def get_initial_condition_data(data_source, worksheet, flag, print_info): """Convenience function to extract the initial condition data (and initial_condition_additions) from the xls worksheet The needs have become more elaborate over time, e.g. to support combining 2 line files into a polygon Given a character string referring to a quantity which has initial conditions in the xls worksheet (e.g. 'Elevation'), extract the associated data from the 'data_source' (an AnugaXls object) on worksheet 'worksheet' This assumes a particular format in the excel sheet """ # Read the polygon / value pairs quantity_data = data_source.get_paired_list( worksheet, flag, [1, 1], post_process=string_or_float) # If the polygon is a wildcard, assume it matches 2 lines, read them in, # and join them to make a polygon. This is a convenient shorthand # for when we have lkl for i in range(len(quantity_data)): polygon_files = glob.glob(quantity_data[i][0]) # Check it only matches 0 or 1 or 2 files msg = 'Polygon:' + str(i) + ' : ' + quantity_data[i][0] + \ ' for ' + flag + ' on worksheet' + \ worksheet + ' matches > 2 files. We can join at most 2 lines' + \ 'to make a polygon' assert len(polygon_files) <= 2, msg if len(polygon_files) == 0: # Check it is valid msg = 'Polygon:' + str(i) + ' : ' + quantity_data[i][0] + \ ' for ' + flag + ' on worksheet' + \ worksheet + ' matches no files, and is not All or None ' + \ 'or Extent (for a raster)' assert ((quantity_data[i][0] == 'All') | (quantity_data[i][0] is None) | (quantity_data[i][0] == 'Extent')), msg elif len(polygon_files) == 2: # If it matches 2, try to combine to 1. # This is often required to use pairs of breaklines as polygons # Do this by: # 1) Setting up the 2 lines as though they were in a # breakline object # 2) Using su.polygon_from_matching_breakLines print_info.append('Initial ' + flag) print_info.append('Combining these files to a polygon: ') print_info.append(str(polygon_files)) print_info.append('') l0 = su.read_polygon(polygon_files[0]) l1 = su.read_polygon(polygon_files[1]) fake_breakline = {polygon_files[0]: l0, polygon_files[1]: l1} fake_match = quantity_data[i][0].split('*')[0] out_poly = su.polygon_from_matching_breaklines( fake_match, fake_breakline) quantity_data[i][0] = out_poly # Get the clip_range for each polygon / function pair quantity_clip_range = data_source.get_fixed_size_subtable_by_columns( worksheet, flag, dimensions=[2, len(quantity_data)], offset=[3, 1], post_process=string_or_float) new_quantity_clip_range = reformat_clip_range(quantity_clip_range) # Get sub-grid size for spatial averaging, if applicable spatial_average = data_source.get_var( worksheet, flag, [5, 1], post_process=string_or_float) if type(spatial_average) == str: spatial_average = None return quantity_data, new_quantity_clip_range, spatial_average