def load_field(self, filename,unmask=True,timeslice=None, fieldname=None,check_for_grid_info=False, grid_info=None,grid_type='HD',**grid_kwargs): """Load a field from a unformatted fortran file using f2py Arguments: filename: string; full path of the file to load grid_type: string; keyword specifying what type of grid to use **grid_kwargs: keyword dictionary; keyword arguments giving parameters of the grid fieldname, timeslice, unmask, check_for_grid_info and grid_info are not used by this method and included only in order to match the abstract of method of IOFileHelper Returns: A numpy array containing the field Loads a field using a specifically written fortran routine through f2py controlled by a f2py_manager instance. A certain amount of manipulation of the field is required to ensure the field is output in the same orientation as other loading methods. """ grid = gd.makeGrid(grid_type,**grid_kwargs) print("Reading input from {0}".format(filename)) mgnr = f2py_mg.f2py_manager(path.join(fortran_source_path, "mod_topo_io.f90"), func_name="read_topo") data = mgnr.run_current_function_or_subroutine(filename,*grid.get_grid_dimensions()) #rotate the array 90 clockwise (i.e. 270 degrees anticlockwise); also flip #to match symmetry of other loading methods return np.fliplr(np.rot90(data,k=3))
def replace_zeros_with_highest_valued_neighbor(self, data): """Replace any zeros with the value of the highest value out of any (non negative) neighboring cell Arguments: field: 2d ndarray; ndarray containing the field to replace the zeros in Returns: field: 2d ndarray; ndarray containing the field with the zeros replaced by the highest (non negative) neighboring cells value If all neighboring cell are negative or zero then a zero valued cell is left unchanged """ data = np.insert(data, obj=(0, np.size(data, axis=0)), values=-1.0, axis=0) f2py_mngr = f2py_mg.f2py_manager( path.join(fortran_source_path, 'mod_res_size_init_kernel.f90'), func_name='latlongrid_res_size_init_krnl') data = ndi.generic_filter(data, f2py_mngr.run_current_function_or_subroutine, size=(3, 3), mode='wrap') return data[1:-1, :]
def mark_river_mouths(self, flow_direction_data): """Mark all sea points that a river flows into as river mouth points Arguments: flow_direction_data: ndarray; field of flow direction data to mark river mouths on Returns: ndarray of flow direction data with the river mouths marked Calls a fortran module as the kernel to ndimage generic filter in order to actually mark the river mouths """ #For an unknown reason insert only works as required if the array axis are #swapped so axis 0 is being inserted into. flow_direction_data = flow_direction_data.swapaxes(0, 1) flow_direction_data = np.insert(flow_direction_data, obj=(0, np.size(flow_direction_data, axis=0)), values=(flow_direction_data[-1, :], flow_direction_data[0, :]), axis=0) flow_direction_data = flow_direction_data.swapaxes(0, 1) f2py_mngr = f2py_mg.f2py_manager( path.join(fortran_source_path, 'mod_river_mouth_kernels.f90'), func_name='latlongrid_river_mouth_kernel') flow_direction_data = ndi.generic_filter( flow_direction_data, f2py_mngr.run_current_function_or_subroutine, size=(3, 3), mode='constant', cval=float(self.sea_point_flow_direction_value)) return flow_direction_data[:, 1:-1]
def compute_catchments(field, loop_logfile, circ_flow_check_period=1000): """Compute the unordered catchments on all grid points Input: field: numpy-like; field of river flow directions loop_logfile: string; the path to a file in which to store any loops found circ_flow_check_period (optional): integer; how often (in terms of step along a river) to check if the river is going in a loop Output: An tuple consisting of an numpy array of the number of each of the six catchment types found and a second numpy array containing the unordered catchments calculated themselves. Use a fortran module to calculate the unordered catchements (labelling them by order of discovery) on all the grid points in the supplied array of river flow direction. Store any loops found in the named logfile. (The axis swap is required to ensure the data is processed in the correct orientation). """ f2py_mngr = f2py_manager.f2py_manager(path.join( fortran_source_path, "mod_compute_catchments.f90"), func_name="compute_catchments") field = np.swapaxes(np.asarray(field, dtype=np.int64), 0, 1) print("Writing circular flow log file to {0}".format(loop_logfile)) catchment_types,catchments = \ f2py_mngr.run_current_function_or_subroutine(field, circ_flow_check_period, loop_logfile) catchments = np.ma.array(np.swapaxes(catchments, 0, 1)) return catchment_types, catchments
def run_cotat_plus(fine_rdirs_field,fine_total_cumulative_flow_field,cotat_plus_parameters_filepath, coarse_grid_type,**coarse_grid_kwargs): """Run the cotat plus fortran code using f2py for a lat-lon field Arguments: fine_rdirs_field: 2d ndarray; the fine river directions to be upscaled in 1-9 keypad format fine_total_cumulative_flow_field: 2d ndarray; the fine total cumulative flow (created from the fine_rdirs_field) to be used in upscaling cotat_plus_parameter_filepath: string; the file path containing the namelist with the parameters for the cotat plus upscaling algorithm coarse_grid_type: string; code for the coarse grid type to be upscaled to **coarse_grid_kwargs(optional): keyword dictionary; the parameter of the coarse grid to upscale to (if required) Return: 2d ndarray; the upscaled river direction on the coarse grid Compiles and runs the COTAT plus algorithm in Fortran using f2py for a lat-lon field """ additional_fortran_filenames = ["base/area_mod.o", "base/coords_mod.o", "algorithms/cotat_parameters_mod.o", "algorithms/cotat_plus.o", "base/doubly_linked_list_mod.o", "base/doubly_linked_list_link_mod.o", "base/field_section_mod.o", "base/precision_mod.o", "base/subfield_mod.o", "algorithms/map_non_coincident_grids_mod.o", "base/unstructured_grid_mod.o"] additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\ additional_fortran_filenames] f2py_mngr = f2py_manager.f2py_manager(path.join(fortran_project_source_path, "drivers", "cotat_plus_driver_mod.f90"), func_name="cotat_plus_latlon_f2py_wrapper", additional_fortran_files=additional_fortran_filepaths, include_path=fortran_project_include_path) coarse_grid = grid.makeGrid(coarse_grid_type,**coarse_grid_kwargs) if using_mpi(): comm = MPI.COMM_WORLD comm.bcast(MPICommands.RUNCOTATPLUS, root=0) coarse_rdirs_field_raw = f2py_mngr.\ run_current_function_or_subroutine(fine_rdirs_field.get_data().astype(np.int64,order='F'), fine_total_cumulative_flow_field.get_data().astype(np.int64,order='F'), cotat_plus_parameters_filepath, coarse_grid.get_grid_dimensions()[0], coarse_grid.get_grid_dimensions()[1]) coarse_rdirs_field = field.makeField(coarse_rdirs_field_raw.astype(np.float64),'RiverDirections',coarse_grid_type, **coarse_grid_kwargs) if fine_rdirs_field.grid_has_coordinates(): nlat_fine,nlon_fine = fine_rdirs_field.get_grid_dimensions() lat_pts_fine,lon_pts_fine = fine_rdirs_field.get_grid_coordinates() nlat_coarse,nlon_coarse = coarse_grid.get_grid_dimensions() lat_pts_coarse,lon_pts_coarse = \ coordinate_scaling_utilities.generate_coarse_pts(nlat_fine,nlon_fine, lat_pts_fine,lon_pts_fine, nlat_coarse,nlon_coarse) coarse_rdirs_field.set_grid_coordinates([lat_pts_coarse,lon_pts_coarse]) return coarse_rdirs_field
def run_cotat_plus(self): f2py_mngr = f2py_manager.f2py_manager( path.join(fortran_project_source_path, "drivers", "cotat_plus_driver_mod.f90"), func_name="cotat_plus_latlon_f2py_worker_wrapper", no_compile=True) f2py_mngr.\ run_current_function_or_subroutine()
def setUp(self): """Unit test setUp function""" self.catchment_field = np.zeros((9, 9), order='F', dtype=np.int32) self.grid_points_in_catchment = np.zeros((2, 9 * 9), order='F', dtype=np.int32) self.f2py_mngr = f2py_mg.f2py_manager(os.path.join( fortran_source_path, "mod_compute_catchments.f90"), func_name="label_catchment")
def run_loop_breaker(coarse_rdirs,coarse_cumulative_flow,coarse_catchments,fine_rdirs_field, fine_cumulative_flow_field,loop_nums_list,coarse_grid_type,**coarse_grid_kwargs): """Run the Fortarn complex loop breaking code via f2py Arguments: coarse_rdirs: 2d ndarray; the coarse river directions to remove the loop from coarse_cumulative_flow: 2d ndarray; the coarse cumulative flow derived from the coarse river directions coarse_catchments: 2d ndarray; the set of catchments generated from the coarse river flow directions fine_rdirs_field: 2d ndarray; the original fine river directions the coarse river direction were upscaled from fine_cumulative_flow_field: 2d ndarray; the fine cumulative flow derived from the original fine river directions loop_nums_list: list of integers; the catchment numbers of catchments that have loops in them that need to be removed coarse_grid_type: string; code of the grid type of the coarse grid **coarse_grid_kwargs: keyword dictionary; key word arguments specifying parameters of the coarse grid Returns: the 2d ndarray of coarse river directions with the specified loops removed This will compile and run code that will remove the specified set of more complex loops (that may contain three or more cells). """ additional_fortran_filenames = ["base/coords_mod.o", "base/doubly_linked_list_mod.o", "algorithms/break_loops_mod.o", "base/doubly_linked_list_link_mod.o", "base/field_section_mod.o", "algorithms/loop_breaker_mod.o", "base/unstructured_grid_mod.o", "algorithms/map_non_coincident_grids_mod.o", "base/subfield_mod.o"] additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\ additional_fortran_filenames] f2py_mngr = f2py_manager.f2py_manager(path.join(fortran_project_source_path, "drivers", "break_loops_driver_mod.f90"), func_name="break_loops_latlon_f2py_wrapper", additional_fortran_files=additional_fortran_filepaths, include_path=fortran_project_include_path) loop_nums_list_array = np.asarray(loop_nums_list) if len(loop_nums_list) == 0: print("List of loops to remove is empty!") return coarse_rdirs coarse_rdirs_field_raw = coarse_rdirs.get_data().astype(np.int32,order='F') f2py_mngr.run_current_function_or_subroutine(coarse_rdirs_field_raw, coarse_cumulative_flow.get_data().astype(np.int64,order='F'), coarse_catchments.get_data().astype(np.int64,order='F'), fine_rdirs_field.get_data().astype(np.int64,order='F'), fine_cumulative_flow_field.get_data().astype(np.int64,order='F'), loop_nums_list_array.astype(np.int64,order='F')) coarse_rdirs_field = field.RiverDirections(coarse_rdirs_field_raw.astype(np.float64), grid=coarse_rdirs.get_grid()) return coarse_rdirs_field
def testRenumbering(self): """Test the core Fortran renumbering subroutine""" f2py_mngr = f2py_mg.f2py_manager(os.path.join( fortran_source_path, "mod_compute_catchments.f90"), func_name="relabel_catchments") input_catchments = np.copy(self.original_catchments) f2py_mngr.run_current_function_or_subroutine(input_catchments, self.old_to_new_label_map) np.testing.assert_equal( input_catchments, self.expected_results, "Catchment renumbering not producing expected results")
def setUp(self): """Unit test setUp function""" self.f2py_mngr = f2py_mg.f2py_manager(os.path.join( fortran_source_path, "mod_compute_catchments.f90"), func_name="follow_river") self.catchment_number = np.asarray(-18) self.catchment_field = np.zeros((9, 9), order='F', dtype=np.int32) try: os.remove(self.loop_logfile) except OSError: pass with open(self.loop_logfile, 'w') as f: f.write('Loops found in catchments:\n')
def find_all_local_minima(self, data): data = np.insert(data, obj=(0, np.size(data, axis=0)), values=float('inf'), axis=0) f2py_mngr = f2py_mg.f2py_manager(path.join( fortran_source_path, 'mod_local_minima_finding_kernels.f90'), func_name='latlon_local_min') local_minima_finding_kernel = f2py_mngr.run_current_function_or_subroutine local_minima = ndi.generic_filter(data, local_minima_finding_kernel, size=(3, 3), mode='wrap') return local_minima[1:-1].astype(bool)
def setUp(self): """Unit test setUp function""" self.f2py_mngr = f2py_mg.f2py_manager( os.path.join(fortran_source_path, "mod_compute_catchments.f90"), func_name="compute_next_grid_cell") self.i = 52 self.j = 47 self.iasarray = np.array(self.i) self.jasarray = np.array(self.j) self.expected_output_coords = [ [ [base + mod for base, mod in zip([self.i, self.j], coordmods) ] #@UndefinedVariable for coordmods in row ] for row in self.expected_output_coords_changes ] #@UnusedVariable
def renumber_catchments_by_size(catchments, loop_logfile=None): """Renumber catchments according to there size in terms of number of cells Input: catchments: numpy-like array; catchments to be relabelled catchments loop_logfile: string; the full path to the existing loop_logfile (optional) Returns: Relabelled catchements Label catchments in order of desceding size using a fortran function to assist a time critical part of the procedure that can't be done in numpy effectively. Also opens the existing loop logfile and changes the catchment numbers with loops to reflect the new catchment labelling. """ f2py_mngr = f2py_manager.f2py_manager(path.join( fortran_source_path, "mod_compute_catchments.f90"), func_name="relabel_catchments") catch_nums = np.arange(np.amax(catchments) + 1) counts = np.bincount(catchments.flatten()) catchments_sizes = np.empty(len(catch_nums), dtype=[('catch_nums', int), ('new_catch_nums', int), ('counts', int)]) catchments_sizes['catch_nums'] = catch_nums catchments_sizes['counts'] = counts catchments_sizes.sort(order='counts') catchments_sizes['new_catch_nums'] = np.arange( len(catchments_sizes['catch_nums']), 0, -1) catchments = np.asfortranarray(catchments, np.int32) old_to_new_label_map = np.asfortranarray( np.copy(np.sort(catchments_sizes, order='catch_nums'))['new_catch_nums'], np.int32) f2py_mngr.run_current_function_or_subroutine(catchments, old_to_new_label_map) if loop_logfile is not None: with open(loop_logfile, 'r') as f: next(f) loops = [int(line.strip()) for line in f] #-1 to account for differing array offset between Fortran and python loops = [ str(old_to_new_label_map[old_loop_num]) + '\n' for old_loop_num in loops ] with open(loop_logfile, 'w') as f: f.write('Loops found in catchments:\n') f.writelines(loops) return catchments
def get_number_of_masked_neighbors(self, data): """Compute the number of neighbors that are masked of each masked cell""" #insert extra rows across the top and bottom of the grid to ensure correct #treatmen of boundaries data = np.insert(data, obj=(0, np.size(data, axis=0)), values=False, axis=0) f2py_mngr = f2py_mg.f2py_manager(path.join( fortran_source_path, 'mod_grid_m_nbrs_kernels.f90'), func_name='HDgrid_masked_nbrs_kernel') nbrs_kernel = f2py_mngr.run_current_function_or_subroutine nbrs_num = ndi.generic_filter(data.astype(np.float64), nbrs_kernel, size=(3, 3), mode='wrap') return nbrs_num[1:-1].astype(int)
def compute_flow_directions(self, data, use_fortran_kernel=True): """Compute the flow direction for a given orograpy Arguments: data: numpy array-like object; the orography to compute the flow direction for use_fortran_kernel: boolean: a flag to choose between using a python kernel (False) or using a Fortran kernel (True) Returns: A numpy array of flow directions as integers Inserts extra rows to ensure there is no cross pole flow; then flip the field to get the same first minima as Stefan uses for easy comparison. Setup the kernel function to use depending on the input flag. Then run a generic filter from the numpy image processing module that considers the 3 by 3 area around each grid cell; wrapping around the globe in the east-west direction. Return the output as an integer after flipping it back the right way up and removing the extra rows added at the top and bottom """ #insert extra rows across the top and bottom of the grid to ensure correct #treatmen of boundaries data = np.insert(data, obj=(0, np.size(data, axis=0)), values=self.pole_boundary_condition, axis=0) #processing the data upside down to ensure that the handling of the case where two neighbours #have exactly the same height matches that of Stefan's scripts data = np.flipud(data) if use_fortran_kernel: f2py_mngr = f2py_mg.f2py_manager(path.join( fortran_source_path, 'mod_grid_flow_direction_kernels.f90'), func_name='HDgrid_fdir_kernel') flow_direction_kernel = f2py_mngr.run_current_function_or_subroutine else: flow_direction_kernel = self.flow_direction_kernel flow_directions_as_float = ndi.generic_filter(data, flow_direction_kernel, size=(3, 3), mode='wrap') flow_directions_as_float = np.flipud(flow_directions_as_float) return flow_directions_as_float[1:-1].astype(int)
def run_flow(input_fine_river_directions, input_fine_total_cumulative_flow, cotat_parameters_filepath, coarse_grid_type, **coarse_grid_kwargs): additional_fortran_filenames = [ "base/area_mod.o", "base/coords_mod.o", "algorithms/cotat_parameters_mod.o", "algorithms/flow.o", "base/doubly_linked_list_mod.o", "base/doubly_linked_list_link_mod.o", "base/field_section_mod.o", "base/precision_mod.o", "base/subfield_mod.o" ] additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\ additional_fortran_filenames] f2py_mngr = f2py_manager.f2py_manager( path.join(fortran_project_source_path, "drivers", "flow_driver_mod.f90"), func_name="flow_latlon_f2py_wrapper", additional_fortran_files=additional_fortran_filepaths, include_path=fortran_project_include_path) output_coarse_river_directions_latlon_indices = f2py_mngr.\ run_current_function_or_subroutine(),
def write_field(self, filename, field,griddescfile=None,fieldname=None): """Write a field to an unformatted fortran file using f2py Arguments: filename: string; full path of the file to write to field: A Field (or Field subclass object); field to write griddescfile is ignored by this function but included to match the abstract function it is overriding fieldname is not used by this method and included only in order to match the abstract of method of IOFileHelper Writes a field using a specifically written fortran routine through f2py controlled by a f2py_manager instance. The manipulation applied to the field in the load field method is reversed for consistency. """ print("Writing output to {0}".format(filename)) mgnr = f2py_mg.f2py_manager(path.join(fortran_source_path, "mod_topo_io.f90"), func_name="write_topo") #reverse the manipulation in the load_field method data = np.rot90(np.fliplr(field.get_data())) mgnr.run_current_function_or_subroutine(filename,data)
def create_hypothetical_river_paths_map(riv_dirs, lsmask=None, use_f2py_func=True, use_f2py_sparse_iterator=False, nlat=360, nlong=720, sparse_fraction=0.5, use_new_method=False): """Create map of cumulative flow to cell from a field of river directions Inputs: riv_dirs: numpy array; an array of river flow directions lsmask(optional): numpy array; a landsea mask (with sea points masked) use_f2py_func (optional): boolean; Whether to use an iterator written in Fortran (True) or in python (False) use_f2py_sparse_iterator (optional): boolean; Whether to use a sparse iterator (that works on a vector containing the remaining points to process rather than the whole river direction array) written in Fortran. Independent of use_f2py_func. nlat: integer; number of latitude points in the input arrays nlong:integer; number of longitude points in the input arrays sparse_fraction (optional): float; threshold (as a fraction of the points in the array remaining to be processed) for switching to the sparse iterator (if that option is selected) Returns: Generated flow to cell path map as a numpy array Perform a certain amount of preperation of the input arrays (adding a one cell border across the top and bottom of the edge of the river direction array and the land sea mask and also applying the landsea mask if that option is selected). Then select an iterator and iterate over the initially empty flow map till it is completely filled; switching to the the sparse iterator when the given threshold is reached if that option is selected. """ riv_dirs = np.insert(riv_dirs, obj=0, values=np.zeros(nlong), axis=0) #nlat+1 because the array is now already nlat+1 elements wide so you want to place #the new row after the last row riv_dirs = np.insert(riv_dirs, obj=nlat + 1, values=np.zeros(nlong), axis=0) if lsmask is not None: lsmask = np.insert(lsmask, obj=0, values=np.ones(nlong, dtype=bool), axis=0) #nlat+1 because the array is now already nlat+1 elements wide so you want to place #the new row after the last row lsmask = np.insert(lsmask, obj=nlat + 1, values=np.ones(nlong, dtype=bool), axis=0) riv_dirs = np.ma.array(riv_dirs, mask=lsmask, copy=True, dtype=int).filled(0) else: riv_dirs = np.array(riv_dirs, copy=True, dtype=int) paths_map = np.zeros((nlat + 2, nlong), dtype=np.int32, order='F') if use_f2py_func and use_new_method: additional_fortran_filenames = [ "algorithms/accumulate_flow_mod.o", "base/coords_mod.o", "algorithms/flow_accumulation_algorithm_mod.o", "base/convert_rdirs_to_indices.o", "base/doubly_linked_list_mod.o", "base/doubly_linked_list_link_mod.o", "base/subfield_mod.o", "base/unstructured_grid_mod.o", "base/precision_mod.o" ] additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\ additional_fortran_filenames] f2py_mngr = f2py_mg.f2py_manager( path.join(fortran_project_source_path, "drivers", "accumulate_flow_driver_mod.f90"), func_name="accumulate_flow_latlon_f2py_wrapper", additional_fortran_files=additional_fortran_filepaths, include_path=fortran_project_include_path) paths_map = f2py_mngr.\ run_current_function_or_subroutine(np.asfortranarray(riv_dirs), *riv_dirs.shape) #Make a minor postprocessing correction paths_map[np.logical_and(np.logical_or(riv_dirs == 5, riv_dirs == 0), paths_map == 0)] = 1 else: if use_f2py_func: f2py_kernel = f2py_mg.f2py_manager(path.join( fortran_source_path, 'mod_iterate_paths_map.f90'), func_name='iterate_paths_map') iterate_paths_map_function = f2py_kernel.run_current_function_or_subroutine else: iterate_paths_map_function = iterate_paths_map while iterate_paths_map_function(riv_dirs, paths_map, nlat, nlong): remaining_points = paths_map.size - np.count_nonzero(paths_map) if use_f2py_sparse_iterator and remaining_points / float( paths_map.size) < sparse_fraction: f2py_sparse_iterator = f2py_mg.f2py_manager( path.join(fortran_source_path, 'mod_iterate_paths_map.f90'), func_name='sparse_iterator') f2py_sparse_iterator.run_current_function_or_subroutine( riv_dirs, paths_map, nlat, nlong) break return paths_map[1:-1, :]
def setUp(self): """Unit test setup function""" self.f2py_mngr = f2py_manager.f2py_manager(os.path.join(fortran_source_path,'mod_iterate_paths_map.f90'))
def setUp(self): """Unit test setUp function""" self.f2py_mngr = f2py_mg.f2py_manager(os.path.join( fortran_source_path, "mod_compute_catchments.f90"), func_name="compute_catchments")