def load_field(self, filename,unmask=True,timeslice=None,
fieldname=None,check_for_grid_info=False,
grid_info=None,grid_type='HD',**grid_kwargs):
"""Load a field from a unformatted fortran file using f2py
Arguments:
filename: string; full path of the file to load
grid_type: string; keyword specifying what type of grid to use
**grid_kwargs: keyword dictionary; keyword arguments giving parameters of the grid
fieldname, timeslice, unmask, check_for_grid_info and grid_info are not used by this
method and included only in order to match the abstract of method of IOFileHelper
Returns:
A numpy array containing the field
Loads a field using a specifically written fortran routine through f2py controlled
by a f2py_manager instance. A certain amount of manipulation of the field is required
to ensure the field is output in the same orientation as other loading methods.
"""
grid = gd.makeGrid(grid_type,**grid_kwargs)
print("Reading input from {0}".format(filename))
mgnr = f2py_mg.f2py_manager(path.join(fortran_source_path,
"mod_topo_io.f90"), func_name="read_topo")
data = mgnr.run_current_function_or_subroutine(filename,*grid.get_grid_dimensions())
#rotate the array 90 clockwise (i.e. 270 degrees anticlockwise); also flip
#to match symmetry of other loading methods
return np.fliplr(np.rot90(data,k=3))
def replace_zeros_with_highest_valued_neighbor(self, data):
"""Replace any zeros with the value of the highest value out of any (non negative) neighboring cell
Arguments:
field: 2d ndarray; ndarray containing the field to replace the zeros in
Returns:
field: 2d ndarray; ndarray containing the field with the zeros replaced
by the highest (non negative) neighboring cells value
If all neighboring cell are negative or zero then a zero valued cell is left unchanged
"""
data = np.insert(data,
obj=(0, np.size(data, axis=0)),
values=-1.0,
axis=0)
f2py_mngr = f2py_mg.f2py_manager(
path.join(fortran_source_path, 'mod_res_size_init_kernel.f90'),
func_name='latlongrid_res_size_init_krnl')
data = ndi.generic_filter(data,
f2py_mngr.run_current_function_or_subroutine,
size=(3, 3),
mode='wrap')
return data[1:-1, :]
def mark_river_mouths(self, flow_direction_data):
"""Mark all sea points that a river flows into as river mouth points
Arguments:
flow_direction_data: ndarray; field of flow direction data to mark river mouths on
Returns: ndarray of flow direction data with the river mouths marked
Calls a fortran module as the kernel to ndimage generic filter in order to actually
mark the river mouths
"""
#For an unknown reason insert only works as required if the array axis are
#swapped so axis 0 is being inserted into.
flow_direction_data = flow_direction_data.swapaxes(0, 1)
flow_direction_data = np.insert(flow_direction_data,
obj=(0,
np.size(flow_direction_data,
axis=0)),
values=(flow_direction_data[-1, :],
flow_direction_data[0, :]),
axis=0)
flow_direction_data = flow_direction_data.swapaxes(0, 1)
f2py_mngr = f2py_mg.f2py_manager(
path.join(fortran_source_path, 'mod_river_mouth_kernels.f90'),
func_name='latlongrid_river_mouth_kernel')
flow_direction_data = ndi.generic_filter(
flow_direction_data,
f2py_mngr.run_current_function_or_subroutine,
size=(3, 3),
mode='constant',
cval=float(self.sea_point_flow_direction_value))
return flow_direction_data[:, 1:-1]
def compute_catchments(field, loop_logfile, circ_flow_check_period=1000):
"""Compute the unordered catchments on all grid points
Input:
field: numpy-like; field of river flow directions
loop_logfile: string; the path to a file in which to store any loops found
circ_flow_check_period (optional): integer; how often (in terms of step
along a river) to check if the river is going in a loop
Output:
An tuple consisting of an numpy array of the number of each of the six catchment
types found and a second numpy array containing the unordered catchments calculated
themselves.
Use a fortran module to calculate the unordered catchements (labelling them by
order of discovery) on all the grid points in the supplied array of river flow
direction. Store any loops found in the named logfile. (The axis swap is required
to ensure the data is processed in the correct orientation).
"""
f2py_mngr = f2py_manager.f2py_manager(path.join(
fortran_source_path, "mod_compute_catchments.f90"),
func_name="compute_catchments")
field = np.swapaxes(np.asarray(field, dtype=np.int64), 0, 1)
print("Writing circular flow log file to {0}".format(loop_logfile))
catchment_types,catchments = \
f2py_mngr.run_current_function_or_subroutine(field,
circ_flow_check_period,
loop_logfile)
catchments = np.ma.array(np.swapaxes(catchments, 0, 1))
return catchment_types, catchments
def run_cotat_plus(fine_rdirs_field,fine_total_cumulative_flow_field,cotat_plus_parameters_filepath,
coarse_grid_type,**coarse_grid_kwargs):
"""Run the cotat plus fortran code using f2py for a lat-lon field
Arguments:
fine_rdirs_field: 2d ndarray; the fine river directions to be upscaled in 1-9 keypad format
fine_total_cumulative_flow_field: 2d ndarray; the fine total cumulative flow (created from
the fine_rdirs_field) to be used in upscaling
cotat_plus_parameter_filepath: string; the file path containing the namelist with the parameters
for the cotat plus upscaling algorithm
coarse_grid_type: string; code for the coarse grid type to be upscaled to
**coarse_grid_kwargs(optional): keyword dictionary; the parameter of the coarse grid to
upscale to (if required)
Return: 2d ndarray; the upscaled river direction on the coarse grid
Compiles and runs the COTAT plus algorithm in Fortran using f2py for a lat-lon field
"""
additional_fortran_filenames = ["base/area_mod.o",
"base/coords_mod.o",
"algorithms/cotat_parameters_mod.o",
"algorithms/cotat_plus.o",
"base/doubly_linked_list_mod.o",
"base/doubly_linked_list_link_mod.o",
"base/field_section_mod.o",
"base/precision_mod.o",
"base/subfield_mod.o",
"algorithms/map_non_coincident_grids_mod.o",
"base/unstructured_grid_mod.o"]
additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\
additional_fortran_filenames]
f2py_mngr = f2py_manager.f2py_manager(path.join(fortran_project_source_path,
"drivers",
"cotat_plus_driver_mod.f90"),
func_name="cotat_plus_latlon_f2py_wrapper",
additional_fortran_files=additional_fortran_filepaths,
include_path=fortran_project_include_path)
coarse_grid = grid.makeGrid(coarse_grid_type,**coarse_grid_kwargs)
if using_mpi():
comm = MPI.COMM_WORLD
comm.bcast(MPICommands.RUNCOTATPLUS, root=0)
coarse_rdirs_field_raw = f2py_mngr.\
run_current_function_or_subroutine(fine_rdirs_field.get_data().astype(np.int64,order='F'),
fine_total_cumulative_flow_field.get_data().astype(np.int64,order='F'),
cotat_plus_parameters_filepath,
coarse_grid.get_grid_dimensions()[0],
coarse_grid.get_grid_dimensions()[1])
coarse_rdirs_field = field.makeField(coarse_rdirs_field_raw.astype(np.float64),'RiverDirections',coarse_grid_type,
**coarse_grid_kwargs)
if fine_rdirs_field.grid_has_coordinates():
nlat_fine,nlon_fine = fine_rdirs_field.get_grid_dimensions()
lat_pts_fine,lon_pts_fine = fine_rdirs_field.get_grid_coordinates()
nlat_coarse,nlon_coarse = coarse_grid.get_grid_dimensions()
lat_pts_coarse,lon_pts_coarse = \
coordinate_scaling_utilities.generate_coarse_pts(nlat_fine,nlon_fine,
lat_pts_fine,lon_pts_fine,
nlat_coarse,nlon_coarse)
coarse_rdirs_field.set_grid_coordinates([lat_pts_coarse,lon_pts_coarse])
return coarse_rdirs_field
def run_cotat_plus(self):
f2py_mngr = f2py_manager.f2py_manager(
path.join(fortran_project_source_path, "drivers",
"cotat_plus_driver_mod.f90"),
func_name="cotat_plus_latlon_f2py_worker_wrapper",
no_compile=True)
f2py_mngr.\
run_current_function_or_subroutine()
def setUp(self):
"""Unit test setUp function"""
self.catchment_field = np.zeros((9, 9), order='F', dtype=np.int32)
self.grid_points_in_catchment = np.zeros((2, 9 * 9),
order='F',
dtype=np.int32)
self.f2py_mngr = f2py_mg.f2py_manager(os.path.join(
fortran_source_path, "mod_compute_catchments.f90"),
func_name="label_catchment")
def run_loop_breaker(coarse_rdirs,coarse_cumulative_flow,coarse_catchments,fine_rdirs_field,
fine_cumulative_flow_field,loop_nums_list,coarse_grid_type,**coarse_grid_kwargs):
"""Run the Fortarn complex loop breaking code via f2py
Arguments:
coarse_rdirs: 2d ndarray; the coarse river directions to remove the loop from
coarse_cumulative_flow: 2d ndarray; the coarse cumulative flow derived from the coarse
river directions
coarse_catchments: 2d ndarray; the set of catchments generated from the coarse river flow
directions
fine_rdirs_field: 2d ndarray; the original fine river directions the coarse river direction
were upscaled from
fine_cumulative_flow_field: 2d ndarray; the fine cumulative flow derived from the original
fine river directions
loop_nums_list: list of integers; the catchment numbers of catchments that have loops in them
that need to be removed
coarse_grid_type: string; code of the grid type of the coarse grid
**coarse_grid_kwargs: keyword dictionary; key word arguments specifying parameters of the coarse
grid
Returns: the 2d ndarray of coarse river directions with the specified loops removed
This will compile and run code that will remove the specified set of more complex loops (that
may contain three or more cells).
"""
additional_fortran_filenames = ["base/coords_mod.o",
"base/doubly_linked_list_mod.o",
"algorithms/break_loops_mod.o",
"base/doubly_linked_list_link_mod.o",
"base/field_section_mod.o",
"algorithms/loop_breaker_mod.o",
"base/unstructured_grid_mod.o",
"algorithms/map_non_coincident_grids_mod.o",
"base/subfield_mod.o"]
additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\
additional_fortran_filenames]
f2py_mngr = f2py_manager.f2py_manager(path.join(fortran_project_source_path,
"drivers",
"break_loops_driver_mod.f90"),
func_name="break_loops_latlon_f2py_wrapper",
additional_fortran_files=additional_fortran_filepaths,
include_path=fortran_project_include_path)
loop_nums_list_array = np.asarray(loop_nums_list)
if len(loop_nums_list) == 0:
print("List of loops to remove is empty!")
return coarse_rdirs
coarse_rdirs_field_raw = coarse_rdirs.get_data().astype(np.int32,order='F')
f2py_mngr.run_current_function_or_subroutine(coarse_rdirs_field_raw,
coarse_cumulative_flow.get_data().astype(np.int64,order='F'),
coarse_catchments.get_data().astype(np.int64,order='F'),
fine_rdirs_field.get_data().astype(np.int64,order='F'),
fine_cumulative_flow_field.get_data().astype(np.int64,order='F'),
loop_nums_list_array.astype(np.int64,order='F'))
coarse_rdirs_field = field.RiverDirections(coarse_rdirs_field_raw.astype(np.float64),
grid=coarse_rdirs.get_grid())
return coarse_rdirs_field
def testRenumbering(self):
"""Test the core Fortran renumbering subroutine"""
f2py_mngr = f2py_mg.f2py_manager(os.path.join(
fortran_source_path, "mod_compute_catchments.f90"),
func_name="relabel_catchments")
input_catchments = np.copy(self.original_catchments)
f2py_mngr.run_current_function_or_subroutine(input_catchments,
self.old_to_new_label_map)
np.testing.assert_equal(
input_catchments, self.expected_results,
"Catchment renumbering not producing expected results")
def setUp(self):
"""Unit test setUp function"""
self.f2py_mngr = f2py_mg.f2py_manager(os.path.join(
fortran_source_path, "mod_compute_catchments.f90"),
func_name="follow_river")
self.catchment_number = np.asarray(-18)
self.catchment_field = np.zeros((9, 9), order='F', dtype=np.int32)
try:
os.remove(self.loop_logfile)
except OSError:
pass
with open(self.loop_logfile, 'w') as f:
f.write('Loops found in catchments:\n')
def find_all_local_minima(self, data):
data = np.insert(data,
obj=(0, np.size(data, axis=0)),
values=float('inf'),
axis=0)
f2py_mngr = f2py_mg.f2py_manager(path.join(
fortran_source_path, 'mod_local_minima_finding_kernels.f90'),
func_name='latlon_local_min')
local_minima_finding_kernel = f2py_mngr.run_current_function_or_subroutine
local_minima = ndi.generic_filter(data,
local_minima_finding_kernel,
size=(3, 3),
mode='wrap')
return local_minima[1:-1].astype(bool)
def setUp(self):
"""Unit test setUp function"""
self.f2py_mngr = f2py_mg.f2py_manager(
os.path.join(fortran_source_path, "mod_compute_catchments.f90"),
func_name="compute_next_grid_cell")
self.i = 52
self.j = 47
self.iasarray = np.array(self.i)
self.jasarray = np.array(self.j)
self.expected_output_coords = [
[
[base + mod for base, mod in zip([self.i, self.j], coordmods)
] #@UndefinedVariable
for coordmods in row
] for row in self.expected_output_coords_changes
] #@UnusedVariable
def renumber_catchments_by_size(catchments, loop_logfile=None):
"""Renumber catchments according to there size in terms of number of cells
Input:
catchments: numpy-like array; catchments to be relabelled catchments
loop_logfile: string; the full path to the existing loop_logfile (optional)
Returns: Relabelled catchements
Label catchments in order of desceding size using a fortran function
to assist a time critical part of the procedure that can't be done in
numpy effectively. Also opens the existing loop logfile and changes the
catchment numbers with loops to reflect the new catchment labelling.
"""
f2py_mngr = f2py_manager.f2py_manager(path.join(
fortran_source_path, "mod_compute_catchments.f90"),
func_name="relabel_catchments")
catch_nums = np.arange(np.amax(catchments) + 1)
counts = np.bincount(catchments.flatten())
catchments_sizes = np.empty(len(catch_nums),
dtype=[('catch_nums', int),
('new_catch_nums', int),
('counts', int)])
catchments_sizes['catch_nums'] = catch_nums
catchments_sizes['counts'] = counts
catchments_sizes.sort(order='counts')
catchments_sizes['new_catch_nums'] = np.arange(
len(catchments_sizes['catch_nums']), 0, -1)
catchments = np.asfortranarray(catchments, np.int32)
old_to_new_label_map = np.asfortranarray(
np.copy(np.sort(catchments_sizes,
order='catch_nums'))['new_catch_nums'], np.int32)
f2py_mngr.run_current_function_or_subroutine(catchments,
old_to_new_label_map)
if loop_logfile is not None:
with open(loop_logfile, 'r') as f:
next(f)
loops = [int(line.strip()) for line in f]
#-1 to account for differing array offset between Fortran and python
loops = [
str(old_to_new_label_map[old_loop_num]) + '\n'
for old_loop_num in loops
]
with open(loop_logfile, 'w') as f:
f.write('Loops found in catchments:\n')
f.writelines(loops)
return catchments
def get_number_of_masked_neighbors(self, data):
"""Compute the number of neighbors that are masked of each masked cell"""
#insert extra rows across the top and bottom of the grid to ensure correct
#treatmen of boundaries
data = np.insert(data,
obj=(0, np.size(data, axis=0)),
values=False,
axis=0)
f2py_mngr = f2py_mg.f2py_manager(path.join(
fortran_source_path, 'mod_grid_m_nbrs_kernels.f90'),
func_name='HDgrid_masked_nbrs_kernel')
nbrs_kernel = f2py_mngr.run_current_function_or_subroutine
nbrs_num = ndi.generic_filter(data.astype(np.float64),
nbrs_kernel,
size=(3, 3),
mode='wrap')
return nbrs_num[1:-1].astype(int)
def compute_flow_directions(self, data, use_fortran_kernel=True):
"""Compute the flow direction for a given orograpy
Arguments:
data: numpy array-like object; the orography to compute the flow direction for
use_fortran_kernel: boolean: a flag to choose between using a python kernel (False)
or using a Fortran kernel (True)
Returns:
A numpy array of flow directions as integers
Inserts extra rows to ensure there is no cross pole flow; then flip the field to get the same first
minima as Stefan uses for easy comparison. Setup the kernel function to use depending on the input
flag. Then run a generic filter from the numpy image processing module that considers the 3 by 3
area around each grid cell; wrapping around the globe in the east-west direction. Return the output
as an integer after flipping it back the right way up and removing the extra rows added at the top
and bottom
"""
#insert extra rows across the top and bottom of the grid to ensure correct
#treatmen of boundaries
data = np.insert(data,
obj=(0, np.size(data, axis=0)),
values=self.pole_boundary_condition,
axis=0)
#processing the data upside down to ensure that the handling of the case where two neighbours
#have exactly the same height matches that of Stefan's scripts
data = np.flipud(data)
if use_fortran_kernel:
f2py_mngr = f2py_mg.f2py_manager(path.join(
fortran_source_path, 'mod_grid_flow_direction_kernels.f90'),
func_name='HDgrid_fdir_kernel')
flow_direction_kernel = f2py_mngr.run_current_function_or_subroutine
else:
flow_direction_kernel = self.flow_direction_kernel
flow_directions_as_float = ndi.generic_filter(data,
flow_direction_kernel,
size=(3, 3),
mode='wrap')
flow_directions_as_float = np.flipud(flow_directions_as_float)
return flow_directions_as_float[1:-1].astype(int)
def run_flow(input_fine_river_directions, input_fine_total_cumulative_flow,
cotat_parameters_filepath, coarse_grid_type,
**coarse_grid_kwargs):
additional_fortran_filenames = [
"base/area_mod.o", "base/coords_mod.o",
"algorithms/cotat_parameters_mod.o", "algorithms/flow.o",
"base/doubly_linked_list_mod.o", "base/doubly_linked_list_link_mod.o",
"base/field_section_mod.o", "base/precision_mod.o",
"base/subfield_mod.o"
]
additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\
additional_fortran_filenames]
f2py_mngr = f2py_manager.f2py_manager(
path.join(fortran_project_source_path, "drivers",
"flow_driver_mod.f90"),
func_name="flow_latlon_f2py_wrapper",
additional_fortran_files=additional_fortran_filepaths,
include_path=fortran_project_include_path)
output_coarse_river_directions_latlon_indices = f2py_mngr.\
run_current_function_or_subroutine(),
def write_field(self, filename, field,griddescfile=None,fieldname=None):
"""Write a field to an unformatted fortran file using f2py
Arguments:
filename: string; full path of the file to write to
field: A Field (or Field subclass object); field to write
griddescfile is ignored by this function but included to match the abstract
function it is overriding
fieldname is not used by this method and included only in order to match the
abstract of method of IOFileHelper
Writes a field using a specifically written fortran routine through f2py controlled
by a f2py_manager instance. The manipulation applied to the field in the load field
method is reversed for consistency.
"""
print("Writing output to {0}".format(filename))
mgnr = f2py_mg.f2py_manager(path.join(fortran_source_path,
"mod_topo_io.f90"), func_name="write_topo")
#reverse the manipulation in the load_field method
data = np.rot90(np.fliplr(field.get_data()))
mgnr.run_current_function_or_subroutine(filename,data)
def create_hypothetical_river_paths_map(riv_dirs,
lsmask=None,
use_f2py_func=True,
use_f2py_sparse_iterator=False,
nlat=360,
nlong=720,
sparse_fraction=0.5,
use_new_method=False):
"""Create map of cumulative flow to cell from a field of river directions
Inputs:
riv_dirs: numpy array; an array of river flow directions
lsmask(optional): numpy array; a landsea mask (with sea points masked)
use_f2py_func (optional): boolean; Whether to use an iterator written in
Fortran (True) or in python (False)
use_f2py_sparse_iterator (optional): boolean; Whether to use a sparse iterator
(that works on a vector containing the remaining points to process
rather than the whole river direction array) written in Fortran.
Independent of use_f2py_func.
nlat: integer; number of latitude points in the input arrays
nlong:integer; number of longitude points in the input arrays
sparse_fraction (optional): float; threshold (as a fraction of the
points in the array remaining to be processed) for switching
to the sparse iterator (if that option is selected)
Returns: Generated flow to cell path map as a numpy array
Perform a certain amount of preperation of the input arrays (adding a
one cell border across the top and bottom of the edge of the river
direction array and the land sea mask and also applying the landsea
mask if that option is selected). Then select an iterator and iterate
over the initially empty flow map till it is completely filled; switching
to the the sparse iterator when the given threshold is reached if that
option is selected.
"""
riv_dirs = np.insert(riv_dirs, obj=0, values=np.zeros(nlong), axis=0)
#nlat+1 because the array is now already nlat+1 elements wide so you want to place
#the new row after the last row
riv_dirs = np.insert(riv_dirs,
obj=nlat + 1,
values=np.zeros(nlong),
axis=0)
if lsmask is not None:
lsmask = np.insert(lsmask,
obj=0,
values=np.ones(nlong, dtype=bool),
axis=0)
#nlat+1 because the array is now already nlat+1 elements wide so you want to place
#the new row after the last row
lsmask = np.insert(lsmask,
obj=nlat + 1,
values=np.ones(nlong, dtype=bool),
axis=0)
riv_dirs = np.ma.array(riv_dirs, mask=lsmask, copy=True,
dtype=int).filled(0)
else:
riv_dirs = np.array(riv_dirs, copy=True, dtype=int)
paths_map = np.zeros((nlat + 2, nlong), dtype=np.int32, order='F')
if use_f2py_func and use_new_method:
additional_fortran_filenames = [
"algorithms/accumulate_flow_mod.o", "base/coords_mod.o",
"algorithms/flow_accumulation_algorithm_mod.o",
"base/convert_rdirs_to_indices.o", "base/doubly_linked_list_mod.o",
"base/doubly_linked_list_link_mod.o", "base/subfield_mod.o",
"base/unstructured_grid_mod.o", "base/precision_mod.o"
]
additional_fortran_filepaths = [path.join(fortran_project_object_path,filename) for filename in\
additional_fortran_filenames]
f2py_mngr = f2py_mg.f2py_manager(
path.join(fortran_project_source_path, "drivers",
"accumulate_flow_driver_mod.f90"),
func_name="accumulate_flow_latlon_f2py_wrapper",
additional_fortran_files=additional_fortran_filepaths,
include_path=fortran_project_include_path)
paths_map = f2py_mngr.\
run_current_function_or_subroutine(np.asfortranarray(riv_dirs),
*riv_dirs.shape)
#Make a minor postprocessing correction
paths_map[np.logical_and(np.logical_or(riv_dirs == 5, riv_dirs == 0),
paths_map == 0)] = 1
else:
if use_f2py_func:
f2py_kernel = f2py_mg.f2py_manager(path.join(
fortran_source_path, 'mod_iterate_paths_map.f90'),
func_name='iterate_paths_map')
iterate_paths_map_function = f2py_kernel.run_current_function_or_subroutine
else:
iterate_paths_map_function = iterate_paths_map
while iterate_paths_map_function(riv_dirs, paths_map, nlat, nlong):
remaining_points = paths_map.size - np.count_nonzero(paths_map)
if use_f2py_sparse_iterator and remaining_points / float(
paths_map.size) < sparse_fraction:
f2py_sparse_iterator = f2py_mg.f2py_manager(
path.join(fortran_source_path,
'mod_iterate_paths_map.f90'),
func_name='sparse_iterator')
f2py_sparse_iterator.run_current_function_or_subroutine(
riv_dirs, paths_map, nlat, nlong)
break
return paths_map[1:-1, :]
def setUp(self):
"""Unit test setup function"""
self.f2py_mngr = f2py_manager.f2py_manager(os.path.join(fortran_source_path,'mod_iterate_paths_map.f90'))
def setUp(self):
"""Unit test setUp function"""
self.f2py_mngr = f2py_mg.f2py_manager(os.path.join(
fortran_source_path, "mod_compute_catchments.f90"),
func_name="compute_catchments")
