def testFieldWritingAndLoadingWithLatLongFloatingPointField(self):
example_field = field.makeEmptyField('Generic',
dtype=np.float64,
grid_type='HD')
lat_points = np.linspace(89.75, -89.75, 360, endpoint=True)
lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True)
example_field.set_grid_coordinates((lat_points, lon_points))
example_field.data[20, 20] = 1.5
example_field.data[200, 20] = 2.5
example_field.data[20, 200] = 3.5
example_field.data[200, 200] = 4.5
iodriver.advanced_field_writer(os.path.join(
self.directory, "advancedfieldwritingandloadingtest.nc"),
example_field,
fieldname='test_field',
clobber=True)
loaded_field = iodriver.advanced_field_loader(os.path.join(
self.directory, "advancedfieldwritingandloadingtest.nc"),
fieldname='test_field')
np.testing.assert_array_equal(example_field.get_data(),
loaded_field.get_data())
self.assertEqual(loaded_field.data[20, 20], 1.5)
self.assertEqual(loaded_field.data[200, 20], 2.5)
self.assertEqual(loaded_field.data[20, 200], 3.5)
self.assertEqual(loaded_field.data[200, 200], 4.5)
def testFieldWritingAndLoadingWithLatLongNoClobber(self):
example_field = field.makeEmptyField('Generic',
dtype=np.float64,
grid_type='HD')
lat_points = np.linspace(89.75, -89.75, 360, endpoint=True)
lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True)
example_field.set_grid_coordinates((lat_points, lon_points))
example_field.data[20, 20] = 1.5
example_field.data[200, 20] = 2.5
example_field.data[20, 200] = 3.5
example_field.data[200, 200] = 4.5
iodriver.advanced_field_writer(os.path.join(
self.directory, "advancedfieldwritingandloadingtest.nc"),
example_field,
fieldname='test_field',
clobber=True)
with self.assertRaisesRegex(
RuntimeError,
r"Target file /Users/thomasriddick/Documents/data/temp/advancedfieldwritingandloadingtest.nc already exists and clobbering is not set"
):
iodriver.advanced_field_writer(os.path.join(
self.directory, "advancedfieldwritingandloadingtest.nc"),
example_field,
fieldname='test_field',
clobber=False)
def extract_lake_volumes(flood_volume_thresholds,
basin_catchment_numbers,merge_types):
lake_mask = basin_catchment_numbers.greater_than_value(0)
lake_volumes = field.makeEmptyField("Generic",np.float64,
grid_type=lake_mask.get_grid())
lake_mask_reduced = lake_mask.copy()
lake_mask_reduced.change_dtype(np.int32)
lake_operators_wrapper.reduce_connected_areas_to_points(lake_mask_reduced.get_data(),True)
lake_mask_inv = lake_mask.copy()
lake_mask_inv.invert_data()
orography = field.makeEmptyField("Generic",np.float64,
grid_type=lake_mask.get_grid())
orography.get_data()[lake_mask_inv.get_data()] = 3.0
orography.get_data()[lake_mask.get_data()] = 1.0
orography.get_data()[lake_mask_reduced.get_data().astype(bool)] = 0.0
lake_mask_inv.change_dtype(np.int32)
rdirs = determine_river_directions.determine_river_directions(orography,
lake_mask_reduced,
truesinks=None,
always_flow_to_sea=True,
use_diagonal_nbrs=True,
mark_pits_as_true_sinks=False)
rdirs.get_data()[lake_mask_inv.get_data()] = -1
simplified_basin_catchment_numbers = None
temporary_file = tempfile.NamedTemporaryFile(delete = False)
try:
temporary_file.close()
dummy_loop_log_filepath = temporary_file.name
simplified_basin_catchment_numbers = \
field.Field(cc.compute_catchments_cpp(rdirs.get_data(),loop_logfile=dummy_loop_log_filepath),
grid=lake_mask.get_grid())
finally:
os.remove(temporary_file.name)
for i in range(1,simplified_basin_catchment_numbers.find_maximum()+1):
if i in simplified_basin_catchment_numbers.get_data():
single_lake_mask = lake_mask.copy()
single_lake_mask.\
get_data()[simplified_basin_catchment_numbers.get_data() != i] = False
single_lake_flood_volume_thresholds = flood_volume_thresholds.copy()
single_lake_flood_volume_thresholds.\
get_data()[np.logical_not(single_lake_mask.get_data())] = 0.0
single_lake_flood_volume_thresholds.\
get_data()[np.logical_and(merge_types.get_data() != 10,
merge_types.get_data() != 11)] = 0.0
lake_volumes.get_data()[single_lake_mask.get_data()] = np.sum(single_lake_flood_volume_thresholds.get_data())
return lake_volumes
def determine_river_directions(orography,lsmask,truesinks=None,
always_flow_to_sea=True,
use_diagonal_nbrs=True,
mark_pits_as_true_sinks=True):
rdirs = field.makeEmptyField('RiverDirections',np.float64,orography.get_grid())
if truesinks is None:
truesinks = field.makeEmptyField('Generic',np.float64,orography.get_grid())
determine_river_directions_wrapper.determine_river_directions(rdirs.get_data(),
orography.get_data(),
lsmask.get_data().astype(dtype=np.int32,
order='C',
copy=False),
truesinks.get_data().astype(dtype=np.int32,
order='C',
copy=False),
always_flow_to_sea,
use_diagonal_nbrs,
mark_pits_as_true_sinks)
return rdirs
def advanced_connected_lsmask_creation_driver(
input_lsmask_filename,
output_lsmask_filename,
input_lsmask_fieldname,
output_lsmask_fieldname,
input_ls_seed_points_filename=None,
input_ls_seed_points_fieldname=None,
input_ls_seed_points_list_filename=None,
use_diagonals_in=True,
rotate_seeds_about_polar_axis=False,
flip_seeds_ud=False):
lsmask = iodriver.advanced_field_loader(input_lsmask_filename,
field_type='Generic',
fieldname=input_lsmask_fieldname)
if input_ls_seed_points_filename:
input_ls_seedpts = iodriver.advanced_field_loader(input_ls_seed_points_filename,
field_type='Generic',
fieldname=\
input_ls_seed_points_fieldname,
adjust_orientation=False)
else:
input_ls_seedpts = field.makeEmptyField('Generic', np.int32)
if input_ls_seed_points_list_filename:
points_list = []
print("Reading input from {0}".format(
input_ls_seed_points_list_filename))
comment_line_pattern = re.compile(r"^ *#.*$")
with open(input_ls_seed_points_list_filename) as f:
if f.readline().strip() != 'LatLong':
raise RuntimeError(
"List of landsea points being loaded is not for correct grid-type"
)
for line in f:
if comment_line_pattern.match(line):
continue
points_list.append(
tuple(int(coord) for coord in line.strip().split(",")))
input_ls_seedpts.flag_listed_points(points_list)
if rotate_seeds_about_polar_axis:
input_ls_seedpts.rotate_field_by_a_hundred_and_eighty_degrees()
if flip_seeds_ud:
input_ls_seedpts.flip_data_ud()
if input_ls_seedpts.get_grid().has_orientation_information():
input_ls_seedpts.orient_data()
lsmask.change_dtype(np.int32)
cc_lsmask_wrapper.create_connected_ls_mask(
lsmask.get_data(),
input_ls_seedpts.get_data().astype(dtype=np.int32,
order='C',
copy=False), use_diagonals_in)
iodriver.advanced_field_writer(output_lsmask_filename,
lsmask,
fieldname=output_lsmask_fieldname)
def find_ocean_basin_catchments(rdirs, catchments, areas=[]):
ocean_catchments = catchments.copy()
ocean_basin_catchments = field.makeEmptyField(
"Generic", grid_type=catchments.get_grid(), dtype=np.int64)
ocean_catchments.get_data()[np.logical_not(
np.logical_and(rdirs.get_data() == 0,
catchments.get_data() != 0))] = 0
for i, area in enumerate(areas, 1):
ocean_catchments_area = ocean_catchments.get_data()[
area['min_lat']:area['max_lat'] + 1,
area['min_lon']:area['max_lon'] + 1]
ocean_catchment_list = np.unique(ocean_catchments_area)
for catchment in ocean_catchment_list:
if catchment == 0:
continue
ocean_basin_catchments.get_data()[catchments.get_data() ==
catchment] = i
return ocean_basin_catchments
def advanced_orography_creation_driver(landsea_mask_filename,
inclines_filename, orography_filename,
landsea_mask_fieldname,
inclines_fieldname,
orography_fieldname):
landsea_in = iodriver.advanced_field_loader(
landsea_mask_filename,
field_type="Generic",
fieldname=landsea_mask_fieldname,
adjust_orientation=True)
inclines_in = iodriver.advanced_field_loader(inclines_filename,
field_type="Generic",
fieldname=inclines_fieldname,
adjust_orientation=True)
orography_in = field.makeEmptyField('Generic', np.float64,
landsea_in.get_grid())
create_orography_wrapper.create_orography(
np.ascontiguousarray(landsea_in.get_data(), dtype=np.int32),
np.ascontiguousarray(inclines_in.get_data(), dtype=np.float64),
np.ascontiguousarray(orography_in.get_data(), dtype=np.float64))
iodriver.advanced_field_writer(orography_filename,
orography_in,
fieldname=orography_fieldname)
def testFieldWritingAndLoadingWithLatLongFieldWithFlip(self):
example_field = field.makeEmptyField('Generic',
dtype=np.int64,
grid_type='HD')
lat_points = np.linspace(89.75, -89.75, 360, endpoint=True)
lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True)
example_field.set_grid_coordinates((lat_points, lon_points))
example_field.data[20, 20] = 1
example_field.data[200, 20] = 2
example_field.data[20, 200] = 3
example_field.data[200, 200] = 4
example_field.flip_data_ud()
iodriver.advanced_field_writer(os.path.join(
self.directory, "advancedfieldwritingandloadingtest.nc"),
example_field,
fieldname='test_field',
clobber=True)
loaded_field = iodriver.advanced_field_loader(os.path.join(
self.directory, "advancedfieldwritingandloadingtest.nc"),
fieldname='test_field',
adjust_orientation=False)
np.testing.assert_array_equal(example_field.get_data(),
loaded_field.get_data())
def no_intermediaries_combine_hydrosheds_plus_rdirs_from_corrected_orog_driver(
self):
"""Combines Hydrosheds river direction with those from a corrected orography and possibly removes sinks"""
config = self._read_and_validate_config()
final_loops_log_filename = path.splitext(
self.output_catchments_filename)[0] + '_loops.log'
if self.rerun_post_processing_only is not None:
final_rdirs = \
iodriver.advanced_field_loader(self.rerun_post_processing_only,
field_type="RiverDirections",
fieldname=config.get("input_fieldname_options",
"river_directions_to_reprocess_fieldname"))
else:
orography_filename = config.get(
"input_options", "ten_minute_corrected_orography_filename")
orography = iodriver.advanced_field_loader(
orography_filename,
field_type='Orography',
fieldname=config.get(
"input_fieldname_options",
"ten_minute_corrected_orography_fieldname"))
hydrosheds_rdirs_au_af_sa_10min_filename = \
config.get("input_options","ten_minute_hydrosheds_au_auf_sa_river_directions_filename")
hydrosheds_rdirs = \
iodriver.advanced_field_loader(hydrosheds_rdirs_au_af_sa_10min_filename,
field_type="RiverDirections",
fieldname=config.get("input_fieldname_options",
"ten_minute_hydrosheds_au_auf_sa_river_directions_fieldname"))
second_intermediate_loops_log_filename = tempfile.mkstemp(
suffix=".txt", prefix="loops_log_second_int", dir="")[1]
truesinks_dummy = field.makeEmptyField("Generic",
np.bool_,
grid_type="LatLong10min")
truesinks_dummy.set_all(False)
if self.true_sinks_filename is not None:
use_true_sinks = True
truesinks = iodriver.advanced_field_loader(
self.true_sinks_filename,
field_type="Generic",
fieldname=config.get("input_fieldname_options",
"input_truesinks_fieldname"))
else:
use_true_sinks = False
if config.getboolean(
"general_options",
"replace_internal_basins_with_rdirs_with_truesinks"):
warnings.warn(
"Option replace_internal_basins_with_rdirs_with_truesinks "
"ignored when no true sinks file is specified")
first_intermediate_rdirs = field.makeEmptyField(
"RiverDirections", np.float64, grid_type="LatLong10min")
if use_true_sinks:
first_intermediate_rdirs_no_sinks = field.makeEmptyField(
"RiverDirections", np.float64, grid_type="LatLong10min")
ls_mask = iodriver.advanced_field_loader(
self.ls_mask_filename,
field_type="Generic",
fieldname=config.get("input_fieldname_options",
"input_landsea_mask_fieldname"))
ls_mask.set_data(
np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32))
next_cell_lat_index_in_dummy = np.zeros(ls_mask.get_data().shape,
dtype=np.int32,
order='C')
next_cell_lon_index_in_dummy = np.zeros(ls_mask.get_data().shape,
dtype=np.int32,
order='C')
catchment_nums_dummy = np.zeros(ls_mask.get_data().shape,
dtype=np.int32,
order='C')
fill_sinks_wrapper.fill_sinks_cpp_func(orography_array=
np.ascontiguousarray(orography.get_data(), #@UndefinedVariable
dtype=np.float64),
method = 4,
use_ls_mask = True,
landsea_in = np.ascontiguousarray(ls_mask.get_data(),
dtype=np.int32),
set_ls_as_no_data_flag = False,
use_true_sinks = False,
true_sinks_in = np.ascontiguousarray(truesinks_dummy.\
get_data(),
dtype=np.int32),
next_cell_lat_index_in = next_cell_lat_index_in_dummy,
next_cell_lon_index_in = next_cell_lon_index_in_dummy,
rdirs_in =
first_intermediate_rdirs.get_data() if not use_true_sinks else
first_intermediate_rdirs_no_sinks.get_data(),
catchment_nums_in = catchment_nums_dummy,
prefer_non_diagonal_initial_dirs = False)
if use_true_sinks:
fill_sinks_wrapper.fill_sinks_cpp_func(
orography_array=np.ascontiguousarray(
orography.get_data(), #@UndefinedVariable
dtype=np.float64),
method=4,
use_ls_mask=True,
landsea_in=np.ascontiguousarray(ls_mask.get_data(),
dtype=np.int32),
set_ls_as_no_data_flag=False,
use_true_sinks=True,
true_sinks_in=np.ascontiguousarray(truesinks.get_data(),
dtype=np.int32),
next_cell_lat_index_in=next_cell_lat_index_in_dummy,
next_cell_lon_index_in=next_cell_lon_index_in_dummy,
rdirs_in=first_intermediate_rdirs.get_data(),
catchment_nums_in=catchment_nums_dummy,
prefer_non_diagonal_initial_dirs=False)
second_intermediate_rdirs = utilities.splice_rdirs(
rdirs_matching_ls_mask=first_intermediate_rdirs,
ls_mask=ls_mask,
other_rdirs=hydrosheds_rdirs)
second_intermediate_catchments = comp_catchs.compute_catchments_cpp(
second_intermediate_rdirs.get_data(),
second_intermediate_loops_log_filename)
second_intermediate_catchments = field.Field(comp_catchs.\
renumber_catchments_by_size(second_intermediate_catchments,
second_intermediate_loops_log_filename),
grid="LatLong10min")
if config.getboolean("general_options",
"keep_all_internal_basins"):
third_intermediate_rdirs = second_intermediate_rdirs
else:
third_intermediate_rdirs = \
utilities.remove_endorheic_basins(rdirs=second_intermediate_rdirs,
catchments=second_intermediate_catchments,
rdirs_without_endorheic_basins=
first_intermediate_rdirs_no_sinks if
(use_true_sinks and not config.getboolean('general_options',
'replace_internal_basins_with_rdirs_with_truesinks'))
else first_intermediate_rdirs,
replace_only_catchments=([int(value) for value in
config.get('general_options',
'replace_only_catchments').\
split(",")]
if config.get('general_options',
'replace_only_catchments')
else []),
exclude_catchments=([int(value) for value in
config.get('general_options',
'exclude_catchments').\
split(",")]
if config.get('general_options',
'exclude_catchments')
else []) )
third_intermediate_flowtocell = field.\
CumulativeFlow(create_hypothetical_river_paths_map(riv_dirs=third_intermediate_rdirs.get_data(),
lsmask=None,
use_f2py_func=True,
use_f2py_sparse_iterator=True,
nlat=1080,nlong=2160),
grid='LatLong10min')
third_intermediate_rdirs.make_contiguous()
third_intermediate_flowtocell.make_contiguous()
first_intermediate_rdirs.make_contiguous()
if use_true_sinks:
first_intermediate_rdirs_no_sinks.make_contiguous()
final_rdirs = utilities.replace_streams_downstream_from_loop(
third_intermediate_rdirs,
cumulative_flow=third_intermediate_flowtocell,
other_rdirs=first_intermediate_rdirs
if not use_true_sinks else first_intermediate_rdirs_no_sinks)
final_catchments = comp_catchs.compute_catchments_cpp(
final_rdirs.get_data(), final_loops_log_filename)
final_catchments = field.Field(comp_catchs.\
renumber_catchments_by_size(final_catchments,
final_loops_log_filename),
grid="LatLong10min")
final_flowtocell = field.CumulativeFlow(
create_hypothetical_river_paths_map(
riv_dirs=final_rdirs.get_data(),
lsmask=None,
use_f2py_func=True,
use_f2py_sparse_iterator=True,
nlat=1080,
nlong=2160),
grid='LatLong10min')
if self.rerun_post_processing_only is None:
iodriver.advanced_field_writer(
self.output_rdirs_filename,
final_rdirs,
fieldname=config.get("output_fieldname_options",
"output_river_directions_fieldname"))
iodriver.advanced_field_writer(self.output_catchments_filename,
final_catchments,
fieldname=config.get(
"output_fieldname_options",
"output_catchments_fieldname"))
iodriver.advanced_field_writer(self.output_cumulative_flow_filename,
final_flowtocell,
fieldname=config.get(
"output_fieldname_options",
"output_cumulative_flow_fieldname"))
if self.rerun_post_processing_only is None:
os.remove(second_intermediate_loops_log_filename)
def advanced_sinkless_flow_directions_generator(filename,output_filename,fieldname,
output_fieldname,ls_mask_filename=None,
truesinks_filename=None,
catchment_nums_filename=None,
ls_mask_fieldname=None,
truesinks_fieldname=None,
catchment_fieldname=None):
orography = iodriver.advanced_field_loader(filename,
field_type='Orography',
fieldname=fieldname)
grid_dims=orography.get_grid().get_grid_dimensions()
grid_num_points = orography.get_grid().get_npoints()
threshold_for_low_mem_version_usage = 500000000
if (grid_num_points < threshold_for_low_mem_version_usage):
rdirs = np.empty(grid_dims,dtype=np.float64,order='C')
if not truesinks_filename:
truesinks = Field(np.empty((1,1),dtype=np.int32),grid='HD')
use_true_sinks = False;
else:
use_true_sinks = True;
truesinks = iodriver.advanced_field_loader(truesinks_filename,
field_type='Generic',
fieldname=truesinks_fieldname)
if ls_mask_filename is None:
use_ls_mask = False
ls_mask = field.makeEmptyField(field_type='Generic',dtype=np.int32,grid_type='HD')
else:
use_ls_mask = True
ls_mask = iodriver.advanced_field_loader(ls_mask_filename,
field_type='Generic',
fieldname=ls_mask_fieldname)
catchment_nums = np.zeros(grid_dims,dtype=np.int32,order='C')
next_cell_lat_index_in = np.zeros(grid_dims,dtype=np.int32,order='C')
next_cell_lon_index_in = np.zeros(grid_dims,dtype=np.int32,order='C')
if (grid_num_points >= threshold_for_low_mem_version_usage):
rdirs = fill_sinks_wrapper_low_mem.fill_sinks_cpp_func(orography_array=np.ascontiguousarray(orography.get_data(), #@UndefinedVariable
dtype=np.float64),
method = 4,
use_ls_mask = use_ls_mask,
landsea_in = np.ascontiguousarray(ls_mask.get_data(),
dtype=np.int32),
set_ls_as_no_data_flag = False,
use_true_sinks = use_true_sinks,
true_sinks_in = np.ascontiguousarray(truesinks.get_data(),
dtype=np.int32),
next_cell_lat_index_in = next_cell_lat_index_in,
next_cell_lon_index_in = next_cell_lon_index_in,
catchment_nums_in = catchment_nums,
prefer_non_diagonal_initial_dirs = False)
else:
fill_sinks_wrapper.fill_sinks_cpp_func(orography_array=np.ascontiguousarray(orography.get_data(), #@UndefinedVariable
dtype=np.float64),
method = 4,
use_ls_mask = use_ls_mask,
landsea_in = np.ascontiguousarray(ls_mask.get_data(),
dtype=np.int32),
set_ls_as_no_data_flag = False,
use_true_sinks = use_true_sinks,
true_sinks_in = np.ascontiguousarray(truesinks.get_data(),
dtype=np.int32),
next_cell_lat_index_in = next_cell_lat_index_in,
next_cell_lon_index_in = next_cell_lon_index_in,
rdirs_in = rdirs,
catchment_nums_in = catchment_nums,
prefer_non_diagonal_initial_dirs = False)
iodriver.advanced_field_writer(output_filename,Field(rdirs,grid=orography.get_grid()),
fieldname=output_fieldname)
if catchment_nums_filename:
iodriver.advanced_field_writer(catchment_nums_filename,
field=Field(catchment_nums,grid=orography.get_grid()),
fieldname=catchment_fieldname)
def drive_connected_lsmask_creation(input_lsmask_filename,
output_lsmask_filename,
input_ls_seed_points_filename=None,
input_ls_seed_points_list_filename=None,
use_diagonals_in=True,
rotate_seeds_about_polar_axis=False,
flip_seeds_ud=False,
flip_input_mask_ud=False,
grid_type='HD',
**grid_kwargs):
"""Drives the creation of a fully connected land-sea mask
Argument:
input_lsmask_filename: string; the full path of the input land-sea mask which
may contain disconnected sea points
output_lsmask_filename: string; the full path of the output land-sea mask without
any disconnected sea point
input_ls_seed_points_filename(optional): string; the full path of the file with an array with
at least one point in the center of each ocean/sea to be included
input_ls_seed_points_list_filename(optional): string; the full path of the file with a
list of at least one point in the center of each ocean/sea to be included - these
will be added to the contents of input_ls_seed_points_filename, normally only one of the
two arguments would be used to initialise the seed points
use_diagonals_in: boolean; if true count a diagonal connection as being connected
flip_input_mask_ud: boolean; if true flip the input mask up down before processing
rotate_seeds_about_polar_axis: if true then rotate the seeds about the polar axis by 180
degrees, i.e. shift the position of the zero longitude by 180 degrees when reading the
seed points
flip_seeds_upside_down: boolean; if true then flip the seeds upside down (i.e. about
the equator)
grid_type: string; the code for this grid type
**grid_kwargs: dictionary; key word arguments specifying parameters of the grid
Returns: Nothing
Creates a land-sea masked based on the supplied land-sea mask with any sea points not connected
to the supplied ocean seeds points (by other sea points) change to land points
"""
lsmask = iodriver.load_field(input_lsmask_filename,
file_type=iodriver.\
get_file_extension(input_lsmask_filename),
field_type='Generic',
grid_type=grid_type,**grid_kwargs)
if flip_input_mask_ud:
lsmask.flip_data_ud()
if input_ls_seed_points_filename:
input_ls_seedpts = iodriver.load_field(input_ls_seed_points_filename,
file_type=iodriver.\
get_file_extension(input_ls_seed_points_filename),
field_type='Generic',
grid_type=grid_type,**grid_kwargs)
else:
input_ls_seedpts = field.makeEmptyField('Generic', np.int32, grid_type,
**grid_kwargs)
if input_ls_seed_points_list_filename:
points_list = []
print("Reading input from {0}".format(
input_ls_seed_points_list_filename))
comment_line_pattern = re.compile(r"^ *#.*$")
with open(input_ls_seed_points_list_filename) as f:
if f.readline().strip() != grid_type:
raise RuntimeError(
"List of landsea points being loaded is not for correct grid-type"
)
for line in f:
if comment_line_pattern.match(line):
continue
points_list.append(
tuple(int(coord) for coord in line.strip().split(",")))
input_ls_seedpts.flag_listed_points(points_list)
if rotate_seeds_about_polar_axis:
input_ls_seedpts.rotate_field_by_a_hundred_and_eighty_degrees()
if flip_seeds_ud:
input_ls_seedpts.flip_data_ud()
lsmask.change_dtype(np.int32)
cc_lsmask_wrapper.create_connected_ls_mask(
lsmask.get_data(),
input_ls_seedpts.get_data().astype(dtype=np.int32,
order='C',
copy=False), use_diagonals_in)
iodriver.write_field(output_lsmask_filename,lsmask,
file_type=iodriver.\
get_file_extension(output_lsmask_filename))
def connect_coarse_lake_catchments(coarse_catchments,lake_centers,basin_catchment_numbers,
flood_next_cell_index_lat,flood_next_cell_index_lon,
flood_redirect_lat,flood_redirect_lon,
additional_flood_redirect_lat,additional_flood_redirect_lon,
local_redirect,additional_local_redirect,
merge_types,river_directions,scale_factor = 3,
correct_cumulative_flow=False,
cumulative_flow=None):
if correct_cumulative_flow:
if cumulative_flow is None or river_directions is None:
raise RuntimeError("Required input files for cumulative flow correction not provided")
old_coarse_catchments = coarse_catchments.copy()
lake_centers_array = np.argwhere(lake_centers.get_data())
lake_centers_list = [lake_centers_array[i,:].tolist()
for i in range(lake_centers_array.shape[0])]
overflow_catchments = field.makeEmptyField(field_type='Generic',dtype=np.int64,
grid_type=lake_centers.get_grid())
overflow_coords_lats = field.makeEmptyField(field_type='Generic',dtype=np.int64,
grid_type=lake_centers.get_grid())
overflow_coords_lons = field.makeEmptyField(field_type='Generic',dtype=np.int64,
grid_type=lake_centers.get_grid())
for lake_center_coords in lake_centers_list:
basin_number = basin_catchment_numbers.get_data()[tuple(lake_center_coords)]
while True:
secondary_merge_coords = np.argwhere(
np.logical_and(
np.logical_or(merge_types.get_data() == 10,
merge_types.get_data() == 11),
basin_catchment_numbers.get_data() == basin_number))[0,:].tolist()
double_merge = np.any(np.logical_and(merge_types.get_data() == 11,
basin_catchment_numbers.get_data() == basin_number))
basin_number = basin_catchment_numbers.get_data()[flood_next_cell_index_lat.get_data()[tuple(secondary_merge_coords)],
flood_next_cell_index_lon.get_data()[tuple(secondary_merge_coords)]]
if basin_number == 0:
is_local_redirect = ((local_redirect.get_data()[tuple(secondary_merge_coords)]) if not double_merge else
(additional_local_redirect.get_data()[tuple(secondary_merge_coords)]))
if is_local_redirect:
if double_merge:
basin_number = \
basin_catchment_numbers.\
get_data()[additional_flood_redirect_lat.get_data()[tuple(secondary_merge_coords)],
additional_flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]]
else:
basin_number = \
basin_catchment_numbers.\
get_data()[flood_redirect_lat.get_data()[tuple(secondary_merge_coords)],
flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]]
else:
if double_merge:
overflow_catchment = \
coarse_catchments.get_data()[additional_flood_redirect_lat.get_data()[tuple(secondary_merge_coords)],
additional_flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]]
overflow_coords = (additional_flood_redirect_lat.get_data()[tuple(secondary_merge_coords)],
additional_flood_redirect_lon.get_data()[tuple(secondary_merge_coords)])
else:
overflow_catchment = \
coarse_catchments.get_data()[flood_redirect_lat.get_data()[tuple(secondary_merge_coords)],
flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]]
overflow_coords = (flood_redirect_lat.get_data()[tuple(secondary_merge_coords)],
flood_redirect_lon.get_data()[tuple(secondary_merge_coords)])
break
overflow_catchments.get_data()[tuple(lake_center_coords)] = overflow_catchment
overflow_coords_lats.get_data()[tuple(lake_center_coords)] = overflow_coords[0]
overflow_coords_lons.get_data()[tuple(lake_center_coords)] = overflow_coords[1]
#specific to latlon grid
sink_points_array = np.argwhere(np.logical_or(river_directions.get_data() == 5,
river_directions.get_data() == -2))
sink_points_list = [sink_points_array[i,:].tolist()
for i in range(sink_points_array.shape[0])]
catchment_trees = CatchmentTrees()
sink_point_cumulative_flow_redirect_lat = field.makeEmptyField(field_type='Generic',dtype=np.int64,
grid_type=coarse_catchments.get_grid())
sink_point_cumulative_flow_redirect_lon = field.makeEmptyField(field_type='Generic',dtype=np.int64,
grid_type=coarse_catchments.get_grid())
for sink_point in sink_points_list:
sink_point_coarse_catchment = coarse_catchments.get_data()[tuple(sink_point)]
#Specific to lat-lon grids
overflow_catch_fine_cells_in_coarse_cell = overflow_catchments.get_data()[sink_point[0]*scale_factor:(sink_point[0]+1)*scale_factor,
sink_point[1]*scale_factor:(sink_point[1]+1)*scale_factor]
overflow_coords_lat_fine_cells_in_coarse_cell = overflow_coords_lats.get_data()[sink_point[0]*scale_factor:(sink_point[0]+1)*scale_factor,
sink_point[1]*scale_factor:(sink_point[1]+1)*scale_factor]
overflow_coords_lon_fine_cells_in_coarse_cell = overflow_coords_lons.get_data()[sink_point[0]*scale_factor:(sink_point[0]+1)*scale_factor,
sink_point[1]*scale_factor:(sink_point[1]+1)*scale_factor]
overflow_catchment_list = overflow_catch_fine_cells_in_coarse_cell[overflow_catch_fine_cells_in_coarse_cell != 0].tolist()
if not overflow_catchment_list:
continue
overflow_catchment_counters = Counter(overflow_catchment_list)
highest_count = max(overflow_catchment_counters.values())
overflow_catchment = [ value for value,count in overflow_catchment_counters.items() if count == highest_count][0]
if sink_point_coarse_catchment == overflow_catchment:
continue
overflow_catchment_fine_coords_within_coarse_cell = \
tuple(np.argwhere(overflow_catch_fine_cells_in_coarse_cell == overflow_catchment)[0,:].tolist())
sink_point_cumulative_flow_redirect_lat.get_data()[tuple(sink_point)] = \
overflow_coords_lat_fine_cells_in_coarse_cell[overflow_catchment_fine_coords_within_coarse_cell]
sink_point_cumulative_flow_redirect_lon.get_data()[tuple(sink_point)] = \
overflow_coords_lon_fine_cells_in_coarse_cell[overflow_catchment_fine_coords_within_coarse_cell]
catchment_trees.add_link(sink_point_coarse_catchment,overflow_catchment)
for supercatchment_number,tree in catchment_trees.primary_catchments.items():
for subcatchments_num in tree.get_all_subcatchment_nums():
coarse_catchments.get_data()[subcatchments_num == coarse_catchments.get_data()] = \
supercatchment_number
if correct_cumulative_flow:
while catchment_trees.all_catchments:
upstream_catchments = catchment_trees.pop_leaves()
for upstream_catchment in upstream_catchments:
if np.any(np.logical_and(np.logical_or(river_directions.get_data() == 5,
river_directions.get_data() == -2),
old_coarse_catchments.get_data() == upstream_catchment)):
upstream_catchment_center = \
tuple(np.argwhere(np.logical_and(np.logical_or(river_directions.get_data() == 5,
river_directions.get_data() == -2),
old_coarse_catchments.get_data() == upstream_catchment))[0,:].tolist())
update_cumulative_flow(upstream_catchment_center,
(sink_point_cumulative_flow_redirect_lat.get_data()[upstream_catchment_center],
sink_point_cumulative_flow_redirect_lon.get_data()[upstream_catchment_center]),
cumulative_flow,river_directions)
return field.Field(cc.renumber_catchments_by_size(coarse_catchments.get_data()),type="Generic",
grid=coarse_catchments.get_grid()),cumulative_flow
return field.Field(cc.renumber_catchments_by_size(coarse_catchments.get_data()),type="Generic",
grid=coarse_catchments.get_grid()),None
def drive_orography_upscaling(input_fine_orography_file,
output_coarse_orography_file,
landsea_file=None,
true_sinks_file=None,
upscaling_parameters_filename=None,
fine_grid_type='LatLong10min',
coarse_grid_type='HD',
input_orography_field_name=None,
flip_landsea=False,
rotate_landsea=False,
flip_true_sinks=False,
rotate_true_sinks=False,
fine_grid_kwargs={},
**coarse_grid_kwargs):
"""Drive the C++ sink filling code base to make a tarasov-like orography upscaling
Arguments:
input_fine_orography_file: string; full path to input fine orography file
output_coarse_orography_file: string; full path of target output coarse orography file
landsea_file: string; full path to input fine landsea mask file (optional)
true_sinks_file: string; full path to input fine true sinks file (optional)
upscaling_parameters_filename: string; full path to the orography upscaling parameter
file (optional)
fine_grid_type: string; code for the fine grid type to be upscaled from (optional)
coarse_grid_type: string; code for the coarse grid type to be upscaled to (optional)
input_orography_field_name: string; name of field in the input orography file (optional)
flip_landsea: bool; flip the input landsea mask upside down
rotate_landsea: bool; rotate the input landsea mask by 180 degrees along the horizontal axis
flip_true_sinks: bool; flip the input true sinks field upside down
rotate_true_sinks: bool; rotate the input true sinks field by 180 degrees along the
horizontal axis
fine_grid_kwargs: keyword dictionary; the parameter of the fine grid to upscale
from (if required)
**coarse_grid_kwargs: keyword dictionary; the parameters of the coarse grid to upscale
to (if required)
Returns: Nothing.
"""
if upscaling_parameters_filename:
config = read_and_validate_config(upscaling_parameters_filename)
method = config.getint("orography_upscaling_parameters", "method")
add_slope_in = config.getboolean("orography_upscaling_parameters",
"add_slope_in")
epsilon_in = config.getfloat("orography_upscaling_parameters",
"epsilon_in")
tarasov_separation_threshold_for_returning_to_same_edge_in =\
config.getint("orography_upscaling_parameters",
"tarasov_separation_threshold_for_returning_to_same_edge_in")
tarasov_min_path_length_in = config.getfloat(
"orography_upscaling_parameters", "tarasov_min_path_length_in")
tarasov_include_corners_in_same_edge_criteria_in = \
config.getboolean("orography_upscaling_parameters",
"tarasov_include_corners_in_same_edge_criteria_in")
else:
#use defaults
method = 1
add_slope_in = False
epsilon_in = 0.1
tarasov_separation_threshold_for_returning_to_same_edge_in = 5
tarasov_min_path_length_in = 2.0
tarasov_include_corners_in_same_edge_criteria_in = False
output_orography = field.makeEmptyField(field_type='Orography',
dtype=np.float64,
grid_type=coarse_grid_type,
**coarse_grid_kwargs)
input_orography = iodriver.load_field(
input_fine_orography_file,
file_type=get_file_extension(input_fine_orography_file),
field_type='Orography',
unmask=True,
fieldname=input_orography_field_name,
grid_type=fine_grid_type,
**fine_grid_kwargs)
if landsea_file:
landsea_mask = iodriver.load_field(
landsea_file,
file_type=get_file_extension(landsea_file),
field_type='Generic',
unmask=True,
grid_type=fine_grid_type,
**fine_grid_kwargs)
if flip_landsea:
landsea_mask.flip_data_ud()
if rotate_landsea:
landsea_mask.rotate_field_by_a_hundred_and_eighty_degrees()
else:
landsea_mask = field.makeEmptyField(field_type='Generic',
dtype=np.int32,
grid_type=fine_grid_type,
**fine_grid_kwargs)
if true_sinks_file:
true_sinks = iodriver.load_field(
true_sinks_file,
file_type=get_file_extension(true_sinks_file),
field_type='Generic',
unmask=True,
grid_type=fine_grid_type,
**fine_grid_kwargs)
if flip_true_sinks:
true_sinks.flip_data_ud()
if rotate_true_sinks:
true_sinks.rotate_field_by_a_hundred_and_eighty_degrees()
else:
true_sinks = field.makeEmptyField(field_type='Generic',
dtype=np.int32,
grid_type=fine_grid_type,
**fine_grid_kwargs)
if not np.issubdtype(input_orography.get_data().dtype, np.float64()):
input_orography.change_dtype(np.float64)
#Make sure old data type array is flushed out of memory immediately
gc.collect()
upscale_orography_wrapper.upscale_orography(orography_in=input_orography.get_data(),
orography_out=output_orography.get_data(),
method=method,landsea_in=landsea_mask.get_data(),
true_sinks_in=true_sinks.get_data(),
add_slope_in=add_slope_in, epsilon_in=epsilon_in,
tarasov_separation_threshold_for_returning_to_same_edge_in=\
tarasov_separation_threshold_for_returning_to_same_edge_in,
tarasov_min_path_length_in=tarasov_min_path_length_in,
tarasov_include_corners_in_same_edge_criteria_in=\
tarasov_include_corners_in_same_edge_criteria_in)
iodriver.write_field(
output_coarse_orography_file,
output_orography,
file_type=get_file_extension(output_coarse_orography_file))
def advanced_drive_orography_upscaling(input_fine_orography_file,
output_coarse_orography_file,
input_orography_fieldname,
output_coarse_orography_fieldname,
landsea_file=None,
true_sinks_file=None,
landsea_fieldname=None,
true_sinks_fieldname=None,
upscaling_parameters_filename=None,
scaling_factor=3):
if upscaling_parameters_filename:
config = read_and_validate_config(upscaling_parameters_filename)
method = config.getint("orography_upscaling_parameters", "method")
add_slope_in = config.getboolean("orography_upscaling_parameters",
"add_slope_in")
epsilon_in = config.getfloat("orography_upscaling_parameters",
"epsilon_in")
tarasov_separation_threshold_for_returning_to_same_edge_in =\
config.getint("orography_upscaling_parameters",
"tarasov_separation_threshold_for_returning_to_same_edge_in")
tarasov_min_path_length_in = config.getfloat(
"orography_upscaling_parameters", "tarasov_min_path_length_in")
tarasov_include_corners_in_same_edge_criteria_in = \
config.getboolean("orography_upscaling_parameters",
"tarasov_include_corners_in_same_edge_criteria_in")
else:
#use defaults
method = 1
add_slope_in = False
epsilon_in = 0.1
tarasov_separation_threshold_for_returning_to_same_edge_in = 5
tarasov_min_path_length_in = 2.0
tarasov_include_corners_in_same_edge_criteria_in = False
input_orography = iodriver.advanced_field_loader(
input_fine_orography_file,
field_type='Orography',
fieldname=input_orography_fieldname)
nlat_fine, nlon_fine = input_orography.get_grid_dimensions()
lat_pts_fine, lon_pts_fine = input_orography.get_grid_coordinates()
nlat_coarse,nlon_coarse,lat_pts_coarse,lon_pts_coarse = \
coordinate_scaling_utilities.generate_coarse_coords(nlat_fine,nlon_fine,
lat_pts_fine,lon_pts_fine,
scaling_factor)
output_orography = field.makeEmptyField(field_type='Orography',
dtype=np.float64,
grid_type='LatLong',
nlat=nlat_coarse,
nlong=nlon_coarse)
output_orography.set_grid_coordinates([lat_pts_coarse, lon_pts_coarse])
if landsea_file:
landsea_mask = iodriver.advanced_field_loader(
landsea_file, field_type='Generic', fieldname=landsea_fieldname)
else:
landsea_mask = field.makeEmptyField(field_type='Generic',
dtype=np.int32,
grid_type='LatLong',
nlat=nlat_fine,
nlong=nlon_fine)
if true_sinks_file:
true_sinks = iodriver.advanced_field_loader(
true_sinks_file,
field_type='Generic',
fieldname=true_sinks_fieldname)
else:
true_sinks = field.makeEmptyField(field_type='Generic',
dtype=np.int32,
grid_type='LatLong',
nlat=nlat_fine,
nlong=nlon_fine)
if not input_orography.get_data().dtype is np.float64():
input_orography.change_dtype(np.float64)
#Make sure old data type array is flushed out of memory immediately
gc.collect()
upscale_orography_wrapper.upscale_orography(orography_in=input_orography.get_data(),
orography_out=output_orography.get_data(),
method=method,landsea_in=landsea_mask.get_data(),
true_sinks_in=true_sinks.get_data(),
add_slope_in=add_slope_in, epsilon_in=epsilon_in,
tarasov_separation_threshold_for_returning_to_same_edge_in=\
tarasov_separation_threshold_for_returning_to_same_edge_in,
tarasov_min_path_length_in=tarasov_min_path_length_in,
tarasov_include_corners_in_same_edge_criteria_in=\
tarasov_include_corners_in_same_edge_criteria_in)
iodriver.advanced_field_writer(output_coarse_orography_file,
output_orography)
