def testFieldWritingAndLoadingWithLatLongFloatingPointField(self): example_field = field.makeEmptyField('Generic', dtype=np.float64, grid_type='HD') lat_points = np.linspace(89.75, -89.75, 360, endpoint=True) lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True) example_field.set_grid_coordinates((lat_points, lon_points)) example_field.data[20, 20] = 1.5 example_field.data[200, 20] = 2.5 example_field.data[20, 200] = 3.5 example_field.data[200, 200] = 4.5 iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=True) loaded_field = iodriver.advanced_field_loader(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), fieldname='test_field') np.testing.assert_array_equal(example_field.get_data(), loaded_field.get_data()) self.assertEqual(loaded_field.data[20, 20], 1.5) self.assertEqual(loaded_field.data[200, 20], 2.5) self.assertEqual(loaded_field.data[20, 200], 3.5) self.assertEqual(loaded_field.data[200, 200], 4.5)
def testFieldWritingAndLoadingWithLatLongNoClobber(self): example_field = field.makeEmptyField('Generic', dtype=np.float64, grid_type='HD') lat_points = np.linspace(89.75, -89.75, 360, endpoint=True) lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True) example_field.set_grid_coordinates((lat_points, lon_points)) example_field.data[20, 20] = 1.5 example_field.data[200, 20] = 2.5 example_field.data[20, 200] = 3.5 example_field.data[200, 200] = 4.5 iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=True) with self.assertRaisesRegex( RuntimeError, r"Target file /Users/thomasriddick/Documents/data/temp/advancedfieldwritingandloadingtest.nc already exists and clobbering is not set" ): iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=False)
def extract_lake_volumes(flood_volume_thresholds, basin_catchment_numbers,merge_types): lake_mask = basin_catchment_numbers.greater_than_value(0) lake_volumes = field.makeEmptyField("Generic",np.float64, grid_type=lake_mask.get_grid()) lake_mask_reduced = lake_mask.copy() lake_mask_reduced.change_dtype(np.int32) lake_operators_wrapper.reduce_connected_areas_to_points(lake_mask_reduced.get_data(),True) lake_mask_inv = lake_mask.copy() lake_mask_inv.invert_data() orography = field.makeEmptyField("Generic",np.float64, grid_type=lake_mask.get_grid()) orography.get_data()[lake_mask_inv.get_data()] = 3.0 orography.get_data()[lake_mask.get_data()] = 1.0 orography.get_data()[lake_mask_reduced.get_data().astype(bool)] = 0.0 lake_mask_inv.change_dtype(np.int32) rdirs = determine_river_directions.determine_river_directions(orography, lake_mask_reduced, truesinks=None, always_flow_to_sea=True, use_diagonal_nbrs=True, mark_pits_as_true_sinks=False) rdirs.get_data()[lake_mask_inv.get_data()] = -1 simplified_basin_catchment_numbers = None temporary_file = tempfile.NamedTemporaryFile(delete = False) try: temporary_file.close() dummy_loop_log_filepath = temporary_file.name simplified_basin_catchment_numbers = \ field.Field(cc.compute_catchments_cpp(rdirs.get_data(),loop_logfile=dummy_loop_log_filepath), grid=lake_mask.get_grid()) finally: os.remove(temporary_file.name) for i in range(1,simplified_basin_catchment_numbers.find_maximum()+1): if i in simplified_basin_catchment_numbers.get_data(): single_lake_mask = lake_mask.copy() single_lake_mask.\ get_data()[simplified_basin_catchment_numbers.get_data() != i] = False single_lake_flood_volume_thresholds = flood_volume_thresholds.copy() single_lake_flood_volume_thresholds.\ get_data()[np.logical_not(single_lake_mask.get_data())] = 0.0 single_lake_flood_volume_thresholds.\ get_data()[np.logical_and(merge_types.get_data() != 10, merge_types.get_data() != 11)] = 0.0 lake_volumes.get_data()[single_lake_mask.get_data()] = np.sum(single_lake_flood_volume_thresholds.get_data()) return lake_volumes
def determine_river_directions(orography,lsmask,truesinks=None, always_flow_to_sea=True, use_diagonal_nbrs=True, mark_pits_as_true_sinks=True): rdirs = field.makeEmptyField('RiverDirections',np.float64,orography.get_grid()) if truesinks is None: truesinks = field.makeEmptyField('Generic',np.float64,orography.get_grid()) determine_river_directions_wrapper.determine_river_directions(rdirs.get_data(), orography.get_data(), lsmask.get_data().astype(dtype=np.int32, order='C', copy=False), truesinks.get_data().astype(dtype=np.int32, order='C', copy=False), always_flow_to_sea, use_diagonal_nbrs, mark_pits_as_true_sinks) return rdirs
def advanced_connected_lsmask_creation_driver( input_lsmask_filename, output_lsmask_filename, input_lsmask_fieldname, output_lsmask_fieldname, input_ls_seed_points_filename=None, input_ls_seed_points_fieldname=None, input_ls_seed_points_list_filename=None, use_diagonals_in=True, rotate_seeds_about_polar_axis=False, flip_seeds_ud=False): lsmask = iodriver.advanced_field_loader(input_lsmask_filename, field_type='Generic', fieldname=input_lsmask_fieldname) if input_ls_seed_points_filename: input_ls_seedpts = iodriver.advanced_field_loader(input_ls_seed_points_filename, field_type='Generic', fieldname=\ input_ls_seed_points_fieldname, adjust_orientation=False) else: input_ls_seedpts = field.makeEmptyField('Generic', np.int32) if input_ls_seed_points_list_filename: points_list = [] print("Reading input from {0}".format( input_ls_seed_points_list_filename)) comment_line_pattern = re.compile(r"^ *#.*$") with open(input_ls_seed_points_list_filename) as f: if f.readline().strip() != 'LatLong': raise RuntimeError( "List of landsea points being loaded is not for correct grid-type" ) for line in f: if comment_line_pattern.match(line): continue points_list.append( tuple(int(coord) for coord in line.strip().split(","))) input_ls_seedpts.flag_listed_points(points_list) if rotate_seeds_about_polar_axis: input_ls_seedpts.rotate_field_by_a_hundred_and_eighty_degrees() if flip_seeds_ud: input_ls_seedpts.flip_data_ud() if input_ls_seedpts.get_grid().has_orientation_information(): input_ls_seedpts.orient_data() lsmask.change_dtype(np.int32) cc_lsmask_wrapper.create_connected_ls_mask( lsmask.get_data(), input_ls_seedpts.get_data().astype(dtype=np.int32, order='C', copy=False), use_diagonals_in) iodriver.advanced_field_writer(output_lsmask_filename, lsmask, fieldname=output_lsmask_fieldname)
def find_ocean_basin_catchments(rdirs, catchments, areas=[]): ocean_catchments = catchments.copy() ocean_basin_catchments = field.makeEmptyField( "Generic", grid_type=catchments.get_grid(), dtype=np.int64) ocean_catchments.get_data()[np.logical_not( np.logical_and(rdirs.get_data() == 0, catchments.get_data() != 0))] = 0 for i, area in enumerate(areas, 1): ocean_catchments_area = ocean_catchments.get_data()[ area['min_lat']:area['max_lat'] + 1, area['min_lon']:area['max_lon'] + 1] ocean_catchment_list = np.unique(ocean_catchments_area) for catchment in ocean_catchment_list: if catchment == 0: continue ocean_basin_catchments.get_data()[catchments.get_data() == catchment] = i return ocean_basin_catchments
def advanced_orography_creation_driver(landsea_mask_filename, inclines_filename, orography_filename, landsea_mask_fieldname, inclines_fieldname, orography_fieldname): landsea_in = iodriver.advanced_field_loader( landsea_mask_filename, field_type="Generic", fieldname=landsea_mask_fieldname, adjust_orientation=True) inclines_in = iodriver.advanced_field_loader(inclines_filename, field_type="Generic", fieldname=inclines_fieldname, adjust_orientation=True) orography_in = field.makeEmptyField('Generic', np.float64, landsea_in.get_grid()) create_orography_wrapper.create_orography( np.ascontiguousarray(landsea_in.get_data(), dtype=np.int32), np.ascontiguousarray(inclines_in.get_data(), dtype=np.float64), np.ascontiguousarray(orography_in.get_data(), dtype=np.float64)) iodriver.advanced_field_writer(orography_filename, orography_in, fieldname=orography_fieldname)
def testFieldWritingAndLoadingWithLatLongFieldWithFlip(self): example_field = field.makeEmptyField('Generic', dtype=np.int64, grid_type='HD') lat_points = np.linspace(89.75, -89.75, 360, endpoint=True) lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True) example_field.set_grid_coordinates((lat_points, lon_points)) example_field.data[20, 20] = 1 example_field.data[200, 20] = 2 example_field.data[20, 200] = 3 example_field.data[200, 200] = 4 example_field.flip_data_ud() iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=True) loaded_field = iodriver.advanced_field_loader(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), fieldname='test_field', adjust_orientation=False) np.testing.assert_array_equal(example_field.get_data(), loaded_field.get_data())
def no_intermediaries_combine_hydrosheds_plus_rdirs_from_corrected_orog_driver( self): """Combines Hydrosheds river direction with those from a corrected orography and possibly removes sinks""" config = self._read_and_validate_config() final_loops_log_filename = path.splitext( self.output_catchments_filename)[0] + '_loops.log' if self.rerun_post_processing_only is not None: final_rdirs = \ iodriver.advanced_field_loader(self.rerun_post_processing_only, field_type="RiverDirections", fieldname=config.get("input_fieldname_options", "river_directions_to_reprocess_fieldname")) else: orography_filename = config.get( "input_options", "ten_minute_corrected_orography_filename") orography = iodriver.advanced_field_loader( orography_filename, field_type='Orography', fieldname=config.get( "input_fieldname_options", "ten_minute_corrected_orography_fieldname")) hydrosheds_rdirs_au_af_sa_10min_filename = \ config.get("input_options","ten_minute_hydrosheds_au_auf_sa_river_directions_filename") hydrosheds_rdirs = \ iodriver.advanced_field_loader(hydrosheds_rdirs_au_af_sa_10min_filename, field_type="RiverDirections", fieldname=config.get("input_fieldname_options", "ten_minute_hydrosheds_au_auf_sa_river_directions_fieldname")) second_intermediate_loops_log_filename = tempfile.mkstemp( suffix=".txt", prefix="loops_log_second_int", dir="")[1] truesinks_dummy = field.makeEmptyField("Generic", np.bool_, grid_type="LatLong10min") truesinks_dummy.set_all(False) if self.true_sinks_filename is not None: use_true_sinks = True truesinks = iodriver.advanced_field_loader( self.true_sinks_filename, field_type="Generic", fieldname=config.get("input_fieldname_options", "input_truesinks_fieldname")) else: use_true_sinks = False if config.getboolean( "general_options", "replace_internal_basins_with_rdirs_with_truesinks"): warnings.warn( "Option replace_internal_basins_with_rdirs_with_truesinks " "ignored when no true sinks file is specified") first_intermediate_rdirs = field.makeEmptyField( "RiverDirections", np.float64, grid_type="LatLong10min") if use_true_sinks: first_intermediate_rdirs_no_sinks = field.makeEmptyField( "RiverDirections", np.float64, grid_type="LatLong10min") ls_mask = iodriver.advanced_field_loader( self.ls_mask_filename, field_type="Generic", fieldname=config.get("input_fieldname_options", "input_landsea_mask_fieldname")) ls_mask.set_data( np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32)) next_cell_lat_index_in_dummy = np.zeros(ls_mask.get_data().shape, dtype=np.int32, order='C') next_cell_lon_index_in_dummy = np.zeros(ls_mask.get_data().shape, dtype=np.int32, order='C') catchment_nums_dummy = np.zeros(ls_mask.get_data().shape, dtype=np.int32, order='C') fill_sinks_wrapper.fill_sinks_cpp_func(orography_array= np.ascontiguousarray(orography.get_data(), #@UndefinedVariable dtype=np.float64), method = 4, use_ls_mask = True, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = False, true_sinks_in = np.ascontiguousarray(truesinks_dummy.\ get_data(), dtype=np.int32), next_cell_lat_index_in = next_cell_lat_index_in_dummy, next_cell_lon_index_in = next_cell_lon_index_in_dummy, rdirs_in = first_intermediate_rdirs.get_data() if not use_true_sinks else first_intermediate_rdirs_no_sinks.get_data(), catchment_nums_in = catchment_nums_dummy, prefer_non_diagonal_initial_dirs = False) if use_true_sinks: fill_sinks_wrapper.fill_sinks_cpp_func( orography_array=np.ascontiguousarray( orography.get_data(), #@UndefinedVariable dtype=np.float64), method=4, use_ls_mask=True, landsea_in=np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag=False, use_true_sinks=True, true_sinks_in=np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), next_cell_lat_index_in=next_cell_lat_index_in_dummy, next_cell_lon_index_in=next_cell_lon_index_in_dummy, rdirs_in=first_intermediate_rdirs.get_data(), catchment_nums_in=catchment_nums_dummy, prefer_non_diagonal_initial_dirs=False) second_intermediate_rdirs = utilities.splice_rdirs( rdirs_matching_ls_mask=first_intermediate_rdirs, ls_mask=ls_mask, other_rdirs=hydrosheds_rdirs) second_intermediate_catchments = comp_catchs.compute_catchments_cpp( second_intermediate_rdirs.get_data(), second_intermediate_loops_log_filename) second_intermediate_catchments = field.Field(comp_catchs.\ renumber_catchments_by_size(second_intermediate_catchments, second_intermediate_loops_log_filename), grid="LatLong10min") if config.getboolean("general_options", "keep_all_internal_basins"): third_intermediate_rdirs = second_intermediate_rdirs else: third_intermediate_rdirs = \ utilities.remove_endorheic_basins(rdirs=second_intermediate_rdirs, catchments=second_intermediate_catchments, rdirs_without_endorheic_basins= first_intermediate_rdirs_no_sinks if (use_true_sinks and not config.getboolean('general_options', 'replace_internal_basins_with_rdirs_with_truesinks')) else first_intermediate_rdirs, replace_only_catchments=([int(value) for value in config.get('general_options', 'replace_only_catchments').\ split(",")] if config.get('general_options', 'replace_only_catchments') else []), exclude_catchments=([int(value) for value in config.get('general_options', 'exclude_catchments').\ split(",")] if config.get('general_options', 'exclude_catchments') else []) ) third_intermediate_flowtocell = field.\ CumulativeFlow(create_hypothetical_river_paths_map(riv_dirs=third_intermediate_rdirs.get_data(), lsmask=None, use_f2py_func=True, use_f2py_sparse_iterator=True, nlat=1080,nlong=2160), grid='LatLong10min') third_intermediate_rdirs.make_contiguous() third_intermediate_flowtocell.make_contiguous() first_intermediate_rdirs.make_contiguous() if use_true_sinks: first_intermediate_rdirs_no_sinks.make_contiguous() final_rdirs = utilities.replace_streams_downstream_from_loop( third_intermediate_rdirs, cumulative_flow=third_intermediate_flowtocell, other_rdirs=first_intermediate_rdirs if not use_true_sinks else first_intermediate_rdirs_no_sinks) final_catchments = comp_catchs.compute_catchments_cpp( final_rdirs.get_data(), final_loops_log_filename) final_catchments = field.Field(comp_catchs.\ renumber_catchments_by_size(final_catchments, final_loops_log_filename), grid="LatLong10min") final_flowtocell = field.CumulativeFlow( create_hypothetical_river_paths_map( riv_dirs=final_rdirs.get_data(), lsmask=None, use_f2py_func=True, use_f2py_sparse_iterator=True, nlat=1080, nlong=2160), grid='LatLong10min') if self.rerun_post_processing_only is None: iodriver.advanced_field_writer( self.output_rdirs_filename, final_rdirs, fieldname=config.get("output_fieldname_options", "output_river_directions_fieldname")) iodriver.advanced_field_writer(self.output_catchments_filename, final_catchments, fieldname=config.get( "output_fieldname_options", "output_catchments_fieldname")) iodriver.advanced_field_writer(self.output_cumulative_flow_filename, final_flowtocell, fieldname=config.get( "output_fieldname_options", "output_cumulative_flow_fieldname")) if self.rerun_post_processing_only is None: os.remove(second_intermediate_loops_log_filename)
def advanced_sinkless_flow_directions_generator(filename,output_filename,fieldname, output_fieldname,ls_mask_filename=None, truesinks_filename=None, catchment_nums_filename=None, ls_mask_fieldname=None, truesinks_fieldname=None, catchment_fieldname=None): orography = iodriver.advanced_field_loader(filename, field_type='Orography', fieldname=fieldname) grid_dims=orography.get_grid().get_grid_dimensions() grid_num_points = orography.get_grid().get_npoints() threshold_for_low_mem_version_usage = 500000000 if (grid_num_points < threshold_for_low_mem_version_usage): rdirs = np.empty(grid_dims,dtype=np.float64,order='C') if not truesinks_filename: truesinks = Field(np.empty((1,1),dtype=np.int32),grid='HD') use_true_sinks = False; else: use_true_sinks = True; truesinks = iodriver.advanced_field_loader(truesinks_filename, field_type='Generic', fieldname=truesinks_fieldname) if ls_mask_filename is None: use_ls_mask = False ls_mask = field.makeEmptyField(field_type='Generic',dtype=np.int32,grid_type='HD') else: use_ls_mask = True ls_mask = iodriver.advanced_field_loader(ls_mask_filename, field_type='Generic', fieldname=ls_mask_fieldname) catchment_nums = np.zeros(grid_dims,dtype=np.int32,order='C') next_cell_lat_index_in = np.zeros(grid_dims,dtype=np.int32,order='C') next_cell_lon_index_in = np.zeros(grid_dims,dtype=np.int32,order='C') if (grid_num_points >= threshold_for_low_mem_version_usage): rdirs = fill_sinks_wrapper_low_mem.fill_sinks_cpp_func(orography_array=np.ascontiguousarray(orography.get_data(), #@UndefinedVariable dtype=np.float64), method = 4, use_ls_mask = use_ls_mask, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = use_true_sinks, true_sinks_in = np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), next_cell_lat_index_in = next_cell_lat_index_in, next_cell_lon_index_in = next_cell_lon_index_in, catchment_nums_in = catchment_nums, prefer_non_diagonal_initial_dirs = False) else: fill_sinks_wrapper.fill_sinks_cpp_func(orography_array=np.ascontiguousarray(orography.get_data(), #@UndefinedVariable dtype=np.float64), method = 4, use_ls_mask = use_ls_mask, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = use_true_sinks, true_sinks_in = np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), next_cell_lat_index_in = next_cell_lat_index_in, next_cell_lon_index_in = next_cell_lon_index_in, rdirs_in = rdirs, catchment_nums_in = catchment_nums, prefer_non_diagonal_initial_dirs = False) iodriver.advanced_field_writer(output_filename,Field(rdirs,grid=orography.get_grid()), fieldname=output_fieldname) if catchment_nums_filename: iodriver.advanced_field_writer(catchment_nums_filename, field=Field(catchment_nums,grid=orography.get_grid()), fieldname=catchment_fieldname)
def drive_connected_lsmask_creation(input_lsmask_filename, output_lsmask_filename, input_ls_seed_points_filename=None, input_ls_seed_points_list_filename=None, use_diagonals_in=True, rotate_seeds_about_polar_axis=False, flip_seeds_ud=False, flip_input_mask_ud=False, grid_type='HD', **grid_kwargs): """Drives the creation of a fully connected land-sea mask Argument: input_lsmask_filename: string; the full path of the input land-sea mask which may contain disconnected sea points output_lsmask_filename: string; the full path of the output land-sea mask without any disconnected sea point input_ls_seed_points_filename(optional): string; the full path of the file with an array with at least one point in the center of each ocean/sea to be included input_ls_seed_points_list_filename(optional): string; the full path of the file with a list of at least one point in the center of each ocean/sea to be included - these will be added to the contents of input_ls_seed_points_filename, normally only one of the two arguments would be used to initialise the seed points use_diagonals_in: boolean; if true count a diagonal connection as being connected flip_input_mask_ud: boolean; if true flip the input mask up down before processing rotate_seeds_about_polar_axis: if true then rotate the seeds about the polar axis by 180 degrees, i.e. shift the position of the zero longitude by 180 degrees when reading the seed points flip_seeds_upside_down: boolean; if true then flip the seeds upside down (i.e. about the equator) grid_type: string; the code for this grid type **grid_kwargs: dictionary; key word arguments specifying parameters of the grid Returns: Nothing Creates a land-sea masked based on the supplied land-sea mask with any sea points not connected to the supplied ocean seeds points (by other sea points) change to land points """ lsmask = iodriver.load_field(input_lsmask_filename, file_type=iodriver.\ get_file_extension(input_lsmask_filename), field_type='Generic', grid_type=grid_type,**grid_kwargs) if flip_input_mask_ud: lsmask.flip_data_ud() if input_ls_seed_points_filename: input_ls_seedpts = iodriver.load_field(input_ls_seed_points_filename, file_type=iodriver.\ get_file_extension(input_ls_seed_points_filename), field_type='Generic', grid_type=grid_type,**grid_kwargs) else: input_ls_seedpts = field.makeEmptyField('Generic', np.int32, grid_type, **grid_kwargs) if input_ls_seed_points_list_filename: points_list = [] print("Reading input from {0}".format( input_ls_seed_points_list_filename)) comment_line_pattern = re.compile(r"^ *#.*$") with open(input_ls_seed_points_list_filename) as f: if f.readline().strip() != grid_type: raise RuntimeError( "List of landsea points being loaded is not for correct grid-type" ) for line in f: if comment_line_pattern.match(line): continue points_list.append( tuple(int(coord) for coord in line.strip().split(","))) input_ls_seedpts.flag_listed_points(points_list) if rotate_seeds_about_polar_axis: input_ls_seedpts.rotate_field_by_a_hundred_and_eighty_degrees() if flip_seeds_ud: input_ls_seedpts.flip_data_ud() lsmask.change_dtype(np.int32) cc_lsmask_wrapper.create_connected_ls_mask( lsmask.get_data(), input_ls_seedpts.get_data().astype(dtype=np.int32, order='C', copy=False), use_diagonals_in) iodriver.write_field(output_lsmask_filename,lsmask, file_type=iodriver.\ get_file_extension(output_lsmask_filename))
def connect_coarse_lake_catchments(coarse_catchments,lake_centers,basin_catchment_numbers, flood_next_cell_index_lat,flood_next_cell_index_lon, flood_redirect_lat,flood_redirect_lon, additional_flood_redirect_lat,additional_flood_redirect_lon, local_redirect,additional_local_redirect, merge_types,river_directions,scale_factor = 3, correct_cumulative_flow=False, cumulative_flow=None): if correct_cumulative_flow: if cumulative_flow is None or river_directions is None: raise RuntimeError("Required input files for cumulative flow correction not provided") old_coarse_catchments = coarse_catchments.copy() lake_centers_array = np.argwhere(lake_centers.get_data()) lake_centers_list = [lake_centers_array[i,:].tolist() for i in range(lake_centers_array.shape[0])] overflow_catchments = field.makeEmptyField(field_type='Generic',dtype=np.int64, grid_type=lake_centers.get_grid()) overflow_coords_lats = field.makeEmptyField(field_type='Generic',dtype=np.int64, grid_type=lake_centers.get_grid()) overflow_coords_lons = field.makeEmptyField(field_type='Generic',dtype=np.int64, grid_type=lake_centers.get_grid()) for lake_center_coords in lake_centers_list: basin_number = basin_catchment_numbers.get_data()[tuple(lake_center_coords)] while True: secondary_merge_coords = np.argwhere( np.logical_and( np.logical_or(merge_types.get_data() == 10, merge_types.get_data() == 11), basin_catchment_numbers.get_data() == basin_number))[0,:].tolist() double_merge = np.any(np.logical_and(merge_types.get_data() == 11, basin_catchment_numbers.get_data() == basin_number)) basin_number = basin_catchment_numbers.get_data()[flood_next_cell_index_lat.get_data()[tuple(secondary_merge_coords)], flood_next_cell_index_lon.get_data()[tuple(secondary_merge_coords)]] if basin_number == 0: is_local_redirect = ((local_redirect.get_data()[tuple(secondary_merge_coords)]) if not double_merge else (additional_local_redirect.get_data()[tuple(secondary_merge_coords)])) if is_local_redirect: if double_merge: basin_number = \ basin_catchment_numbers.\ get_data()[additional_flood_redirect_lat.get_data()[tuple(secondary_merge_coords)], additional_flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]] else: basin_number = \ basin_catchment_numbers.\ get_data()[flood_redirect_lat.get_data()[tuple(secondary_merge_coords)], flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]] else: if double_merge: overflow_catchment = \ coarse_catchments.get_data()[additional_flood_redirect_lat.get_data()[tuple(secondary_merge_coords)], additional_flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]] overflow_coords = (additional_flood_redirect_lat.get_data()[tuple(secondary_merge_coords)], additional_flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]) else: overflow_catchment = \ coarse_catchments.get_data()[flood_redirect_lat.get_data()[tuple(secondary_merge_coords)], flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]] overflow_coords = (flood_redirect_lat.get_data()[tuple(secondary_merge_coords)], flood_redirect_lon.get_data()[tuple(secondary_merge_coords)]) break overflow_catchments.get_data()[tuple(lake_center_coords)] = overflow_catchment overflow_coords_lats.get_data()[tuple(lake_center_coords)] = overflow_coords[0] overflow_coords_lons.get_data()[tuple(lake_center_coords)] = overflow_coords[1] #specific to latlon grid sink_points_array = np.argwhere(np.logical_or(river_directions.get_data() == 5, river_directions.get_data() == -2)) sink_points_list = [sink_points_array[i,:].tolist() for i in range(sink_points_array.shape[0])] catchment_trees = CatchmentTrees() sink_point_cumulative_flow_redirect_lat = field.makeEmptyField(field_type='Generic',dtype=np.int64, grid_type=coarse_catchments.get_grid()) sink_point_cumulative_flow_redirect_lon = field.makeEmptyField(field_type='Generic',dtype=np.int64, grid_type=coarse_catchments.get_grid()) for sink_point in sink_points_list: sink_point_coarse_catchment = coarse_catchments.get_data()[tuple(sink_point)] #Specific to lat-lon grids overflow_catch_fine_cells_in_coarse_cell = overflow_catchments.get_data()[sink_point[0]*scale_factor:(sink_point[0]+1)*scale_factor, sink_point[1]*scale_factor:(sink_point[1]+1)*scale_factor] overflow_coords_lat_fine_cells_in_coarse_cell = overflow_coords_lats.get_data()[sink_point[0]*scale_factor:(sink_point[0]+1)*scale_factor, sink_point[1]*scale_factor:(sink_point[1]+1)*scale_factor] overflow_coords_lon_fine_cells_in_coarse_cell = overflow_coords_lons.get_data()[sink_point[0]*scale_factor:(sink_point[0]+1)*scale_factor, sink_point[1]*scale_factor:(sink_point[1]+1)*scale_factor] overflow_catchment_list = overflow_catch_fine_cells_in_coarse_cell[overflow_catch_fine_cells_in_coarse_cell != 0].tolist() if not overflow_catchment_list: continue overflow_catchment_counters = Counter(overflow_catchment_list) highest_count = max(overflow_catchment_counters.values()) overflow_catchment = [ value for value,count in overflow_catchment_counters.items() if count == highest_count][0] if sink_point_coarse_catchment == overflow_catchment: continue overflow_catchment_fine_coords_within_coarse_cell = \ tuple(np.argwhere(overflow_catch_fine_cells_in_coarse_cell == overflow_catchment)[0,:].tolist()) sink_point_cumulative_flow_redirect_lat.get_data()[tuple(sink_point)] = \ overflow_coords_lat_fine_cells_in_coarse_cell[overflow_catchment_fine_coords_within_coarse_cell] sink_point_cumulative_flow_redirect_lon.get_data()[tuple(sink_point)] = \ overflow_coords_lon_fine_cells_in_coarse_cell[overflow_catchment_fine_coords_within_coarse_cell] catchment_trees.add_link(sink_point_coarse_catchment,overflow_catchment) for supercatchment_number,tree in catchment_trees.primary_catchments.items(): for subcatchments_num in tree.get_all_subcatchment_nums(): coarse_catchments.get_data()[subcatchments_num == coarse_catchments.get_data()] = \ supercatchment_number if correct_cumulative_flow: while catchment_trees.all_catchments: upstream_catchments = catchment_trees.pop_leaves() for upstream_catchment in upstream_catchments: if np.any(np.logical_and(np.logical_or(river_directions.get_data() == 5, river_directions.get_data() == -2), old_coarse_catchments.get_data() == upstream_catchment)): upstream_catchment_center = \ tuple(np.argwhere(np.logical_and(np.logical_or(river_directions.get_data() == 5, river_directions.get_data() == -2), old_coarse_catchments.get_data() == upstream_catchment))[0,:].tolist()) update_cumulative_flow(upstream_catchment_center, (sink_point_cumulative_flow_redirect_lat.get_data()[upstream_catchment_center], sink_point_cumulative_flow_redirect_lon.get_data()[upstream_catchment_center]), cumulative_flow,river_directions) return field.Field(cc.renumber_catchments_by_size(coarse_catchments.get_data()),type="Generic", grid=coarse_catchments.get_grid()),cumulative_flow return field.Field(cc.renumber_catchments_by_size(coarse_catchments.get_data()),type="Generic", grid=coarse_catchments.get_grid()),None
def drive_orography_upscaling(input_fine_orography_file, output_coarse_orography_file, landsea_file=None, true_sinks_file=None, upscaling_parameters_filename=None, fine_grid_type='LatLong10min', coarse_grid_type='HD', input_orography_field_name=None, flip_landsea=False, rotate_landsea=False, flip_true_sinks=False, rotate_true_sinks=False, fine_grid_kwargs={}, **coarse_grid_kwargs): """Drive the C++ sink filling code base to make a tarasov-like orography upscaling Arguments: input_fine_orography_file: string; full path to input fine orography file output_coarse_orography_file: string; full path of target output coarse orography file landsea_file: string; full path to input fine landsea mask file (optional) true_sinks_file: string; full path to input fine true sinks file (optional) upscaling_parameters_filename: string; full path to the orography upscaling parameter file (optional) fine_grid_type: string; code for the fine grid type to be upscaled from (optional) coarse_grid_type: string; code for the coarse grid type to be upscaled to (optional) input_orography_field_name: string; name of field in the input orography file (optional) flip_landsea: bool; flip the input landsea mask upside down rotate_landsea: bool; rotate the input landsea mask by 180 degrees along the horizontal axis flip_true_sinks: bool; flip the input true sinks field upside down rotate_true_sinks: bool; rotate the input true sinks field by 180 degrees along the horizontal axis fine_grid_kwargs: keyword dictionary; the parameter of the fine grid to upscale from (if required) **coarse_grid_kwargs: keyword dictionary; the parameters of the coarse grid to upscale to (if required) Returns: Nothing. """ if upscaling_parameters_filename: config = read_and_validate_config(upscaling_parameters_filename) method = config.getint("orography_upscaling_parameters", "method") add_slope_in = config.getboolean("orography_upscaling_parameters", "add_slope_in") epsilon_in = config.getfloat("orography_upscaling_parameters", "epsilon_in") tarasov_separation_threshold_for_returning_to_same_edge_in =\ config.getint("orography_upscaling_parameters", "tarasov_separation_threshold_for_returning_to_same_edge_in") tarasov_min_path_length_in = config.getfloat( "orography_upscaling_parameters", "tarasov_min_path_length_in") tarasov_include_corners_in_same_edge_criteria_in = \ config.getboolean("orography_upscaling_parameters", "tarasov_include_corners_in_same_edge_criteria_in") else: #use defaults method = 1 add_slope_in = False epsilon_in = 0.1 tarasov_separation_threshold_for_returning_to_same_edge_in = 5 tarasov_min_path_length_in = 2.0 tarasov_include_corners_in_same_edge_criteria_in = False output_orography = field.makeEmptyField(field_type='Orography', dtype=np.float64, grid_type=coarse_grid_type, **coarse_grid_kwargs) input_orography = iodriver.load_field( input_fine_orography_file, file_type=get_file_extension(input_fine_orography_file), field_type='Orography', unmask=True, fieldname=input_orography_field_name, grid_type=fine_grid_type, **fine_grid_kwargs) if landsea_file: landsea_mask = iodriver.load_field( landsea_file, file_type=get_file_extension(landsea_file), field_type='Generic', unmask=True, grid_type=fine_grid_type, **fine_grid_kwargs) if flip_landsea: landsea_mask.flip_data_ud() if rotate_landsea: landsea_mask.rotate_field_by_a_hundred_and_eighty_degrees() else: landsea_mask = field.makeEmptyField(field_type='Generic', dtype=np.int32, grid_type=fine_grid_type, **fine_grid_kwargs) if true_sinks_file: true_sinks = iodriver.load_field( true_sinks_file, file_type=get_file_extension(true_sinks_file), field_type='Generic', unmask=True, grid_type=fine_grid_type, **fine_grid_kwargs) if flip_true_sinks: true_sinks.flip_data_ud() if rotate_true_sinks: true_sinks.rotate_field_by_a_hundred_and_eighty_degrees() else: true_sinks = field.makeEmptyField(field_type='Generic', dtype=np.int32, grid_type=fine_grid_type, **fine_grid_kwargs) if not np.issubdtype(input_orography.get_data().dtype, np.float64()): input_orography.change_dtype(np.float64) #Make sure old data type array is flushed out of memory immediately gc.collect() upscale_orography_wrapper.upscale_orography(orography_in=input_orography.get_data(), orography_out=output_orography.get_data(), method=method,landsea_in=landsea_mask.get_data(), true_sinks_in=true_sinks.get_data(), add_slope_in=add_slope_in, epsilon_in=epsilon_in, tarasov_separation_threshold_for_returning_to_same_edge_in=\ tarasov_separation_threshold_for_returning_to_same_edge_in, tarasov_min_path_length_in=tarasov_min_path_length_in, tarasov_include_corners_in_same_edge_criteria_in=\ tarasov_include_corners_in_same_edge_criteria_in) iodriver.write_field( output_coarse_orography_file, output_orography, file_type=get_file_extension(output_coarse_orography_file))
def advanced_drive_orography_upscaling(input_fine_orography_file, output_coarse_orography_file, input_orography_fieldname, output_coarse_orography_fieldname, landsea_file=None, true_sinks_file=None, landsea_fieldname=None, true_sinks_fieldname=None, upscaling_parameters_filename=None, scaling_factor=3): if upscaling_parameters_filename: config = read_and_validate_config(upscaling_parameters_filename) method = config.getint("orography_upscaling_parameters", "method") add_slope_in = config.getboolean("orography_upscaling_parameters", "add_slope_in") epsilon_in = config.getfloat("orography_upscaling_parameters", "epsilon_in") tarasov_separation_threshold_for_returning_to_same_edge_in =\ config.getint("orography_upscaling_parameters", "tarasov_separation_threshold_for_returning_to_same_edge_in") tarasov_min_path_length_in = config.getfloat( "orography_upscaling_parameters", "tarasov_min_path_length_in") tarasov_include_corners_in_same_edge_criteria_in = \ config.getboolean("orography_upscaling_parameters", "tarasov_include_corners_in_same_edge_criteria_in") else: #use defaults method = 1 add_slope_in = False epsilon_in = 0.1 tarasov_separation_threshold_for_returning_to_same_edge_in = 5 tarasov_min_path_length_in = 2.0 tarasov_include_corners_in_same_edge_criteria_in = False input_orography = iodriver.advanced_field_loader( input_fine_orography_file, field_type='Orography', fieldname=input_orography_fieldname) nlat_fine, nlon_fine = input_orography.get_grid_dimensions() lat_pts_fine, lon_pts_fine = input_orography.get_grid_coordinates() nlat_coarse,nlon_coarse,lat_pts_coarse,lon_pts_coarse = \ coordinate_scaling_utilities.generate_coarse_coords(nlat_fine,nlon_fine, lat_pts_fine,lon_pts_fine, scaling_factor) output_orography = field.makeEmptyField(field_type='Orography', dtype=np.float64, grid_type='LatLong', nlat=nlat_coarse, nlong=nlon_coarse) output_orography.set_grid_coordinates([lat_pts_coarse, lon_pts_coarse]) if landsea_file: landsea_mask = iodriver.advanced_field_loader( landsea_file, field_type='Generic', fieldname=landsea_fieldname) else: landsea_mask = field.makeEmptyField(field_type='Generic', dtype=np.int32, grid_type='LatLong', nlat=nlat_fine, nlong=nlon_fine) if true_sinks_file: true_sinks = iodriver.advanced_field_loader( true_sinks_file, field_type='Generic', fieldname=true_sinks_fieldname) else: true_sinks = field.makeEmptyField(field_type='Generic', dtype=np.int32, grid_type='LatLong', nlat=nlat_fine, nlong=nlon_fine) if not input_orography.get_data().dtype is np.float64(): input_orography.change_dtype(np.float64) #Make sure old data type array is flushed out of memory immediately gc.collect() upscale_orography_wrapper.upscale_orography(orography_in=input_orography.get_data(), orography_out=output_orography.get_data(), method=method,landsea_in=landsea_mask.get_data(), true_sinks_in=true_sinks.get_data(), add_slope_in=add_slope_in, epsilon_in=epsilon_in, tarasov_separation_threshold_for_returning_to_same_edge_in=\ tarasov_separation_threshold_for_returning_to_same_edge_in, tarasov_min_path_length_in=tarasov_min_path_length_in, tarasov_include_corners_in_same_edge_criteria_in=\ tarasov_include_corners_in_same_edge_criteria_in) iodriver.advanced_field_writer(output_coarse_orography_file, output_orography)