def advanced_river_direction_determination_driver(rdirs_filename, orography_filename, lsmask_filename, truesinks_filename=None, rdirs_fieldname=None, orography_fieldname=None, lsmask_fieldname=None, truesinks_fieldname=None, always_flow_to_sea=True, use_diagonal_nbrs=True, mark_pits_as_true_sinks=True): orography = iodriver.advanced_field_loader(orography_filename, field_type='Orography', fieldname=orography_fieldname, adjust_orientation=True) lsmask = iodriver.advanced_field_loader(lsmask_filename, field_type='Generic', fieldname=lsmask_fieldname, adjust_orientation=True) if truesinks_filename is None: truesinks = None else: truesinks = iodriver.advanced_field_loader(truesinks_filename, field_type='Generic', fieldname=truesinks_filename, adjust_orientation=True) rdirs = determine_river_directions(orography, lsmask, truesinks, always_flow_to_sea, use_diagonal_nbrs, mark_pits_as_true_sinks) iodriver.advanced_field_writer(rdirs_filename,rdirs, fieldname=rdirs_fieldname)
def advanced_narrow_lake_filtering_driver(input_unfilled_orography_file, input_unfilled_orography_fieldname, input_filled_orography_file, input_filled_orography_fieldname, output_unfilled_orography_file, output_unfilled_orography_fieldname, interior_cell_min_masked_neighbors=5, edge_cell_max_masked_neighbors=4, max_range=5, iterations=5): input_unfilled_orography = \ iodriver.advanced_field_loader(input_unfilled_orography_file, field_type='Generic', fieldname=input_unfilled_orography_fieldname) input_filled_orography = \ iodriver.advanced_field_loader(input_filled_orography_file, field_type='Generic', fieldname=input_filled_orography_fieldname) output_unfilled_orography = filter_narrow_lakes( input_unfilled_orography, input_filled_orography, interior_cell_min_masked_neighbors, edge_cell_max_masked_neighbors, max_range, iterations) iodriver.advanced_field_writer( output_unfilled_orography_file, output_unfilled_orography, fieldname=output_unfilled_orography_fieldname)
def advanced_cotat_plus_driver(input_fine_rdirs_filepath,input_fine_total_cumulative_flow_path, output_coarse_rdirs_filepath, input_fine_rdirs_fieldname, input_fine_total_cumulative_flow_fieldname, output_coarse_rdirs_fieldname, cotat_plus_parameters_filepath,scaling_factor): fine_rdirs_field = iodriver.advanced_field_loader(input_fine_rdirs_filepath, field_type='RiverDirections', fieldname=input_fine_rdirs_fieldname) fine_total_cumulative_flow_field =\ iodriver.advanced_field_loader(input_fine_total_cumulative_flow_path, field_type='CumulativeFlow', fieldname=input_fine_total_cumulative_flow_fieldname) nlat_fine,nlon_fine = fine_rdirs_field.get_grid_dimensions() lat_pts_fine,lon_pts_fine = fine_rdirs_field.get_grid_coordinates() nlat_coarse,nlon_coarse,lat_pts_coarse,lon_pts_coarse = \ coordinate_scaling_utilities.generate_coarse_coords(nlat_fine,nlon_fine, lat_pts_fine,lon_pts_fine, scaling_factor) coarse_rdirs_field = run_cotat_plus(fine_rdirs_field, fine_total_cumulative_flow_field, cotat_plus_parameters_filepath, coarse_grid_type="LatLong",nlat=nlat_coarse, nlong=nlon_coarse) coarse_rdirs_field.set_grid_coordinates([lat_pts_coarse,lon_pts_coarse]) iodriver.advanced_field_writer(output_coarse_rdirs_filepath, coarse_rdirs_field, fieldname=output_coarse_rdirs_fieldname)
def advanced_shallow_lake_filtering_driver( input_unfilled_orography_file, input_unfilled_orography_fieldname, input_filled_orography_file, input_filled_orography_fieldname, output_unfilled_orography_file, output_unfilled_orography_fieldname, minimum_depth_threshold): input_unfilled_orography = \ iodriver.advanced_field_loader(input_unfilled_orography_file, field_type='Orography', fieldname=input_unfilled_orography_fieldname) input_filled_orography = \ iodriver.advanced_field_loader(input_filled_orography_file, field_type='Orography', fieldname=input_filled_orography_fieldname) output_unfilled_orography = \ field.Field(np.ascontiguousarray(input_unfilled_orography.get_data(), dtype=np.float64), grid=input_unfilled_orography.get_grid()) lake_operators_wrapper.filter_out_shallow_lakes(output_unfilled_orography.get_data(), np.ascontiguousarray(input_filled_orography.\ get_data(), dtype=np.float64), minimum_depth_threshold) iodriver.advanced_field_writer( output_unfilled_orography_file, output_unfilled_orography, fieldname=output_unfilled_orography_fieldname)
def testFieldWritingAndLoadingWithLatLongFloatingPointField(self): example_field = field.makeEmptyField('Generic', dtype=np.float64, grid_type='HD') lat_points = np.linspace(89.75, -89.75, 360, endpoint=True) lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True) example_field.set_grid_coordinates((lat_points, lon_points)) example_field.data[20, 20] = 1.5 example_field.data[200, 20] = 2.5 example_field.data[20, 200] = 3.5 example_field.data[200, 200] = 4.5 iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=True) loaded_field = iodriver.advanced_field_loader(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), fieldname='test_field') np.testing.assert_array_equal(example_field.get_data(), loaded_field.get_data()) self.assertEqual(loaded_field.data[20, 20], 1.5) self.assertEqual(loaded_field.data[200, 20], 2.5) self.assertEqual(loaded_field.data[20, 200], 3.5) self.assertEqual(loaded_field.data[200, 200], 4.5)
def testFieldWritingAndLoadingWithLatLongNoClobber(self): example_field = field.makeEmptyField('Generic', dtype=np.float64, grid_type='HD') lat_points = np.linspace(89.75, -89.75, 360, endpoint=True) lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True) example_field.set_grid_coordinates((lat_points, lon_points)) example_field.data[20, 20] = 1.5 example_field.data[200, 20] = 2.5 example_field.data[20, 200] = 3.5 example_field.data[200, 200] = 4.5 iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=True) with self.assertRaisesRegex( RuntimeError, r"Target file /Users/thomasriddick/Documents/data/temp/advancedfieldwritingandloadingtest.nc already exists and clobbering is not set" ): iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=False)
def advanced_fill_lakes_driver(input_minima_file, input_minima_fieldname, input_lakemask_file, input_lakemask_fieldname, input_orography_file, input_orography_fieldname, output_orography_file, output_orography_fieldname, use_highest_possible_lake_water_level=True): input_minima = iodriver.advanced_field_loader( input_minima_file, field_type='Orography', fieldname=input_minima_fieldname) input_lakemask = iodriver.advanced_field_loader( input_lakemask_file, field_type='Orography', fieldname=input_lakemask_fieldname) input_orography = iodriver.advanced_field_loader( input_orography_file, field_type='Orography', fieldname=input_orography_fieldname) lake_operators_wrapper.fill_lakes(input_minima.get_data(), input_lakemask.get_data(), input_orography.get_data(), use_highest_possible_lake_water_level) iodriver.advanced_field_writer(output_orography_file, input_orography, fieldname=output_orography_fieldname)
def prepare_true_sinks(self,initial_true_sinks_filename, output_true_sinks_filename, true_sinks_list_filename): if self.working_directory is None: raise RuntimeError("Applying tweaks requires a working directory to be specified") print("Note - All adjustments must be relative to a 180 degree W to 180 degree E," "90 degree N to 90 degree S grid") true_sinks_field = iodriver.advanced_field_loader(initial_true_sinks_filename, field_type='Generic', fieldname="true_sinks") true_sinks_list = [] true_sinks_to_remove_list = [] with open(true_sinks_list_filename,"r") as true_sinks_list_file: if not self.true_sinks_first_line_pattern.\ match(true_sinks_list_file.readline().strip()): raise RuntimeError("List of extra true sinks being loaded has" " incorrect header format") for line in true_sinks_list_file: if self.comment_line_match.match(line): continue elif self.remove_sink_line_match.match(line): true_sinks_to_remove_list.append(tuple(int(value) for value in line.strip().split(",")[1:])) else: true_sinks_list.append(tuple(int(value) for value in line.strip().split(","))) for lat,lon in true_sinks_list: print("Adding true sinks at lat={0},lon={1}".format(lat,lon)) true_sinks_field.get_data()[lat,lon] = True for lat,lon in true_sinks_to_remove_list: print("Removing true sinks at lat={0},lon={1}".format(lat,lon)) true_sinks_field.get_data()[lat,lon] = False iodriver.advanced_field_writer(output_true_sinks_filename, true_sinks_field, fieldname="true_sinks")
def advanced_local_minima_finding_driver(input_orography_file, input_orography_fieldname, output_local_minima_file, output_local_minima_fieldname): input_orography = iodriver.advanced_field_loader( input_orography_file, field_type='Orography', fieldname=input_orography_fieldname) output_local_minima = input_orography.find_all_local_minima() iodriver.advanced_field_writer(output_local_minima_file, output_local_minima, fieldname=output_local_minima_fieldname)
def advanced_connected_lsmask_creation_driver( input_lsmask_filename, output_lsmask_filename, input_lsmask_fieldname, output_lsmask_fieldname, input_ls_seed_points_filename=None, input_ls_seed_points_fieldname=None, input_ls_seed_points_list_filename=None, use_diagonals_in=True, rotate_seeds_about_polar_axis=False, flip_seeds_ud=False): lsmask = iodriver.advanced_field_loader(input_lsmask_filename, field_type='Generic', fieldname=input_lsmask_fieldname) if input_ls_seed_points_filename: input_ls_seedpts = iodriver.advanced_field_loader(input_ls_seed_points_filename, field_type='Generic', fieldname=\ input_ls_seed_points_fieldname, adjust_orientation=False) else: input_ls_seedpts = field.makeEmptyField('Generic', np.int32) if input_ls_seed_points_list_filename: points_list = [] print("Reading input from {0}".format( input_ls_seed_points_list_filename)) comment_line_pattern = re.compile(r"^ *#.*$") with open(input_ls_seed_points_list_filename) as f: if f.readline().strip() != 'LatLong': raise RuntimeError( "List of landsea points being loaded is not for correct grid-type" ) for line in f: if comment_line_pattern.match(line): continue points_list.append( tuple(int(coord) for coord in line.strip().split(","))) input_ls_seedpts.flag_listed_points(points_list) if rotate_seeds_about_polar_axis: input_ls_seedpts.rotate_field_by_a_hundred_and_eighty_degrees() if flip_seeds_ud: input_ls_seedpts.flip_data_ud() if input_ls_seedpts.get_grid().has_orientation_information(): input_ls_seedpts.orient_data() lsmask.change_dtype(np.int32) cc_lsmask_wrapper.create_connected_ls_mask( lsmask.get_data(), input_ls_seedpts.get_data().astype(dtype=np.int32, order='C', copy=False), use_diagonals_in) iodriver.advanced_field_writer(output_lsmask_filename, lsmask, fieldname=output_lsmask_fieldname)
def generate_orography_with_sinks_filled_advanced_driver(input_orography_filename, output_orography_filename, input_orography_fieldname, output_orography_fieldname, ls_mask_filename=None, truesinks_filename=None, ls_mask_fieldname=None, truesinks_fieldname=None, add_slight_slope_when_filling_sinks=True, slope_param=0.1): orography = iodriver.advanced_field_loader(input_orography_filename, field_type='Orography', fieldname=input_orography_fieldname) nlat,nlon = orography.get_grid_dimensions() lat_pts,lon_pts = orography.get_grid_coordinates() if not truesinks_filename: truesinks = Field(np.empty((1,1),dtype=np.int32),grid='LatLong', nlat=nlat,nlong=nlon) use_true_sinks = False; else: use_true_sinks = True; truesinks = iodriver.advanced_field_loader(truesinks_filename, field_type='Generic', fieldname=truesinks_fieldname) if ls_mask_filename is None: use_ls_mask = False ls_mask = Field(np.zeros((nlat,nlon),dtype=np.int32,order='C'), grid='LatLong',nlat=nlat,nlong=nlon) else: use_ls_mask = True ls_mask = iodriver.advanced_field_loader(ls_mask_filename, field_type='Generic', fieldname=ls_mask_fieldname) #Dtype change and ascontinguousarray must be applied here so orography keeps the correct #reference when running the C++ code orography.change_dtype(np.float64) orography.make_contiguous() fill_sinks_wrapper.fill_sinks_cpp_func(orography_array=orography.get_data(), method = 1, use_ls_mask = use_ls_mask, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = use_true_sinks, true_sinks_in = np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), add_slope =add_slight_slope_when_filling_sinks, epsilon = slope_param) orography.set_grid_dimensions([nlat,nlon]) orography.set_grid_coordinates([lat_pts,lon_pts]) iodriver.advanced_field_writer(output_orography_filename,orography, fieldname=output_orography_fieldname)
def lake_volume_extraction_driver(lake_parameters_filepath, basin_catchment_numbers_filepath, lake_volumes_out_filepath): flood_volume_thresholds = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname="flood_volume_thresholds") merge_types = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname="merge_points") basin_catchment_numbers = iodriver.advanced_field_loader(basin_catchment_numbers_filepath, field_type="Generic", fieldname="basin_catchment_numbers") lake_volumes = extract_lake_volumes(flood_volume_thresholds, basin_catchment_numbers,merge_types) iodriver.advanced_field_writer(lake_volumes_out_filepath,lake_volumes, fieldname="lake_volume")
def reduce_connected_areas_to_points(input_minima_file, input_minima_fieldname, output_minima_file, output_minima_fieldname, use_diagonals=True): input_minima = iodriver.advanced_field_loader( input_minima_file, field_type='Generic', fieldname=input_minima_fieldname) minima_array = np.ascontiguousarray(input_minima.get_data(), dtype=np.int32) lake_operators_wrapper.reduce_connected_areas_to_points( minima_array, use_diagonals) output_minima = field.Field(minima_array, grid=input_minima.get_grid()) iodriver.advanced_field_writer(output_minima_file, output_minima, fieldname=output_minima_fieldname)
def advanced_main(rdirs_filename, output_filename, rdirs_fieldname, output_fieldname): rdirs = iodriver.advanced_field_loader(rdirs_filename, field_type="Generic", fieldname=rdirs_fieldname) nlat, nlong = rdirs.get_grid().get_grid_dimensions() paths_map = field.Field(create_hypothetical_river_paths_map( riv_dirs=rdirs.get_data(), lsmask=None, use_f2py_func=True, use_f2py_sparse_iterator=True, nlat=nlat, nlong=nlong), grid=rdirs.get_grid()) iodriver.advanced_field_writer(target_filename=output_filename, field=paths_map, fieldname=output_fieldname)
def advanced_burn_carved_rivers_driver( input_orography_file, input_orography_fieldname, input_rdirs_file, input_rdirs_fieldname, input_minima_file, input_minima_fieldname, input_lakemask_file, input_lakemask_fieldname, output_orography_file, output_orography_fieldname, add_slope=False, max_exploration_range=0, minimum_height_change_threshold=0.0, short_path_threshold=0, short_minimum_height_change_threshold=0.0): input_orography = iodriver.advanced_field_loader( input_orography_file, field_type='Orography', fieldname=input_orography_fieldname) input_rdirs = iodriver.advanced_field_loader( input_rdirs_file, field_type='Orography', fieldname=input_rdirs_fieldname) input_minima = iodriver.advanced_field_loader( input_minima_file, field_type='Orography', fieldname=input_minima_fieldname) input_lakemask = iodriver.advanced_field_loader( input_lakemask_file, field_type='Generic', fieldname=input_lakemask_fieldname) output_orography = field.Field(np.ascontiguousarray( input_orography.get_data(), dtype=np.float64), grid=input_orography.get_grid()) lake_operators_wrapper.burn_carved_rivers( output_orography.get_data(), np.ascontiguousarray(input_rdirs.get_data(), dtype=np.float64), np.ascontiguousarray(input_minima.get_data(), dtype=np.int32), np.ascontiguousarray(input_lakemask.get_data(), dtype=np.int32), add_slope, max_exploration_range, minimum_height_change_threshold, short_path_threshold, short_minimum_height_change_threshold) iodriver.advanced_field_writer(output_orography_file, output_orography, fieldname=output_orography_fieldname)
def prepare_index_fields(): nlat= 48 nlon = 96 lat_index_filename= "/Users/thomasriddick/Documents/data/temp/lat_index_file.nc" lon_index_filename= "/Users/thomasriddick/Documents/data/temp/lon_index_file.nc" lat_indices = np.arange(1,nlat+1) lon_indices = np.arange(1,nlon+1) lat_indices_field = np.transpose(np.tile(lat_indices,(nlon,1))) lon_indices_field = np.tile(lon_indices,(nlat,1)) gaussian_grid = grid.makeGrid("T31") iodriver.advanced_field_writer(lat_index_filename, Field(lat_indices_field, grid=gaussian_grid), fieldname="lat_index") iodriver.advanced_field_writer(lon_index_filename, Field(lon_indices_field, grid=gaussian_grid), fieldname="lon_index")
def advanced_water_redistribution_driver( input_lake_numbers_file, input_lake_numbers_fieldname, input_lake_centers_file, input_lake_centers_fieldname, input_water_to_redistribute_file, input_water_to_redistribute_fieldname, output_water_redistributed_to_lakes_file, output_water_redistributed_to_lakes_fieldname, output_water_redistributed_to_rivers_file, output_water_redistributed_to_rivers_fieldname, coarse_grid_type, **coarse_grid_kwargs): lake_numbers = iodriver.advanced_field_loader( input_lake_numbers_file, field_type='Generic', fieldname=input_lake_numbers_fieldname) lake_centers = iodriver.advanced_field_loader( input_lake_centers_file, field_type='Generic', fieldname=input_lake_centers_fieldname) water_to_redistribute = \ iodriver.advanced_field_loader(input_water_to_redistribute_file, field_type='Generic', fieldname=input_water_to_redistribute_fieldname) fine_grid = lake_numbers.get_grid() fine_shape = lake_numbers.get_data().shape coarse_grid = grid.makeGrid(coarse_grid_type, **coarse_grid_kwargs) water_redistributed_to_lakes = field.Field( np.zeros(fine_shape, dtype=np.float64, order='C'), fine_grid) water_redistributed_to_rivers = field.Field( coarse_grid.create_empty_field(np.float64), coarse_grid) lake_operators_wrapper.redistribute_water( np.ascontiguousarray(lake_numbers.get_data(), dtype=np.int32), np.ascontiguousarray(lake_centers.get_data(), dtype=np.int32), np.ascontiguousarray(water_to_redistribute.get_data(), dtype=np.float64), water_redistributed_to_lakes.get_data(), water_redistributed_to_rivers.get_data()) iodriver.advanced_field_writer( output_water_redistributed_to_lakes_file, water_redistributed_to_lakes, fieldname=output_water_redistributed_to_lakes_fieldname) iodriver.advanced_field_writer( output_water_redistributed_to_rivers_file, water_redistributed_to_rivers, fieldname=output_water_redistributed_to_rivers_fieldname)
def advanced_river_mouth_marking_driver(input_river_directions_filename, output_river_directions_filename, input_river_directions_fieldname, output_river_directions_fieldname, lsmask_filename=None, lsmask_fieldname=None): rdirs = iodriver.advanced_field_loader(input_river_directions_filename, field_type="RiverDirections", fieldname=input_river_directions_fieldname) if lsmask_filename is not None: lsmask = iodriver.advanced_field_loader(lsmask_filename, field_type="Generic", fieldname=lsmask_fieldname) else: lsmask=None rdirs.mark_river_mouths(lsmask) iodriver.advanced_field_writer(output_river_directions_filename, field=rdirs, fieldname=output_river_directions_fieldname)
def add_lake_bathymetry_driver(input_orography_file, input_orography_fieldname, input_bathymetry_file, input_bathymetry_fieldname, lake_mask_file, lake_mask_fieldname, output_orography_file, output_orography_fieldname): input_orography = iodriver.advanced_field_loader( input_orography_file, field_type='Generic', fieldname=input_orography_fieldname) input_bathymetry = iodriver.advanced_field_loader( input_bathymetry_file, field_type='Generic', fieldname=input_bathymetry_fieldname) lake_mask = iodriver.advanced_field_loader(lake_mask_file, field_type='Generic', fieldname=lake_mask_fieldname) lake_mask.change_dtype(np.bool) output_orography = add_lake_bathymetry(input_orography, input_bathymetry, lake_mask) iodriver.advanced_field_writer(output_orography_file, output_orography, fieldname=output_orography_fieldname)
def advanced_flow_to_rivermouth_calculation_driver(input_river_directions_filename, input_flow_to_cell_filename, output_flow_to_river_mouths_filename, input_river_directions_fieldname, input_flow_to_cell_fieldname, output_flow_to_river_mouths_fieldname): rdirs_field = iodriver.advanced_field_loader(input_river_directions_filename, field_type="RiverDirections", fieldname=input_river_directions_fieldname) flowtocell_field = iodriver.advanced_field_loader(input_flow_to_cell_filename, field_type="CumulativeFlow", fieldname=input_flow_to_cell_fieldname) rivermouth_field = field.makeField(rdirs_field.get_river_mouths(),'Generic', grid_type=rdirs_field.get_grid()) flowtorivermouths_field = field.makeField(flowtocell_field.\ find_cumulative_flow_at_outlets(rivermouth_field.\ get_data()), field_type="Generic", grid_type=rdirs_field.get_grid()) iodriver.advanced_field_writer(output_flow_to_river_mouths_filename, field=flowtorivermouths_field, fieldname=output_flow_to_river_mouths_fieldname)
def testFieldWritingAndLoadingWithLatLongFieldWithFlip(self): example_field = field.makeEmptyField('Generic', dtype=np.int64, grid_type='HD') lat_points = np.linspace(89.75, -89.75, 360, endpoint=True) lon_points = np.linspace(-179.75, 179.75, 720, endpoint=True) example_field.set_grid_coordinates((lat_points, lon_points)) example_field.data[20, 20] = 1 example_field.data[200, 20] = 2 example_field.data[20, 200] = 3 example_field.data[200, 200] = 4 example_field.flip_data_ud() iodriver.advanced_field_writer(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), example_field, fieldname='test_field', clobber=True) loaded_field = iodriver.advanced_field_loader(os.path.join( self.directory, "advancedfieldwritingandloadingtest.nc"), fieldname='test_field', adjust_orientation=False) np.testing.assert_array_equal(example_field.get_data(), loaded_field.get_data())
def advanced_orography_creation_driver(landsea_mask_filename, inclines_filename, orography_filename, landsea_mask_fieldname, inclines_fieldname, orography_fieldname): landsea_in = iodriver.advanced_field_loader( landsea_mask_filename, field_type="Generic", fieldname=landsea_mask_fieldname, adjust_orientation=True) inclines_in = iodriver.advanced_field_loader(inclines_filename, field_type="Generic", fieldname=inclines_fieldname, adjust_orientation=True) orography_in = field.makeEmptyField('Generic', np.float64, landsea_in.get_grid()) create_orography_wrapper.create_orography( np.ascontiguousarray(landsea_in.get_data(), dtype=np.int32), np.ascontiguousarray(inclines_in.get_data(), dtype=np.float64), np.ascontiguousarray(orography_in.get_data(), dtype=np.float64)) iodriver.advanced_field_writer(orography_filename, orography_in, fieldname=orography_fieldname)
def advanced_main(filename, fieldname, output_filename, output_fieldname, loop_logfile, use_cpp_alg=True): rdirs = iodriver.advanced_field_loader(filename, field_type='Generic', fieldname=fieldname) nlat, nlon = rdirs.get_grid_dimensions() if use_cpp_alg: catchments = compute_catchments_cpp(rdirs.get_data(), loop_logfile) else: catchment_types, catchments = compute_catchments( rdirs.get_data(), loop_logfile) check_catchment_types(catchment_types, logfile=path.splitext(output_filename)[0] + ".log") numbered_catchments = field.Field(renumber_catchments_by_size( catchments, loop_logfile), grid=rdirs.get_grid()) iodriver.advanced_field_writer(target_filename=output_filename, field=numbered_catchments, fieldname=output_fieldname)
def advanced_drive_orography_upscaling(input_fine_orography_file, output_coarse_orography_file, input_orography_fieldname, output_coarse_orography_fieldname, landsea_file=None, true_sinks_file=None, landsea_fieldname=None, true_sinks_fieldname=None, upscaling_parameters_filename=None, scaling_factor=3): if upscaling_parameters_filename: config = read_and_validate_config(upscaling_parameters_filename) method = config.getint("orography_upscaling_parameters", "method") add_slope_in = config.getboolean("orography_upscaling_parameters", "add_slope_in") epsilon_in = config.getfloat("orography_upscaling_parameters", "epsilon_in") tarasov_separation_threshold_for_returning_to_same_edge_in =\ config.getint("orography_upscaling_parameters", "tarasov_separation_threshold_for_returning_to_same_edge_in") tarasov_min_path_length_in = config.getfloat( "orography_upscaling_parameters", "tarasov_min_path_length_in") tarasov_include_corners_in_same_edge_criteria_in = \ config.getboolean("orography_upscaling_parameters", "tarasov_include_corners_in_same_edge_criteria_in") else: #use defaults method = 1 add_slope_in = False epsilon_in = 0.1 tarasov_separation_threshold_for_returning_to_same_edge_in = 5 tarasov_min_path_length_in = 2.0 tarasov_include_corners_in_same_edge_criteria_in = False input_orography = iodriver.advanced_field_loader( input_fine_orography_file, field_type='Orography', fieldname=input_orography_fieldname) nlat_fine, nlon_fine = input_orography.get_grid_dimensions() lat_pts_fine, lon_pts_fine = input_orography.get_grid_coordinates() nlat_coarse,nlon_coarse,lat_pts_coarse,lon_pts_coarse = \ coordinate_scaling_utilities.generate_coarse_coords(nlat_fine,nlon_fine, lat_pts_fine,lon_pts_fine, scaling_factor) output_orography = field.makeEmptyField(field_type='Orography', dtype=np.float64, grid_type='LatLong', nlat=nlat_coarse, nlong=nlon_coarse) output_orography.set_grid_coordinates([lat_pts_coarse, lon_pts_coarse]) if landsea_file: landsea_mask = iodriver.advanced_field_loader( landsea_file, field_type='Generic', fieldname=landsea_fieldname) else: landsea_mask = field.makeEmptyField(field_type='Generic', dtype=np.int32, grid_type='LatLong', nlat=nlat_fine, nlong=nlon_fine) if true_sinks_file: true_sinks = iodriver.advanced_field_loader( true_sinks_file, field_type='Generic', fieldname=true_sinks_fieldname) else: true_sinks = field.makeEmptyField(field_type='Generic', dtype=np.int32, grid_type='LatLong', nlat=nlat_fine, nlong=nlon_fine) if not input_orography.get_data().dtype is np.float64(): input_orography.change_dtype(np.float64) #Make sure old data type array is flushed out of memory immediately gc.collect() upscale_orography_wrapper.upscale_orography(orography_in=input_orography.get_data(), orography_out=output_orography.get_data(), method=method,landsea_in=landsea_mask.get_data(), true_sinks_in=true_sinks.get_data(), add_slope_in=add_slope_in, epsilon_in=epsilon_in, tarasov_separation_threshold_for_returning_to_same_edge_in=\ tarasov_separation_threshold_for_returning_to_same_edge_in, tarasov_min_path_length_in=tarasov_min_path_length_in, tarasov_include_corners_in_same_edge_criteria_in=\ tarasov_include_corners_in_same_edge_criteria_in) iodriver.advanced_field_writer(output_coarse_orography_file, output_orography)
def apply_tweaks(self, initial_corrections_filename, output_corrections_filename, corrections_list_filename, corrections_type): if self.working_directory is None: raise RuntimeError("Applying tweaks requires a working directory to be specified") print("Note - All adjustments must be relative to a 180 degree W to 180 degree E," "90 degree N to 90 degree S grid") original_orography_filename = join(self.orography_path, "ice5g_v1_2_00_0k_10min.nc") intermediary_orography_filename = join(self.working_directory, "pre_{}_tweak_orography.nc".\ format("final" if corrections_type == CorrectionTypes.FINAL else "preliminary")) corrected_orography_filename = join(self.working_directory, "post_{}_tweak_orography.nc".\ format("final" if corrections_type == CorrectionTypes.FINAL else "preliminary")) utilities.advanced_apply_orog_correction_field(original_orography_filename= original_orography_filename, orography_corrections_filename= initial_corrections_filename, corrected_orography_filename= intermediary_orography_filename, original_orography_fieldname="orog", orography_corrections_fieldname="orog", corrected_orography_fieldname="orog") orography_field = iodriver.advanced_field_loader(intermediary_orography_filename, field_type='Orography', fieldname="orog") read_lines_from_file=False corrections_list = [] with open(corrections_list_filename,"r") as corr_list_file: if not self.first_line_pattern.match(corr_list_file.readline().strip()): raise RuntimeError("List of corrections being loaded has" " incorrect header format") for line in corr_list_file: if self.prelim_match.match(line): if corrections_type == CorrectionTypes.PRELIMINARY: read_lines_from_file=True else: read_lines_from_file=False elif self.final_match.match(line): if corrections_type == CorrectionTypes.FINAL: read_lines_from_file=True else: read_lines_from_file=False elif self.comment_line_match.match(line): continue elif read_lines_from_file: corrections_list.append(tuple(int(value) if i < 2 else float(value) \ for i,value in enumerate(line.strip().split(",")))) for lat,lon,height in corrections_list: print("Correcting height of lat={0},lon={1} to {2} m".format(lat,lon,height)) orography_field.get_data()[lat,lon] = height iodriver.advanced_field_writer(corrected_orography_filename, orography_field, fieldname="orog") utilities.advanced_orog_correction_field_generator(original_orography_filename= original_orography_filename, corrected_orography_filename= corrected_orography_filename, orography_corrections_filename= output_corrections_filename, original_orography_fieldname= "orog", corrected_orography_fieldname= "orog", orography_corrections_fieldname= "orog")
def advanced_loop_breaker_driver(input_coarse_rdirs_filepath,input_coarse_cumulative_flow_filepath, input_coarse_catchments_filepath,input_fine_rdirs_filepath, input_fine_cumulative_flow_filepath,output_updated_coarse_rdirs_filepath, input_coarse_rdirs_fieldname,input_coarse_cumulative_flow_fieldname, input_coarse_catchments_fieldname,input_fine_rdirs_fieldname, input_fine_cumulative_flow_fieldname,output_updated_coarse_rdirs_fieldname, loop_nums_list_filepath, scaling_factor): """Drive the FORTRAN code to remove more complex loops from a field of river directions Arguments: input_coarse_rdirs_filepath: string, full path to input coarse river directions to remove loops from input_coarse_cumulative_flow_filepath: string, full path to the input coarse cumulative flow file input_coarse_catchments_filepath: string, full path to the coarse input catchments file input_fine_rdirs_filepath: string, full path to the fine input river directions the coarse input river directions were upscaled from input_fine_cumulative_flow_filepath: string, full path to the catchments generated from the fine input river directions output_updated_coarse_rdirs_filepath: string, full path to write the coarse river direction with the specified loops removed too loop_nums_list_filepath: string, full path to the file contain the catchment numbers of the loops to remove, one per line, see code below for correct format for the first line Returns: nothing """ input_coarse_rdirs_field = iodriver.advanced_field_loader(input_coarse_rdirs_filepath, field_type='RiverDirections', fieldname=input_coarse_rdirs_fieldname) coarse_cumulative_flow_field =\ iodriver.advanced_field_loader(input_coarse_cumulative_flow_filepath, field_type='CumulativeFlow', fieldname=input_coarse_cumulative_flow_fieldname) coarse_catchments_field =\ iodriver.advanced_field_loader(input_coarse_catchments_filepath, field_type='Generic', fieldname=input_coarse_catchments_fieldname) fine_rdirs_field = iodriver.advanced_field_loader(input_fine_rdirs_filepath, field_type='RiverDirections', fieldname=input_fine_rdirs_fieldname) fine_cumulative_flow_field =\ iodriver.advanced_field_loader(input_fine_cumulative_flow_filepath, field_type='CumulativeFlow', fieldname=input_fine_cumulative_flow_fieldname) loop_nums_list = [] first_line_pattern = re.compile(r"^Loops found in catchments:$") with open(loop_nums_list_filepath,'r') as f: if not first_line_pattern.match(f.readline().strip()): raise RuntimeError("Format of the file with list of catchments to remove loops from" " is invalid") for line in f: loop_nums_list.append(int(line.strip())) print('Removing loops from catchments: ' + ", ".join(str(value) for value in loop_nums_list)) nlat_fine,nlon_fine = fine_rdirs_field.get_grid_dimensions() lat_pts_fine,lon_pts_fine = fine_rdirs_field.get_grid_coordinates() nlat_coarse,nlon_coarse,lat_pts_coarse,lon_pts_coarse = \ coordinate_scaling_utilities.generate_coarse_coords(nlat_fine,nlon_fine, lat_pts_fine,lon_pts_fine, scaling_factor) output_coarse_rdirs_field = run_loop_breaker(input_coarse_rdirs_field,coarse_cumulative_flow_field, coarse_catchments_field,fine_rdirs_field, fine_cumulative_flow_field,loop_nums_list, coarse_grid_type="LatLong",nlat=nlat_coarse, nlong=nlon_coarse) output_coarse_rdirs_field.set_grid_coordinates([lat_pts_coarse,lon_pts_coarse]) iodriver.advanced_field_writer(output_updated_coarse_rdirs_filepath, output_coarse_rdirs_field, fieldname=output_updated_coarse_rdirs_fieldname)
def advanced_basin_evaluation_driver( input_minima_file, input_minima_fieldname, input_raw_orography_file, input_raw_orography_fieldname, input_corrected_orography_file, input_corrected_orography_fieldname, input_cell_areas_file, input_cell_areas_fieldname, input_prior_fine_rdirs_file, input_prior_fine_rdirs_fieldname, input_prior_fine_catchments_file, input_prior_fine_catchments_fieldname, input_coarse_catchment_nums_file, input_coarse_catchment_nums_fieldname, input_coarse_rdirs_file, input_coarse_rdirs_fieldname, combined_output_filename, output_filepath, output_filelabel, output_basin_catchment_nums_filepath=None): input_minima = iodriver.advanced_field_loader( input_minima_file, field_type='Generic', fieldname=input_minima_fieldname) input_raw_orography = iodriver.advanced_field_loader( input_raw_orography_file, field_type='Orography', fieldname=input_raw_orography_fieldname) input_corrected_orography = iodriver.advanced_field_loader( input_corrected_orography_file, field_type='Orography', fieldname=input_corrected_orography_fieldname) input_cell_areas = iodriver.advanced_field_loader( input_cell_areas_file, field_type='Generic', fieldname=input_cell_areas_fieldname) input_prior_fine_rdirs = iodriver.advanced_field_loader( input_prior_fine_rdirs_file, field_type='RiverDirections', fieldname=input_prior_fine_rdirs_fieldname) input_prior_fine_catchments = iodriver.advanced_field_loader( input_prior_fine_catchments_file, field_type='Generic', fieldname=input_prior_fine_catchments_fieldname) input_coarse_catchment_nums = iodriver.advanced_field_loader( input_coarse_catchment_nums_file, field_type='Generic', fieldname=input_coarse_catchment_nums_fieldname) input_coarse_rdirs = iodriver.advanced_field_loader( input_coarse_rdirs_file, field_type='Generic', fieldname=input_coarse_rdirs_fieldname) fine_grid = input_raw_orography.get_grid() fine_shape = input_raw_orography.get_data().shape connection_volume_thresholds = field.Field( np.zeros(fine_shape, dtype=np.float64, order='C'), fine_grid) flood_volume_thresholds = field.Field( np.zeros(fine_shape, dtype=np.float64, order='C'), fine_grid) flood_next_cell_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) flood_next_cell_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_next_cell_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_next_cell_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) flood_force_merge_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) flood_force_merge_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_force_merge_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_force_merge_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) flood_redirect_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) flood_redirect_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_redirect_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_redirect_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) additional_flood_redirect_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) additional_flood_redirect_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) additional_connect_redirect_lat_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) additional_connect_redirect_lon_index = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) flood_local_redirect = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) connect_local_redirect = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) additional_flood_local_redirect = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) additional_connect_local_redirect = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) merge_points = field.Field(np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) if output_basin_catchment_nums_filepath is not None: basin_catchment_numbers = field.Field( np.zeros(fine_shape, dtype=np.int32, order='C'), fine_grid) else: basin_catchment_numbers = None evaluate_basins_wrapper.evaluate_basins( minima_in_int=np.ascontiguousarray(input_minima.get_data(), dtype=np.int32), raw_orography_in=np.ascontiguousarray(input_raw_orography.get_data(), dtype=np.float64), corrected_orography_in=np.ascontiguousarray( input_corrected_orography.get_data(), dtype=np.float64), cell_areas_in=np.ascontiguousarray(input_cell_areas.get_data(), dtype=np.float64), connection_volume_thresholds_in=connection_volume_thresholds.get_data( ), flood_volume_thresholds_in=flood_volume_thresholds.get_data(), prior_fine_rdirs_in=np.ascontiguousarray( input_prior_fine_rdirs.get_data(), dtype=np.float64), prior_coarse_rdirs_in=np.ascontiguousarray( input_coarse_rdirs.get_data(), dtype=np.float64), prior_fine_catchments_in=np.ascontiguousarray( input_prior_fine_catchments.get_data(), dtype=np.int32), coarse_catchment_nums_in=np.ascontiguousarray( input_coarse_catchment_nums.get_data(), dtype=np.int32), flood_next_cell_lat_index_in=flood_next_cell_lat_index.get_data(), flood_next_cell_lon_index_in=flood_next_cell_lon_index.get_data(), connect_next_cell_lat_index_in=connect_next_cell_lat_index.get_data(), connect_next_cell_lon_index_in=connect_next_cell_lon_index.get_data(), flood_force_merge_lat_index_in=flood_force_merge_lat_index.get_data(), flood_force_merge_lon_index_in=flood_force_merge_lon_index.get_data(), connect_force_merge_lat_index_in=connect_force_merge_lat_index. get_data(), connect_force_merge_lon_index_in=connect_force_merge_lon_index. get_data(), flood_redirect_lat_index_in=flood_redirect_lat_index.get_data(), flood_redirect_lon_index_in=flood_redirect_lon_index.get_data(), connect_redirect_lat_index_in=connect_redirect_lat_index.get_data(), connect_redirect_lon_index_in=connect_redirect_lon_index.get_data(), additional_flood_redirect_lat_index_in= additional_flood_redirect_lat_index.get_data(), additional_flood_redirect_lon_index_in= additional_flood_redirect_lon_index.get_data(), additional_connect_redirect_lat_index_in= additional_connect_redirect_lat_index.get_data(), additional_connect_redirect_lon_index_in= additional_connect_redirect_lon_index.get_data(), flood_local_redirect_out_int=flood_local_redirect.get_data(), connect_local_redirect_out_int=connect_local_redirect.get_data(), additional_flood_local_redirect_out_int=additional_flood_local_redirect .get_data(), additional_connect_local_redirect_out_int= additional_connect_local_redirect.get_data(), merge_points_out_int=merge_points.get_data(), basin_catchment_numbers_in=basin_catchment_numbers.get_data()) connection_volume_thresholds_filename = path.join( output_filepath, "connect_vts_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connection_volume_thresholds_filename, connection_volume_thresholds, fieldname='connection_volume_thresholds') flood_volume_thresholds_filename = path.join( output_filepath, "flood_vts_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_volume_thresholds_filename, flood_volume_thresholds, fieldname='flood_volume_thresholds') flood_next_cell_lat_index_filename = path.join( output_filepath, "flood_nci_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_next_cell_lat_index_filename, flood_next_cell_lat_index, fieldname='flood_next_cell_lat_index') flood_next_cell_lon_index_filename = path.join( output_filepath, "flood_nci_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_next_cell_lon_index_filename, flood_next_cell_lon_index, fieldname='flood_next_cell_lon_index') connect_next_cell_lat_index_filename = path.join( output_filepath, "connect_nci_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_next_cell_lat_index_filename, connect_next_cell_lat_index, fieldname='connect_next_cell_lat_index') connect_next_cell_lon_index_filename = path.join( output_filepath, "connect_nci_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_next_cell_lon_index_filename, connect_next_cell_lon_index, fieldname='connect_next_cell_lon_index') flood_force_merge_lat_index_filename = path.join( output_filepath, "flood_fmi_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_force_merge_lat_index_filename, flood_force_merge_lat_index, fieldname='flood_force_merge_lat_index') flood_force_merge_lon_index_filename = path.join( output_filepath, "flood_fmi_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_force_merge_lon_index_filename, flood_force_merge_lon_index, fieldname='flood_force_merge_lon_index') connect_force_merge_lat_index_filename = path.join( output_filepath, "connect_fmi_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_force_merge_lat_index_filename, connect_force_merge_lat_index, fieldname='connect_force_merge_lat_index') connect_force_merge_lon_index_filename = path.join( output_filepath, "connect_fmi_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_force_merge_lon_index_filename, connect_force_merge_lon_index, fieldname='connect_force_merge_lon_index') flood_redirect_lat_index_filename = path.join( output_filepath, "flood_ri_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_redirect_lat_index_filename, flood_redirect_lat_index, fieldname='flood_redirect_lat_index') flood_redirect_lon_index_filename = path.join( output_filepath, "flood_ri_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_redirect_lon_index_filename, flood_redirect_lon_index, fieldname='flood_redirect_lon_index') connect_redirect_lat_index_filename = path.join( output_filepath, "connect_ri_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_redirect_lat_index_filename, connect_redirect_lat_index, fieldname='connect_redirect_lat_index') connect_redirect_lon_index_filename = path.join( output_filepath, "connect_ri_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_redirect_lon_index_filename, connect_redirect_lon_index, fieldname='connect_redirect_lon_index') additional_flood_redirect_lat_index_filename = path.join( output_filepath, "additional_flood_ri_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer( additional_flood_redirect_lat_index_filename, additional_flood_redirect_lat_index, fieldname='additional_flood_redirect_lat_index') additional_flood_redirect_lon_index_filename = path.join( output_filepath, "additional_flood_ri_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer( additional_flood_redirect_lon_index_filename, additional_flood_redirect_lon_index, fieldname='additional_flood_redirect_lon_index') additional_connect_redirect_lat_index_filename = path.join( output_filepath, "additional_connect_ri_lat_" + output_filelabel + ".nc") iodriver.advanced_field_writer( additional_connect_redirect_lat_index_filename, additional_connect_redirect_lat_index, fieldname='additional_connect_redirect_lat_index') additional_connect_redirect_lon_index_filename = path.join( output_filepath, "additional_connect_ri_lon_" + output_filelabel + ".nc") iodriver.advanced_field_writer( additional_connect_redirect_lon_index_filename, additional_connect_redirect_lon_index, fieldname='additional_connect_redirect_lon_index') flood_local_redirect_filename = path.join( output_filepath, "flood_local_r_" + output_filelabel + ".nc") iodriver.advanced_field_writer(flood_local_redirect_filename, flood_local_redirect, fieldname='flood_local_redirect') connect_local_redirect_filename = path.join( output_filepath, "connect_local_r_" + output_filelabel + ".nc") iodriver.advanced_field_writer(connect_local_redirect_filename, connect_local_redirect, fieldname='connect_local_redirect') additional_flood_local_redirect_filename = path.join( output_filepath, "additional_flood_local_r_" + output_filelabel + ".nc") iodriver.advanced_field_writer(additional_flood_local_redirect_filename, additional_flood_local_redirect, fieldname='additional_flood_local_redirect') additional_connect_local_redirect_filename = path.join( output_filepath, "additional_connect_local_r_" + output_filelabel + ".nc") iodriver.advanced_field_writer( additional_connect_local_redirect_filename, additional_connect_local_redirect, fieldname='additional_connect_local_redirect') merge_points_filename = path.join( output_filepath, "merge_points_" + output_filelabel + ".nc") iodriver.advanced_field_writer(merge_points_filename, merge_points, fieldname='merge_points') lake_centers_filename = path.join( output_filepath, "lake_centers_" + output_filelabel + ".nc") iodriver.advanced_field_writer(lake_centers_filename, input_minima, fieldname="lake_centers") individual_field_filenames = [ connection_volume_thresholds_filename, flood_volume_thresholds_filename, flood_next_cell_lat_index_filename, flood_next_cell_lon_index_filename, connect_next_cell_lat_index_filename, connect_next_cell_lon_index_filename, flood_force_merge_lat_index_filename, flood_force_merge_lon_index_filename, connect_force_merge_lat_index_filename, connect_force_merge_lon_index_filename, flood_redirect_lat_index_filename, flood_redirect_lon_index_filename, connect_redirect_lat_index_filename, connect_redirect_lon_index_filename, additional_flood_redirect_lat_index_filename, additional_flood_redirect_lon_index_filename, additional_connect_redirect_lat_index_filename, additional_connect_redirect_lon_index_filename, flood_local_redirect_filename, connect_local_redirect_filename, additional_flood_local_redirect_filename, additional_connect_local_redirect_filename, merge_points_filename, lake_centers_filename ] cdo_inst = cdo.Cdo() cdo_inst.merge(input=" ".join(individual_field_filenames), output=combined_output_filename) for individual_field_filename in individual_field_filenames: os.remove(individual_field_filename) if output_basin_catchment_nums_filepath is not None: iodriver.advanced_field_writer(output_basin_catchment_nums_filepath, basin_catchment_numbers, fieldname="basin_catchment_numbers")
def advanced_sinkless_flow_directions_generator(filename,output_filename,fieldname, output_fieldname,ls_mask_filename=None, truesinks_filename=None, catchment_nums_filename=None, ls_mask_fieldname=None, truesinks_fieldname=None, catchment_fieldname=None): orography = iodriver.advanced_field_loader(filename, field_type='Orography', fieldname=fieldname) grid_dims=orography.get_grid().get_grid_dimensions() grid_num_points = orography.get_grid().get_npoints() threshold_for_low_mem_version_usage = 500000000 if (grid_num_points < threshold_for_low_mem_version_usage): rdirs = np.empty(grid_dims,dtype=np.float64,order='C') if not truesinks_filename: truesinks = Field(np.empty((1,1),dtype=np.int32),grid='HD') use_true_sinks = False; else: use_true_sinks = True; truesinks = iodriver.advanced_field_loader(truesinks_filename, field_type='Generic', fieldname=truesinks_fieldname) if ls_mask_filename is None: use_ls_mask = False ls_mask = field.makeEmptyField(field_type='Generic',dtype=np.int32,grid_type='HD') else: use_ls_mask = True ls_mask = iodriver.advanced_field_loader(ls_mask_filename, field_type='Generic', fieldname=ls_mask_fieldname) catchment_nums = np.zeros(grid_dims,dtype=np.int32,order='C') next_cell_lat_index_in = np.zeros(grid_dims,dtype=np.int32,order='C') next_cell_lon_index_in = np.zeros(grid_dims,dtype=np.int32,order='C') if (grid_num_points >= threshold_for_low_mem_version_usage): rdirs = fill_sinks_wrapper_low_mem.fill_sinks_cpp_func(orography_array=np.ascontiguousarray(orography.get_data(), #@UndefinedVariable dtype=np.float64), method = 4, use_ls_mask = use_ls_mask, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = use_true_sinks, true_sinks_in = np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), next_cell_lat_index_in = next_cell_lat_index_in, next_cell_lon_index_in = next_cell_lon_index_in, catchment_nums_in = catchment_nums, prefer_non_diagonal_initial_dirs = False) else: fill_sinks_wrapper.fill_sinks_cpp_func(orography_array=np.ascontiguousarray(orography.get_data(), #@UndefinedVariable dtype=np.float64), method = 4, use_ls_mask = use_ls_mask, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = use_true_sinks, true_sinks_in = np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), next_cell_lat_index_in = next_cell_lat_index_in, next_cell_lon_index_in = next_cell_lon_index_in, rdirs_in = rdirs, catchment_nums_in = catchment_nums, prefer_non_diagonal_initial_dirs = False) iodriver.advanced_field_writer(output_filename,Field(rdirs,grid=orography.get_grid()), fieldname=output_fieldname) if catchment_nums_filename: iodriver.advanced_field_writer(catchment_nums_filename, field=Field(catchment_nums,grid=orography.get_grid()), fieldname=catchment_fieldname)
def no_intermediaries_combine_hydrosheds_plus_rdirs_from_corrected_orog_driver( self): """Combines Hydrosheds river direction with those from a corrected orography and possibly removes sinks""" config = self._read_and_validate_config() final_loops_log_filename = path.splitext( self.output_catchments_filename)[0] + '_loops.log' if self.rerun_post_processing_only is not None: final_rdirs = \ iodriver.advanced_field_loader(self.rerun_post_processing_only, field_type="RiverDirections", fieldname=config.get("input_fieldname_options", "river_directions_to_reprocess_fieldname")) else: orography_filename = config.get( "input_options", "ten_minute_corrected_orography_filename") orography = iodriver.advanced_field_loader( orography_filename, field_type='Orography', fieldname=config.get( "input_fieldname_options", "ten_minute_corrected_orography_fieldname")) hydrosheds_rdirs_au_af_sa_10min_filename = \ config.get("input_options","ten_minute_hydrosheds_au_auf_sa_river_directions_filename") hydrosheds_rdirs = \ iodriver.advanced_field_loader(hydrosheds_rdirs_au_af_sa_10min_filename, field_type="RiverDirections", fieldname=config.get("input_fieldname_options", "ten_minute_hydrosheds_au_auf_sa_river_directions_fieldname")) second_intermediate_loops_log_filename = tempfile.mkstemp( suffix=".txt", prefix="loops_log_second_int", dir="")[1] truesinks_dummy = field.makeEmptyField("Generic", np.bool_, grid_type="LatLong10min") truesinks_dummy.set_all(False) if self.true_sinks_filename is not None: use_true_sinks = True truesinks = iodriver.advanced_field_loader( self.true_sinks_filename, field_type="Generic", fieldname=config.get("input_fieldname_options", "input_truesinks_fieldname")) else: use_true_sinks = False if config.getboolean( "general_options", "replace_internal_basins_with_rdirs_with_truesinks"): warnings.warn( "Option replace_internal_basins_with_rdirs_with_truesinks " "ignored when no true sinks file is specified") first_intermediate_rdirs = field.makeEmptyField( "RiverDirections", np.float64, grid_type="LatLong10min") if use_true_sinks: first_intermediate_rdirs_no_sinks = field.makeEmptyField( "RiverDirections", np.float64, grid_type="LatLong10min") ls_mask = iodriver.advanced_field_loader( self.ls_mask_filename, field_type="Generic", fieldname=config.get("input_fieldname_options", "input_landsea_mask_fieldname")) ls_mask.set_data( np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32)) next_cell_lat_index_in_dummy = np.zeros(ls_mask.get_data().shape, dtype=np.int32, order='C') next_cell_lon_index_in_dummy = np.zeros(ls_mask.get_data().shape, dtype=np.int32, order='C') catchment_nums_dummy = np.zeros(ls_mask.get_data().shape, dtype=np.int32, order='C') fill_sinks_wrapper.fill_sinks_cpp_func(orography_array= np.ascontiguousarray(orography.get_data(), #@UndefinedVariable dtype=np.float64), method = 4, use_ls_mask = True, landsea_in = np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag = False, use_true_sinks = False, true_sinks_in = np.ascontiguousarray(truesinks_dummy.\ get_data(), dtype=np.int32), next_cell_lat_index_in = next_cell_lat_index_in_dummy, next_cell_lon_index_in = next_cell_lon_index_in_dummy, rdirs_in = first_intermediate_rdirs.get_data() if not use_true_sinks else first_intermediate_rdirs_no_sinks.get_data(), catchment_nums_in = catchment_nums_dummy, prefer_non_diagonal_initial_dirs = False) if use_true_sinks: fill_sinks_wrapper.fill_sinks_cpp_func( orography_array=np.ascontiguousarray( orography.get_data(), #@UndefinedVariable dtype=np.float64), method=4, use_ls_mask=True, landsea_in=np.ascontiguousarray(ls_mask.get_data(), dtype=np.int32), set_ls_as_no_data_flag=False, use_true_sinks=True, true_sinks_in=np.ascontiguousarray(truesinks.get_data(), dtype=np.int32), next_cell_lat_index_in=next_cell_lat_index_in_dummy, next_cell_lon_index_in=next_cell_lon_index_in_dummy, rdirs_in=first_intermediate_rdirs.get_data(), catchment_nums_in=catchment_nums_dummy, prefer_non_diagonal_initial_dirs=False) second_intermediate_rdirs = utilities.splice_rdirs( rdirs_matching_ls_mask=first_intermediate_rdirs, ls_mask=ls_mask, other_rdirs=hydrosheds_rdirs) second_intermediate_catchments = comp_catchs.compute_catchments_cpp( second_intermediate_rdirs.get_data(), second_intermediate_loops_log_filename) second_intermediate_catchments = field.Field(comp_catchs.\ renumber_catchments_by_size(second_intermediate_catchments, second_intermediate_loops_log_filename), grid="LatLong10min") if config.getboolean("general_options", "keep_all_internal_basins"): third_intermediate_rdirs = second_intermediate_rdirs else: third_intermediate_rdirs = \ utilities.remove_endorheic_basins(rdirs=second_intermediate_rdirs, catchments=second_intermediate_catchments, rdirs_without_endorheic_basins= first_intermediate_rdirs_no_sinks if (use_true_sinks and not config.getboolean('general_options', 'replace_internal_basins_with_rdirs_with_truesinks')) else first_intermediate_rdirs, replace_only_catchments=([int(value) for value in config.get('general_options', 'replace_only_catchments').\ split(",")] if config.get('general_options', 'replace_only_catchments') else []), exclude_catchments=([int(value) for value in config.get('general_options', 'exclude_catchments').\ split(",")] if config.get('general_options', 'exclude_catchments') else []) ) third_intermediate_flowtocell = field.\ CumulativeFlow(create_hypothetical_river_paths_map(riv_dirs=third_intermediate_rdirs.get_data(), lsmask=None, use_f2py_func=True, use_f2py_sparse_iterator=True, nlat=1080,nlong=2160), grid='LatLong10min') third_intermediate_rdirs.make_contiguous() third_intermediate_flowtocell.make_contiguous() first_intermediate_rdirs.make_contiguous() if use_true_sinks: first_intermediate_rdirs_no_sinks.make_contiguous() final_rdirs = utilities.replace_streams_downstream_from_loop( third_intermediate_rdirs, cumulative_flow=third_intermediate_flowtocell, other_rdirs=first_intermediate_rdirs if not use_true_sinks else first_intermediate_rdirs_no_sinks) final_catchments = comp_catchs.compute_catchments_cpp( final_rdirs.get_data(), final_loops_log_filename) final_catchments = field.Field(comp_catchs.\ renumber_catchments_by_size(final_catchments, final_loops_log_filename), grid="LatLong10min") final_flowtocell = field.CumulativeFlow( create_hypothetical_river_paths_map( riv_dirs=final_rdirs.get_data(), lsmask=None, use_f2py_func=True, use_f2py_sparse_iterator=True, nlat=1080, nlong=2160), grid='LatLong10min') if self.rerun_post_processing_only is None: iodriver.advanced_field_writer( self.output_rdirs_filename, final_rdirs, fieldname=config.get("output_fieldname_options", "output_river_directions_fieldname")) iodriver.advanced_field_writer(self.output_catchments_filename, final_catchments, fieldname=config.get( "output_fieldname_options", "output_catchments_fieldname")) iodriver.advanced_field_writer(self.output_cumulative_flow_filename, final_flowtocell, fieldname=config.get( "output_fieldname_options", "output_cumulative_flow_fieldname")) if self.rerun_post_processing_only is None: os.remove(second_intermediate_loops_log_filename)
def connect_coarse_lake_catchments_driver(coarse_catchments_filepath, lake_parameters_filepath, basin_catchment_numbers_filepath, river_directions_filepath, connected_coarse_catchments_out_filename, coarse_catchments_fieldname, connected_coarse_catchments_out_fieldname, basin_catchment_numbers_fieldname, river_directions_fieldname, cumulative_flow_filepath = None, connected_cumulative_flow_out_filepath=None, cumulative_flow_fieldname = None, connected_cumulative_flow_out_fieldname=None, scale_factor = 3): coarse_catchments = iodriver.advanced_field_loader(coarse_catchments_filepath, field_type='Generic', fieldname=\ coarse_catchments_fieldname) basin_catchment_numbers = iodriver.advanced_field_loader(basin_catchment_numbers_filepath, field_type='Generic', fieldname=\ basin_catchment_numbers_fieldname) lake_centers = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "lake_centers") flood_next_cell_index_lat = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "flood_next_cell_lat_index") flood_next_cell_index_lon = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "flood_next_cell_lon_index") flood_redirect_lat = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "flood_redirect_lat_index") flood_redirect_lon = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "flood_redirect_lon_index") additional_flood_redirect_lat = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "additional_flood_redirect_lat_index") additional_flood_redirect_lon = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname= "additional_flood_redirect_lon_index") local_redirect = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname=\ "flood_local_redirect") additional_local_redirect = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname=\ "additional_flood_local_redirect") merge_types = iodriver.advanced_field_loader(lake_parameters_filepath, field_type='Generic', fieldname=\ "merge_points") river_directions = iodriver.advanced_field_loader(river_directions_filepath, field_type='Generic', fieldname=\ river_directions_fieldname) if cumulative_flow_filepath is not None: cumulative_flow = iodriver.advanced_field_loader(cumulative_flow_filepath, field_type='Generic', fieldname=\ cumulative_flow_fieldname) catchments, corrected_cumulative_flow = \ connect_coarse_lake_catchments(coarse_catchments,lake_centers,basin_catchment_numbers, flood_next_cell_index_lat,flood_next_cell_index_lon, flood_redirect_lat,flood_redirect_lon, additional_flood_redirect_lat, additional_flood_redirect_lon, local_redirect,additional_local_redirect, merge_types,river_directions,scale_factor, cumulative_flow=(cumulative_flow if cumulative_flow_filepath is not None else None), correct_cumulative_flow=(True if cumulative_flow_filepath is not None else False)) iodriver.advanced_field_writer(connected_coarse_catchments_out_filename, field=catchments, fieldname=connected_coarse_catchments_out_fieldname) if cumulative_flow_filepath is not None: iodriver.advanced_field_writer(connected_cumulative_flow_out_filepath, field=corrected_cumulative_flow, fieldname=connected_cumulative_flow_out_fieldname)