def __init__(self, \ gpu_ctx, \ eta0, hu0, hv0, H, \ nx, ny, \ dx, dy, dt, \ g, f, r, \ angle=np.array([[0]], dtype=np.float32), \ t=0.0, \ theta=1.3, rk_order=2, \ coriolis_beta=0.0, \ max_wind_direction_perturbation = 0, \ wind_stress=WindStress.WindStress(), \ boundary_conditions=Common.BoundaryConditions(), \ boundary_conditions_data=Common.BoundaryConditionsData(), \ small_scale_perturbation=False, \ small_scale_perturbation_amplitude=None, \ small_scale_perturbation_interpolation_factor = 1, \ model_time_step=None, reportGeostrophicEquilibrium=False, \ use_lcg=False, \ write_netcdf=False, \ comm=None, \ netcdf_filename=None, \ ignore_ghostcells=False, \ courant_number=0.8, \ offset_x=0, offset_y=0, \ flux_slope_eps = 1.0e-1, \ desingularization_eps = 1.0e-1, \ depth_cutoff = 1.0e-5, \ block_width=32, block_height=8, num_threads_dt=256, block_width_model_error=16, block_height_model_error=16): """ Initialization routine eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells H: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners nx: Number of cells along x-axis ny: Number of cells along y-axis dx: Grid cell spacing along x-axis (20 000 m) dy: Grid cell spacing along y-axis (20 000 m) dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y r: Bottom friction coefficient (2.4e-3 m/s) angle: Angle of rotation from North to y-axis t: Start simulation at time t theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme rk_order: Order of Runge Kutta method {1,2*,3} coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0) max_wind_direction_perturbation: Large-scale model error emulation by per-time-step perturbation of wind direction by +/- max_wind_direction_perturbation (degrees) wind_stress: Wind stress parameters boundary_conditions: Boundary condition object small_scale_perturbation: Boolean value for applying a stochastic model error small_scale_perturbation_amplitude: Amplitude (q0 coefficient) for model error small_scale_perturbation_interpolation_factor: Width factor for correlation in model error model_time_step: The size of a data assimilation model step (default same as dt) reportGeostrophicEquilibrium: Calculate the Geostrophic Equilibrium variables for each superstep use_lcg: Use LCG as the random number generator. Default is False, which means using curand. write_netcdf: Write the results after each superstep to a netCDF file comm: MPI communicator desingularization_eps: Used for desingularizing hu/h flux_slope_eps: Used for setting zero flux for symmetric Riemann fan depth_cutoff: Used for defining dry cells netcdf_filename: Use this filename. (If not defined, a filename will be generated by SimWriter.) """ self.logger = logging.getLogger(__name__) assert( rk_order < 4 or rk_order > 0 ), "Only 1st, 2nd and 3rd order Runge Kutta supported" if (rk_order == 3): assert(r == 0.0), "3rd order Runge Kutta supported only without friction" # Sort out internally represented ghost_cells in the presence of given # boundary conditions ghost_cells_x = 2 ghost_cells_y = 2 #Coriolis at "first" cell x_zero_reference_cell = ghost_cells_x y_zero_reference_cell = ghost_cells_y # In order to pass it to the super constructor # Boundary conditions self.boundary_conditions = boundary_conditions if (boundary_conditions.isSponge()): nx = nx + boundary_conditions.spongeCells[1] + boundary_conditions.spongeCells[3] - 2*ghost_cells_x ny = ny + boundary_conditions.spongeCells[0] + boundary_conditions.spongeCells[2] - 2*ghost_cells_y x_zero_reference_cell += boundary_conditions.spongeCells[3] y_zero_reference_cell += boundary_conditions.spongeCells[2] #Compensate f for reference cell (first cell in internal of domain) north = np.array([np.sin(angle[0,0]), np.cos(angle[0,0])]) f = f - coriolis_beta * (x_zero_reference_cell*dx*north[0] + y_zero_reference_cell*dy*north[1]) x_zero_reference_cell = 0 y_zero_reference_cell = 0 A = None self.max_wind_direction_perturbation = max_wind_direction_perturbation super(CDKLM16, self).__init__(gpu_ctx, \ nx, ny, \ ghost_cells_x, \ ghost_cells_y, \ dx, dy, dt, \ g, f, r, A, \ t, \ theta, rk_order, \ coriolis_beta, \ y_zero_reference_cell, \ wind_stress, \ write_netcdf, \ ignore_ghostcells, \ offset_x, offset_y, \ comm, \ block_width, block_height) # Index range for interior domain (north, east, south, west) # so that interior domain of eta is # eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \ # self.interior_domain_indices[3]:self.interior_domain_indices[1] ] self.interior_domain_indices = np.array([-2,-2,2,2]) self._set_interior_domain_from_sponge_cells() defines={'block_width': block_width, 'block_height': block_height, 'KPSIMULATOR_DESING_EPS': str(desingularization_eps)+'f', 'KPSIMULATOR_FLUX_SLOPE_EPS': str(flux_slope_eps)+'f', 'KPSIMULATOR_DEPTH_CUTOFF': str(depth_cutoff)+'f'} #Get kernels self.kernel = gpu_ctx.get_kernel("CDKLM16_kernel.cu", defines=defines, compile_args={ # default, fast_math, optimal 'options' : ["--ftz=true", # false, true, true "--prec-div=false", # true, false, false, "--prec-sqrt=false", # true, false, false "--fmad=false"] # true, true, false #'options': ["--use_fast_math"] #'options': ["--generate-line-info"], #nvcc_options=["--maxrregcount=39"], #'arch': "compute_50", #'code': "sm_50" }, jit_compile_args={ #jit_options=[(cuda.jit_option.MAX_REGISTERS, 39)] } ) # Get CUDA functions and define data types for prepared_{async_}call() self.cdklm_swe_2D = self.kernel.get_function("cdklm_swe_2D") self.cdklm_swe_2D.prepare("iiffffffffiiPiPiPiPiPiPiPiPiffi") self.update_wind_stress(self.kernel, self.cdklm_swe_2D) # CUDA functions for finding max time step size: self.num_threads_dt = num_threads_dt self.num_blocks_dt = np.int32(self.global_size[0]*self.global_size[1]) self.update_dt_kernels = gpu_ctx.get_kernel("max_dt.cu", defines={'block_width': block_width, 'block_height': block_height, 'NUM_THREADS': self.num_threads_dt}) self.per_block_max_dt_kernel = self.update_dt_kernels.get_function("per_block_max_dt") self.per_block_max_dt_kernel.prepare("iifffPiPiPiPifPi") self.max_dt_reduction_kernel = self.update_dt_kernels.get_function("max_dt_reduction") self.max_dt_reduction_kernel.prepare("iPP") # Bathymetry self.bathymetry = Common.Bathymetry(gpu_ctx, self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, H, boundary_conditions) # Adjust eta for possible dry states Hm = self.downloadBathymetry()[1] eta0 = np.maximum(eta0, -Hm) # Create data by uploading to device self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0) # Allocate memory for calculating maximum timestep host_dt = np.zeros((self.global_size[1], self.global_size[0]), dtype=np.float32) self.device_dt = Common.CUDAArray2D(self.gpu_stream, self.global_size[0], self.global_size[1], 0, 0, host_dt) host_max_dt_buffer = np.zeros((1,1), dtype=np.float32) self.max_dt_buffer = Common.CUDAArray2D(self.gpu_stream, 1, 1, 0, 0, host_max_dt_buffer) self.courant_number = courant_number ## Allocating memory for geostrophical equilibrium variables self.reportGeostrophicEquilibrium = np.int32(reportGeostrophicEquilibrium) self.geoEq_uxpvy = None self.geoEq_Kx = None self.geoEq_Ly = None if self.reportGeostrophicEquilibrium: dummy_zero_array = np.zeros((ny+2*ghost_cells_y, nx+2*ghost_cells_x), dtype=np.float32, order='C') self.geoEq_uxpvy = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.geoEq_Kx = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.geoEq_Ly = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.constant_equilibrium_depth = np.max(H) self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \ self.nx, \ self.ny, \ ghost_cells_x, \ ghost_cells_y, \ self.boundary_conditions, \ boundary_conditions_data, \ ) # Small scale perturbation: self.small_scale_perturbation = small_scale_perturbation self.small_scale_model_error = None self.small_scale_perturbation_interpolation_factor = small_scale_perturbation_interpolation_factor if small_scale_perturbation: if small_scale_perturbation_amplitude is None: self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self, interpolation_factor=small_scale_perturbation_interpolation_factor, use_lcg=use_lcg, block_width=block_width_model_error, block_height=block_height_model_error) else: self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self, soar_q0=small_scale_perturbation_amplitude, interpolation_factor=small_scale_perturbation_interpolation_factor, use_lcg=use_lcg, block_width=block_width_model_error, block_height=block_height_model_error) # Data assimilation model step size self.model_time_step = model_time_step if model_time_step is None: self.model_time_step = self.dt self.total_time_steps = 0 if self.write_netcdf: self.sim_writer = SimWriter.SimNetCDFWriter(self, filename=netcdf_filename, ignore_ghostcells=self.ignore_ghostcells, \ offset_x=self.offset_x, offset_y=self.offset_y) #Upload data to GPU and bind to texture reference self.angle_texref = self.kernel.get_texref("angle_tex") self.angle_texref.set_array(cuda.np_to_array(np.ascontiguousarray(angle, dtype=np.float32), order="C")) # Set texture parameters self.angle_texref.set_filter_mode(cuda.filter_mode.LINEAR) #bilinear interpolation self.angle_texref.set_address_mode(0, cuda.address_mode.CLAMP) #no indexing outside domain self.angle_texref.set_address_mode(1, cuda.address_mode.CLAMP) self.angle_texref.set_flags(cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing
def __init__(self, \ gpu_ctx, \ eta0, H, hu0, hv0, \ nx, ny, \ dx, dy, dt, \ g, f=0.0, r=0.0, \ t=0.0, \ theta=1.3, use_rk2=True, coriolis_beta=0.0, \ y_zero_reference_cell = 0, \ wind_stress=WindStress.WindStress(), \ boundary_conditions=Common.BoundaryConditions(), \ write_netcdf=False, \ comm=None, \ ignore_ghostcells=False, \ offset_x=0, offset_y=0, \ flux_slope_eps = 1.0e-1, \ depth_cutoff = 1.0e-5, \ block_width=32, block_height=16): """ Initialization routine eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells H: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners nx: Number of cells along x-axis ny: Number of cells along y-axis dx: Grid cell spacing along x-axis (20 000 m) dy: Grid cell spacing along y-axis (20 000 m) dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y r: Bottom friction coefficient (2.4e-3 m/s) t: Start simulation at time t theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme use_rk2: Boolean if to use 2nd order Runge-Kutta (false -> 1st order forward Euler) coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0) y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell . wind_stress: Wind stress parameters boundary_conditions: Boundary condition object write_netcdf: Write the results after each superstep to a netCDF file comm: MPI communicator depth_cutoff: Used for defining dry cells flux_slope_eps: Used for desingularization with dry cells """ ghost_cells_x = 2 ghost_cells_y = 2 y_zero_reference_cell = 2.0 + y_zero_reference_cell # Boundary conditions self.boundary_conditions = boundary_conditions # Extend the computational domain if the boundary conditions # require it if (boundary_conditions.isSponge()): nx = nx + boundary_conditions.spongeCells[ 1] + boundary_conditions.spongeCells[3] - 2 * ghost_cells_x ny = ny + boundary_conditions.spongeCells[ 0] + boundary_conditions.spongeCells[2] - 2 * ghost_cells_y y_zero_reference_cell = boundary_conditions.spongeCells[ 2] + y_zero_reference_cell self.use_rk2 = use_rk2 rk_order = np.int32(use_rk2 + 1) A = None super(KP07, self).__init__(gpu_ctx, \ nx, ny, \ ghost_cells_x, \ ghost_cells_y, \ dx, dy, dt, \ g, f, r, A, \ t, \ theta, rk_order, \ coriolis_beta, \ y_zero_reference_cell, \ wind_stress, \ write_netcdf, \ ignore_ghostcells, \ offset_x, offset_y, \ comm, \ block_width, block_height) # Index range for interior domain (north, east, south, west) # so that interior domain of eta is # eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \ # self.interior_domain_indices[3]:self.interior_domain_indices[1] ] self.interior_domain_indices = np.array([-2, -2, 2, 2]) self._set_interior_domain_from_sponge_cells() # The ocean simulators and the swashes cases are defined on # completely different scales. We therefore specify a different # desingularization parameter if we run a swashes case. # Typical values: #ifndef SWASHES #define KPSIMULATOR_FLUX_SLOPE_EPS 1e-1f #define KPSIMULATOR_FLUX_SLOPE_EPS_4 1.0e-4f #else #define KPSIMULATOR_FLUX_SLOPE_EPS 1.0e-4f #define KPSIMULATOR_FLUX_SLOPE_EPS_4 1.0e-16f #endif defines = { 'block_width': block_width, 'block_height': block_height, 'KPSIMULATOR_FLUX_SLOPE_EPS': str(flux_slope_eps) + 'f', 'KPSIMULATOR_FLUX_SLOPE_EPS_4': str(flux_slope_eps**4) + 'f', 'KPSIMULATOR_DEPTH_CUTOFF': str(depth_cutoff) + 'f' } #Get kernels self.kp07_kernel = gpu_ctx.get_kernel( "KP07_kernel.cu", defines=defines, compile_args={ # default, fast_math, optimal 'options': [ "--ftz=true", # false, true, true "--prec-div=false", # true, false, false, "--prec-sqrt=false", # true, false, false "--fmad=false" ] # true, true, false #'options': ["--use_fast_math"] #'options': ["--generate-line-info"], #nvcc_options=["--maxrregcount=39"], #'arch': "compute_50", #'code': "sm_50" }, jit_compile_args={ #jit_options=[(cuda.jit_option.MAX_REGISTERS, 39)] }) # Get CUDA functions and define data types for prepared_{async_}call() self.swe_2D = self.kp07_kernel.get_function("swe_2D") self.swe_2D.prepare("iifffffffffiPiPiPiPiPiPiPiPiiiiif") self.update_wind_stress(self.kp07_kernel, self.swe_2D) # Upload Bathymetry self.bathymetry = Common.Bathymetry(self.gpu_ctx, self.gpu_stream, \ nx, ny, ghost_cells_x, ghost_cells_y, H, boundary_conditions) # Adjust eta for possible dry states Hm = self.downloadBathymetry()[1] eta0 = np.maximum(eta0, -Hm) #Create data by uploading to device self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0) self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \ self.nx, \ self.ny, \ ghost_cells_x, \ ghost_cells_y, \ self.boundary_conditions) if self.write_netcdf: self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \ offset_x=self.offset_x, offset_y=self.offset_y)
def __init__(self, \ gpu_ctx, \ eta0, Hi, hu0, hv0, \ nx, ny, \ dx, dy, dt, \ g, f=0.0, r=0.0, \ t=0.0, \ theta=1.3, use_rk2=True, coriolis_beta=0.0, \ y_zero_reference_cell = 0, \ wind_stress=WindStress.WindStress(), \ boundary_conditions=Common.BoundaryConditions(), \ write_netcdf=False, \ ignore_ghostcells=False, \ offset_x=0, offset_y=0, \ block_width=32, block_height=16): """ Initialization routine eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells Hi: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners nx: Number of cells along x-axis ny: Number of cells along y-axis dx: Grid cell spacing along x-axis (20 000 m) dy: Grid cell spacing along y-axis (20 000 m) dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y r: Bottom friction coefficient (2.4e-3 m/s) t: Start simulation at time t theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme use_rk2: Boolean if to use 2nd order Runge-Kutta (false -> 1st order forward Euler) coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0) y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell . wind_stress: Wind stress parameters boundary_conditions: Boundary condition object write_netcdf: Write the results after each superstep to a netCDF file """ ## After changing from (h, B) to (eta, H), several of the simulator settings used are wrong. This check will help detect that. if ( np.sum(eta0 - Hi[:-1, :-1] > 0) > nx): assert(False), "It seems you are using water depth/elevation h and bottom topography B, while you should use water level eta and equillibrium depth H." ghost_cells_x = 2 ghost_cells_y = 2 y_zero_reference_cell = 2.0 + y_zero_reference_cell # Boundary conditions self.boundary_conditions = boundary_conditions # Extend the computational domain if the boundary conditions # require it if (boundary_conditions.isSponge()): nx = nx + boundary_conditions.spongeCells[1] + boundary_conditions.spongeCells[3] - 2*ghost_cells_x ny = ny + boundary_conditions.spongeCells[0] + boundary_conditions.spongeCells[2] - 2*ghost_cells_y y_zero_reference_cell = boundary_conditions.spongeCells[2] + y_zero_reference_cell self.use_rk2 = use_rk2 rk_order = np.int32(use_rk2 + 1) A = None super(KP07, self).__init__(gpu_ctx, \ nx, ny, \ ghost_cells_x, \ ghost_cells_y, \ dx, dy, dt, \ g, f, r, A, \ t, \ theta, rk_order, \ coriolis_beta, \ y_zero_reference_cell, \ wind_stress, \ write_netcdf, \ ignore_ghostcells, \ offset_x, offset_y, \ block_width, block_height) # Index range for interior domain (north, east, south, west) # so that interior domain of eta is # eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \ # self.interior_domain_indices[3]:self.interior_domain_indices[1] ] self.interior_domain_indices = np.array([-2,-2,2,2]) self._set_interior_domain_from_sponge_cells() #Get kernels self.kp07_kernel = gpu_ctx.get_kernel("KP07_kernel.cu", defines={'block_width': block_width, 'block_height': block_height}) # Get CUDA functions and define data types for prepared_{async_}call() self.swe_2D = self.kp07_kernel.get_function("swe_2D") self.swe_2D.prepare("iifffffffffiPiPiPiPiPiPiPiPiiiiif") self.update_wind_stress(self.kp07_kernel, self.swe_2D) #Create data by uploading to device self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0) #Bathymetry self.bathymetry = Common.Bathymetry(self.gpu_ctx, self.gpu_stream, \ nx, ny, ghost_cells_x, ghost_cells_y, Hi, boundary_conditions) self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \ self.nx, \ self.ny, \ ghost_cells_x, \ ghost_cells_y, \ self.boundary_conditions) if self.write_netcdf: self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \ offset_x=self.offset_x, offset_y=self.offset_y)
def __init__(self, \ gpu_ctx, \ eta0, hu0, hv0, H, \ nx, ny, \ dx, dy, dt, \ g, f, r, \ subsample_f=10, \ angle=np.array([[0]], dtype=np.float32), \ subsample_angle=10, \ latitude=None, \ t=0.0, \ theta=1.3, rk_order=2, \ coriolis_beta=0.0, \ max_wind_direction_perturbation = 0, \ wind_stress=WindStress.WindStress(), \ boundary_conditions=Common.BoundaryConditions(), \ boundary_conditions_data=Common.BoundaryConditionsData(), \ small_scale_perturbation=False, \ small_scale_perturbation_amplitude=None, \ small_scale_perturbation_interpolation_factor = 1, \ model_time_step=None, reportGeostrophicEquilibrium=False, \ use_lcg=False, \ write_netcdf=False, \ comm=None, \ local_particle_id=0, \ super_dir_name=None, \ netcdf_filename=None, \ ignore_ghostcells=False, \ courant_number=0.8, \ offset_x=0, offset_y=0, \ flux_slope_eps = 1.0e-1, \ desingularization_eps = 1.0e-1, \ depth_cutoff = 1.0e-5, \ block_width=12, block_height=32, num_threads_dt=256, block_width_model_error=16, block_height_model_error=16): """ Initialization routine eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells H: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners nx: Number of cells along x-axis ny: Number of cells along y-axis dx: Grid cell spacing along x-axis (20 000 m) dy: Grid cell spacing along y-axis (20 000 m) dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y r: Bottom friction coefficient (2.4e-3 m/s) subsample_f: Subsample the coriolis f when creating texture by factor angle: Angle of rotation from North to y-axis as a texture (cuda.Array) or numpy array (in radians) subsample_angle: Subsample the angles given as input when creating texture by factor latitude: Specify latitude. This will override any f and beta plane already set (in radians) t: Start simulation at time t theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme rk_order: Order of Runge Kutta method {1,2*,3} coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0) max_wind_direction_perturbation: Large-scale model error emulation by per-time-step perturbation of wind direction by +/- max_wind_direction_perturbation (degrees) wind_stress: Wind stress parameters boundary_conditions: Boundary condition object small_scale_perturbation: Boolean value for applying a stochastic model error small_scale_perturbation_amplitude: Amplitude (q0 coefficient) for model error small_scale_perturbation_interpolation_factor: Width factor for correlation in model error model_time_step: The size of a data assimilation model step (default same as dt) reportGeostrophicEquilibrium: Calculate the Geostrophic Equilibrium variables for each superstep use_lcg: Use LCG as the random number generator. Default is False, which means using curand. write_netcdf: Write the results after each superstep to a netCDF file comm: MPI communicator local_particle_id: Local (for each MPI process) particle id desingularization_eps: Used for desingularizing hu/h flux_slope_eps: Used for setting zero flux for symmetric Riemann fan depth_cutoff: Used for defining dry cells super_dir_name: Directory to write netcdf files to netcdf_filename: Use this filename. (If not defined, a filename will be generated by SimWriter.) """ self.logger = logging.getLogger(__name__) assert (rk_order < 4 or rk_order > 0 ), "Only 1st, 2nd and 3rd order Runge Kutta supported" if (rk_order == 3): assert (r == 0.0 ), "3rd order Runge Kutta supported only without friction" # Sort out internally represented ghost_cells in the presence of given # boundary conditions ghost_cells_x = 2 ghost_cells_y = 2 #Coriolis at "first" cell x_zero_reference_cell = ghost_cells_x y_zero_reference_cell = ghost_cells_y # In order to pass it to the super constructor # Boundary conditions self.boundary_conditions = boundary_conditions #Compensate f for reference cell (first cell in internal of domain) north = np.array([np.sin(angle[0, 0]), np.cos(angle[0, 0])]) f = f - coriolis_beta * (x_zero_reference_cell * dx * north[0] + y_zero_reference_cell * dy * north[1]) x_zero_reference_cell = 0 y_zero_reference_cell = 0 A = None self.max_wind_direction_perturbation = max_wind_direction_perturbation super(CDKLM16, self).__init__(gpu_ctx, \ nx, ny, \ ghost_cells_x, \ ghost_cells_y, \ dx, dy, dt, \ g, f, r, A, \ t, \ theta, rk_order, \ coriolis_beta, \ y_zero_reference_cell, \ wind_stress, \ write_netcdf, \ ignore_ghostcells, \ offset_x, offset_y, \ comm, \ block_width, block_height, local_particle_id=local_particle_id) # Index range for interior domain (north, east, south, west) # so that interior domain of eta is # eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \ # self.interior_domain_indices[3]:self.interior_domain_indices[1] ] self.interior_domain_indices = np.array([-2, -2, 2, 2]) defines = { 'block_width': block_width, 'block_height': block_height, 'KPSIMULATOR_DESING_EPS': "{:.12f}f".format(desingularization_eps), 'KPSIMULATOR_FLUX_SLOPE_EPS': "{:.12f}f".format(flux_slope_eps), 'KPSIMULATOR_DEPTH_CUTOFF': "{:.12f}f".format(depth_cutoff), 'THETA': "{:.12f}f".format(self.theta), 'RK_ORDER': int(self.rk_order), 'NX': int(self.nx), 'NY': int(self.ny), 'DX': "{:.12f}f".format(self.dx), 'DY': "{:.12f}f".format(self.dy), 'GRAV': "{:.12f}f".format(self.g), 'FRIC': "{:.12f}f".format(self.r) } #Get kernels self.kernel = gpu_ctx.get_kernel( "CDKLM16_kernel.cu", defines=defines, compile_args={ # default, fast_math, optimal 'options': [ "--ftz=true", # false, true, true "--prec-div=false", # true, false, false, "--prec-sqrt=false", # true, false, false "--fmad=false" ] # true, true, false #'options': ["--use_fast_math"] #'options': ["--generate-line-info"], #nvcc_options=["--maxrregcount=39"], #'arch': "compute_50", #'code': "sm_50" }, jit_compile_args={ #jit_options=[(cuda.jit_option.MAX_REGISTERS, 39)] }) # Get CUDA functions and define data types for prepared_{async_}call() self.cdklm_swe_2D = self.kernel.get_function("cdklm_swe_2D") self.cdklm_swe_2D.prepare("fiPiPiPiPiPiPiPiPiffi") self.update_wind_stress(self.kernel, self.cdklm_swe_2D) # CUDA functions for finding max time step size: self.num_threads_dt = num_threads_dt self.num_blocks_dt = np.int32(self.global_size[0] * self.global_size[1]) self.update_dt_kernels = gpu_ctx.get_kernel("max_dt.cu", defines={ 'block_width': block_width, 'block_height': block_height, 'NUM_THREADS': self.num_threads_dt }) self.per_block_max_dt_kernel = self.update_dt_kernels.get_function( "per_block_max_dt") self.per_block_max_dt_kernel.prepare("iifffPiPiPiPifPi") self.max_dt_reduction_kernel = self.update_dt_kernels.get_function( "max_dt_reduction") self.max_dt_reduction_kernel.prepare("iPP") # Bathymetry self.bathymetry = Common.Bathymetry(gpu_ctx, self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, H, boundary_conditions) # Adjust eta for possible dry states Hm = self.downloadBathymetry()[1] eta0 = np.maximum(eta0, -Hm) # Create data by uploading to device self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0) # Allocate memory for calculating maximum timestep host_dt = np.zeros((self.global_size[1], self.global_size[0]), dtype=np.float32) self.device_dt = Common.CUDAArray2D(self.gpu_stream, self.global_size[0], self.global_size[1], 0, 0, host_dt) host_max_dt_buffer = np.zeros((1, 1), dtype=np.float32) self.max_dt_buffer = Common.CUDAArray2D(self.gpu_stream, 1, 1, 0, 0, host_max_dt_buffer) self.courant_number = courant_number ## Allocating memory for geostrophical equilibrium variables self.reportGeostrophicEquilibrium = np.int32( reportGeostrophicEquilibrium) self.geoEq_uxpvy = None self.geoEq_Kx = None self.geoEq_Ly = None if self.reportGeostrophicEquilibrium: dummy_zero_array = np.zeros( (ny + 2 * ghost_cells_y, nx + 2 * ghost_cells_x), dtype=np.float32, order='C') self.geoEq_uxpvy = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.geoEq_Kx = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.geoEq_Ly = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.constant_equilibrium_depth = np.max(H) self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \ self.nx, \ self.ny, \ ghost_cells_x, \ ghost_cells_y, \ self.boundary_conditions, \ boundary_conditions_data, \ ) def subsample_texture(data, factor): ny, nx = data.shape dx, dy = 1 / nx, 1 / ny I = interp2d(np.linspace(0.5 * dx, 1 - 0.5 * dx, nx), np.linspace(0.5 * dy, 1 - 0.5 * dy, ny), data, kind='linear') new_nx, new_ny = max(2, nx // factor), max(2, ny // factor) new_dx, new_dy = 1 / new_nx, 1 / new_ny x_new = np.linspace(0.5 * new_dx, 1 - 0.5 * new_dx, new_nx) y_new = np.linspace(0.5 * new_dy, 1 - 0.5 * new_dy, new_ny) return I(x_new, y_new) # Texture for angle self.angle_texref = self.kernel.get_texref("angle_tex") if isinstance(angle, cuda.Array): # angle is already a texture, so we just set the texture reference self.angle_texref.set_array(angle) else: #Upload data to GPU and bind to texture reference if (subsample_angle and angle.size >= eta0.size): self.logger.info("Subsampling angle texture by factor " + str(subsample_angle)) self.logger.warning( "This will give inaccurate angle along the border!") angle = subsample_texture(angle, subsample_angle) self.angle_texref.set_array( cuda.np_to_array(np.ascontiguousarray(angle, dtype=np.float32), order="C")) # Set texture parameters self.angle_texref.set_filter_mode( cuda.filter_mode.LINEAR) #bilinear interpolation self.angle_texref.set_address_mode( 0, cuda.address_mode.CLAMP) #no indexing outside domain self.angle_texref.set_address_mode(1, cuda.address_mode.CLAMP) self.angle_texref.set_flags( cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing # Texture for coriolis f self.coriolis_texref = self.kernel.get_texref("coriolis_f_tex") # Create the CPU coriolis if (latitude is not None): if (self.f != 0.0): raise RuntimeError( "Cannot specify both latitude and f. Make your mind up.") coriolis_f, _ = OceanographicUtilities.calcCoriolisParams(latitude) coriolis_f = coriolis_f.astype(np.float32) else: if (self.coriolis_beta != 0.0): if (angle.size != 1): raise RuntimeError( "non-constant angle cannot be combined with beta plane model (makes no sense)" ) #Generate coordinates for all cells, including ghost cells from center to center # [-3/2dx, nx+3/2dx] for ghost_cells_x == 2 x = np.linspace((-self.ghost_cells_x + 0.5) * self.dx, (self.nx + self.ghost_cells_x - 0.5) * self.dx, self.nx + 2 * self.ghost_cells_x) y = np.linspace((-self.ghost_cells_y + 0.5) * self.dy, (self.ny + self.ghost_cells_y - 0.5) * self.dy, self.ny + 2 * self.ghost_cells_x) self.logger.info( "Using latitude to create Coriolis f texture ({:f}x{:f} cells)" .format(x.size, y.size)) x, y = np.meshgrid(x, y) n = x * np.sin(angle[0, 0]) + y * np.cos( angle[0, 0]) #North vector coriolis_f = self.f + self.coriolis_beta * n else: if (self.f.size == 1): coriolis_f = np.array([[self.f]], dtype=np.float32) elif (self.f.shape == eta0.shape): coriolis_f = np.array(self.f, dtype=np.float32) else: raise RuntimeError( "The shape of f should match up with eta or be scalar." ) if (subsample_f and coriolis_f.size >= eta0.size): self.logger.info("Subsampling coriolis texture by factor " + str(subsample_f)) self.logger.warning( "This will give inaccurate coriolis along the border!") coriolis_f = subsample_texture(coriolis_f, subsample_f) #Upload data to GPU and bind to texture reference self.coriolis_texref.set_array( cuda.np_to_array(np.ascontiguousarray(coriolis_f, dtype=np.float32), order="C")) # Set texture parameters self.coriolis_texref.set_filter_mode( cuda.filter_mode.LINEAR) #bilinear interpolation self.coriolis_texref.set_address_mode( 0, cuda.address_mode.CLAMP) #no indexing outside domain self.coriolis_texref.set_address_mode(1, cuda.address_mode.CLAMP) self.coriolis_texref.set_flags( cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing # Small scale perturbation: self.small_scale_perturbation = small_scale_perturbation self.small_scale_model_error = None self.small_scale_perturbation_interpolation_factor = small_scale_perturbation_interpolation_factor if small_scale_perturbation: self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim( self, soar_q0=small_scale_perturbation_amplitude, interpolation_factor= small_scale_perturbation_interpolation_factor, use_lcg=use_lcg, block_width=block_width_model_error, block_height=block_height_model_error) # Data assimilation model step size self.model_time_step = model_time_step self.total_time_steps = 0 if model_time_step is None: self.model_time_step = self.dt if self.write_netcdf: self.sim_writer = SimWriter.SimNetCDFWriter(self, super_dir_name=super_dir_name, filename=netcdf_filename, \ ignore_ghostcells=self.ignore_ghostcells, offset_x=self.offset_x, offset_y=self.offset_y) # Update timestep if dt is given as zero if self.dt <= 0: self.updateDt()
def __init__(self, \ gpu_ctx, \ eta0, hu0, hv0, Hi, \ nx, ny, \ dx, dy, dt, \ g, f, r, \ t=0.0, \ theta=1.3, rk_order=2, \ coriolis_beta=0.0, \ y_zero_reference_cell = 0, \ max_wind_direction_perturbation = 0, \ wind_stress=WindStress.WindStress(), \ boundary_conditions=Common.BoundaryConditions(), \ small_scale_perturbation=False, \ small_scale_perturbation_amplitude=None, \ h0AsWaterElevation=False, \ reportGeostrophicEquilibrium=False, \ write_netcdf=False, \ ignore_ghostcells=False, \ offset_x=0, offset_y=0, \ block_width=32, block_height=4): """ Initialization routine eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells Hi: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners nx: Number of cells along x-axis ny: Number of cells along y-axis dx: Grid cell spacing along x-axis (20 000 m) dy: Grid cell spacing along y-axis (20 000 m) dt: Size of each timestep (90 s) g: Gravitational accelleration (9.81 m/s^2) f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y r: Bottom friction coefficient (2.4e-3 m/s) t: Start simulation at time t theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme rk_order: Order of Runge Kutta method {1,2*,3} coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0) y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell . max_wind_direction_perturbation: Large-scale model error emulation by per-time-step perturbation of wind direction by +/- max_wind_direction_perturbation (degrees) wind_stress: Wind stress parameters boundary_conditions: Boundary condition object h0AsWaterElevation: True if h0 is described by the surface elevation, and false if h0 is described by water depth reportGeostrophicEquilibrium: Calculate the Geostrophic Equilibrium variables for each superstep write_netcdf: Write the results after each superstep to a netCDF file """ ## After changing from (h, B) to (eta, H), several of the simulator settings used are wrong. This check will help detect that. if ( np.sum(eta0 - Hi[:-1, :-1] > 0) > nx): assert(False), "It seems you are using water depth/elevation h and bottom topography B, while you should use water level eta and equillibrium depth H." assert( rk_order < 4 or rk_order > 0 ), "Only 1st, 2nd and 3rd order Runge Kutta supported" if (rk_order == 3): assert(r == 0.0), "3rd order Runge Kutta supported only without friction" # Sort out internally represented ghost_cells in the presence of given # boundary conditions ghost_cells_x = 2 ghost_cells_y = 2 y_zero_reference_cell = 2 + y_zero_reference_cell # Boundary conditions self.boundary_conditions = boundary_conditions if (boundary_conditions.isSponge()): nx = nx + boundary_conditions.spongeCells[1] + boundary_conditions.spongeCells[3] - 2*ghost_cells_x ny = ny + boundary_conditions.spongeCells[0] + boundary_conditions.spongeCells[2] - 2*ghost_cells_y y_zero_reference_cell = boundary_conditions.spongeCells[2] + y_zero_reference_cell A = None self.max_wind_direction_perturbation = max_wind_direction_perturbation super(CDKLM16, self).__init__(gpu_ctx, \ nx, ny, \ ghost_cells_x, \ ghost_cells_y, \ dx, dy, dt, \ g, f, r, A, \ t, \ theta, rk_order, \ coriolis_beta, \ y_zero_reference_cell, \ wind_stress, \ write_netcdf, \ ignore_ghostcells, \ offset_x, offset_y, \ block_width, block_height) # Index range for interior domain (north, east, south, west) # so that interior domain of eta is # eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \ # self.interior_domain_indices[3]:self.interior_domain_indices[1] ] self.interior_domain_indices = np.array([-2,-2,2,2]) self._set_interior_domain_from_sponge_cells() #Get kernels self.kernel = gpu_ctx.get_kernel("CDKLM16_kernel.cu", defines={'block_width': block_width, 'block_height': block_height}) # Get CUDA functions and define data types for prepared_{async_}call() self.swe_2D = self.kernel.get_function("swe_2D") self.swe_2D.prepare("iifffffffffiiPiPiPiPiPiPiPiPifiiiiiPiPiPi") self.update_wind_stress(self.kernel, self.swe_2D) #Create data by uploading to device self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0) ## Allocating memory for geostrophical equilibrium variables self.reportGeostrophicEquilibrium = np.int32(reportGeostrophicEquilibrium) dummy_zero_array = np.zeros((ny+2*ghost_cells_y, nx+2*ghost_cells_x), dtype=np.float32, order='C') self.geoEq_uxpvy = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.geoEq_Kx = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) self.geoEq_Ly = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array) #Bathymetry self.bathymetry = Common.Bathymetry(gpu_ctx, self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, Hi, boundary_conditions) self.h0AsWaterElevation = h0AsWaterElevation if self.h0AsWaterElevation: self.bathymetry.waterElevationToDepth(self.gpu_data.h0) self.constant_equilibrium_depth = np.max(Hi) self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \ self.nx, \ self.ny, \ ghost_cells_x, \ ghost_cells_y, \ self.boundary_conditions, \ ) # Small scale perturbation: self.small_scale_perturbation = small_scale_perturbation self.small_scale_model_error = None if small_scale_perturbation: if small_scale_perturbation_amplitude is None: self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self) else: self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self, soar_q0=small_scale_perturbation_amplitude) if self.write_netcdf: self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \ offset_x=self.offset_x, offset_y=self.offset_y)