def allocateBuffers(self, HCPU):
host_buffer = np.zeros((self.ny, self.nx))
self.eta = Common.CUDAArray2D(self.gpu_stream, self.nx, self.ny, 0, 0,
host_buffer)
self.hu = Common.CUDAArray2D(self.gpu_stream, self.nx, self.ny, 0, 0,
host_buffer)
self.hv = Common.CUDAArray2D(self.gpu_stream, self.nx, self.ny, 0, 0,
host_buffer)
self.H = Common.CUDAArray2D(self.gpu_stream, self.nx + 1, self.ny + 1,
0, 0, HCPU)
del host_buffer
def _setupGPU(self):
"""
Setting up kernel for reading observations, along with host and device buffers
"""
# Create observation buffer!
if self.observation_type == dautils.ObservationType.UnderlyingFlow or \
self.observation_type == dautils.ObservationType.DirectUnderlyingFlow:
zeros = np.zeros((self.driftersPerOceanModel, 2),
dtype=np.float32,
order='C')
self.observation_buffer = Common.CUDAArray2D(self.gpu_stream, \
2, self.driftersPerOceanModel, 0, 0, \
zeros)
# Generate kernels
self.observation_kernels = self.gpu_ctx.get_kernel("observationKernels.cu", \
defines={})
# Get CUDA functions and define data types for prepared_{async_}call()
self.observeUnderlyingFlowKernel = self.observation_kernels.get_function(
"observeUnderlyingFlow")
self.observeUnderlyingFlowKernel.prepare("iiffiiPiPiPifiPiPi")
self.local_size = (int(self.driftersPerOceanModel), 1, 1)
self.global_size = (1, 1)
def test_copy_buffer(self):
clarray2 = Common.CUDAArray2D(self.gpu_stream, \
self.nx, self.ny, self.nx_halo, self.ny_halo, \
self.buf3)
host_data_pre_copy = self.cudaarray.download(self.gpu_stream)
self.assertEqual(host_data_pre_copy.tolist(), self.buf1.tolist())
self.cudaarray.copyBuffer(self.gpu_stream, clarray2)
host_data_post_copy = self.cudaarray.download(self.gpu_stream)
self.assertEqual(host_data_post_copy.tolist(), self.buf3.tolist())
self.tests_failed = False
def __init__(self, gpu_ctx, numDrifters, \
observation_variance=0.01, \
boundaryConditions=Common.BoundaryConditions(), \
initialization_cov_drifters=None, \
domain_size_x=1.0, domain_size_y=1.0, \
gpu_stream=None, \
block_width = 64):
super(GPUDrifterCollection, self).__init__(numDrifters,
observation_variance=observation_variance,
boundaryConditions=boundaryConditions,
domain_size_x=domain_size_x,
domain_size_y=domain_size_y)
# Define CUDA environment:
self.gpu_ctx = gpu_ctx
self.block_width = block_width
self.block_height = 1
# TODO: Where should the cl_queue come from?
# For sure, the drifter and the ocean simulator should use
# the same queue...
self.gpu_stream = gpu_stream
if self.gpu_stream is None:
self.gpu_stream = cuda.Stream()
self.sensitivity = 1.0
self.driftersHost = np.zeros((self.getNumDrifters() + 1, 2)).astype(np.float32, order='C')
self.driftersDevice = Common.CUDAArray2D(self.gpu_stream, \
2, self.getNumDrifters()+1, 0, 0, \
self.driftersHost)
self.drift_kernels = gpu_ctx.get_kernel("driftKernels.cu", \
defines={'block_width': self.block_width, 'block_height': self.block_height})
# Get CUDA functions and define data types for prepared_{async_}call()
self.passiveDrifterKernel = self.drift_kernels.get_function("passiveDrifterKernel")
self.passiveDrifterKernel.prepare("iifffiiPiPiPifiiiPif")
self.enforceBoundaryConditionsKernel = self.drift_kernels.get_function("enforceBoundaryConditions")
self.enforceBoundaryConditionsKernel.prepare("ffiiiPi")
self.local_size = (self.block_width, self.block_height, 1)
self.global_size = (\
int(np.ceil((self.getNumDrifters() + 2)/float(self.block_width))), \
1)
# Initialize drifters:
self.uniformly_distribute_drifters(initialization_cov_drifters=initialization_cov_drifters)
def test_cross_precision_copy_buffer(self):
self.init_double()
single_cudaarray2 = Common.CUDAArray2D(self.gpu_stream, \
self.nx, self.ny, \
self.nx_halo, self.ny_halo, \
self.buf3)
host_data_pre_copy = self.double_cudaarray.download(self.gpu_stream)
self.assertEqual(host_data_pre_copy.tolist(), self.dbuf1.tolist())
with self.assertRaises(AssertionError):
self.double_cudaarray.copyBuffer(self.gpu_stream,
single_cudaarray2)
self.tests_failed = False
def test_double_copy_buffer(self):
self.init_double()
double_cudaarray2 = Common.CUDAArray2D(self.gpu_stream, \
self.nx, self.ny, \
self.nx_halo, self.ny_halo, \
self.dbuf3, \
double_precision=True)
host_data_pre_copy = self.double_cudaarray.download(self.gpu_stream)
self.assertEqual(host_data_pre_copy.tolist(), self.dbuf1.tolist())
self.double_cudaarray.copyBuffer(self.gpu_stream, double_cudaarray2)
host_data_post_copy = self.double_cudaarray.download(self.gpu_stream)
self.assertEqual(host_data_post_copy.tolist(), self.dbuf3.tolist())
self.tests_failed = False
def setUp(self):
#Set which CL device to use, and disable kernel caching
self.gpu_ctx = Common.CUDAContext()
# Make some host data which we can play with
self.nx = 3
self.ny = 5
self.nx_halo = 1
self.ny_halo = 2
self.dataShape = (self.ny + 2 * self.ny_halo,
self.nx + 2 * self.nx_halo)
self.buf1 = np.zeros(self.dataShape, dtype=np.float32, order='C')
self.dbuf1 = np.zeros(self.dataShape)
self.buf3 = np.zeros(self.dataShape, dtype=np.float32, order='C')
self.dbuf3 = np.zeros(self.dataShape)
for j in range(self.dataShape[0]):
for i in range(self.dataShape[1]):
self.buf1[j, i] = i * 100 + j
self.dbuf1[j, i] = self.buf1[j, i]
self.buf3[j, i] = j * 1000 - i
self.dbuf3[j, i] = self.buf3[j, i]
self.explicit_free = False
self.device_name = self.gpu_ctx.cuda_device.name()
self.gpu_stream = cuda.Stream()
self.tests_failed = True
self.cudaarray = Common.CUDAArray2D(self.gpu_stream, \
self.nx, self.ny, \
self.nx_halo, self.ny_halo, \
self.buf1)
self.double_cudaarray = None
def __init__(self,
gpu_ctx,
gpu_stream,
nx,
ny,
dx,
dy,
boundaryConditions,
staggered,
soar_q0=None,
soar_L=None,
interpolation_factor=1,
use_lcg=False,
angle=np.array([[0]], dtype=np.float32),
coriolis_f=np.array([[0]], dtype=np.float32),
block_width=16,
block_height=16):
"""
Initiates a class that generates small scale geostrophically balanced perturbations of
the ocean state.
(nx, ny): number of internal grid cells in the domain
(dx, dy): size of each grid cell
soar_q0: amplitude parameter for the perturbation, default: dx*1e-5
soar_L: length scale of the perturbation covariance, default: 0.74*dx*interpolation_factor
interpolation_factor: indicates that the perturbation of eta should be generated on a coarse mesh,
and then interpolated down to the computational mesh. The coarse mesh will then have
(nx/interpolation_factor, ny/interpolation_factor) grid cells.
use_lcg: LCG is a linear algorithm for generating a serie of pseudo-random numbers
angle: Angle of rotation from North to y-axis as a texture (cuda.Array) or numpy array
(block_width, block_height): The size of each GPU block
"""
self.use_lcg = use_lcg
# Set numpy random state
self.random_state = np.random.RandomState()
# Make sure that all variables initialized within ifs are defined
self.random_numbers = None
self.rng = None
self.seed = None
self.host_seed = None
self.gpu_ctx = gpu_ctx
self.gpu_stream = gpu_stream
self.nx = np.int32(nx)
self.ny = np.int32(ny)
self.dx = np.float32(dx)
self.dy = np.float32(dy)
self.staggered = np.int(0)
if staggered:
self.staggered = np.int(1)
# The cutoff parameter is hard-coded.
# The size of the cutoff determines the computational radius in the
# SOAR function. Hence, the size of the local memory in the OpenCL
# kernels has to be hard-coded.
self.cutoff = np.int32(config.soar_cutoff)
# Check that the interpolation factor plays well with the grid size:
assert (interpolation_factor > 0 and interpolation_factor % 2
== 1), 'interpolation_factor must be a positive odd integer'
assert (nx % interpolation_factor == 0
), 'nx must be divisible by the interpolation factor'
assert (ny % interpolation_factor == 0
), 'ny must be divisible by the interpolation factor'
self.interpolation_factor = np.int32(interpolation_factor)
# The size of the coarse grid
self.coarse_nx = np.int32(nx / self.interpolation_factor)
self.coarse_ny = np.int32(ny / self.interpolation_factor)
self.coarse_dx = np.float32(dx * self.interpolation_factor)
self.coarse_dy = np.float32(dy * self.interpolation_factor)
self.periodicNorthSouth = np.int32(
boundaryConditions.isPeriodicNorthSouth())
self.periodicEastWest = np.int32(
boundaryConditions.isPeriodicEastWest())
# Size of random field and seed
# The SOAR function is a stencil which requires cutoff number of grid cells,
# and the interpolation operator requires further 2 ghost cell values in each direction.
# The random field must therefore be created with 2 + cutoff number of ghost cells.
self.rand_ghost_cells_x = np.int32(2 + self.cutoff)
self.rand_ghost_cells_y = np.int32(2 + self.cutoff)
if self.periodicEastWest:
self.rand_ghost_cells_x = np.int32(0)
if self.periodicNorthSouth:
self.rand_ghost_cells_y = np.int32(0)
self.rand_nx = np.int32(self.coarse_nx + 2 * self.rand_ghost_cells_x)
self.rand_ny = np.int32(self.coarse_ny + 2 * self.rand_ghost_cells_y)
# Since normal distributed numbers are generated in pairs, we need to store half the number of
# of seed values compared to the number of random numbers.
self.seed_ny = np.int32(self.rand_ny)
self.seed_nx = np.int32(np.ceil(self.rand_nx / 2))
# Generate seed:
self.floatMax = 2147483648.0
if self.use_lcg:
self.host_seed = self.random_state.rand(
self.seed_ny, self.seed_nx) * self.floatMax
self.host_seed = self.host_seed.astype(np.uint64, order='C')
if not self.use_lcg:
self.rng = XORWOWRandomNumberGenerator()
else:
self.seed = Common.CUDAArray2D(gpu_stream,
self.seed_nx,
self.seed_ny,
0,
0,
self.host_seed,
double_precision=True,
integers=True)
# Constants for the SOAR function:
self.soar_q0 = np.float32(self.dx / 100000)
if soar_q0 is not None:
self.soar_q0 = np.float32(soar_q0)
self.soar_L = np.float32(0.75 * self.coarse_dx)
if soar_L is not None:
self.soar_L = np.float32(soar_L)
# Allocate memory for random numbers (xi)
self.random_numbers_host = np.zeros((self.rand_ny, self.rand_nx),
dtype=np.float32,
order='C')
self.random_numbers = Common.CUDAArray2D(self.gpu_stream, self.rand_nx,
self.rand_ny, 0, 0,
self.random_numbers_host)
# Allocate a second buffer for random numbers (nu)
self.perpendicular_random_numbers_host = np.zeros(
(self.rand_ny, self.rand_nx), dtype=np.float32, order='C')
self.perpendicular_random_numbers = Common.CUDAArray2D(
self.gpu_stream, self.rand_nx, self.rand_ny, 0, 0,
self.random_numbers_host)
# Allocate memory for coarse buffer if needed
# Two ghost cells in each direction needed for bicubic interpolation
self.coarse_buffer_host = np.zeros(
(self.coarse_ny + 4, self.coarse_nx + 4),
dtype=np.float32,
order='C')
self.coarse_buffer = Common.CUDAArray2D(self.gpu_stream,
self.coarse_nx, self.coarse_ny,
2, 2, self.coarse_buffer_host)
# Allocate extra memory needed for reduction kernels.
# Currently: A single GPU buffer with 3x1 elements: [xi^T * xi, nu^T * nu, xi^T * nu]
self.reduction_buffer = None
reduction_buffer_host = np.zeros((1, 3), dtype=np.float32)
self.reduction_buffer = Common.CUDAArray2D(self.gpu_stream, 3, 1, 0, 0,
reduction_buffer_host)
# Generate kernels
self.kernels = gpu_ctx.get_kernel("ocean_noise.cu", \
defines={'block_width': block_width, 'block_height': block_height},
compile_args={
'options': ["--use_fast_math",
"--maxrregcount=32"]
})
self.reduction_kernels = self.gpu_ctx.get_kernel("reductions.cu", \
defines={})
# Get CUDA functions and define data types for prepared_{async_}call()
# Generate kernels
self.squareSumKernel = self.reduction_kernels.get_function("squareSum")
self.squareSumKernel.prepare("iiPP")
self.squareSumDoubleKernel = self.reduction_kernels.get_function(
"squareSumDouble")
self.squareSumDoubleKernel.prepare("iiPPP")
self.makePerpendicularKernel = self.kernels.get_function(
"makePerpendicular")
self.makePerpendicularKernel.prepare("iiPiPiP")
self.uniformDistributionKernel = self.kernels.get_function(
"uniformDistribution")
self.uniformDistributionKernel.prepare("iiiPiPi")
self.normalDistributionKernel = None
if self.use_lcg:
self.normalDistributionKernel = self.kernels.get_function(
"normalDistribution")
self.normalDistributionKernel.prepare("iiiPiPi")
self.soarKernel = self.kernels.get_function("SOAR")
self.soarKernel.prepare("iifffffiiPiPii")
self.geostrophicBalanceKernel = self.kernels.get_function(
"geostrophicBalance")
self.geostrophicBalanceKernel.prepare("iiffiiffffPiPiPiPiPif")
self.bicubicInterpolationKernel = self.kernels.get_function(
"bicubicInterpolation")
self.bicubicInterpolationKernel.prepare(
"iiiiffiiiiffiiffffPiPiPiPiPif")
#Compute kernel launch parameters
self.local_size = (block_width, block_height, 1)
self.local_size_reductions = (128, 1, 1)
self.global_size_reductions = (1, 1)
# Launch one thread for each seed, which in turns generates two iid N(0,1)
self.global_size_random_numbers = ( \
int(np.ceil(self.seed_nx / float(self.local_size[0]))), \
int(np.ceil(self.seed_ny / float(self.local_size[1]))) \
)
# Launch on thread for each random number (in order to create perpendicular random numbers)
self.global_size_perpendicular = ( \
int(np.ceil(self.rand_nx / float(self.local_size[0]))), \
int(np.ceil(self.rand_ny / float(self.local_size[1]))) \
)
# Launch one thread per SOAR-correlated result - need to write to two ghost
# cells in order to do bicubic interpolation based on the result
self.global_size_SOAR = ( \
int(np.ceil( (self.coarse_nx+4)/float(self.local_size[0]))), \
int(np.ceil( (self.coarse_ny+4)/float(self.local_size[1]))) \
)
# One thread per resulting perturbed grid cell
self.global_size_geo_balance = ( \
int(np.ceil( (self.nx)/float(self.local_size[0]))), \
int(np.ceil( (self.ny)/float(self.local_size[1]))) \
)
# Texture for coriolis field
self.coriolis_texref = self.kernels.get_texref("coriolis_f_tex")
if isinstance(coriolis_f, cuda.Array):
# coriolis_f is already a texture, so we just set the reference
self.coriolis_texref.set_array(coriolis_f)
else:
#Upload data to GPU and bind to texture reference
self.coriolis_texref.set_array(
cuda.np_to_array(np.ascontiguousarray(coriolis_f,
dtype=np.float32),
order="C"))
# Set texture parameters
self.coriolis_texref.set_filter_mode(
cuda.filter_mode.LINEAR) #bilinear interpolation
self.coriolis_texref.set_address_mode(
0, cuda.address_mode.CLAMP) #no indexing outside domain
self.coriolis_texref.set_address_mode(1, cuda.address_mode.CLAMP)
self.coriolis_texref.set_flags(
cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing
# FIXME! Allow different versions of coriolis, similar to CDKLM
# Texture for angle towards north
self.angle_texref = self.kernels.get_texref("angle_tex")
if isinstance(angle, cuda.Array):
# angle is already a texture, so we just set the reference
self.angle_texref.set_array(angle)
else:
#Upload data to GPU and bind to texture reference
self.angle_texref.set_array(
cuda.np_to_array(np.ascontiguousarray(angle, dtype=np.float32),
order="C"))
# Set texture parameters
self.angle_texref.set_filter_mode(
cuda.filter_mode.LINEAR) #bilinear interpolation
self.angle_texref.set_address_mode(
0, cuda.address_mode.CLAMP) #no indexing outside domain
self.angle_texref.set_address_mode(1, cuda.address_mode.CLAMP)
self.angle_texref.set_flags(
cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing
def __init__(self, \
gpu_ctx, \
eta0, hu0, hv0, H, \
nx, ny, \
dx, dy, dt, \
g, f, r, \
angle=np.array([[0]], dtype=np.float32), \
t=0.0, \
theta=1.3, rk_order=2, \
coriolis_beta=0.0, \
max_wind_direction_perturbation = 0, \
wind_stress=WindStress.WindStress(), \
boundary_conditions=Common.BoundaryConditions(), \
boundary_conditions_data=Common.BoundaryConditionsData(), \
small_scale_perturbation=False, \
small_scale_perturbation_amplitude=None, \
small_scale_perturbation_interpolation_factor = 1, \
model_time_step=None,
reportGeostrophicEquilibrium=False, \
use_lcg=False, \
write_netcdf=False, \
comm=None, \
netcdf_filename=None, \
ignore_ghostcells=False, \
courant_number=0.8, \
offset_x=0, offset_y=0, \
flux_slope_eps = 1.0e-1, \
desingularization_eps = 1.0e-1, \
depth_cutoff = 1.0e-5, \
block_width=32, block_height=8, num_threads_dt=256,
block_width_model_error=16, block_height_model_error=16):
"""
Initialization routine
eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells
hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells
hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells
H: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners
nx: Number of cells along x-axis
ny: Number of cells along y-axis
dx: Grid cell spacing along x-axis (20 000 m)
dy: Grid cell spacing along y-axis (20 000 m)
dt: Size of each timestep (90 s)
g: Gravitational accelleration (9.81 m/s^2)
f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y
r: Bottom friction coefficient (2.4e-3 m/s)
angle: Angle of rotation from North to y-axis
t: Start simulation at time t
theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme
rk_order: Order of Runge Kutta method {1,2*,3}
coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0)
max_wind_direction_perturbation: Large-scale model error emulation by per-time-step perturbation of wind direction by +/- max_wind_direction_perturbation (degrees)
wind_stress: Wind stress parameters
boundary_conditions: Boundary condition object
small_scale_perturbation: Boolean value for applying a stochastic model error
small_scale_perturbation_amplitude: Amplitude (q0 coefficient) for model error
small_scale_perturbation_interpolation_factor: Width factor for correlation in model error
model_time_step: The size of a data assimilation model step (default same as dt)
reportGeostrophicEquilibrium: Calculate the Geostrophic Equilibrium variables for each superstep
use_lcg: Use LCG as the random number generator. Default is False, which means using curand.
write_netcdf: Write the results after each superstep to a netCDF file
comm: MPI communicator
desingularization_eps: Used for desingularizing hu/h
flux_slope_eps: Used for setting zero flux for symmetric Riemann fan
depth_cutoff: Used for defining dry cells
netcdf_filename: Use this filename. (If not defined, a filename will be generated by SimWriter.)
"""
self.logger = logging.getLogger(__name__)
assert( rk_order < 4 or rk_order > 0 ), "Only 1st, 2nd and 3rd order Runge Kutta supported"
if (rk_order == 3):
assert(r == 0.0), "3rd order Runge Kutta supported only without friction"
# Sort out internally represented ghost_cells in the presence of given
# boundary conditions
ghost_cells_x = 2
ghost_cells_y = 2
#Coriolis at "first" cell
x_zero_reference_cell = ghost_cells_x
y_zero_reference_cell = ghost_cells_y # In order to pass it to the super constructor
# Boundary conditions
self.boundary_conditions = boundary_conditions
if (boundary_conditions.isSponge()):
nx = nx + boundary_conditions.spongeCells[1] + boundary_conditions.spongeCells[3] - 2*ghost_cells_x
ny = ny + boundary_conditions.spongeCells[0] + boundary_conditions.spongeCells[2] - 2*ghost_cells_y
x_zero_reference_cell += boundary_conditions.spongeCells[3]
y_zero_reference_cell += boundary_conditions.spongeCells[2]
#Compensate f for reference cell (first cell in internal of domain)
north = np.array([np.sin(angle[0,0]), np.cos(angle[0,0])])
f = f - coriolis_beta * (x_zero_reference_cell*dx*north[0] + y_zero_reference_cell*dy*north[1])
x_zero_reference_cell = 0
y_zero_reference_cell = 0
A = None
self.max_wind_direction_perturbation = max_wind_direction_perturbation
super(CDKLM16, self).__init__(gpu_ctx, \
nx, ny, \
ghost_cells_x, \
ghost_cells_y, \
dx, dy, dt, \
g, f, r, A, \
t, \
theta, rk_order, \
coriolis_beta, \
y_zero_reference_cell, \
wind_stress, \
write_netcdf, \
ignore_ghostcells, \
offset_x, offset_y, \
comm, \
block_width, block_height)
# Index range for interior domain (north, east, south, west)
# so that interior domain of eta is
# eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \
# self.interior_domain_indices[3]:self.interior_domain_indices[1] ]
self.interior_domain_indices = np.array([-2,-2,2,2])
self._set_interior_domain_from_sponge_cells()
defines={'block_width': block_width, 'block_height': block_height,
'KPSIMULATOR_DESING_EPS': str(desingularization_eps)+'f',
'KPSIMULATOR_FLUX_SLOPE_EPS': str(flux_slope_eps)+'f',
'KPSIMULATOR_DEPTH_CUTOFF': str(depth_cutoff)+'f'}
#Get kernels
self.kernel = gpu_ctx.get_kernel("CDKLM16_kernel.cu",
defines=defines,
compile_args={ # default, fast_math, optimal
'options' : ["--ftz=true", # false, true, true
"--prec-div=false", # true, false, false,
"--prec-sqrt=false", # true, false, false
"--fmad=false"] # true, true, false
#'options': ["--use_fast_math"]
#'options': ["--generate-line-info"],
#nvcc_options=["--maxrregcount=39"],
#'arch': "compute_50",
#'code': "sm_50"
},
jit_compile_args={
#jit_options=[(cuda.jit_option.MAX_REGISTERS, 39)]
}
)
# Get CUDA functions and define data types for prepared_{async_}call()
self.cdklm_swe_2D = self.kernel.get_function("cdklm_swe_2D")
self.cdklm_swe_2D.prepare("iiffffffffiiPiPiPiPiPiPiPiPiffi")
self.update_wind_stress(self.kernel, self.cdklm_swe_2D)
# CUDA functions for finding max time step size:
self.num_threads_dt = num_threads_dt
self.num_blocks_dt = np.int32(self.global_size[0]*self.global_size[1])
self.update_dt_kernels = gpu_ctx.get_kernel("max_dt.cu",
defines={'block_width': block_width,
'block_height': block_height,
'NUM_THREADS': self.num_threads_dt})
self.per_block_max_dt_kernel = self.update_dt_kernels.get_function("per_block_max_dt")
self.per_block_max_dt_kernel.prepare("iifffPiPiPiPifPi")
self.max_dt_reduction_kernel = self.update_dt_kernels.get_function("max_dt_reduction")
self.max_dt_reduction_kernel.prepare("iPP")
# Bathymetry
self.bathymetry = Common.Bathymetry(gpu_ctx, self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, H, boundary_conditions)
# Adjust eta for possible dry states
Hm = self.downloadBathymetry()[1]
eta0 = np.maximum(eta0, -Hm)
# Create data by uploading to device
self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0)
# Allocate memory for calculating maximum timestep
host_dt = np.zeros((self.global_size[1], self.global_size[0]), dtype=np.float32)
self.device_dt = Common.CUDAArray2D(self.gpu_stream, self.global_size[0], self.global_size[1],
0, 0, host_dt)
host_max_dt_buffer = np.zeros((1,1), dtype=np.float32)
self.max_dt_buffer = Common.CUDAArray2D(self.gpu_stream, 1, 1, 0, 0, host_max_dt_buffer)
self.courant_number = courant_number
## Allocating memory for geostrophical equilibrium variables
self.reportGeostrophicEquilibrium = np.int32(reportGeostrophicEquilibrium)
self.geoEq_uxpvy = None
self.geoEq_Kx = None
self.geoEq_Ly = None
if self.reportGeostrophicEquilibrium:
dummy_zero_array = np.zeros((ny+2*ghost_cells_y, nx+2*ghost_cells_x), dtype=np.float32, order='C')
self.geoEq_uxpvy = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array)
self.geoEq_Kx = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array)
self.geoEq_Ly = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array)
self.constant_equilibrium_depth = np.max(H)
self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \
self.nx, \
self.ny, \
ghost_cells_x, \
ghost_cells_y, \
self.boundary_conditions, \
boundary_conditions_data, \
)
# Small scale perturbation:
self.small_scale_perturbation = small_scale_perturbation
self.small_scale_model_error = None
self.small_scale_perturbation_interpolation_factor = small_scale_perturbation_interpolation_factor
if small_scale_perturbation:
if small_scale_perturbation_amplitude is None:
self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self,
interpolation_factor=small_scale_perturbation_interpolation_factor,
use_lcg=use_lcg,
block_width=block_width_model_error,
block_height=block_height_model_error)
else:
self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self,
soar_q0=small_scale_perturbation_amplitude,
interpolation_factor=small_scale_perturbation_interpolation_factor,
use_lcg=use_lcg,
block_width=block_width_model_error,
block_height=block_height_model_error)
# Data assimilation model step size
self.model_time_step = model_time_step
if model_time_step is None:
self.model_time_step = self.dt
self.total_time_steps = 0
if self.write_netcdf:
self.sim_writer = SimWriter.SimNetCDFWriter(self, filename=netcdf_filename, ignore_ghostcells=self.ignore_ghostcells, \
offset_x=self.offset_x, offset_y=self.offset_y)
#Upload data to GPU and bind to texture reference
self.angle_texref = self.kernel.get_texref("angle_tex")
self.angle_texref.set_array(cuda.np_to_array(np.ascontiguousarray(angle, dtype=np.float32), order="C"))
# Set texture parameters
self.angle_texref.set_filter_mode(cuda.filter_mode.LINEAR) #bilinear interpolation
self.angle_texref.set_address_mode(0, cuda.address_mode.CLAMP) #no indexing outside domain
self.angle_texref.set_address_mode(1, cuda.address_mode.CLAMP)
self.angle_texref.set_flags(cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing
def __init__(self, \
gpu_ctx, \
eta0, hu0, hv0, H, \
nx, ny, \
dx, dy, dt, \
g, f, r, \
subsample_f=10, \
angle=np.array([[0]], dtype=np.float32), \
subsample_angle=10, \
latitude=None, \
t=0.0, \
theta=1.3, rk_order=2, \
coriolis_beta=0.0, \
max_wind_direction_perturbation = 0, \
wind_stress=WindStress.WindStress(), \
boundary_conditions=Common.BoundaryConditions(), \
boundary_conditions_data=Common.BoundaryConditionsData(), \
small_scale_perturbation=False, \
small_scale_perturbation_amplitude=None, \
small_scale_perturbation_interpolation_factor = 1, \
model_time_step=None,
reportGeostrophicEquilibrium=False, \
use_lcg=False, \
write_netcdf=False, \
comm=None, \
local_particle_id=0, \
super_dir_name=None, \
netcdf_filename=None, \
ignore_ghostcells=False, \
courant_number=0.8, \
offset_x=0, offset_y=0, \
flux_slope_eps = 1.0e-1, \
desingularization_eps = 1.0e-1, \
depth_cutoff = 1.0e-5, \
block_width=12, block_height=32, num_threads_dt=256,
block_width_model_error=16, block_height_model_error=16):
"""
Initialization routine
eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells
hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells
hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells
H: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners
nx: Number of cells along x-axis
ny: Number of cells along y-axis
dx: Grid cell spacing along x-axis (20 000 m)
dy: Grid cell spacing along y-axis (20 000 m)
dt: Size of each timestep (90 s)
g: Gravitational accelleration (9.81 m/s^2)
f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y
r: Bottom friction coefficient (2.4e-3 m/s)
subsample_f: Subsample the coriolis f when creating texture by factor
angle: Angle of rotation from North to y-axis as a texture (cuda.Array) or numpy array (in radians)
subsample_angle: Subsample the angles given as input when creating texture by factor
latitude: Specify latitude. This will override any f and beta plane already set (in radians)
t: Start simulation at time t
theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme
rk_order: Order of Runge Kutta method {1,2*,3}
coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0)
max_wind_direction_perturbation: Large-scale model error emulation by per-time-step perturbation of wind direction by +/- max_wind_direction_perturbation (degrees)
wind_stress: Wind stress parameters
boundary_conditions: Boundary condition object
small_scale_perturbation: Boolean value for applying a stochastic model error
small_scale_perturbation_amplitude: Amplitude (q0 coefficient) for model error
small_scale_perturbation_interpolation_factor: Width factor for correlation in model error
model_time_step: The size of a data assimilation model step (default same as dt)
reportGeostrophicEquilibrium: Calculate the Geostrophic Equilibrium variables for each superstep
use_lcg: Use LCG as the random number generator. Default is False, which means using curand.
write_netcdf: Write the results after each superstep to a netCDF file
comm: MPI communicator
local_particle_id: Local (for each MPI process) particle id
desingularization_eps: Used for desingularizing hu/h
flux_slope_eps: Used for setting zero flux for symmetric Riemann fan
depth_cutoff: Used for defining dry cells
super_dir_name: Directory to write netcdf files to
netcdf_filename: Use this filename. (If not defined, a filename will be generated by SimWriter.)
"""
self.logger = logging.getLogger(__name__)
assert (rk_order < 4 or rk_order > 0
), "Only 1st, 2nd and 3rd order Runge Kutta supported"
if (rk_order == 3):
assert (r == 0.0
), "3rd order Runge Kutta supported only without friction"
# Sort out internally represented ghost_cells in the presence of given
# boundary conditions
ghost_cells_x = 2
ghost_cells_y = 2
#Coriolis at "first" cell
x_zero_reference_cell = ghost_cells_x
y_zero_reference_cell = ghost_cells_y # In order to pass it to the super constructor
# Boundary conditions
self.boundary_conditions = boundary_conditions
#Compensate f for reference cell (first cell in internal of domain)
north = np.array([np.sin(angle[0, 0]), np.cos(angle[0, 0])])
f = f - coriolis_beta * (x_zero_reference_cell * dx * north[0] +
y_zero_reference_cell * dy * north[1])
x_zero_reference_cell = 0
y_zero_reference_cell = 0
A = None
self.max_wind_direction_perturbation = max_wind_direction_perturbation
super(CDKLM16, self).__init__(gpu_ctx, \
nx, ny, \
ghost_cells_x, \
ghost_cells_y, \
dx, dy, dt, \
g, f, r, A, \
t, \
theta, rk_order, \
coriolis_beta, \
y_zero_reference_cell, \
wind_stress, \
write_netcdf, \
ignore_ghostcells, \
offset_x, offset_y, \
comm, \
block_width, block_height,
local_particle_id=local_particle_id)
# Index range for interior domain (north, east, south, west)
# so that interior domain of eta is
# eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \
# self.interior_domain_indices[3]:self.interior_domain_indices[1] ]
self.interior_domain_indices = np.array([-2, -2, 2, 2])
defines = {
'block_width': block_width,
'block_height': block_height,
'KPSIMULATOR_DESING_EPS': "{:.12f}f".format(desingularization_eps),
'KPSIMULATOR_FLUX_SLOPE_EPS': "{:.12f}f".format(flux_slope_eps),
'KPSIMULATOR_DEPTH_CUTOFF': "{:.12f}f".format(depth_cutoff),
'THETA': "{:.12f}f".format(self.theta),
'RK_ORDER': int(self.rk_order),
'NX': int(self.nx),
'NY': int(self.ny),
'DX': "{:.12f}f".format(self.dx),
'DY': "{:.12f}f".format(self.dy),
'GRAV': "{:.12f}f".format(self.g),
'FRIC': "{:.12f}f".format(self.r)
}
#Get kernels
self.kernel = gpu_ctx.get_kernel(
"CDKLM16_kernel.cu",
defines=defines,
compile_args={ # default, fast_math, optimal
'options': [
"--ftz=true", # false, true, true
"--prec-div=false", # true, false, false,
"--prec-sqrt=false", # true, false, false
"--fmad=false"
] # true, true, false
#'options': ["--use_fast_math"]
#'options': ["--generate-line-info"],
#nvcc_options=["--maxrregcount=39"],
#'arch': "compute_50",
#'code': "sm_50"
},
jit_compile_args={
#jit_options=[(cuda.jit_option.MAX_REGISTERS, 39)]
})
# Get CUDA functions and define data types for prepared_{async_}call()
self.cdklm_swe_2D = self.kernel.get_function("cdklm_swe_2D")
self.cdklm_swe_2D.prepare("fiPiPiPiPiPiPiPiPiffi")
self.update_wind_stress(self.kernel, self.cdklm_swe_2D)
# CUDA functions for finding max time step size:
self.num_threads_dt = num_threads_dt
self.num_blocks_dt = np.int32(self.global_size[0] *
self.global_size[1])
self.update_dt_kernels = gpu_ctx.get_kernel("max_dt.cu",
defines={
'block_width':
block_width,
'block_height':
block_height,
'NUM_THREADS':
self.num_threads_dt
})
self.per_block_max_dt_kernel = self.update_dt_kernels.get_function(
"per_block_max_dt")
self.per_block_max_dt_kernel.prepare("iifffPiPiPiPifPi")
self.max_dt_reduction_kernel = self.update_dt_kernels.get_function(
"max_dt_reduction")
self.max_dt_reduction_kernel.prepare("iPP")
# Bathymetry
self.bathymetry = Common.Bathymetry(gpu_ctx, self.gpu_stream, nx, ny,
ghost_cells_x, ghost_cells_y, H,
boundary_conditions)
# Adjust eta for possible dry states
Hm = self.downloadBathymetry()[1]
eta0 = np.maximum(eta0, -Hm)
# Create data by uploading to device
self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny,
ghost_cells_x, ghost_cells_y,
eta0, hu0, hv0)
# Allocate memory for calculating maximum timestep
host_dt = np.zeros((self.global_size[1], self.global_size[0]),
dtype=np.float32)
self.device_dt = Common.CUDAArray2D(self.gpu_stream,
self.global_size[0],
self.global_size[1], 0, 0, host_dt)
host_max_dt_buffer = np.zeros((1, 1), dtype=np.float32)
self.max_dt_buffer = Common.CUDAArray2D(self.gpu_stream, 1, 1, 0, 0,
host_max_dt_buffer)
self.courant_number = courant_number
## Allocating memory for geostrophical equilibrium variables
self.reportGeostrophicEquilibrium = np.int32(
reportGeostrophicEquilibrium)
self.geoEq_uxpvy = None
self.geoEq_Kx = None
self.geoEq_Ly = None
if self.reportGeostrophicEquilibrium:
dummy_zero_array = np.zeros(
(ny + 2 * ghost_cells_y, nx + 2 * ghost_cells_x),
dtype=np.float32,
order='C')
self.geoEq_uxpvy = Common.CUDAArray2D(self.gpu_stream, nx, ny,
ghost_cells_x, ghost_cells_y,
dummy_zero_array)
self.geoEq_Kx = Common.CUDAArray2D(self.gpu_stream, nx, ny,
ghost_cells_x, ghost_cells_y,
dummy_zero_array)
self.geoEq_Ly = Common.CUDAArray2D(self.gpu_stream, nx, ny,
ghost_cells_x, ghost_cells_y,
dummy_zero_array)
self.constant_equilibrium_depth = np.max(H)
self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \
self.nx, \
self.ny, \
ghost_cells_x, \
ghost_cells_y, \
self.boundary_conditions, \
boundary_conditions_data, \
)
def subsample_texture(data, factor):
ny, nx = data.shape
dx, dy = 1 / nx, 1 / ny
I = interp2d(np.linspace(0.5 * dx, 1 - 0.5 * dx, nx),
np.linspace(0.5 * dy, 1 - 0.5 * dy, ny),
data,
kind='linear')
new_nx, new_ny = max(2, nx // factor), max(2, ny // factor)
new_dx, new_dy = 1 / new_nx, 1 / new_ny
x_new = np.linspace(0.5 * new_dx, 1 - 0.5 * new_dx, new_nx)
y_new = np.linspace(0.5 * new_dy, 1 - 0.5 * new_dy, new_ny)
return I(x_new, y_new)
# Texture for angle
self.angle_texref = self.kernel.get_texref("angle_tex")
if isinstance(angle, cuda.Array):
# angle is already a texture, so we just set the texture reference
self.angle_texref.set_array(angle)
else:
#Upload data to GPU and bind to texture reference
if (subsample_angle and angle.size >= eta0.size):
self.logger.info("Subsampling angle texture by factor " +
str(subsample_angle))
self.logger.warning(
"This will give inaccurate angle along the border!")
angle = subsample_texture(angle, subsample_angle)
self.angle_texref.set_array(
cuda.np_to_array(np.ascontiguousarray(angle, dtype=np.float32),
order="C"))
# Set texture parameters
self.angle_texref.set_filter_mode(
cuda.filter_mode.LINEAR) #bilinear interpolation
self.angle_texref.set_address_mode(
0, cuda.address_mode.CLAMP) #no indexing outside domain
self.angle_texref.set_address_mode(1, cuda.address_mode.CLAMP)
self.angle_texref.set_flags(
cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing
# Texture for coriolis f
self.coriolis_texref = self.kernel.get_texref("coriolis_f_tex")
# Create the CPU coriolis
if (latitude is not None):
if (self.f != 0.0):
raise RuntimeError(
"Cannot specify both latitude and f. Make your mind up.")
coriolis_f, _ = OceanographicUtilities.calcCoriolisParams(latitude)
coriolis_f = coriolis_f.astype(np.float32)
else:
if (self.coriolis_beta != 0.0):
if (angle.size != 1):
raise RuntimeError(
"non-constant angle cannot be combined with beta plane model (makes no sense)"
)
#Generate coordinates for all cells, including ghost cells from center to center
# [-3/2dx, nx+3/2dx] for ghost_cells_x == 2
x = np.linspace((-self.ghost_cells_x + 0.5) * self.dx,
(self.nx + self.ghost_cells_x - 0.5) * self.dx,
self.nx + 2 * self.ghost_cells_x)
y = np.linspace((-self.ghost_cells_y + 0.5) * self.dy,
(self.ny + self.ghost_cells_y - 0.5) * self.dy,
self.ny + 2 * self.ghost_cells_x)
self.logger.info(
"Using latitude to create Coriolis f texture ({:f}x{:f} cells)"
.format(x.size, y.size))
x, y = np.meshgrid(x, y)
n = x * np.sin(angle[0, 0]) + y * np.cos(
angle[0, 0]) #North vector
coriolis_f = self.f + self.coriolis_beta * n
else:
if (self.f.size == 1):
coriolis_f = np.array([[self.f]], dtype=np.float32)
elif (self.f.shape == eta0.shape):
coriolis_f = np.array(self.f, dtype=np.float32)
else:
raise RuntimeError(
"The shape of f should match up with eta or be scalar."
)
if (subsample_f and coriolis_f.size >= eta0.size):
self.logger.info("Subsampling coriolis texture by factor " +
str(subsample_f))
self.logger.warning(
"This will give inaccurate coriolis along the border!")
coriolis_f = subsample_texture(coriolis_f, subsample_f)
#Upload data to GPU and bind to texture reference
self.coriolis_texref.set_array(
cuda.np_to_array(np.ascontiguousarray(coriolis_f,
dtype=np.float32),
order="C"))
# Set texture parameters
self.coriolis_texref.set_filter_mode(
cuda.filter_mode.LINEAR) #bilinear interpolation
self.coriolis_texref.set_address_mode(
0, cuda.address_mode.CLAMP) #no indexing outside domain
self.coriolis_texref.set_address_mode(1, cuda.address_mode.CLAMP)
self.coriolis_texref.set_flags(
cuda.TRSF_NORMALIZED_COORDINATES) #Use [0, 1] indexing
# Small scale perturbation:
self.small_scale_perturbation = small_scale_perturbation
self.small_scale_model_error = None
self.small_scale_perturbation_interpolation_factor = small_scale_perturbation_interpolation_factor
if small_scale_perturbation:
self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(
self,
soar_q0=small_scale_perturbation_amplitude,
interpolation_factor=
small_scale_perturbation_interpolation_factor,
use_lcg=use_lcg,
block_width=block_width_model_error,
block_height=block_height_model_error)
# Data assimilation model step size
self.model_time_step = model_time_step
self.total_time_steps = 0
if model_time_step is None:
self.model_time_step = self.dt
if self.write_netcdf:
self.sim_writer = SimWriter.SimNetCDFWriter(self, super_dir_name=super_dir_name, filename=netcdf_filename, \
ignore_ghostcells=self.ignore_ghostcells, offset_x=self.offset_x, offset_y=self.offset_y)
# Update timestep if dt is given as zero
if self.dt <= 0:
self.updateDt()
def __init__(self, \
gpu_ctx, \
H, eta0, hu0, hv0, \
nx, ny, \
dx, dy, dt, \
g, f, r, \
t=0.0, \
coriolis_beta=0.0, \
y_zero_reference_cell = 0, \
wind_stress=WindStress.WindStress(), \
boundary_conditions=Common.BoundaryConditions(), \
write_netcdf=False, \
ignore_ghostcells=False, \
offset_x=0, offset_y=0, \
block_width=16, block_height=16):
"""
Initialization routine
H: Water depth incl ghost cells, (nx+2)*(ny+2) cells
eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells
hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells
hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells
nx: Number of cells along x-axis
ny: Number of cells along y-axis
dx: Grid cell spacing along x-axis (20 000 m)
dy: Grid cell spacing along y-axis (20 000 m)
dt: Size of each timestep (90 s)
g: Gravitational accelleration (9.81 m/s^2)
f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y
r: Bottom friction coefficient (2.4e-3 m/s)
coriolis_beta: Coriolis linear factor -> f = f + beta*y
y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell .
wind_stress: Wind stress parameters
boundary_conditions: Boundary condition object
write_netcdf: Write the results after each superstep to a netCDF file
"""
#Create data by uploading to device
ghost_cells_x = 0
ghost_cells_y = 0
y_zero_reference_cell = y_zero_reference_cell
self.asym_ghost_cells = [0, 0, 0, 0] # [N, E, S, W]
# Index range for interior domain (north, east, south, west)
# so that interior domain of eta is
# eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \
# self.interior_domain_indices[3]:self.interior_domain_indices[1] ]
self.interior_domain_indices = np.array([None, None, 0, 0])
self.boundary_conditions = boundary_conditions
# Add asym ghost cell if periodic boundary condition:
if (self.boundary_conditions.north == 2) or \
(self.boundary_conditions.south == 2):
self.asym_ghost_cells[0] = 1
self.interior_domain_indices[0] = -1
if (self.boundary_conditions.east == 2) or \
(self.boundary_conditions.west == 2):
self.asym_ghost_cells[1] = 1
self.interior_domain_indices[1] = -1
if boundary_conditions.isSponge():
nx = nx + boundary_conditions.spongeCells[
1] + boundary_conditions.spongeCells[
3] # - self.asym_ghost_cells[1] - self.asym_ghost_cells[3]
ny = ny + boundary_conditions.spongeCells[
0] + boundary_conditions.spongeCells[
2] # - self.asym_ghost_cells[0] - self.asym_ghost_cells[2]
y_zero_reference_cell = y_zero_reference_cell + boundary_conditions.spongeCells[
2]
rk_order = None
theta = None
A = None
super(FBL, self).__init__(gpu_ctx, \
nx, ny, \
ghost_cells_x, \
ghost_cells_y, \
dx, dy, dt, \
g, f, r, A, \
t, \
theta, rk_order, \
coriolis_beta, \
y_zero_reference_cell, \
wind_stress, \
write_netcdf, \
ignore_ghostcells, \
offset_x, offset_y, \
block_width, block_height)
self._set_interior_domain_from_sponge_cells()
#Get kernels
self.u_kernel = gpu_ctx.get_kernel("FBL_U_kernel.cu",
defines={
'block_width': block_width,
'block_height': block_height
})
self.v_kernel = gpu_ctx.get_kernel("FBL_V_kernel.cu",
defines={
'block_width': block_width,
'block_height': block_height
})
self.eta_kernel = gpu_ctx.get_kernel("FBL_eta_kernel.cu",
defines={
'block_width': block_width,
'block_height': block_height
})
# Get CUDA functions
self.computeUKernel = self.u_kernel.get_function("computeUKernel")
self.computeVKernel = self.v_kernel.get_function("computeVKernel")
self.computeEtaKernel = self.eta_kernel.get_function(
"computeEtaKernel")
# Prepare kernel lauches
self.computeUKernel.prepare("iiffffffffPiPiPiPif")
self.computeVKernel.prepare("iiffffffffPiPiPiPif")
self.computeEtaKernel.prepare("iiffffffffPiPiPiPi")
# Set up textures
self.update_wind_stress(self.u_kernel, self.computeUKernel)
self.update_wind_stress(self.v_kernel, self.computeVKernel)
self.H = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x,
ghost_cells_y, H, self.asym_ghost_cells)
self.gpu_data = Common.SWEDataArakawaC(self.gpu_stream, nx, ny,
ghost_cells_x, ghost_cells_y,
eta0, hu0, hv0,
self.asym_ghost_cells)
# Overwrite halo with asymetric ghost cells
self.nx_halo = np.int32(nx + self.asym_ghost_cells[1] +
self.asym_ghost_cells[3])
self.ny_halo = np.int32(ny + self.asym_ghost_cells[0] +
self.asym_ghost_cells[2])
self.bc_kernel = FBL_periodic_boundary(self.gpu_ctx, \
self.nx, \
self.ny, \
self.boundary_conditions, \
self.asym_ghost_cells
)
self.totalNumIterations = 0
if self.write_netcdf:
self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \
staggered_grid=True, offset_x=self.offset_x, offset_y=self.offset_y)
def __init__(self, \
gpu_ctx, \
H, eta0, hu0, hv0, \
nx, ny, \
dx, dy, dt, \
g, f, r, A=0.0, \
t=0.0, \
coriolis_beta=0.0, \
y_zero_reference_cell = 0, \
wind_stress=WindStress.WindStress(), \
boundary_conditions=Common.BoundaryConditions(), \
write_netcdf=False, \
ignore_ghostcells=False, \
offset_x=0, offset_y=0, \
block_width=16, block_height=16):
"""
Initialization routine
H: Water depth incl ghost cells, (nx+2)*(ny+2) cells
eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells
hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells
hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells
nx: Number of cells along x-axis
ny: Number of cells along y-axis
dx: Grid cell spacing along x-axis (20 000 m)
dy: Grid cell spacing along y-axis (20 000 m)
dt: Size of each timestep (90 s)
g: Gravitational accelleration (9.81 m/s^2)
f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y
r: Bottom friction coefficient (2.4e-3 m/s)
A: Eddy viscosity coefficient (O(dx))
t: Start simulation at time t
coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0)
y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell .
wind_stress: Wind stress parameters
boundary_conditions: Boundary condition object
write_netcdf: Write the results after each superstep to a netCDF file
"""
# Sort out internally represented ghost_cells in the presence of given
# boundary conditions
halo_x = 1
halo_y = 1
ghost_cells_x = 1
ghost_cells_y = 1
y_zero_reference_cell = y_zero_reference_cell + 1
self.boundary_conditions = boundary_conditions
if boundary_conditions.isSponge():
nx = nx + boundary_conditions.spongeCells[
1] + boundary_conditions.spongeCells[3] - 2 * ghost_cells_x
ny = ny + boundary_conditions.spongeCells[
0] + boundary_conditions.spongeCells[2] - 2 * ghost_cells_y
y_zero_reference_cell = y_zero_reference_cell + boundary_conditions.spongeCells[
2]
# self.<parameters> are sat in parent constructor:
rk_order = None
theta = None
super(CTCS, self).__init__(gpu_ctx, \
nx, ny, \
ghost_cells_x, \
ghost_cells_y, \
dx, dy, dt, \
g, f, r, A, \
t, \
theta, rk_order, \
coriolis_beta, \
y_zero_reference_cell, \
wind_stress, \
write_netcdf, \
ignore_ghostcells, \
offset_x, offset_y, \
block_width, block_height)
# Index range for interior domain (north, east, south, west)
# so that interior domain of eta is
# eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \
# self.interior_domain_indices[3]:self.interior_domain_indices[1] ]
self.interior_domain_indices = np.array([-1, -1, 1, 1])
self._set_interior_domain_from_sponge_cells()
#Get kernels
self.u_kernel = gpu_ctx.get_kernel("CTCS_U_kernel.cu",
defines={
'block_width': block_width,
'block_height': block_height
})
self.v_kernel = gpu_ctx.get_kernel("CTCS_V_kernel.cu",
defines={
'block_width': block_width,
'block_height': block_height
})
self.eta_kernel = gpu_ctx.get_kernel("CTCS_eta_kernel.cu",
defines={
'block_width': block_width,
'block_height': block_height
})
# Get CUDA functions
self.computeUKernel = self.u_kernel.get_function("computeUKernel")
self.computeVKernel = self.v_kernel.get_function("computeVKernel")
self.computeEtaKernel = self.eta_kernel.get_function(
"computeEtaKernel")
# Prepare kernel lauches
self.computeUKernel.prepare("iiiifffffffffPiPiPiPiPif")
self.computeVKernel.prepare("iiiifffffffffPiPiPiPiPif")
self.computeEtaKernel.prepare("iiffffffffPiPiPi")
# Set up textures
self.update_wind_stress(self.u_kernel, self.computeUKernel)
self.update_wind_stress(self.v_kernel, self.computeVKernel)
#Create data by uploading to device
self.H = Common.CUDAArray2D(self.gpu_stream, nx, ny, halo_x, halo_y, H)
self.gpu_data = Common.SWEDataArakawaC(self.gpu_stream, nx, ny, halo_x,
halo_y, eta0, hu0, hv0)
# Global size needs to be larger than the default from parent.__init__
self.global_size = ( \
int(np.ceil((self.nx+2*halo_x) / float(self.local_size[0]))), \
int(np.ceil((self.ny+2*halo_y) / float(self.local_size[1]))) \
)
self.bc_kernel = CTCS_boundary_condition(gpu_ctx, \
self.nx, \
self.ny, \
self.boundary_conditions, \
halo_x, halo_y \
)
if self.write_netcdf:
self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \
staggered_grid=True, offset_x=self.offset_x, offset_y=self.offset_y)
def __init__(self, \
gpu_ctx, \
eta0, hu0, hv0, Hi, \
nx, ny, \
dx, dy, dt, \
g, f, r, \
t=0.0, \
theta=1.3, rk_order=2, \
coriolis_beta=0.0, \
y_zero_reference_cell = 0, \
max_wind_direction_perturbation = 0, \
wind_stress=WindStress.WindStress(), \
boundary_conditions=Common.BoundaryConditions(), \
small_scale_perturbation=False, \
small_scale_perturbation_amplitude=None, \
h0AsWaterElevation=False, \
reportGeostrophicEquilibrium=False, \
write_netcdf=False, \
ignore_ghostcells=False, \
offset_x=0, offset_y=0, \
block_width=32, block_height=4):
"""
Initialization routine
eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells
hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells
hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+1) cells
Hi: Depth from equilibrium defined on cell corners, (nx+5)*(ny+5) corners
nx: Number of cells along x-axis
ny: Number of cells along y-axis
dx: Grid cell spacing along x-axis (20 000 m)
dy: Grid cell spacing along y-axis (20 000 m)
dt: Size of each timestep (90 s)
g: Gravitational accelleration (9.81 m/s^2)
f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y
r: Bottom friction coefficient (2.4e-3 m/s)
t: Start simulation at time t
theta: MINMOD theta used the reconstructions of the derivatives in the numerical scheme
rk_order: Order of Runge Kutta method {1,2*,3}
coriolis_beta: Coriolis linear factor -> f = f + beta*(y-y_0)
y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell .
max_wind_direction_perturbation: Large-scale model error emulation by per-time-step perturbation of wind direction by +/- max_wind_direction_perturbation (degrees)
wind_stress: Wind stress parameters
boundary_conditions: Boundary condition object
h0AsWaterElevation: True if h0 is described by the surface elevation, and false if h0 is described by water depth
reportGeostrophicEquilibrium: Calculate the Geostrophic Equilibrium variables for each superstep
write_netcdf: Write the results after each superstep to a netCDF file
"""
## After changing from (h, B) to (eta, H), several of the simulator settings used are wrong. This check will help detect that.
if ( np.sum(eta0 - Hi[:-1, :-1] > 0) > nx):
assert(False), "It seems you are using water depth/elevation h and bottom topography B, while you should use water level eta and equillibrium depth H."
assert( rk_order < 4 or rk_order > 0 ), "Only 1st, 2nd and 3rd order Runge Kutta supported"
if (rk_order == 3):
assert(r == 0.0), "3rd order Runge Kutta supported only without friction"
# Sort out internally represented ghost_cells in the presence of given
# boundary conditions
ghost_cells_x = 2
ghost_cells_y = 2
y_zero_reference_cell = 2 + y_zero_reference_cell
# Boundary conditions
self.boundary_conditions = boundary_conditions
if (boundary_conditions.isSponge()):
nx = nx + boundary_conditions.spongeCells[1] + boundary_conditions.spongeCells[3] - 2*ghost_cells_x
ny = ny + boundary_conditions.spongeCells[0] + boundary_conditions.spongeCells[2] - 2*ghost_cells_y
y_zero_reference_cell = boundary_conditions.spongeCells[2] + y_zero_reference_cell
A = None
self.max_wind_direction_perturbation = max_wind_direction_perturbation
super(CDKLM16, self).__init__(gpu_ctx, \
nx, ny, \
ghost_cells_x, \
ghost_cells_y, \
dx, dy, dt, \
g, f, r, A, \
t, \
theta, rk_order, \
coriolis_beta, \
y_zero_reference_cell, \
wind_stress, \
write_netcdf, \
ignore_ghostcells, \
offset_x, offset_y, \
block_width, block_height)
# Index range for interior domain (north, east, south, west)
# so that interior domain of eta is
# eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \
# self.interior_domain_indices[3]:self.interior_domain_indices[1] ]
self.interior_domain_indices = np.array([-2,-2,2,2])
self._set_interior_domain_from_sponge_cells()
#Get kernels
self.kernel = gpu_ctx.get_kernel("CDKLM16_kernel.cu", defines={'block_width': block_width, 'block_height': block_height})
# Get CUDA functions and define data types for prepared_{async_}call()
self.swe_2D = self.kernel.get_function("swe_2D")
self.swe_2D.prepare("iifffffffffiiPiPiPiPiPiPiPiPifiiiiiPiPiPi")
self.update_wind_stress(self.kernel, self.swe_2D)
#Create data by uploading to device
self.gpu_data = Common.SWEDataArakawaA(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0)
## Allocating memory for geostrophical equilibrium variables
self.reportGeostrophicEquilibrium = np.int32(reportGeostrophicEquilibrium)
dummy_zero_array = np.zeros((ny+2*ghost_cells_y, nx+2*ghost_cells_x), dtype=np.float32, order='C')
self.geoEq_uxpvy = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array)
self.geoEq_Kx = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array)
self.geoEq_Ly = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, dummy_zero_array)
#Bathymetry
self.bathymetry = Common.Bathymetry(gpu_ctx, self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, Hi, boundary_conditions)
self.h0AsWaterElevation = h0AsWaterElevation
if self.h0AsWaterElevation:
self.bathymetry.waterElevationToDepth(self.gpu_data.h0)
self.constant_equilibrium_depth = np.max(Hi)
self.bc_kernel = Common.BoundaryConditionsArakawaA(gpu_ctx, \
self.nx, \
self.ny, \
ghost_cells_x, \
ghost_cells_y, \
self.boundary_conditions, \
)
# Small scale perturbation:
self.small_scale_perturbation = small_scale_perturbation
self.small_scale_model_error = None
if small_scale_perturbation:
if small_scale_perturbation_amplitude is None:
self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self)
else:
self.small_scale_model_error = OceanStateNoise.OceanStateNoise.fromsim(self, soar_q0=small_scale_perturbation_amplitude)
if self.write_netcdf:
self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \
offset_x=self.offset_x, offset_y=self.offset_y)
def __init__(self, \
gpu_ctx, \
H, eta0, hu0, hv0, \
nx, ny, \
dx, dy, dt, \
g, f, r, \
t=0.0, \
coriolis_beta=0.0, \
y_zero_reference_cell = 1, \
wind_stress=WindStress.WindStress(), \
boundary_conditions=Common.BoundaryConditions(), \
write_netcdf=False, \
comm=None, \
ignore_ghostcells=False, \
offset_x=0, offset_y=0, \
block_width=16, block_height=16):
"""
Initialization routine
H: Water depth incl ghost cells, (nx+2)*(ny+2) cells
eta0: Initial deviation from mean sea level incl ghost cells, (nx+2)*(ny+2) cells
hu0: Initial momentum along x-axis incl ghost cells, (nx+1)*(ny+2) cells
hv0: Initial momentum along y-axis incl ghost cells, (nx+2)*(ny+3) cells
nx: Number of cells along x-axis
ny: Number of cells along y-axis
dx: Grid cell spacing along x-axis (20 000 m)
dy: Grid cell spacing along y-axis (20 000 m)
dt: Size of each timestep (90 s)
g: Gravitational accelleration (9.81 m/s^2)
f: Coriolis parameter (1.2e-4 s^1), effectively as f = f + beta*y
r: Bottom friction coefficient (2.4e-3 m/s)
coriolis_beta: Coriolis linear factor -> f = f + beta*y
y_zero_reference_cell: The cell representing y_0 in the above, defined as the lower face of the cell .
wind_stress: Wind stress parameters
boundary_conditions: Boundary condition object
write_netcdf: Write the results after each superstep to a netCDF file
comm: MPI communicator
"""
#### THIS ALLOWS MAKES IT POSSIBLE TO GIVE THE OLD INPUT SHAPES TO NEW GHOST CELL REGIME: Only valid for benchmarking!
if (eta0.shape == (ny, nx)):
new_eta = np.zeros((ny+2, nx+2), dtype=np.float32)
new_eta[:ny, :nx] = eta0.copy()
eta0 = new_eta.copy()
if (H.shape == (ny, nx)):
new_H = np.ones((ny+2, nx+2), dtype=np.float32)*np.max(H)
new_H[:ny,:nx] = H.copy()
H = new_H.copy()
if (hu0.shape == (ny, nx+1)):
new_hu = np.zeros((ny+2, nx+1), dtype=np.float32)
new_hu[:ny, :nx+1] = hu0.copy()
hu0 = new_hu.copy()
if (hv0.shape == (ny+1, nx)):
new_hv = np.zeros((ny+3, nx+2), dtype=np.float32)
new_hv[:ny+1,:nx] = hv0.copy()
hv0 = new_hv.copy()
#Create data by uploading to device
ghost_cells_x = 1
ghost_cells_y = 1
y_zero_reference_cell = y_zero_reference_cell
# Index range for interior domain (north, east, south, west)
# so that interior domain of eta is
# eta[self.interior_domain_indices[2]:self.interior_domain_indices[0], \
# self.interior_domain_indices[3]:self.interior_domain_indices[1] ]
self.interior_domain_indices = np.array([-1, -1, 1, 1])
self.boundary_conditions = boundary_conditions
if boundary_conditions.isSponge():
nx = nx - 2 + boundary_conditions.spongeCells[1] + boundary_conditions.spongeCells[3]
ny = ny - 2 + boundary_conditions.spongeCells[0] + boundary_conditions.spongeCells[2]
y_zero_reference_cell = y_zero_reference_cell + boundary_conditions.spongeCells[2]
rk_order = None
theta = None
A = None
super(FBL, self).__init__(gpu_ctx, \
nx, ny, \
ghost_cells_x, \
ghost_cells_y, \
dx, dy, dt, \
g, f, r, A, \
t, \
theta, rk_order, \
coriolis_beta, \
y_zero_reference_cell, \
wind_stress, \
write_netcdf, \
ignore_ghostcells, \
offset_x, offset_y, \
comm, \
block_width, block_height)
self._set_interior_domain_from_sponge_cells()
#Get kernels
self.step_kernel = gpu_ctx.get_kernel("FBL_step_kernel.cu",
defines={'block_width': block_width, 'block_height': block_height},
compile_args={
'no_extern_c': True,
'options': ["--use_fast_math"],
#'options': ["--generate-line-info"],
#'options': ["--maxrregcount=32"]
#'arch': "compute_50",
#'code': "sm_50"
},
jit_compile_args={
#jit_options=[(cuda.jit_option.MAX_REGISTERS, 39)]
}
)
# Get CUDA functions
self.fblStepKernel = self.step_kernel.get_function("fblStepKernel")
# Prepare kernel lauches
self.fblStepKernel.prepare("iiffffffffPiPiPiPiif")
# Set up textures
self.update_wind_stress(self.step_kernel, self.fblStepKernel)
self.H = Common.CUDAArray2D(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, H)
self.gpu_data = Common.SWEDataArakawaC(self.gpu_stream, nx, ny, ghost_cells_x, ghost_cells_y, eta0, hu0, hv0, fbl=True)
# Domain including ghost cells
self.nx_halo = np.int32(nx + 2)
self.ny_halo = np.int32(ny + 2)
self.bc_kernel = FBL_boundary_conditions(self.gpu_ctx, \
self.nx, \
self.ny, \
self.boundary_conditions
)
# Bit-wise boolean for wall boundary conditions
self.wall_bc = np.int32(0)
if (self.boundary_conditions.north == 1):
self.wall_bc = self.wall_bc | 0x01
if (self.boundary_conditions.east == 1):
self.wall_bc = self.wall_bc | 0x02
if (self.boundary_conditions.south == 1):
self.wall_bc = self.wall_bc | 0x04
if (self.boundary_conditions.west == 1):
self.wall_bc = self.wall_bc | 0x08
if self.write_netcdf:
self.sim_writer = SimWriter.SimNetCDFWriter(self, ignore_ghostcells=self.ignore_ghostcells, \
staggered_grid=True, \
offset_x=self.offset_x, offset_y=self.offset_y)
def __init__(self, gpu_ctx, gpu_stream,
nx, ny, dx, dy,
boundaryConditions, staggered,
soar_q0=None, soar_L=None,
block_width=16, block_height=16):
self.random_numbers = None
self.seed = None
self.gpu_ctx = gpu_ctx
self.gpu_stream = gpu_stream
self.nx = np.int32(nx)
self.ny = np.int32(ny)
self.dx = np.float32(dx)
self.dy = np.float32(dy)
self.staggered = np.int(0)
if staggered:
self.staggered = np.int(1)
# The cutoff parameter is hard-coded.
# The size of the cutoff determines the computational radius in the
# SOAR function. Hence, the size of the local memory in the OpenCL
# kernels has to be hard-coded.
self.cutoff = np.int32(config.soar_cutoff)
self.periodicNorthSouth = np.int32(boundaryConditions.isPeriodicNorthSouth())
self.periodicEastWest = np.int32(boundaryConditions.isPeriodicEastWest())
# Size of random field and seed
self.rand_nx = np.int32(nx + 2*(1+self.cutoff))
self.rand_ny = np.int32(ny + 2*(1+self.cutoff))
if self.periodicEastWest:
self.rand_nx = np.int32(nx)
if self.periodicNorthSouth:
self.rand_ny = np.int32(ny)
self.seed_ny = np.int32(self.rand_ny)
self.seed_nx = np.int32(self.rand_nx/2)
### WHAT IF rand_nx IS ODD??
# For now, we check this by assert
assert(self.rand_nx % 2 == 0), "The OceanStateNoise module might not work with odd Nx, so just to be sure you are not allowed to use odd Nx for now :)"
# Constants for the SOAR function:
self.soar_q0 = np.float32(self.dx/100000)
if soar_q0 is not None:
self.soar_q0 = np.float32(soar_q0)
self.soar_L = np.float32(0.75*self.dx)
if soar_L is not None:
self.soar_L = np.float32(soar_L)
# Generate seed:
self.floatMax = 2147483648.0
self.host_seed = np.random.rand(self.seed_ny, self.seed_nx)*self.floatMax
self.host_seed = self.host_seed.astype(np.uint64, order='C')
self.seed = Common.CUDAArray2D(gpu_stream, self.seed_nx, self.seed_ny, 0, 0, self.host_seed, double_precision=True, integers=True)
# Allocate memory for random numbers
self.random_numbers_host = np.zeros((self.rand_ny, self.rand_nx), dtype=np.float32, order='C')
self.random_numbers = Common.CUDAArray2D(self.gpu_stream, self.rand_nx, self.rand_ny, 0, 0, self.random_numbers_host)
# Generate kernels
self.kernels = gpu_ctx.get_kernel("ocean_noise.cu", \
defines={'block_width': block_width, 'block_height': block_height})
# Get CUDA functions and define data types for prepared_{async_}call()
self.uniformDistributionKernel = self.kernels.get_function("uniformDistribution")
self.uniformDistributionKernel.prepare("iiiPiPi")
self.normalDistributionKernel = self.kernels.get_function("normalDistribution")
self.normalDistributionKernel.prepare("iiiPiPi")
self.perturbOceanKernel = self.kernels.get_function("perturbOcean")
self.perturbOceanKernel.prepare("iiffiiffffffiiPiPiPiPiPi")
#Compute kernel launch parameters
self.local_size = (block_width, block_height, 1)
self.global_size_random_numbers = ( \
int(np.ceil(self.seed_nx / float(self.local_size[0]))), \
int(np.ceil(self.seed_ny / float(self.local_size[1]))) \
)
self.global_size_noise = ( \
int(np.ceil(self.rand_nx / float(self.local_size[0]))), \
int(np.ceil(self.rand_ny / float(self.local_size[1]))) \
)
def init_double(self):
self.double_cudaarray = Common.CUDAArray2D(self.gpu_stream, \
self.nx, self.ny, \
self.nx_halo, self.ny_halo, \
self.dbuf1, \
double_precision=True)
