def setUp(self):
self.nx, self.ny, self.nz = 32, 32, 32
self.omega = 12.
self.comm = MPI.COMM_WORLD
self.fcomm = self.comm.py2f()
self.periodic = numpy.array([True, True, True])
self.filter_type = ('compact', 'compact', 'compact')
self.grid_partition = t3dmod.t3d(self.fcomm, self.nx, self.ny, self.nz,
self.periodic)
self.chunk_3d_size = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_lo = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_hi = numpy.zeros(3, dtype=numpy.int32, order='F')
self.grid_partition.get_sz3d(self.chunk_3d_size)
self.grid_partition.get_st3d(self.chunk_3d_lo)
self.grid_partition.get_en3d(self.chunk_3d_hi)
self.chunk_3d_lo = self.chunk_3d_lo - 1 # Convert to 0 based indexing
self.chunk_3d_hi = self.chunk_3d_hi - 1 # Convert to 0 based indexing
self.fil = Filter(self.grid_partition,
self.filter_type,
periodic_dimensions=self.periodic)
self.x = numpy.linspace(0., 2. * numpy.pi, num=self.nx +
1)[self.chunk_3d_lo[0]:self.chunk_3d_hi[0] + 1]
self.y = numpy.linspace(0., 2. * numpy.pi, num=self.ny +
1)[self.chunk_3d_lo[1]:self.chunk_3d_hi[1] + 1]
self.z = numpy.linspace(0., 2. * numpy.pi, num=self.nz +
1)[self.chunk_3d_lo[2]:self.chunk_3d_hi[2] + 1]
self.x, self.y, self.z = numpy.meshgrid(self.x,
self.y,
self.z,
indexing='ij')
self.x = numpy.asfortranarray(self.x)
self.y = numpy.asfortranarray(self.y)
self.z = numpy.asfortranarray(self.z)
self.f = numpy.sin(self.omega * self.x) * numpy.cos(
self.omega * self.y) * numpy.cos(self.omega * self.z)
self.dx, self.dy, self.dz = 2. * numpy.pi / self.nx, 2. * numpy.pi / self.ny, 2. * numpy.pi / self.nz
k_norm_x = self.omega * self.dx
TF_x = getTransferFunction(k_norm_x)
k_norm_y = self.omega * self.dy
TF_y = getTransferFunction(k_norm_y)
k_norm_z = self.omega * self.dz
TF_z = getTransferFunction(k_norm_z)
self.f_tilde_x_exact = TF_x * self.f
self.f_tilde_y_exact = TF_y * self.f
self.f_tilde_z_exact = TF_z * self.f
self.f_tilde_exact = TF_x * TF_y * TF_z * self.f
def setUp(self):
self.nx, self.ny, self.nz = 64, 64, 1
self.omega = 1.
self.comm = MPI.COMM_WORLD
self.fcomm = self.comm.py2f()
self.periodic = numpy.array([True, True, True])
self.order = (10, 10)
self.dx, self.dy = 2. * numpy.pi / self.nx, 2. * numpy.pi / self.ny
self.grid_partition = t3dmod.t3d(self.fcomm, self.nx, self.ny, self.nz,
self.periodic)
self.chunk_3d_size = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_lo = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_hi = numpy.zeros(3, dtype=numpy.int32, order='F')
self.grid_partition.get_sz3d(self.chunk_3d_size)
self.grid_partition.get_st3d(self.chunk_3d_lo)
self.grid_partition.get_en3d(self.chunk_3d_hi)
self.chunk_3d_lo = self.chunk_3d_lo - 1 # Convert to 0 based indexing
self.chunk_3d_hi = self.chunk_3d_hi - 1 # Convert to 0 based indexing
self.der = CompactDifferentiator(self.grid_partition, (self.dx, self.dy), self.order, 2, \
(self.periodic[0], self.periodic[1]))
self.x = numpy.linspace(0., 2. * numpy.pi, num=self.nx +
1)[self.chunk_3d_lo[0]:self.chunk_3d_hi[0] + 1]
self.y = numpy.linspace(0., 2. * numpy.pi, num=self.ny +
1)[self.chunk_3d_lo[1]:self.chunk_3d_hi[1] + 1]
self.x, self.y = numpy.meshgrid(self.x, self.y, indexing='ij')
self.x = numpy.asfortranarray(self.x)
self.y = numpy.asfortranarray(self.y)
self.f = numpy.sin(self.omega * self.x) * numpy.cos(
self.omega * self.y)
self.dfdx_exact = self.omega * numpy.cos(
self.omega * self.x) * numpy.cos(self.omega * self.y)
self.dfdy_exact = -self.omega * numpy.sin(
self.omega * self.x) * numpy.sin(self.omega * self.y)
self.d2fdx2_exact = -self.omega**2 * numpy.sin(
self.omega * self.x) * numpy.cos(self.omega * self.y)
self.d2fdy2_exact = -self.omega**2 * numpy.sin(
self.omega * self.x) * numpy.cos(self.omega * self.y)
def setUp(self):
self.nx, self.ny, self.nz = 64, 32, 16
self.omega_x, self.omega_y, self.omega_z = 1., 2., 3.
self.comm = MPI.COMM_WORLD
self.fcomm = self.comm.py2f()
self.periodic = numpy.array([True, True, True])
self.dx, self.dy, self.dz = 2.*numpy.pi / self.nx, 2.*numpy.pi / self.ny, 2.*numpy.pi / self.nz
self.grid_partition = t3dmod.t3d(self.fcomm, self.nx, self.ny, self.nz, self.periodic )
self.chunk_3d_size = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_lo = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_hi = numpy.zeros(3, dtype=numpy.int32, order='F')
self.grid_partition.get_sz3d(self.chunk_3d_size)
self.grid_partition.get_st3d(self.chunk_3d_lo)
self.grid_partition.get_en3d(self.chunk_3d_hi)
self.chunk_3d_lo = self.chunk_3d_lo - 1 # Convert to 0 based indexing
self.chunk_3d_hi = self.chunk_3d_hi - 1 # Convert to 0 based indexing
self.x = numpy.linspace(0., 2.*numpy.pi, num=self.nx+1)[self.chunk_3d_lo[0]:self.chunk_3d_hi[0]+1]
self.y = numpy.linspace(0., 2.*numpy.pi, num=self.ny+1)[self.chunk_3d_lo[1]:self.chunk_3d_hi[1]+1]
self.z = numpy.linspace(0., 2.*numpy.pi, num=self.nz+1)[self.chunk_3d_lo[2]:self.chunk_3d_hi[2]+1]
self.x, self.y, self.z = numpy.meshgrid(self.x, self.y, self.z, indexing='ij')
self.x = numpy.asfortranarray(self.x)
self.y = numpy.asfortranarray(self.y)
self.z = numpy.asfortranarray(self.z)
self.f = numpy.sin(self.omega_x*self.x) * numpy.cos(self.omega_y*self.y) * numpy.cos(self.omega_z*self.z)
self.avg_x = red.Reduction(self.grid_partition, ( True, False, False))
self.avg_y = red.Reduction(self.grid_partition, (False, True, False))
self.avg_z = red.Reduction(self.grid_partition, (False, False, True))
self.avg_xy = red.Reduction(self.grid_partition, ( True, True, False))
self.avg_yz = red.Reduction(self.grid_partition, (False, True, True))
self.avg_xz = red.Reduction(self.grid_partition, ( True, False, True))
self.avg_xyz = red.Reduction(self.grid_partition, ( True, True, True))
#------------------------------------ # Set the MPI communicator #------------------------------------- comm = MPI.COMM_WORLD fcomm = MPI.COMM_WORLD.py2f() nx = 8 ny = 16 nz = 32 periodic = numpy.array([True, True, True]) fail = numpy.array([True]) reorder = True # Create the t3d object gp = t3dmod.t3d(fcomm, nx, ny, nz, periodic) # Get # of procs in x, y and z px = gp.px() py = gp.py() pz = gp.pz() # Get cartesian communicators comm_x = MPI.Comm.f2py( gp.commx () ) comm_y = MPI.Comm.f2py( gp.commy () ) comm_z = MPI.Comm.f2py( gp.commz () ) comm_xy = MPI.Comm.f2py( gp.commxy() ) comm_yz = MPI.Comm.f2py( gp.commyz() ) comm_xz = MPI.Comm.f2py( gp.commxz() ) # Get size of subdomain of this processor and start and end of the subdomain
def __init__(self, comm, serial_reader, sub_domain=None, num_ghosts=None):
"""
Constructor of the class.
comm : mpi4py communicator object
serial_reader : a concrete object that extends BaseReader (Do not use this outside of this class)
sub_domain : iterable of size 2 with the first entry being lo and second entry being hi
num_ghosts : numpy integer array of size 3 with the no. of ghost values in the x, y and z directions respectively
"""
if not isinstance(serial_reader, BaseReader):
raise RuntimeError("The given serial data reader is not instance of the base reader!")
if serial_reader.data_order != 'F':
raise RuntimeError("The data order should be 'F'!")
# Set the communicator and it's Fortran value.
self._comm = comm
self._fcomm = comm.py2f()
# Set the serial data reader to use.
self._serial_reader = serial_reader
# Dimensionality of the data set (1D, 2D or 3D)
self._dim = serial_reader.dimension
if self._dim < 0 or self._dim > 3:
raise RuntimeError('Dimension of data should be between 1 and 3!')
self._periodic_dimensions = None
self._domain_size = None
if num_ghosts is None:
self._num_ghosts = numpy.array([0, 0, 0], dtype=numpy.int32)
else:
if self._dim == 1:
if len(num_ghosts) < 1 or len(num_ghosts) > 3:
raise RuntimeError('Dimension of num_ghosts should be between 1 and 3!')
self._num_ghosts = numpy.array([num_ghosts[0], 0, 0], dtype=numpy.int32)
elif self._dim == 2:
if len(num_ghosts) < 2 or len(num_ghosts) > 3:
raise RuntimeError('Dimension of num_ghosts should be between 2 and 3!')
self._num_ghosts = numpy.array([num_ghosts[0], num_ghosts[1], 0], dtype=numpy.int32)
else:
if len(num_ghosts) != 3:
raise RuntimeError('Dimension of num_ghosts should be 3!')
self._num_ghosts = numpy.asarray(num_ghosts, dtype=numpy.int32)
if self._dim == 1:
self._periodic_dimensions = numpy.array([serial_reader.periodic_dimensions[0], True, True])
self._domain_size = numpy.array([serial_reader.domain_size[0], 1, 1])
self._num_ghosts[1:] = 0
elif self._dim == 2:
self._periodic_dimensions = numpy.array([serial_reader.periodic_dimensions[0], \
serial_reader.periodic_dimensions[1] , True])
self._domain_size = numpy.array([serial_reader.domain_size[0], serial_reader.domain_size[1], 1])
self._num_ghosts[2] = 0
else:
self._periodic_dimensions = numpy.asarray(serial_reader.periodic_dimensions)
self._domain_size = numpy.asarray(serial_reader.domain_size)
if sub_domain == None:
self._subdomain_lo = numpy.array([0, 0, 0], dtype=self._domain_size.dtype)
self._subdomain_hi = self._domain_size - 1
self._subdomain_size = self._domain_size
else:
# Need to change periodic_dimensions if only reading in a sub-domain
raise NotImplementedError('Reading in only a partial sub domain is not yet implemented. Sorry!')
try:
lo, hi = sub_domain
except ValueError:
raise ValueError("Pass an iterable of sub_domain with two items!")
for i in range(self._dim):
if lo[i] < 0 or lo[i] > self._domain_size[i]:
raise ValueError('Invalid indices in sub-domain. Cannot be < 0 or > domain size!')
if hi[i] < 0 or hi[i] > self._domain_size[i]:
raise ValueError('Invalid indices in sub-domain. Cannot be < 0 or > domain size!')
if hi[i] < lo[i]:
raise ValueError('Invalid indices in sub-domain. Upper bound cannot be smaller than lower bound!')
self._subdomain_lo = numpy.asarray(lo)
self._subdomain_hi = numpy.asarray(hi)
self._subdomain_size = self._subdomain_hi - self._subdomain_lo + 1
# Create the parallel grid partition object that handles all the communication stuff.
self._grid_partition = t3dmod.t3d(self._fcomm, \
self._subdomain_size[0], self._subdomain_size[1], self._subdomain_size[2], \
self._periodic_dimensions, nghosts=self._num_ghosts )
# Size of the interior chunk of this process.
self._interior_chunk_size = numpy.zeros(3, dtype=numpy.int32, order='F')
# Indices of the start and end of the interior chunk of this process.
self._interior_chunk_lo = numpy.zeros(3, dtype=numpy.int32, order='F')
self._interior_chunk_hi = numpy.zeros(3, dtype=numpy.int32, order='F')
self._grid_partition.get_sz3d(self._interior_chunk_size)
self._grid_partition.get_st3d(self._interior_chunk_lo)
self._grid_partition.get_en3d(self._interior_chunk_hi)
self._interior_chunk_lo = self._interior_chunk_lo - 1 # Convert to 0 based indexing
self._interior_chunk_hi = self._interior_chunk_hi - 1 # Convert to 0 based indexing
# Size of the full chunk of this process.
self._full_chunk_size = numpy.zeros(3, dtype=numpy.int32, order='F')
# Indices of the start and end of the full chunk of this process.
self._full_chunk_lo = numpy.zeros(3, dtype=numpy.int32, order='F')
self._full_chunk_hi = numpy.zeros(3, dtype=numpy.int32, order='F')
self._grid_partition.get_sz3dg(self._full_chunk_size)
self._grid_partition.get_st3dg(self._full_chunk_lo)
self._grid_partition.get_en3dg(self._full_chunk_hi)
self._full_chunk_lo = self._full_chunk_lo - 1 # Convert to 0 based indexing
self._full_chunk_hi = self._full_chunk_hi - 1 # Convert to 0 based indexing
# Set the sub domain to read in using the serial data reader.
self._serial_reader.sub_domain = ( tuple(self._interior_chunk_lo), tuple(self._interior_chunk_hi) )
self._interior = (
slice(self._interior_chunk_lo[0] - self._full_chunk_lo[0],
self._full_chunk_size[0] - (self._full_chunk_hi[0] - self._interior_chunk_hi[0])),
slice(self._interior_chunk_lo[1] - self._full_chunk_lo[1],
self._full_chunk_size[1] - (self._full_chunk_hi[1] - self._interior_chunk_hi[1])),
slice(self._interior_chunk_lo[2] - self._full_chunk_lo[2],
self._full_chunk_size[2] - (self._full_chunk_hi[2] - self._interior_chunk_hi[2])) )
# Set the MPI communicator #------------------------------------- comm = MPI.COMM_WORLD fcomm = MPI.COMM_WORLD.py2f() nx = 8 ny = 16 nz = 32 periodic = numpy.array([True, True, True]) nghosts = numpy.array([1,1,1], dtype=numpy.int32, order='F') fail = numpy.array([True]) reorder = True # gp = t3dmod.t3d(fcomm, nx, ny, nz, px, py, pz, periodic, reorder, fail, nghosts=nghosts, createcrosscommunicators=True) gp = t3dmod.t3d(fcomm, nx, ny, nz, periodic, nghosts=nghosts) # Get # of procs in x, y and z px = gp.px() py = gp.py() pz = gp.pz() # Get cartesian communicators comm_x = MPI.Comm.f2py( gp.commx () ) comm_y = MPI.Comm.f2py( gp.commy () ) comm_z = MPI.Comm.f2py( gp.commz () ) comm_xy = MPI.Comm.f2py( gp.commxy() ) comm_yz = MPI.Comm.f2py( gp.commyz() ) comm_xz = MPI.Comm.f2py( gp.commxz() ) # Get size of interior subdomain of this processor and start and end of the interior subdomain
def __init__(self, nx, ny, nz, dx, dy, dz, periodic):
self.nx = nx
self.ny = ny
self.nz = nz
self.dx = dx
self.dy = dy
self.dz = dz
self.comm = MPI.COMM_WORLD
self.fcomm = self.comm.py2f()
self.periodic = periodic
self.order = (10, 10, 10)
self.filter_type = ('compact', 'compact', 'compact')
self.grid_partition = t3dmod.t3d(self.fcomm, self.nx, self.ny, self.nz,
self.periodic)
self.chunk_3d_size = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_lo = numpy.zeros(3, dtype=numpy.int32, order='F')
self.chunk_3d_hi = numpy.zeros(3, dtype=numpy.int32, order='F')
self.grid_partition.get_sz3d(self.chunk_3d_size)
self.grid_partition.get_st3d(self.chunk_3d_lo)
self.grid_partition.get_en3d(self.chunk_3d_hi)
self.chunk_3d_lo = self.chunk_3d_lo - 1 # Convert to 0 based indexing
self.chunk_3d_hi = self.chunk_3d_hi - 1 # Convert to 0 based indexing
# Set up comms
self.xcom = MPI.Comm.f2py(self.grid_partition.commx())
self.ycom = MPI.Comm.f2py(self.grid_partition.commy())
self.zcom = MPI.Comm.f2py(self.grid_partition.commz())
self.xycom = MPI.Comm.f2py(self.grid_partition.commxy())
self.yzcom = MPI.Comm.f2py(self.grid_partition.commyz())
self.xzcom = MPI.Comm.f2py(self.grid_partition.commxz())
self.der = CompactDerivative(self.grid_partition,
(self.dx, self.dy, self.dz), self.order,
self.periodic)
self.fil = Filter(self.grid_partition,
self.filter_type,
periodic_dimensions=self.periodic)
self.gfil = Filter(self.grid_partition,
('gaussian', 'gaussian', 'gaussian'),
periodic_dimensions=self.periodic)
self.master = False
if self.comm.rank == 0:
self.master = True
self.x1proc = False
if self.xcom.rank == 0:
self.x1proc = True
self.xnproc = False
if self.xcom.rank == self.xcom.size - 1:
self.xnproc = True
self.y1proc = False
if self.ycom.rank == 0:
self.y1proc = True
self.ynproc = False
if self.ycom.rank == self.ycom.size - 1:
self.ynproc = True
