def __init__(self, physical_system, performance_test_flag = False):
"""
Constructor for the nonlinear_solver object. It takes the physical
system object as an argument and uses it in intialization and
evolution of the system in consideration.
Additionally, a performance test flag is also passed which when true,
stores time which is consumed by each of the major solver routines.
This proves particularly useful in analyzing performance bottlenecks
and obtaining benchmarks.
Parameters:
-----------
physical_system: object
The defined physical system object which holds
all the simulation information such as the initial
conditions, and the domain info is passed as an
argument in defining an instance of the
nonlinear_solver. This system is then evolved, and
monitored using the various methods under the
nonlinear_solver class.
performance_test_flag: bool
When set to true, the time elapsed in each of the
solver routines is measured. These performance
stats can be obtained at the end of the run using
the command print_performance_timings, which summarizes
the results in a table.
"""
self.physical_system = physical_system
# Holding Domain Info:
self.q1_start, self.q1_end = physical_system.q1_start,\
physical_system.q1_end
self.q2_start, self.q2_end = physical_system.q2_start,\
physical_system.q2_end
self.p1_start, self.p1_end = physical_system.p1_start,\
physical_system.p1_end
self.p2_start, self.p2_end = physical_system.p2_start,\
physical_system.p2_end
self.p3_start, self.p3_end = physical_system.p3_start,\
physical_system.p3_end
# Holding Domain Resolution:
self.N_q1, self.dq1 = physical_system.N_q1, physical_system.dq1
self.N_q2, self.dq2 = physical_system.N_q2, physical_system.dq2
self.N_p1, self.dp1 = physical_system.N_p1, physical_system.dp1
self.N_p2, self.dp2 = physical_system.N_p2, physical_system.dp2
self.N_p3, self.dp3 = physical_system.N_p3, physical_system.dp3
# Getting number of ghost zones, and the boundary
# conditions that are utilized:
N_g = self.N_ghost = physical_system.N_ghost
self.boundary_conditions = physical_system.boundary_conditions
# MPI Communicator:
self._comm = self.physical_system.mpi_communicator
if(self.physical_system.params.num_devices>1):
rank = self._comm.rank
if (self.physical_system.params.manual_device_allocation == True):
af.set_device(self.physical_system.params.device_allocation[rank])
else:
af.set_device(rank%self.physical_system.params.num_devices)
# Getting number of species:
N_s = self.N_species = self.physical_system.N_species
# TODO: Remove mass and charge from lib
if(type(physical_system.params.mass) == list):
# Having a temporary copy of the lists to copy to af.Array:
list_mass = physical_system.params.mass.copy()
list_charge = physical_system.params.charge.copy()
# Initializing af.Arrays for mass and charge:
# Having the mass and charge along axis 1:
self.physical_system.params.mass = af.constant(0, 1, N_s, dtype = af.Dtype.f64)
self.physical_system.params.charge = af.constant(0, 1, N_s, dtype = af.Dtype.f64)
for i in range(N_s):
self.physical_system.params.mass[0, i] = list_mass[i]
self.physical_system.params.charge[0, i] = list_charge[i]
self.physical_system.params.rank = self._comm.rank
PETSc.Sys.Print('\nBackend Details for Nonlinear Solver:')
# Printing the backend details for each rank/device/node:
PETSc.Sys.syncPrint(indent('Rank ' + str(self._comm.rank) + ' of ' + str(self._comm.size-1)))
PETSc.Sys.syncPrint(indent('On Node: '+ socket.gethostname()))
PETSc.Sys.syncPrint(indent('Device Details:'))
PETSc.Sys.syncPrint(indent(af.info_str(), 2))
PETSc.Sys.syncPrint(indent('Device Bandwidth = ' + str(bandwidth_test(100)) + ' GB / sec'))
PETSc.Sys.syncPrint()
PETSc.Sys.syncFlush()
self.performance_test_flag = performance_test_flag
# Initializing variables which are used to time the components of the solver:
if(performance_test_flag == True):
self.time_ts = 0
self.time_interp2 = 0
self.time_sourcets = 0
self.time_fvm_solver = 0
self.time_reconstruct = 0
self.time_riemann = 0
self.time_fieldstep = 0
self.time_interp3 = 0
self.time_apply_bcs_f = 0
self.time_communicate_f = 0
petsc_bc_in_q1 = 'ghosted'; self.N_g1 = self.N_ghost
petsc_bc_in_q2 = 'ghosted'; self.N_g2 = self.N_ghost
# Only for periodic boundary conditions or shearing-box boundary conditions
# do the boundary conditions passed to the DA need to be changed. PETSc
# automatically handles the application of periodic boundary conditions when
# running in parallel. For shearing box boundary conditions, an interpolation
# operation needs to be applied on top of the periodic boundary conditions.
# In all other cases, ghosted boundaries are used.
if( self.boundary_conditions.in_q1_left == 'periodic'
or self.boundary_conditions.in_q1_left == 'shearing-box'
):
petsc_bc_in_q1 = 'periodic'
if( self.boundary_conditions.in_q2_bottom == 'periodic'
or self.boundary_conditions.in_q2_bottom == 'shearing-box'
):
petsc_bc_in_q2 = 'periodic'
if(self.boundary_conditions.in_q1_left == 'none'):
petsc_bc_in_q1 = 'none'; self.N_g1 = 0
if(self.boundary_conditions.in_q2_bottom == 'none'):
petsc_bc_in_q2 = 'none'; self.N_g2 = 0
if(self.boundary_conditions.in_q1_left == 'periodic'):
try:
assert(self.boundary_conditions.in_q1_right == 'periodic')
except:
raise Exception('Periodic boundary conditions need to be applied to \
both the boundaries of a particular axis'
)
if(self.boundary_conditions.in_q1_left == 'shearing-box'):
try:
assert(self.boundary_conditions.in_q1_right == 'shearing-box')
except:
raise Exception('Shearing box boundary conditions need to be applied to \
both the boundaries of a particular axis'
)
if(self.boundary_conditions.in_q2_bottom == 'periodic'):
try:
assert(self.boundary_conditions.in_q2_top == 'periodic')
except:
raise Exception('Periodic boundary conditions need to be applied to \
both the boundaries of a particular axis'
)
if(self.boundary_conditions.in_q2_bottom == 'shearing-box'):
try:
assert(self.boundary_conditions.in_q2_top == 'shearing-box')
except:
raise Exception('Shearing box boundary conditions need to be applied to \
both the boundaries of a particular axis'
)
if(self.boundary_conditions.in_q1_left == 'none'):
try:
assert(self.boundary_conditions.in_q1_right == 'none')
except:
raise Exception('NONE boundary conditions need to be applied to \
both the boundaries of a particular axis'
)
if(self.boundary_conditions.in_q2_bottom == 'none'):
try:
assert(self.boundary_conditions.in_q2_top == 'none')
except:
raise Exception('NONE boundary conditions need to be applied to \
both the boundaries of a particular axis'
)
self._nproc_in_q1 = PETSc.DECIDE
self._nproc_in_q2 = PETSc.DECIDE
# Break up the domain into manually defined portions
self._ownership_ranges = None
if self.physical_system.params.enable_manual_domain_decomposition:
ownership_q1 = [self.N_q1*item for item in self.physical_system.params.q1_partition]
ownership_q2 = [self.N_q2*item for item in self.physical_system.params.q2_partition]
self._ownership_ranges = (ownership_q1, ownership_q2)
# TODO : Implement error handling and give clean messages
# Since shearing boundary conditions require interpolations which are non-local:
if(self.boundary_conditions.in_q2_bottom == 'shearing-box'):
self._nproc_in_q1 = 1
if(self.boundary_conditions.in_q1_left == 'shearing-box'):
self._nproc_in_q2 = 1
# DMDA is a data structure to handle a distributed structure
# grid and its related core algorithms. It stores metadata of
# how the grid is partitioned when run in parallel which is
# utilized by the various methods of the solver.
self._da_f = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof = ( self.N_species
* self.N_p1
* self.N_p2
* self.N_p3
),
stencil_width = N_g,
boundary_type = (petsc_bc_in_q1,
petsc_bc_in_q2
),
proc_sizes = (self._nproc_in_q1,
self._nproc_in_q2
),
ownership_ranges = self._ownership_ranges,
stencil_type = 1,
comm = self._comm
)
lx, ly = self._da_f.getOwnershipRanges()
# This DA is used by the FileIO routine dump_moments():
# Finding the number of definitions for the moments:
attributes = [a for a in dir(self.physical_system.moments) if not a.startswith('_')]
# Removing utility functions:
if('integral_over_v' in attributes):
attributes.remove('integral_over_v')
self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof = self.N_species
* (len(attributes)-2), # don't countintegral_over_p, and params in moments.py
proc_sizes = (self._nproc_in_q1,
self._nproc_in_q2
),
ownership_ranges = self._ownership_ranges,
comm = self._comm
)
# For dumping aux arrays:
self.dump_aux_arrays_initial_call = 1
# Creation of the local and global vectors from the DA:
# This is for the distribution function
self._glob_f = self._da_f.createGlobalVec()
self._local_f = self._da_f.createLocalVec()
# The following vector is used to dump the data to file:
self._glob_moments = self._da_dump_moments.createGlobalVec()
# Getting the arrays for the above vectors:
self._glob_f_array = self._glob_f.getArray()
self._local_f_array = self._local_f.getArray()
self._glob_moments_array = self._glob_moments.getArray()
# Setting names for the objects which will then be
# used as the key identifiers for the HDF5 files:
PETSc.Object.setName(self._glob_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
# Indexing vars used through out
self.i_q1_start = self.N_g1; self.i_q1_end = -self.N_g1
self.i_q2_start = self.N_g2; self.i_q2_end = -self.N_g2
if (self.N_g1 == 0):
self.i_q1_end = 1
if (self.N_g2 == 0):
self.i_q2_end = 1
# Get start (corner) indices of the local zone wrt global ordering and its size
((_i_q1_start, _i_q2_start), (_N_q1_local, _N_q2_local)) = self._da_f.getCorners()
# Coordinates of the local zone in a global coord system
self.q1_start_local = self.q1_start + _i_q1_start * self.dq1
self.q2_start_local = self.q2_start + _i_q2_start * self.dq2
# TODO : Fix this. Passing into params for use in coords.py
self.physical_system.params.q1_start_local_left = self.q1_start_local
self.physical_system.params.q2_start_local_bottom = self.q2_start_local
print("nonlinear.py: rank = ", self._comm.rank,
"(q1_start_local, q2_start_local) = (",
self.q1_start_local, self.q2_start_local, ")"
)
print("nonlinear.py: rank = ", self._comm.rank,
"(N_q1_local, N_q2_local) = (",
_N_q1_local, _N_q2_local, ")"
)
print("nonlinear.py: rank = ", self._comm.rank,
"ownership_ranges : lx = ", lx, "ly = ", ly
)
self.N_q1_local = _N_q1_local
self.N_q2_local = _N_q2_local
self.N_q1_local_with_Ng = _N_q1_local + 2*self.N_g1
self.N_q2_local_with_Ng = _N_q2_local + 2*self.N_g2
# Obtaining the array values of spatial coordinates:
q_left_bot, q_center_bot, q_left_center, q_center = \
calculate_q(self.q1_start_local,
self.q2_start_local,
self.N_q1_local, self.N_q2_local,
self.N_g1, self.N_g2,
self.dq1, self.dq2
)
self.q1_left_bot = q_left_bot[0]
self.q2_left_bot = q_left_bot[1]
self.q1_center_bot = q_center_bot[0]
self.q2_center_bot = q_center_bot[1]
self.q1_left_center = q_left_center[0]
self.q2_left_center = q_left_center[1]
self.q1_center = q_center[0]
self.q2_center = q_center[1]
self.p1_center, self.p2_center, self.p3_center = \
calculate_p_center(self.p1_start, self.p2_start, self.p3_start,
self.N_p1, self.N_p2, self.N_p3,
self.dp1, self.dp2, self.dp3,
)
self.p1_left, self.p2_bottom, self.p3_back = \
calculate_p_corner(self.p1_start, self.p2_start, self.p3_start,
self.N_p1, self.N_p2, self.N_p3,
self.dp1, self.dp2, self.dp3,
)
# Need to convert the lists dp1, dp2, dp3 to af.Arrays for vector
# computations to work
self.dp1 = af.moddims(af.to_array(np.array(self.dp1)), 1, self.N_species)
self.dp2 = af.moddims(af.to_array(np.array(self.dp2)), 1, self.N_species)
self.dp3 = af.moddims(af.to_array(np.array(self.dp3)), 1, self.N_species)
# Need to do the same for the p1_start/end lists.
self.p1_start = af.moddims(af.to_array(self.p1_start), 1, self.N_species)
self.p2_start = af.moddims(af.to_array(self.p2_start), 1, self.N_species)
self.p3_start = af.moddims(af.to_array(self.p3_start), 1, self.N_species)
self.p1_end = af.moddims(af.to_array(self.p1_end), 1, self.N_species)
self.p2_end = af.moddims(af.to_array(self.p2_end), 1, self.N_species)
self.p3_end = af.moddims(af.to_array(self.p3_end), 1, self.N_species)
self.p2_left = self.p2_center
self.p3_left = self.p3_center
self.p1_bottom = self.p1_center
self.p3_bottom = self.p3_center
self.p1_back = self.p1_center
self.p2_back = self.p2_center
# Initialize according to initial condition provided by user:
self._initialize(physical_system.params)
# Initializing a variable to track time-elapsed:
self.time_elapsed = 0
# Assigning the function objects to methods of the solver:
self._A_q = physical_system.A_q
self._C_q = physical_system.C_q
self._A_p = physical_system.A_p
self._C_p = physical_system.C_p
# Source/Sink term:
self._source = physical_system.source
def build_and_backend() -> str:
return af.info_str().split('\n')[0]
def __init__(self, physical_system):
"""
Constructor for the linear_solver object. It takes the physical
system object as an argument and uses it in intialization and
evolution of the system in consideration.
Parameters
----------
physical_system: The defined physical system object which holds
all the simulation information such as the initial
conditions, and the domain info is passed as an
argument in defining an instance of the
nonlinear_solver. This system is then evolved, and
monitored using the various methods under the
nonlinear_solver class.
"""
self.physical_system = physical_system
# Storing Domain Information:
self.q1_start, self.q1_end = physical_system.q1_start,\
physical_system.q1_end
self.q2_start, self.q2_end = physical_system.q2_start,\
physical_system.q2_end
self.p1_start, self.p1_end = physical_system.p1_start,\
physical_system.p1_end
self.p2_start, self.p2_end = physical_system.p2_start,\
physical_system.p2_end
self.p3_start, self.p3_end = physical_system.p3_start,\
physical_system.p3_end
# Getting Domain Resolution
self.N_q1, self.dq1 = physical_system.N_q1, physical_system.dq1
self.N_q2, self.dq2 = physical_system.N_q2, physical_system.dq2
self.N_p1, self.dp1 = physical_system.N_p1, physical_system.dp1
self.N_p2, self.dp2 = physical_system.N_p2, physical_system.dp2
self.N_p3, self.dp3 = physical_system.N_p3, physical_system.dp3
# Checking that periodic B.C's are utilized:
if( physical_system.boundary_conditions.in_q1_left != 'periodic'
and physical_system.boundary_conditions.in_q1_right != 'periodic'
and physical_system.boundary_conditions.in_q2_bottom != 'periodic'
and physical_system.boundary_conditions.in_q2_top != 'periodic'
):
raise Exception('Only systems with periodic boundary conditions \
can be solved using the linear solver'
)
# Initializing DAs which will be used in file-writing:
self._da_dump_f = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=( self.N_p1
* self.N_p2
* self.N_p3
)
)
self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=len(self.physical_system.\
moment_exponents
)
)
# Printing backend details:
PETSc.Sys.Print('\nBackend Details for Linear Solver:')
PETSc.Sys.Print(indent('On Node: '+ socket.gethostname()))
PETSc.Sys.Print(indent('Device Details:'))
PETSc.Sys.Print(indent(af.info_str(), 2))
PETSc.Sys.Print(indent('Device Bandwidth = ' + str(bandwidth_test(100)) + ' GB / sec'))
PETSc.Sys.Print()
# Creating PETSc Vecs which are used in dumping to file:
self._glob_f = self._da_dump_f.createGlobalVec()
self._glob_f_value = self._da_dump_f.getVecArray(self._glob_f)
self._glob_moments = self._da_dump_moments.createGlobalVec()
self._glob_moments_value = self._da_dump_moments.\
getVecArray(self._glob_moments)
# Setting names for the objects which will then be
# used as the key identifiers for the HDF5 files:
PETSc.Object.setName(self._glob_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
# Intializing position, wavenumber and velocity arrays:
self.q1_center, self.q2_center = self._calculate_q_center()
self.k_q1, self.k_q2 = self._calculate_k()
self.p1, self.p2, self.p3 = self._calculate_p_center()
# Assigning the advection terms along q1 and q2
self._A_q1 = \
self.physical_system.A_q(self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
physical_system.params
)[0]
self._A_q2 = \
self.physical_system.A_q(self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
physical_system.params
)[1]
# Assigning the function objects to methods of the solver:
self._A_p = self.physical_system.A_p
self._source = self.physical_system.source
if(len(signature(self._source).parameters) == 6):
self.single_mode_evolution = True
else:
self.single_mode_evolution = False
# Initializing f, f_hat and the other EM field quantities:
self._initialize(physical_system.params)
def __init__(self, physical_system, performance_test_flag=False):
"""
Constructor for the nonlinear_solver object. It takes the physical
system object as an argument and uses it in intialization and
evolution of the system in consideration.
Additionally, a performance test flag is also passed which when true,
stores time which is consumed by each of the major solver routines.
This proves particularly useful in analyzing performance bottlenecks
and obtaining benchmarks.
Parameters:
-----------
physical_system: The defined physical system object which holds
all the simulation information such as the initial
conditions, and the domain info is passed as an
argument in defining an instance of the
nonlinear_solver. This system is then evolved, and
monitored using the various methods under the
nonlinear_solver class.
"""
self.physical_system = physical_system
# Holding Domain Info:
self.q1_start, self.q1_end = physical_system.q1_start,\
physical_system.q1_end
self.q2_start, self.q2_end = physical_system.q2_start,\
physical_system.q2_end
self.p1_start, self.p1_end = physical_system.p1_start,\
physical_system.p1_end
self.p2_start, self.p2_end = physical_system.p2_start,\
physical_system.p2_end
self.p3_start, self.p3_end = physical_system.p3_start,\
physical_system.p3_end
# Holding Domain Resolution:
self.N_q1, self.dq1 = physical_system.N_q1, physical_system.dq1
self.N_q2, self.dq2 = physical_system.N_q2, physical_system.dq2
self.N_p1, self.dp1 = physical_system.N_p1, physical_system.dp1
self.N_p2, self.dp2 = physical_system.N_p2, physical_system.dp2
self.N_p3, self.dp3 = physical_system.N_p3, physical_system.dp3
# Getting number of ghost zones, and the boundary
# conditions that are utilized:
N_g_q = self.N_ghost_q = physical_system.N_ghost_q
N_g_p = self.N_ghost_p = physical_system.N_ghost_p
self.boundary_conditions = physical_system.boundary_conditions
# Declaring the communicator:
self._comm = PETSc.COMM_WORLD.tompi4py()
if (self.physical_system.params.num_devices > 1):
af.set_device(self._comm.rank %
self.physical_system.params.num_devices)
# Getting number of species:
self.N_species = len(physical_system.params.mass)
# Having the mass and charge along axis 1:
self.physical_system.params.mass = \
af.cast(af.moddims(af.to_array(physical_system.params.mass),
1, self.N_species
),
af.Dtype.f64
)
self.physical_system.params.charge = \
af.cast(af.moddims(af.to_array(physical_system.params.charge),
1, self.N_species
),
af.Dtype.f64
)
PETSc.Sys.Print('\nBackend Details for Nonlinear Solver:')
# Printing the backend details for each rank/device/node:
PETSc.Sys.syncPrint(
indent('Rank ' + str(self._comm.rank) + ' of ' +
str(self._comm.size - 1)))
PETSc.Sys.syncPrint(indent('On Node: ' + socket.gethostname()))
PETSc.Sys.syncPrint(indent('Device Details:'))
PETSc.Sys.syncPrint(indent(af.info_str(), 2))
PETSc.Sys.syncPrint(
indent('Device Bandwidth = ' + str(bandwidth_test(100)) +
' GB / sec'))
PETSc.Sys.syncPrint()
PETSc.Sys.syncFlush()
self.performance_test_flag = performance_test_flag
# Initializing variables which are used to time the components of the solver:
if (performance_test_flag == True):
self.time_ts = 0
self.time_interp2 = 0
self.time_sourcets = 0
self.time_fvm_solver = 0
self.time_reconstruct = 0
self.time_riemann = 0
self.time_fieldstep = 0
self.time_interp3 = 0
self.time_apply_bcs_f = 0
self.time_communicate_f = 0
petsc_bc_in_q1 = 'ghosted'
petsc_bc_in_q2 = 'ghosted'
# Only for periodic boundary conditions or shearing-box boundary conditions
# do the boundary conditions passed to the DA need to be changed. PETSc
# automatically handles the application of periodic boundary conditions when
# running in parallel. For shearing box boundary conditions, an interpolation
# operation needs to be applied on top of the periodic boundary conditions.
# In all other cases, ghosted boundaries are used.
if (self.boundary_conditions.in_q1_left == 'periodic'
or self.boundary_conditions.in_q1_left == 'shearing-box'):
petsc_bc_in_q1 = 'periodic'
if (self.boundary_conditions.in_q2_bottom == 'periodic'
or self.boundary_conditions.in_q2_bottom == 'shearing-box'):
petsc_bc_in_q2 = 'periodic'
if (self.boundary_conditions.in_q1_left == 'periodic'):
try:
assert (self.boundary_conditions.in_q1_right == 'periodic')
except:
raise Exception(
'Periodic boundary conditions need to be applied to \
both the boundaries of a particular axis')
if (self.boundary_conditions.in_q1_left == 'shearing-box'):
try:
assert (self.boundary_conditions.in_q1_right == 'shearing-box')
except:
raise Exception(
'Shearing box boundary conditions need to be applied to \
both the boundaries of a particular axis')
if (self.boundary_conditions.in_q2_bottom == 'periodic'):
try:
assert (self.boundary_conditions.in_q2_top == 'periodic')
except:
raise Exception(
'Periodic boundary conditions need to be applied to \
both the boundaries of a particular axis')
if (self.boundary_conditions.in_q2_bottom == 'shearing-box'):
try:
assert (self.boundary_conditions.in_q2_top == 'shearing-box')
except:
raise Exception(
'Shearing box boundary conditions need to be applied to \
both the boundaries of a particular axis')
nproc_in_q1 = PETSc.DECIDE
nproc_in_q2 = PETSc.DECIDE
# Since shearing boundary conditions require interpolations which are non-local:
if (self.boundary_conditions.in_q2_bottom == 'shearing-box'):
nproc_in_q1 = 1
if (self.boundary_conditions.in_q1_left == 'shearing-box'):
nproc_in_q2 = 1
# DMDA is a data structure to handle a distributed structure
# grid and its related core algorithms. It stores metadata of
# how the grid is partitioned when run in parallel which is
# utilized by the various methods of the solver.
self._da_f = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=(self.N_species * (self.N_p1 + 2 * N_g_p) *
(self.N_p2 + 2 * N_g_p) * (self.N_p3 + 2 * N_g_p)),
stencil_width=N_g_q,
boundary_type=(petsc_bc_in_q1, petsc_bc_in_q2),
proc_sizes=(nproc_in_q1, nproc_in_q2),
stencil_type=1,
comm=self._comm)
# This DA is used by the FileIO routine dump_distribution_function():
self._da_dump_f = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=(self.N_species * self.N_p1 * self.N_p2 * self.N_p3),
stencil_width=N_g_q,
boundary_type=(petsc_bc_in_q1, petsc_bc_in_q2),
proc_sizes=(nproc_in_q1, nproc_in_q2),
stencil_type=1,
comm=self._comm)
# This DA is used by the FileIO routine dump_moments():
# Finding the number of definitions for the moments:
attributes = [
a for a in dir(self.physical_system.moments)
if not a.startswith('_')
]
# Removing utility functions:
if ('integral_over_v' in attributes):
attributes.remove('integral_over_v')
self._da_dump_moments = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=self.N_species * len(attributes),
proc_sizes=(nproc_in_q1, nproc_in_q2),
comm=self._comm)
# Creation of the local and global vectors from the DA:
# This is for the distribution function
self._glob_f = self._da_f.createGlobalVec()
self._local_f = self._da_f.createLocalVec()
# The following vector is used to dump the data to file:
self._glob_dump_f = self._da_dump_f.createGlobalVec()
self._glob_moments = self._da_dump_moments.createGlobalVec()
# Getting the arrays for the above vectors:
self._glob_f_array = self._glob_f.getArray()
self._local_f_array = self._local_f.getArray()
self._glob_moments_array = self._glob_moments.getArray()
self._glob_dump_f_array = self._glob_dump_f.getArray()
# Setting names for the objects which will then be
# used as the key identifiers for the HDF5 files:
PETSc.Object.setName(self._glob_dump_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
# Obtaining the array values of the cannonical variables:
self.q1_center, self.q2_center = self._calculate_q_center()
self.p1_center, self.p2_center, self.p3_center = self._calculate_p_center(
)
# Initialize according to initial condition provided by user:
self._initialize(physical_system.params)
# Obtaining start coordinates for the local zone
# Additionally, we also obtain the size of the local zone
((i_q1_start, i_q2_start), (N_q1_local,
N_q2_local)) = self._da_f.getCorners()
(i_q1_end, i_q2_end) = (i_q1_start + N_q1_local - 1,
i_q2_start + N_q2_local - 1)
# Applying dirichlet boundary conditions:
if (self.physical_system.boundary_conditions.in_q1_left == 'dirichlet'
):
# If local zone includes the left physical boundary:
if (i_q1_start == 0):
self.f[:, :N_g_q] = self.boundary_conditions.\
f_left(self.f, self.q1_center, self.q2_center,
self.p1_center, self.p2_center, self.p3_center,
self.physical_system.params
)[:, :N_g_q]
if (self.physical_system.boundary_conditions.in_q1_right == 'dirichlet'
):
# If local zone includes the right physical boundary:
if (i_q1_end == self.N_q1 - 1):
self.f[:, -N_g_q:] = self.boundary_conditions.\
f_right(self.f, self.q1_center, self.q2_center,
self.p1_center, self.p2_center, self.p3_center,
self.physical_system.params
)[:, -N_g_q:]
if (self.physical_system.boundary_conditions.in_q2_bottom ==
'dirichlet'):
# If local zone includes the bottom physical boundary:
if (i_q2_start == 0):
self.f[:, :, :N_g_q] = self.boundary_conditions.\
f_bot(self.f, self.q1_center, self.q2_center,
self.p1_center, self.p2_center, self.p3_center,
self.physical_system.params
)[:, :, :N_g_q]
if (self.physical_system.boundary_conditions.in_q2_top == 'dirichlet'):
# If local zone includes the top physical boundary:
if (i_q2_end == self.N_q2 - 1):
self.f[:, :, -N_g_q:] = self.boundary_conditions.\
f_top(self.f, self.q1_center, self.q2_center,
self.p1_center, self.p2_center, self.p3_center,
self.physical_system.params
)[:, :, -N_g_q:]
# Assigning the value to the PETSc Vecs(for dump at t = 0):
(af.flat(self.f)).to_ndarray(self._local_f_array)
(af.flat(self.f[:, :, N_g_q:-N_g_q,
N_g_q:-N_g_q])).to_ndarray(self._glob_f_array)
# Assigning the function objects to methods of the solver:
self._A_q = physical_system.A_q
self._C_q = physical_system.C_q
self._A_p = physical_system.A_p
self._C_p = physical_system.C_p
# Source/Sink term:
self._source = physical_system.source
# Initializing a variable to track time-elapsed:
self.time_elapsed = 0
def __init__(self, physical_system):
"""
Constructor for the linear_solver object. It takes the physical
system object as an argument and uses it in intialization and
evolution of the system in consideration.
Parameters
----------
physical_system: The defined physical system object which holds
all the simulation information such as the initial
conditions, and the domain info is passed as an
argument in defining an instance of the linear_solver.
This system is then evolved, and monitored using the
various methods under the linear_solver class.
"""
self.physical_system = physical_system
# Storing Domain Information:
self.q1_start, self.q1_end = physical_system.q1_start,\
physical_system.q1_end
self.q2_start, self.q2_end = physical_system.q2_start,\
physical_system.q2_end
self.p1_start, self.p1_end = physical_system.p1_start,\
physical_system.p1_end
self.p2_start, self.p2_end = physical_system.p2_start,\
physical_system.p2_end
self.p3_start, self.p3_end = physical_system.p3_start,\
physical_system.p3_end
# Getting Domain Resolution
self.N_q1, self.dq1 = physical_system.N_q1, physical_system.dq1
self.N_q2, self.dq2 = physical_system.N_q2, physical_system.dq2
self.N_p1, self.dp1 = physical_system.N_p1, physical_system.dp1
self.N_p2, self.dp2 = physical_system.N_p2, physical_system.dp2
self.N_p3, self.dp3 = physical_system.N_p3, physical_system.dp3
# Getting number of species:
self.N_species = len(physical_system.params.mass)
# Having the mass and charge along axis 1:
self.physical_system.params.mass = \
af.cast(af.moddims(af.to_array(physical_system.params.mass),
1, self.N_species
),
af.Dtype.f64
)
self.physical_system.params.charge = \
af.cast(af.moddims(af.to_array(physical_system.params.charge),
1, self.N_species
),
af.Dtype.f64
)
# Initializing variable to hold time elapsed:
self.time_elapsed = 0
# Checking that periodic B.C's are utilized:
if (physical_system.boundary_conditions.in_q1_left != 'periodic' and
physical_system.boundary_conditions.in_q1_right != 'periodic'
and
physical_system.boundary_conditions.in_q2_bottom != 'periodic'
and
physical_system.boundary_conditions.in_q2_top != 'periodic'):
raise Exception('Only systems with periodic boundary conditions \
can be solved using the linear solver')
# Initializing DAs which will be used in file-writing:
# This is done so that the output format used by the linear solver matches
# with the output format of the nonlinear solver:
self._da_dump_f = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=(self.N_species * self.N_p1 *
self.N_p2 * self.N_p3))
# Getting the number of definitions in moments:
attributes = [
a for a in dir(self.physical_system.moments)
if not a.startswith('_')
]
self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=self.N_species *
len(attributes))
# Printing backend details:
PETSc.Sys.Print('\nBackend Details for Linear Solver:')
PETSc.Sys.Print(indent('On Node: ' + socket.gethostname()))
PETSc.Sys.Print(indent('Device Details:'))
PETSc.Sys.Print(indent(af.info_str(), 2))
PETSc.Sys.Print(
indent('Device Bandwidth = ' + str(bandwidth_test(100)) +
' GB / sec'))
PETSc.Sys.Print()
# Creating PETSc Vecs which are used in dumping to file:
self._glob_f = self._da_dump_f.createGlobalVec()
self._glob_f_array = self._glob_f.getArray()
self._glob_moments = self._da_dump_moments.createGlobalVec()
self._glob_moments_array = self._glob_moments.getArray()
# Setting names for the objects which will then be
# used as the key identifiers for the HDF5 files:
PETSc.Object.setName(self._glob_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
# Intializing position, wavenumber and velocity arrays:
self.q1_center, self.q2_center = self._calculate_q_center()
self.k_q1, self.k_q2 = self._calculate_k()
self.p1, self.p2, self.p3 = self._calculate_p_center()
# Assigning the function objects to methods of the solver:
self._A_q = self.physical_system.A_q
self._A_p = self.physical_system.A_p
self._source = self.physical_system.source
# Initializing f, f_hat and the other EM field quantities:
self._initialize(physical_system.params)
def __init__(self, physical_system, performance_test_flag=False):
"""
Constructor for the nonlinear_solver object. It takes the physical
system object as an argument and uses it in intialization and
evolution of the system in consideration.
Additionally, a performance test flag is also passed which when true
stores time which is consumed by each of the major solver routines.
This proves particularly useful in analyzing performance bottlenecks
and obtaining benchmarks.
Parameters:
-----------
physical_system: The defined physical system object which holds
all the simulation information such as the initial
conditions, and the domain info is passed as an
argument in defining an instance of the
nonlinear_solver. This system is then evolved, and
monitored using the various methods under the
nonlinear_solver class.
"""
self.physical_system = physical_system
# Holding Domain Info:
self.q1_start, self.q1_end = physical_system.q1_start,\
physical_system.q1_end
self.q2_start, self.q2_end = physical_system.q2_start,\
physical_system.q2_end
self.p1_start, self.p1_end = physical_system.p1_start,\
physical_system.p1_end
self.p2_start, self.p2_end = physical_system.p2_start,\
physical_system.p2_end
self.p3_start, self.p3_end = physical_system.p3_start,\
physical_system.p3_end
# Holding Domain Resolution:
self.N_q1, self.dq1 = physical_system.N_q1, physical_system.dq1
self.N_q2, self.dq2 = physical_system.N_q2, physical_system.dq2
self.N_p1, self.dp1 = physical_system.N_p1, physical_system.dp1
self.N_p2, self.dp2 = physical_system.N_p2, physical_system.dp2
self.N_p3, self.dp3 = physical_system.N_p3, physical_system.dp3
# Getting number of ghost zones, and the boundary
# conditions that are utilized:
N_g = self.N_ghost = physical_system.N_ghost
self.boundary_conditions = physical_system.boundary_conditions
# Declaring the communicator:
self._comm = PETSc.COMM_WORLD.tompi4py()
if (self.physical_system.params.num_devices > 1):
af.set_device(self._comm.rank %
self.physical_system.params.num_devices)
PETSc.Sys.Print('\nBackend Details for Nonlinear Solver:')
# Printing the backend details for each rank/device/node:
PETSc.Sys.syncPrint(
indent('Rank ' + str(self._comm.rank) + ' of ' +
str(self._comm.size - 1)))
PETSc.Sys.syncPrint(indent('On Node: ' + socket.gethostname()))
PETSc.Sys.syncPrint(indent('Device Details:'))
PETSc.Sys.syncPrint(indent(af.info_str(), 2))
PETSc.Sys.syncPrint(
indent('Device Bandwidth = ' + str(bandwidth_test(100)) +
' GB / sec'))
PETSc.Sys.syncPrint()
PETSc.Sys.syncFlush()
self.performance_test_flag = performance_test_flag
if (performance_test_flag == True):
self.time_ts = 0
self.time_interp2 = 0
self.time_sourcets = 0
self.time_fvm_solver = 0
self.time_reconstruct = 0
self.time_riemann = 0
self.time_fieldstep = 0
self.time_fieldsolver = 0
self.time_interp3 = 0
self.time_apply_bcs_f = 0
self.time_apply_bcs_fields = 0
self.time_communicate_f = 0
self.time_communicate_fields = 0
petsc_bc_in_q1 = 'ghosted'
petsc_bc_in_q2 = 'ghosted'
# Only for periodic boundary conditions do the boundary
# conditions passed to the DA need to be changed. PETSc
# automatically handles the application of periodic
# boundary conditions when running in parallel. In all other
# cases, ghosted boundaries are used.
if (self.boundary_conditions.in_q1_left == 'periodic'):
petsc_bc_in_q1 = 'periodic'
if (self.boundary_conditions.in_q2_bottom == 'periodic'):
petsc_bc_in_q2 = 'periodic'
# DMDA is a data structure to handle a distributed structure
# grid and its related core algorithms. It stores metadata of
# how the grid is partitioned when run in parallel which is
# utilized by the various methods of the solver.
self._da_f = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=(self.N_p1 * self.N_p2 * self.N_p3),
stencil_width=self.N_ghost,
boundary_type=(petsc_bc_in_q1, petsc_bc_in_q2),
proc_sizes=(PETSc.DECIDE, PETSc.DECIDE),
stencil_type=1,
comm=self._comm)
# This DA object is used in the communication routines for the
# EM field quantities. A DOF of 6 is taken so that the communications,
# and application of B.C's may be carried out in a single call among
# all the field quantities(E1, E2, E3, B1, B2, B3)
self._da_fields = PETSc.DMDA().create([self.N_q1, self.N_q2],
dof=6,
stencil_width=self.N_ghost,
boundary_type=(petsc_bc_in_q1,
petsc_bc_in_q2),
proc_sizes=(PETSc.DECIDE,
PETSc.DECIDE),
stencil_type=1,
comm=self._comm)
# Additionally, a DMDA object also needs to be created for
# the KSP/SNES solver with a DOF of 1. This is used to solve for
# the electrostatic case:
self._da_ksp = PETSc.DMDA().create([self.N_q1, self.N_q2],
stencil_width=self.N_ghost,
boundary_type=(petsc_bc_in_q1,
petsc_bc_in_q2),
proc_sizes=(PETSc.DECIDE,
PETSc.DECIDE),
stencil_type=1,
comm=self._comm)
# This DA is used by the FileIO routine dump_moments():
self._da_dump_moments = PETSc.DMDA().create(
[self.N_q1, self.N_q2],
dof=len(self.physical_system.moment_exponents),
proc_sizes=(PETSc.DECIDE, PETSc.DECIDE),
comm=self._comm)
# Creation of the local and global vectors from the DA:
# This is for the distribution function
self._glob_f = self._da_f.createGlobalVec()
self._local_f = self._da_f.createLocalVec()
# The following global and local vectors are used in
# the communication routines for EM fields
self._glob_fields = self._da_fields.createGlobalVec()
self._local_fields = self._da_fields.createLocalVec()
# The following vector is used to dump the data to file:
self._glob_moments = self._da_dump_moments.createGlobalVec()
# Getting the arrays for the above vectors:
self._glob_f_array = self._glob_f.getArray()
self._local_f_array = self._local_f.getArray()
self._glob_fields_array = self._glob_fields.getArray()
self._local_fields_array = self._local_fields.getArray()
self._glob_moments_array = self._glob_moments.getArray()
# Setting names for the objects which will then be
# used as the key identifiers for the HDF5 files:
PETSc.Object.setName(self._glob_f, 'distribution_function')
PETSc.Object.setName(self._glob_moments, 'moments')
# Obtaining the array values of the cannonical variables:
self.q1_center, self.q2_center = self._calculate_q_center()
self.p1, self.p2, self.p3 = self._calculate_p_center()
# Initialize according to initial condition provided by user:
self._initialize(physical_system.params)
# Obtaining start coordinates for the local zone
# Additionally, we also obtain the size of the local zone
((i_q1_start, i_q2_start), (N_q1_local,
N_q2_local)) = self._da_f.getCorners()
(i_q1_end, i_q2_end) = (i_q1_start + N_q1_local - 1,
i_q2_start + N_q2_local - 1)
# Applying dirichlet boundary conditions:
if (self.physical_system.boundary_conditions.in_q1_left == 'dirichlet'
):
# If local zone includes the left physical boundary:
if (i_q1_start == 0):
self.f[:, :N_g] = self.boundary_conditions.\
f_left(self.f, self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
self.physical_system.params
)[:, :N_g]
if (self.physical_system.boundary_conditions.in_q1_right == 'dirichlet'
):
# If local zone includes the right physical boundary:
if (i_q1_end == self.N_q1 - 1):
self.f[:, -N_g:] = self.boundary_conditions.\
f_right(self.f, self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
self.physical_system.params
)[:, -N_g:]
if (self.physical_system.boundary_conditions.in_q2_bottom ==
'dirichlet'):
# If local zone includes the bottom physical boundary:
if (i_q2_start == 0):
self.f[:, :, :N_g] = self.boundary_conditions.\
f_bot(self.f, self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
self.physical_system.params
)[:, :, :N_g]
if (self.physical_system.boundary_conditions.in_q2_top == 'dirichlet'):
# If local zone includes the top physical boundary:
if (i_q2_end == self.N_q2 - 1):
self.f[:, :, -N_g:] = self.boundary_conditions.\
f_top(self.f, self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
self.physical_system.params
)[:, :, -N_g:]
# Assigning the value to the PETSc Vecs(for dump at t = 0):
(af.flat(self.f)).to_ndarray(self._local_f_array)
(af.flat(self.f[:, N_g:-N_g, N_g:-N_g])).to_ndarray(self._glob_f_array)
# Assigning the advection terms along q1 and q2
self._A_q1 = physical_system.A_q(self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
physical_system.params)[0]
self._A_q2 = physical_system.A_q(self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
physical_system.params)[1]
# Assigning the conservative advection terms along q1 and q2
self._C_q1 = physical_system.C_q(self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
physical_system.params)[0]
self._C_q2 = physical_system.C_q(self.q1_center, self.q2_center,
self.p1, self.p2, self.p3,
physical_system.params)[1]
# Assigning the function objects to methods of the solver:
self._A_p = physical_system.A_p
# Source/Sink term:
self._source = physical_system.source
# Initializing a variable to track time-elapsed:
# This becomes necessary when applying shearing wall
# boundary conditions(WIP):
self.time_elapsed = 0