def setup_matrices(self):
###################
# Create Matrices #
###################
# PM - Pressure Poisson Matrix
# VM - Pressure Poisson Matrix
# DM - Velocity Divergence Matrix
# SM - S-Terms Matrix (RHS to Pressure Poisson)
# GM - Gradient of P Matrix
if self.using_trilinos:
# Square Matrices
from PyTrilinos.Epetra import CrsMatrix, Copy
self.PM = CrsMatrix(Copy, self.PMap, self.max_row_nz)
self.VM = [ CrsMatrix(Copy, vmap, self.max_row_nz) for vmap in self.VMaps ]
# Rectangulars
self.DM = [ CrsMatrix(Copy, self.PMap, 2 ) for vmap in self.VMaps ]
self.ST = [ CrsMatrix(Copy, self.PMap, 2 ) for vmap in self.VMaps ]
self.GM = [ CrsMatrix(Copy, vmap, 2 ) for vmap in self.VMaps ]
else:
# Scipy imported here now
from scipy.sparse import lil_matrix
# Square Matrices
self.PM = lil_matrix( (self.P_dof_num, self.P_dof_num ) )
self.VM = [ lil_matrix( (dof_count, dof_count) ) for dof_count in self.vel_dof_nums ]
# Rectangulars
self.DM = [ lil_matrix( (self.P_dof_num, dof_count) ) for dof_count in self.vel_dof_nums ]
self.ST = [ lil_matrix( (self.P_dof_num, dof_count) ) for dof_count in self.vel_dof_nums ]
self.GM = [ lil_matrix( (dof_count, self.P_dof_num) ) for dof_count in self.vel_dof_nums ]
#################
# Fill Matrices #
#################
# Setup the Pressure Matrix
self.dbprint("Filling the Pressure Poisson Matrix")
self._fill_PM()
# Setup the n Velocity, divergence, and gradient matrices
for dim in range(self.ndim):
self.dbprint( "Setting up Velocity Poisson Matrix (%i)" % dim )
self._fill_VM(dim)
self.dbprint( "Calculating Divergence/Biot Matrix (%i)" % dim )
self._fill_DM(dim)
self.dbprint( "Calculating Gradient Matrix (%i)" % dim )
self._fill_GM(dim)
# Due to the symbolic nature of these, they are done
# All at once
self.dbprint("Calculating S-Terms")
self._fill_S()
##############################
# Convert/Finialize Matrices #
##############################
if self.using_trilinos:
self.dbprint("FillComplete on all Matrices (Waiting For other Threads)")
self.sync("Pre-FillComplete() of matrices")
# Square Matrix Implicit in FillComplete (PM, VM's)
self.PM.FillComplete()
[ m.FillComplete() for m in self.VM ]
# Rectagular need maps defined
[ m.FillComplete(vmap, self.PMap) for m, vmap in zip(self.DM, self.VMaps) ]
[ m.FillComplete(vmap, self.PMap) for m, vmap in zip(self.ST, self.VMaps) ]
[ m.FillComplete(self.PMap, vmap) for m, vmap in zip(self.GM, self.VMaps) ]
else:
self.dbprint("Converting All To CSR")
self.PM = self.PM.tocsr()
self.VM = [m.tocsr() for m in self.VM]
self.DM = [m.tocsr() for m in self.DM]
self.ST = [m.tocsr() for m in self.ST]
self.GM = [m.tocsr() for m in self.GM]
class Solver():
'''This is the setup method of the solver. It instantiates a class
with methods central to solving low Reynolds number flows.
Required Arguments:
'solid' is an n-dimensional array describing geometry of the problem
'dP' is list/tuple of pressure difference across
the n-th dimension of the domain"
'''
def __init__(self, solid_or_filename, dP, sol_method = "default", printing = False, dbcallback = None ):
# Log the starting time
self.start_time = time.time()
# Set debugging flag
self.printlevel = printing
self.dbcallback = dbcallback
# Trilinos setup and iteration requires very different programmatic flow . . .
self.using_trilinos = ( sol_method == "trilinos")
# Trilinos communicator between threads . . .
if self.using_trilinos:
# Epetra Imported Here and all vectors defined
from PyTrilinos import Epetra
self.Comm = Epetra.PyComm()
self.myID = self.Comm.MyPID()
self.cpuCount = self.Comm.NumProc()
self.dbprint("Trilinos Inititated: %s" % gethostname())
else:
# Only one thread
self.myID = 0
self.cpuCount = 1
self.dbprint("Solver Instantiated", level = 2)
# Default direction of pressure drop
self.dP = dP
# I left this in here so you can manually
# disable Biot number based acceleration if desired
self.useBi = True
if sol_method == "default":
self.method = "spsolve"
else:
self.method = sol_method
# Iteration count
self.I = 0
################################################################
# Solver Internal Stuff ## Degree of Freedom Grids and Numbers #
################################################################
# the ndarray of solid
if hasattr(solid_or_filename, "shape"):
self.S = solid_or_filename
self.shape = self.S.shape
self.ndim = len(self.S.shape)
# Pressure (cell centered)
# Get the P degrees of freedom
self.P_dof_grid = ndim_eq.p_dof( self.S )
# Velocities (face centered)
self.vel_dof_grids = ndim_eq.velocity_dofs( self.S )
self.bigMode = False
elif type(solid_or_filename) == str:
if self.myID == 0:
self.setup_dof_cache_faster(solid_or_filename)
self.sync()
self.dbprint("Waiting for cached file to flush to disk.")
time.sleep(1)
self.sync()
self.import_dof_cache()
self.bigMode = True
self.P_dof_num = self.P_dof_grid.max() + 1
self.vel_dof_nums = [int(grid.max() + 1) for grid in self.vel_dof_grids]
self.sync("DOF Config Completion Sync")
self.la_is_setup = False
self.bc_is_setup = False
def force_la_setup(self):
'''Force the solver to set-up the linear algebra bits (vectors, matrices etc.)'''
if not self.la_is_setup:
self.setup_la()
def force_bc_setup(self):
'''Force the solver to set-up the boundary conditions (P_CORR and V_CORRS etc.)'''
if not self.bc_is_setup:
self.setup_bc()
def setup_la(self):
self.dbprint("Starting Setup Routine", 2)
# Everything is ND wrt. the 0th axis
self.h = 1./self.shape[0]
#################
# Stupid checks #
#################
if self.ndim != len(self.dP):
raise ValueError("Solid Array and Pressure Drop do not have matching dimensions:\n\tself.ndim:%i\n\t%s" % (self.ndim, self.dP))
#################
# FYI Printouts #
#################
# DOF Number total
self.dof_number = sum(self.vel_dof_nums) + self.P_dof_num
# print some useful DOF debugging information:
self.dbprint( "Degree of freedom count: %i" % self.dof_number )
self.dbprint( "\tPressure: %i" % self.P_dof_num )
for dim in range(self.ndim):
self.dbprint( "\tVelocity %i: %i" % (dim, self.vel_dof_nums[dim]) )
##########################
# DOF Grabbing Functions #
##########################
# This returns the pressure DOF for a given point
self.pdp = lambda point: int( self.P_dof_grid[tuple(array(point) % array(self.shape))] )
# This returns the velocity DOF for a given dimension and point
self.nvd = lambda dim, point: int( self.vel_dof_grids[dim][tuple(array(point) % array(self.shape))] )
##########################
# Linear Algebra Related #
##########################
# The maximum number of non-zero entries on a Poisson matrix of dimension self.ndim
self.max_row_nz = 1 + (2 * self.ndim)
###########
# Vectors #
###########
# Vector shaped pressure and correction
if self.using_trilinos:
from PyTrilinos import Epetra
self.PMap = Epetra.Map(self.P_dof_num, 0, self.Comm)
self.myP_dof_min = self.PMap.MinMyGID()
self.myP_dof_max = self.PMap.MaxMyGID()
self.P_LHS = Epetra.Vector(self.PMap)
self.P_RHS = Epetra.Vector(self.PMap)
self.P_COR = Epetra.Vector(self.PMap)
self.PTEMP = Epetra.Vector(self.PMap)
# Biot number and Divergence have the same vector size/Map
self.Bi = Epetra.Vector(self.PMap)
self.DIV_MULT = Epetra.Vector(self.PMap)
# Numbers related to the divergence
self.last_abs_div= Epetra.Vector(self.PMap)
self.D_LIN = Epetra.Vector(self.PMap)
self.ABS_D_LIN = Epetra.Vector(self.PMap)
else:
self.myP_dof_min = 0
self.myP_dof_max = self.P_dof_num - 1
# Pressure
self.P_LHS = zeros( self.P_dof_num )
self.P_RHS = zeros( self.P_dof_num )
self.P_COR = zeros( self.P_dof_num )
self.PTEMP = zeros( self.P_dof_num )
# Biot number and Divergence have the same vector size/Map
self.Bi = zeros( self.P_dof_num )
self.DIV_MULT = zeros( self.P_dof_num )
# Numbers related to the divergence
self.last_abs_div= zeros( self.P_dof_num )
self.D_LIN = zeros( self.P_dof_num )
self.ABS_D_LIN = zeros( self.P_dof_num )
# So the convergence loop runs once before assessing that its done
self.max_D = inf
self.last_max_D = inf
# All of these variables are per velocity axis (aka per dimension)
# So, these Lists contain one element for each dimension.
# self.VEL_EDGE - The static adjustment that come from PBCS to the dv/dx
# self.V_LHS - The linear representation of velocity (index is dof #)
# self.VEL_RHS - The matrices that generate the RHS to the velocity equation
# Vector shaped Velocities and
if self.using_trilinos:
self.VMaps = [ Epetra.Map(dof_count, 0, self.Comm) for dof_count in self.vel_dof_nums ]
self.myV_dof_min = [ vmap.MinMyGID() for vmap in self.VMaps ]
self.myV_dof_max = [ vmap.MaxMyGID() for vmap in self.VMaps ]
self.V_LHS = [ Epetra.Vector(vmap) for vmap in self.VMaps ]
self.V_RHS = [ Epetra.Vector(vmap) for vmap in self.VMaps ]
self.V_COR = [ Epetra.Vector(vmap) for vmap in self.VMaps ]
self.VTEMP = [ Epetra.Vector(vmap) for vmap in self.VMaps ]
else:
self.myV_dof_min = [ 0 for dofs in self.vel_dof_nums ]
self.myV_dof_max = [ dofs for dofs in self.vel_dof_nums ]
self.V_LHS = [ zeros( dof_count ) for dof_count in self.vel_dof_nums ]
self.V_RHS = [ zeros( dof_count ) for dof_count in self.vel_dof_nums ]
self.V_COR = [ zeros( dof_count ) for dof_count in self.vel_dof_nums ]
self.VTEMP = [ zeros( dof_count ) for dof_count in self.vel_dof_nums ]
# Print pressure DOFs
for cpu in range(self.cpuCount):
if cpu == self.myID:
self.dbprint("My Pressure Degrees of Freedom. Min:%i Max:%i"
% (self.myP_dof_min, self.myP_dof_max) )
self.sync()
# Print the velocity degrees of freedom
# In order!
for x in range(self.ndim):
for cpu in range(self.cpuCount):
if cpu != self.myID:
continue
self.dbprint("My Velocity (%i) DOFs. Min:%i Max:%i"
% (x, self.myV_dof_min[x], self.myV_dof_max[x]) )
self.sync()
# These lists are the ones you have to step through to cover
# Each cpu's degrees of freedom
# DOF Points for P
self.Get_P_Iterator = lambda : ndimed.pruned_iterator(self.P_dof_grid,
self.myP_dof_min,
self.myP_dof_max)
self.Get_V_Iterator = lambda axis: ndimed.pruned_iterator(self.vel_dof_grids[axis],
self.myV_dof_min[axis],
self.myV_dof_max[axis])
# Setup the matrices!
self.setup_matrices()
# Setup Matrix Solver
self.setup_matrix_solver()
# Get your coffee, were ready to go!
self.setup_time = time.time() - self.start_time
# Mark the linear algebra as setup
self.la_is_setup = True
def setup_dof_cache_faster(self, s_filename):
from tables import openFile
self.dbprint("BIG MODE! Setting up DOF Cache")
# Should only be run by cpu 0!
if self.myID != 0:
raise RuntimeError("Only thread 0 should setup dof cache!")
# Open the file to copy S from
self.dbprint("Opening h5 file to read solid")
source_h5 = openFile(s_filename)
S = source_h5.root.geometry.S[:]
self.dbprint("Success!", 3)
shape = S.shape
ndim = len(shape)
# Write Memory Maps
self.dbprint("Writing memmap files.")
shape_map = memmap("shape.mem", dtype="int64", mode='w+', shape=tuple([ndim]))
s_memmap = memmap("S.mem", dtype="int64", mode='w+', shape=shape)
p_memmap = memmap("P.mem", dtype="int64", mode='w+', shape=shape)
v_memmaps = [memmap("V%i.mem" % x, dtype="int64", mode='w+', shape=shape) for x in range(ndim)]
self.dbprint("Assigning Values.")
# Assign the shape
shape_map[:] = array(shape).astype(int64)[:]
self.dbprint("Shape Done.")
# Assign S
s_memmap[:] = S[:].astype(int64)
self.dbprint("S Done.")
# Assign P's
p_memmap[:] = ndim_eq.p_dof(S).astype(int64)
self.dbprint("P Done.")
# Assign V's
for axis, v_mmap in enumerate(v_memmaps):
v_mmap[:] = ndim_eq.velocity_dof(S, axis).astype(int64)
self.dbprint("VS Done.")
self.dbprint("Loaded Maps . . . Flushing.")
# Flush All to disk.
shape_map.flush()
s_memmap.flush()
p_memmap.flush()
[v_mmap.flush() for v_mmap in v_memmaps]
source_h5.close()
self.dbprint("\tDone Constructing memory mapped files.")
def import_dof_cache(self):
self.dbprint("Opening DOF Cache")
sm = memmap("shape.mem", dtype="int64", mode='r')
self.dbprint("\tShape Done.", 3)
self.shape = tuple(sm[:])
self.ndim = len(self.shape)
self.S = memmap("S.mem", dtype="int64", mode='r', shape=self.shape)
self.dbprint("\tSolid Done.", 3)
self.P_dof_grid = memmap("P.mem", dtype="int64", mode='r', shape=self.shape)
self.dbprint("\tPressure Done.", 3)
self.vel_dof_grids = [ memmap("V%i.mem" % x, dtype="int64", mode='r', shape=self.shape) for x in range(self.ndim) ]
self.dbprint("\tVelocities Done.", 3)
# Get the matrix product between where Mx = b
# Agnostic to whether we are using Trilinos/scipy
def mat_mult(self, M, x, b):
self.dbprint("Matrix Multiply Called", 2)
if ("scipy" in str(type(M))):
b[:] = M * x
elif ("Epetra" in str(type(M))):
self.sync("Pre MM")
self.dbprint("Trilinos MM return: %i" % M.Multiply(False, x, b), 3)
self.sync("Post MM")
else:
raise ValueError("Huh?")
# Debug printer . . . kinda neat to watch
def dbprint(self, string, level = 1):
'''This is a debug printing routine.
Level Indicates Urgency:\n
0:Non-recoverable errors
1:Information
2:Details'''
# Call whatever debug printing function you desire!
time_diff = time.time() - self.start_time
string = "[%02f][%i/%i] %s" % (time_diff, self.myID + 1, self.cpuCount, string)
# This is used to output debugging info locally on headless nodes, etc.
# i.e make a function that cats the strings to a file, or html stream etc.
if self.dbcallback != None:
self.dbcallback(string)
# Screen output regulated on priority, callback not
if level <= self.printlevel:
print string
def setup_matrices(self):
###################
# Create Matrices #
###################
# PM - Pressure Poisson Matrix
# VM - Pressure Poisson Matrix
# DM - Velocity Divergence Matrix
# SM - S-Terms Matrix (RHS to Pressure Poisson)
# GM - Gradient of P Matrix
if self.using_trilinos:
# Square Matrices
from PyTrilinos.Epetra import CrsMatrix, Copy
self.PM = CrsMatrix(Copy, self.PMap, self.max_row_nz)
self.VM = [ CrsMatrix(Copy, vmap, self.max_row_nz) for vmap in self.VMaps ]
# Rectangulars
self.DM = [ CrsMatrix(Copy, self.PMap, 2 ) for vmap in self.VMaps ]
self.ST = [ CrsMatrix(Copy, self.PMap, 2 ) for vmap in self.VMaps ]
self.GM = [ CrsMatrix(Copy, vmap, 2 ) for vmap in self.VMaps ]
else:
# Scipy imported here now
from scipy.sparse import lil_matrix
# Square Matrices
self.PM = lil_matrix( (self.P_dof_num, self.P_dof_num ) )
self.VM = [ lil_matrix( (dof_count, dof_count) ) for dof_count in self.vel_dof_nums ]
# Rectangulars
self.DM = [ lil_matrix( (self.P_dof_num, dof_count) ) for dof_count in self.vel_dof_nums ]
self.ST = [ lil_matrix( (self.P_dof_num, dof_count) ) for dof_count in self.vel_dof_nums ]
self.GM = [ lil_matrix( (dof_count, self.P_dof_num) ) for dof_count in self.vel_dof_nums ]
#################
# Fill Matrices #
#################
# Setup the Pressure Matrix
self.dbprint("Filling the Pressure Poisson Matrix")
self._fill_PM()
# Setup the n Velocity, divergence, and gradient matrices
for dim in range(self.ndim):
self.dbprint( "Setting up Velocity Poisson Matrix (%i)" % dim )
self._fill_VM(dim)
self.dbprint( "Calculating Divergence/Biot Matrix (%i)" % dim )
self._fill_DM(dim)
self.dbprint( "Calculating Gradient Matrix (%i)" % dim )
self._fill_GM(dim)
# Due to the symbolic nature of these, they are done
# All at once
self.dbprint("Calculating S-Terms")
self._fill_S()
##############################
# Convert/Finialize Matrices #
##############################
if self.using_trilinos:
self.dbprint("FillComplete on all Matrices (Waiting For other Threads)")
self.sync("Pre-FillComplete() of matrices")
# Square Matrix Implicit in FillComplete (PM, VM's)
self.PM.FillComplete()
[ m.FillComplete() for m in self.VM ]
# Rectagular need maps defined
[ m.FillComplete(vmap, self.PMap) for m, vmap in zip(self.DM, self.VMaps) ]
[ m.FillComplete(vmap, self.PMap) for m, vmap in zip(self.ST, self.VMaps) ]
[ m.FillComplete(self.PMap, vmap) for m, vmap in zip(self.GM, self.VMaps) ]
else:
self.dbprint("Converting All To CSR")
self.PM = self.PM.tocsr()
self.VM = [m.tocsr() for m in self.VM]
self.DM = [m.tocsr() for m in self.DM]
self.ST = [m.tocsr() for m in self.ST]
self.GM = [m.tocsr() for m in self.GM]
# scipy spsolve
def _spsolve_P(self, *args, **kwargs):
self.P_LHS = self.spsolve( self.PM, self.P_RHS )
def _spsolve_V(self, dim, *args, **kwargs):
self.V_LHS[dim] = self.spsolve( self.VM[dim], self.V_RHS[dim] )
# scipy splu
def _splu_P(self, *args, **kwargs):
self.P_LHS = self.PM_LU.solve(self.P_RHS)
def _splu_V(self, dim, *args, **kwargs):
self.V_LHS[dim] = self.VM_LU[dim].solve(self.V_RHS[dim])
# pyamg
# def _pyamg_P(self, *args, **kwargs):
# self.P_LHS = self.PM_RUBE.solve(self.P_RHS, tol=1e-10)
# def _pyamg_V(self, dim, *args, **kwargs):
# self.V_LHS[dim] = self.VM_RUBE[dim].solve(rhs, tol=1e-10)
# pytrilinos
# I expected it to converge more quickly using the
# Previoud LHS as the first guess, but its def.
# seems to cause problems (zeroing out is the fastest I can find)
def _trilinos_P(self, *args, **kwargs):
self.sync("Pre P Solve" )
self.P_LHS[:] = 0
tril_return = self.t_PSol.Iterate(5000, 1e-12)
self.sync("Post P Solve")
return_string = "Trilinos solve return code: %i" % tril_return
self.dbprint(return_string, 3)
if tril_return > 0:
warnings.warn(return_string)
elif tril_return < 0:
self.dbprint("Non-Zero Trilinos Return. This is BAD!", 0)
# raise RuntimeError(return_string)
def _trilinos_V(self, dim, *args, **kwargs):
self.sync("Pre V Solve (%i)" % dim)
self.V_LHS[dim][:] = 0
tril_return = self.t_VSol[dim].Iterate(5000, 1e-12)
self.sync("Post V Solve (%i)" % dim)
return_string = "Trilinos solve return code: %i" % tril_return
self.dbprint(return_string, 3)
if tril_return > 0:
warnings.warn(return_string)
elif tril_return < 0:
self.dbprint("Non-Zero Trilinos Return. This is BAD!", 0)
# raise RuntimeError(return_string)
def test_matrices(self):
def vec_nnz(vec):
if "Trilinos" in str(type(vec)):
return vec.Norm1()
else:
return abs(vec).sum()
def mat_nnz(mat):
if hasattr(mat, "getnnz"):
return mat.getnnz()
elif hasattr(mat, "NumGlobalNonzeros"):
return mat.NumGlobalNonzeros()
else:
raise ValueError("Not a recognized Matrix Format")
self.dbprint("Matrix Non-Zero Check")
self.dbprint("PM: %i" % mat_nnz(self.PM) )
for dim in range(self.ndim):
self.dbprint("VM (%i): %i" % (dim, mat_nnz(self.VM[dim])) )
self.dbprint("GM (%i): %i" % (dim, mat_nnz(self.GM[dim])) )
self.dbprint("DM (%i): %i" % (dim, mat_nnz(self.DM[dim])) )
self.dbprint("ST (%i): %i" % (dim, mat_nnz(self.ST[dim])) )
self.dbprint("Vector Norm Check")
self.dbprint("P_COR: %i" % vec_nnz(self.P_COR))
for dim in range(self.ndim):
self.dbprint("V_COR (%i): %i" % (dim, vec_nnz(self.V_COR[dim])) )
def setup_matrix_solver(self):
########################
# setup solution methods
########################
# This converts the matrices to a format appropriate to the solution method
# It presents exposes the functions: SOLVE_P(rhs) and SOLVE_V(dim, rhs)
# so solution method is transparent to the main iteration loop
# No Biot number . . . purely derived from
# Dr. Erdmann's thesis (for validation/benchmarking)
if self.method == "nobi":
self.dbprint("Bi Number disabled.", level=2)
# Set the ignore Bi, flag and use spsolve
self.useBi = False
self.method = 'spsolve'
# spsolve is the slowest but has no additional memory overhead
# (kept around for my shitty laptop) Also good for large domains where
# splu blows up memory wise (200x200)+
if self.method == 'spsolve':
from scipy.sparse.linalg import spsolve
self.spsolve = spsolve
self.dbprint("spsolve selected . . . doing nothing", level=2)
self.PM = self.PM.tocsr()
for dim in range(self.ndim):
self.VM[dim] = self.VM[dim].tocsr()
self.SOLVE_P = self._spsolve_P
self.SOLVE_V = self._spsolve_V
try:
from scipy.sparse.linalg import splu
except:
warnings.warn("You do not seem to have UMFpack installed; the use of spsolve will be _very_ slow!")
# This sparse LU decomposition. Good for systems with any version of scipy
# Generally faster for anything above 100x100
# Major memory hog for large systems (200x200 >200Mb)
# Don't even try this on a 3d . . .
elif self.method == 'splu':
from scipy.sparse.linalg import splu
self.dbprint("SPLU'ing Matrices", level=2)
self.dbprint("\t Pressure.tocsc()", level=2)
self.PM = self.PM.tocsc()
self.dbprint("\t Pressure", level=2)
self.PM_LU = splu(self.PM)
self.VM_LU = [None] * self.ndim
for dim in range(self.ndim):
self.dbprint("\t Velocity %i tocsc()" % dim, level=2)
self.VM[dim] = self.VM[dim].tocsc()
self.dbprint("\t Velocity %i -splu" % dim, level=2)
self.VM_LU[dim] = splu(self.VM[dim])
self.SOLVE_P = self._splu_P
self.SOLVE_V = self._splu_V
# Relies on pyamg for AMG solvers. Should be wicked fast breaks for large grids?
# Need to add adjustable parameters for large and small systems . . .
# TODO: Currently crashes so commented out . . .
# elif self.method == 'ruge':
# import pyamg
# self.dbprint("Setting up ruge_stuben_solver(s)", level=2)
# self.dbprint("\t Pressure.tocsr()", level=2)
# self.PM = self.PM.tocsr()
# self.dbprint("\t Pressure Stuben", level=2)
# self.PM_RUBE = pyamg.ruge_stuben_solver( self.PM )
# self.VM_RUBE = [None] * self.ndim
# for dim in range(self.ndim):
# self.dbprint("\t Velocity %i tocsr()" % dim, level=2)
# self.VM[dim] = self.VM[dim].tocsr()
# self.dbprint("\t Velocity %i -rube" % dim, level=2)
# self.VM_RUBE[dim] = pyamg.ruge_stuben_solver( self.VM[dim] )
# self.SOLVE_P = _pyamg_P
# self.SOLVE_V = _pyamg_V
elif self.method == "trilinos":
self.dbprint("Using Trilinos!", level=2)
# Trilinos Solving Options:
MLList = { "max levels" : 10,
"output" : 10,
"smoother: pre or post" : "both",
"smoother: type" : "Chebyshev",
"aggregation: type" : "Uncoupled",
"coarse: type" : "Amesos-KLU" }
# import ML for the preconditioners
from PyTrilinos import ML, AztecOO
# Compute the preconditioner . . .
self.t_PCond = ML.MultiLevelPreconditioner(self.PM, False)
self.t_PCond.SetParameterList(MLList)
self.t_PCond.ComputePreconditioner()
# Setup the Pressure Poisson solver
self.t_PSol = AztecOO.AztecOO(self.PM, self.P_LHS, self.P_RHS)
self.t_PSol.SetPrecOperator(self.t_PCond)
self.t_PSol.SetAztecOption(AztecOO.AZ_solver, AztecOO.AZ_gmres)
self.t_PSol.SetAztecOption(AztecOO.AZ_output, 64)
self.t_VCon = [ML.MultiLevelPreconditioner(mat, False) for mat in self.VM]
self.t_VSol = [AztecOO.AztecOO(self.VM[dim], self.V_LHS[dim], self.V_RHS[dim]) for dim in range(self.ndim)]
for prec, sol in zip(self.t_VCon, self.t_VSol):
# Compute the preconditioner . . .
prec.SetParameterList(MLList)
prec.ComputePreconditioner()
# Apply it and setup the Velocity Poisson solver for each axis
sol.SetPrecOperator(prec)
sol.SetAztecOption(AztecOO.AZ_solver, AztecOO.AZ_gmres)
sol.SetAztecOption(AztecOO.AZ_output, 64)
self.SOLVE_P = self._trilinos_P
self.SOLVE_V = self._trilinos_V
else:
self.dbprint("Solver type '%s' not recognized!!!!" % self.method, level = 0)
raise ValueError("Solver type '%s' not recognized!!!!" % self.method)
def setup_bc(self):
self.dbprint("Calculating Periodic Correction Vectors", 2)
self.dbprint("\tV - RHS Correction", 3)
#############################
# Velocity RHS's Correction #
#############################
for dim in range(self.ndim):
for point in self.Get_V_Iterator(dim):
# This decides when the RHS of the velocity equation
# requires a addition of the pressure drop
# This correction comes from the PBC's and the gradient of the pressure (hence the negative)
if (point[dim] == 0):
vdof = self.nvd(dim, point)
# Map to trilinos value
if self.using_trilinos:
vdof = self.VMaps[dim].LID(vdof)
self.V_COR[dim][vdof] -= self.dP[dim]
###########################
# Pressure RHS Correction #
###########################
for point in self.Get_P_Iterator():
pdof = self.pdp(point)
# This Pins the pressure solution for the 0th DOF
if pdof == 0:
if self.using_trilinos:
pdof = self.PMap.LID(pdof)
self.P_COR[pdof] = 1
continue
# Now check to see if were at the edge
# (have to add a constant to the RHS)
# Check each dimension
point_pressure_correction = 0
for dim in range(self.ndim):
# Shift the Point Forward in this axis
back_point = list(point)
back_point[dim] -= 1
test_back = tuple(back_point)
# Looking back yeilds negative dP
# If point is at the near edge AND
# Isn't adjacent to solid (normal to that edge)
if (point[dim] == 0) and (self.pdp(test_back) != -1):
point_pressure_correction -= self.dP[dim]
# Shift the Point Backward in this axis
shifted_point = list(point)
shifted_point[dim] += 1
fore_point = tuple(shifted_point)
# Looking back yeilds posative dP
# If point is at the far edge AND
# Isn't adjacent to solid (normal to that edge)
if (point[dim] == self.shape[dim] - 1) and (self.pdp(fore_point) != -1):
point_pressure_correction += self.dP[dim]
# If using Trilinos, de-reference to local indexing
if self.using_trilinos:
# pdof = self.PMap.LID(pdof)
self.P_COR.SumIntoGlobalValue(pdof, 0, point_pressure_correction)
else:
self.P_COR[pdof] += point_pressure_correction
# Flag the BC's as setup
self.bc_is_setup = True
def _fill_PM(self):
# ndimed.iter_grid only gits points for each proc.
for point in self.Get_P_Iterator():
dof = self.pdp(point)
# #ignore non-degrees of freedom (no longer necessary?)
# if dof < 0:
# continue
# Pin the solution to obtain
# a unique (non-singular) solution
if dof == 0:
self.PM[dof,dof] = 1
continue
# The trace will be -2 * the number of dimensions
center_value = -2 * self.ndim
# The 'dof' is the row, and we gather the rows and cols to facilitate trilinos/scipy
col = []
val = []
# Oscillate one in each direction
for pp in ndimed.perturb(point):
# Get the DOF number
test_dof = self.pdp(pp)
# If it is a dof . . .
if test_dof != -1:
col.append(test_dof)
val.append(1)
else:
center_value += 1
# Stupid Warning, I don't think it is necessary any more . . .
if center_value == 0:
self.dbprint("Something bad probably just happened! %s" % str(point), 1)
# Add the center point to the list . . .
col.append(dof)
val.append(center_value)
# Populate the matrix
for c, v in zip(col, val):
self.PM[dof,c] = v
def _fill_VM( self, axis ):
#Define self.UM
for point in self.Get_V_Iterator(axis):
dof = self.nvd(axis, point)
# #ignore non-degrees of freedom
# if dof < 0:
# continue
# The trace will be -2 * ndim
center_val = -2 * self.ndim
col = []
val = []
for pp in ndimed.perturb(point):
test_dof = self.nvd(axis, pp)
if test_dof >= 0:
col.append(test_dof)
val.append(1)
elif test_dof ==-3:
center_val -= 1
# Add the center value in the list of stuff to add
col.append(dof)
val.append(center_val)
# Populate the matrix
for c, v in zip(col, val):
self.VM[axis][dof,c] = v
def _fill_DM(self, dim):
# Matrix associated with this dimension
this_DM = self.DM[dim]
rolled_v_dof_grid = roll(self.vel_dof_grids[dim], -1, axis=dim)
# Iterate over the grid
for point in self.Get_P_Iterator():
# P Degree of freedom
pd = self.pdp(point)
# Neg is flowing in
dof = self.vel_dof_grids[dim][point]
if dof >= 0:
if self.using_trilinos:
this_DM.InsertGlobalValues(pd, [-1.], [dof])
else:
this_DM[pd, dof] = -1
# Positive is flowing out
dof = rolled_v_dof_grid[point]
if dof >= 0:
if self.using_trilinos:
this_DM.InsertGlobalValues(pd, [1.], [dof])
else:
this_DM[pd, dof] = 1
def _fill_S(self):
# Iterate over the grid
for point in self.Get_P_Iterator():
# Find the DOF for the current pressure cell
pd = self.pdp(point)
# No Source Term for pinned point
if pd == 0:
continue
# If completely liquid . . . Laplace equation . . . no source
cfg_code = ndim_eq.config_code(self.S, point)
# assert ndim_eq.config_code(self.S, point) == ndim_eq.new_config_code(self.S, point)
if cfg_code == 0:
continue
# This will return a list of n equations
point_equations = ndim_eq.s_term(self.P_dof_grid, self.vel_dof_grids, point)
# For each dimension (equation) populate the corresponding matrix row
for dim, eq in enumerate(point_equations):
for dofn, coeff in eq.iteritems():
if self.using_trilinos:
self.ST[dim].InsertGlobalValues(pd, [coeff], [dofn])
else:
self.ST[dim][pd, dofn] = coeff
def _fill_GM(self, dim):
rolled_p = roll(self.P_dof_grid, 1, axis=dim)
vals_added = 0
for point in ndimed.full_iter_grid(self.P_dof_grid):
vel_dof = self.vel_dof_grids[dim][point]
# Still Necessary!
if vel_dof < 0:
continue
if self.using_trilinos and (not self.VMaps[dim].MyGID(vel_dof)):
continue
p1 = self.P_dof_grid[point]
if p1 >= 0:
if self.using_trilinos:
self.GM[dim].InsertGlobalValues(vel_dof, [1.], [p1])
else:
self.GM[dim][vel_dof, p1] = 1
vals_added += 1
p2 = rolled_p[point]
if p2 >= 0:
if self.using_trilinos:
self.GM[dim].InsertGlobalValues(vel_dof, [-1.], [p2])
else:
self.GM[dim][vel_dof, p2] = -1
vals_added += 1
def update_D(self):
# Track the previous abs values
self.last_abs_div[:] = self.ABS_D_LIN
# If divergence has halted . . . something borked
# If this happend 5x in a row . . .
if self.last_max_D == self.max_D and self.max_D != inf:
self.bork_count += 1
if self.bork_count >= 5: raise ValueError("WTF")
else:
self.bork_count = 0
self.last_max_D = self.max_D
# Zero out the divergence
self.D_LIN *= 0
# Re-accumulate it from the various flow dimensions
for d in range(self.ndim):
self.mat_mult(self.DM[d], self.V_LHS[d], self.PTEMP)
self.D_LIN[:] = self.D_LIN[:] + self.PTEMP
# This is ok for all dimensions as
# edge -> sa
# sa -> vol
# so h factor is constant
self.D_LIN /= self.h
# Calculate. the max value (convergence test)
# Abs divergence vector (for bi optimization)
if self.using_trilinos:
self.ABS_D_LIN.Abs(self.D_LIN)
self.max_D = self.ABS_D_LIN.MaxValue()
else:
self.ABS_D_LIN = abs(self.D_LIN)
self.max_D = self.ABS_D_LIN.max()
def monolithic_solve(self, method = "default"):
self.force_la_setup()
self.force_bc_setup()
self.dbprint( "Starting Monolithic Solve", 1)
from scipy.sparse import lil_matrix
self.MM = lil_matrix((self.dof_number, self.dof_number))
# TODO: using coo you could just add offsets to all the matrices
# involved and cat them together making the setup
# faster and trilinos compliant etc . . .
# DOF numbers
pdof = self.P_dof_num
vns = self.vel_dof_nums #[ndim]
self.dbprint("\tAdding Pressure Laplace", 2)
# Laplace Matrices
# PM-L
xo, yo = (0,0)
self.PM
xi, yi = self.PM.nonzero()
for x, y in zip(xi, yi):
self.MM[x,y] = self.PM[x, y]
xo, yo = self.PM.shape
# VM-L
self.VM
for dim in range(self.ndim):
self.dbprint("\tAdding Velocity Laplace (%i)" % dim, 2)
vel_matrix = self.VM[dim]
xi, yi = vel_matrix.nonzero()
for x, y in zip(xi, yi):
self.MM[xo + x, yo + y] = vel_matrix[x, y]
xo += vel_matrix.shape[0]
yo += vel_matrix.shape[1]
# # Gradient Matrices
xo = self.PM.shape[0]
yo = 0
for dim in range(self.ndim):
self.dbprint("\tAdding Gradient (%i)" % dim, 2)
grad_mat = self.GM[dim]
xi, yi = grad_mat.nonzero()
for x, y in zip(xi, yi):
self.MM[x + xo, y + yo] = - grad_mat[x, y] * self.h
xo += self.VM[dim].shape[0]
# # S-term Matrices
xo = 0
yo = self.PM.shape[0]
for dim in range(self.ndim):
self.dbprint("\tAdding S-terms (%i)"%dim,2)
s_mat = self.ST[dim]
xi, yi = s_mat.nonzero()
for x, y in zip(xi, yi):
self.MM[x + xo, y + yo] = -s_mat[x, y] / self.h
yo += self.VM[dim].shape[0]
self.dbprint("\tAssembling RHS",2)
self.MM_rhs = zeros(self.P_dof_num)
self.MM_rhs[0:len(self.P_COR)] = self.P_COR
for dim in range(self.ndim):
self.MM_rhs = concatenate( (self.MM_rhs, self.V_COR[dim] * self.h) )
# self.dbprint("DEBUG:TODO, Committing matrix and rhs to disk")
# from sparse_to_h5 import storeSparseProblem
# storeSparseProblem(self.MM, self.MM_rhs, "BigMatrixStorage.h5")
self.dbprint("Converting to CSR",2)
self.MM = self.MM.tocsr()
self.dbprint("Solving . . .", 1)
self.solve_start = time.time()
if self.method == "spsolve" or self.method == "nobi":
from scipy.sparse.linalg import spsolve
ans = spsolve(self.MM, self.MM_rhs)
elif self.method == "bicgstab":
ans = self.bicgstab(self.MM, self.MM_rhs)
# elif self.method == "ruge":
# from pyamg import ruge_stuben_solver
# self.dbprint("Setting up ruge_stuben_solver.", level=2)
# self.rss = self.ruge_stuben_solver( self.MM, max_levels=2)
# self.dbprint(self.rss)
# ans = self.rss.solve(self.MM_rhs, tol=1e-10)
else:
self.dbprint("Solver method not supported!",0)
raise ValueError
self.solve_time = time.time() - self.solve_start
self.P_LHS = ans[0:self.P_dof_num]
current_offset = len(self.P_LHS)
for dim in range(self.ndim):
self.V_LHS[dim] = ans[current_offset:current_offset + self.vel_dof_nums[dim]]
current_offset += self.vel_dof_nums[dim]
# The Bi number based acceleration
# TODO: cleanup this, make it actual Bi instead of multiple of the divergence
def update_Bi(self, dn_mult = 0.05, up_mult = 1.001, start_Bi=0.0000005, starting_I = 10 ):
'''This routine does a simple optimization on the Biot number of the Pressure Solution
i.e. adjusting of the element wise paramaters accelerates convergence by reducing
the system stiffness. If the cell-wise divergence goes up, it increases the stringency of BC's at that point
otherwise it lets it loosen slightly.
The default paramaters are _extremely_ conservative, but more aggressive setting can vastly accelerate convergence.
Selection of understable paramaters can lead to irreversable divergence, so adjust with care'''
self.dbprint("Starting Bi Optimization")
mx = 1
if self.I < starting_I:
self.DIV_MULT[:] = 0
elif self.I == starting_I:
self.DIV_MULT[:] = start_Bi
else:
# True for dof's who have a lower divergence this round than last
# Epetra Vectors dont Support fancy slicing, so
lower = array(self.ABS_D_LIN) < array(self.last_abs_div)
dm_copy = array(self.DIV_MULT)
inc_count = sum(1 * lower)
self.dbprint("Bi DOF Count Increased %i" % inc_count, 2 )
# Some Bi numbers go up
dm_copy[lower] *= up_mult
dm_copy[logical_not(lower)] *= dn_mult
# Bi Capped at some value
dm_copy[dm_copy > mx] = mx
self.DIV_MULT[:] = dm_copy
me = mean(self.DIV_MULT)
mi = min(self.DIV_MULT)
mx = max(self.DIV_MULT)
dbinfo = "Bi Optimization Finished - Local Info Mean:%e Min:%e Max:%e" % (me, mi, mx)
self.dbprint( dbinfo, 2 )
def sync(self, place=""):
if self.using_trilinos:
self.dbprint("Trilinos Thread Sync - %s" % place, 3)
self.Comm.Barrier()
self.dbprint("\tDone.", 3)
def iterate(self):
'''This is the main iteration loop that converges the system.'''
# Make sure we are setup properly
self.force_la_setup()
self.force_bc_setup()
self.sync("Beginning of Iteration")
###############################
# Solve the Pressure Equation #
###############################
self.dbprint("\tCalculating P RHS", level = 2)
# This is the Bi contribution
self.P_RHS[:] = self.DIV_MULT * self.D_LIN
for d in range(self.ndim):
self.mat_mult(self.ST[d], self.V_LHS[d], self.PTEMP)
self.P_RHS[:] = self.P_RHS[:] + self.PTEMP
# Divide by h and add PBC pressure corrections
self.P_RHS[:] = (self.P_RHS / self.h) + self.P_COR
# Solve the pressure Eq.
self.dbprint("\tSolving P", level = 2)
self.SOLVE_P()
################################
# Solve the Velocity Equations #
################################
for dim in range(self.ndim):
# Setup the RHS to the Velocity equation
self.dbprint("\tCalculating Grad P (V RHS - %i)" % dim , level = 2)
self.mat_mult(self.GM[dim], self.P_LHS, self.VTEMP[dim])
self.V_RHS[dim][:] = (self.VTEMP[dim] + self.V_COR[dim]) * self.h
# Solve the velocity Equation in the n'th dimension
self.dbprint("\tSolving V - %i" % dim , level = 2)
self.SOLVE_V(dim)
# Sync
self.sync("End of Iteration!")
# Increment the iteration count
self.I += 1
def regrid(self):
'''Convert the linear-algebra formed vectors back into familiar field varibles'''
# Make the 2d/3d arrays to hold the results
self.P = zeros(self.S.shape, dtype=float64)
# One face centered velocity grid for each Velocity Axis
self.V_GRIDS = [ zeros( self.shape) for dim in range(self.ndim) ]
# Convenience handles (P, u, v, w, x)
for name, dim in zip(['u','v','w','x'], range(self.ndim)):
setattr(self, name, self.V_GRIDS[dim])
# Put them back in their respective arrays
self.assign_P_to_obj(self.P)
for axis in range(self.ndim):
self.assign_V_to_obj( axis, self.V_GRIDS[axis] )
def ungrid(self):
'''Take the grid-wise P, u, v, etc and put into the
linear forms used during matrix solution
Useful if you want to seed a solution into the solver'''
# There is a classier way to do this with "take" from numpy, but isn't Trilinos safe
for point in self.P_points_list:
pdof = self.pdp(point)
self.P_LHS[pdof] = self.P[point]
for axis in range(self.ndim):
for point in self.V_points_list[axis]:
dof = self.nvd( axis, point )
self.V_LHS[axis][dof] = self.V_GRIDS[axis][point]
def assign_P_to_obj(self, obj):
'''These two functions are used in saving results
This implementation makes the core not reliant on tables
i.e. hdf5 can open a file, and assign only DOFs from a cerain
CPU at a time.'''
for point in self.Get_P_Iterator():
dof = self.pdp(point)
if self.using_trilinos:
dof = self.PMap.LID(dof)
obj[point] = self.P_LHS[dof]
def assign_V_to_obj(self, axis, obj):
'''These two functions are used in saving results
This implementation makes the core not reliant on tables
i.e. hdf5 can open a file, and assign only DOFs from a cerain
CPU at a time.'''
for point in self.Get_V_Iterator(axis):
dof = self.nvd(axis, point)
if self.using_trilinos:
dof = self.VMaps[axis].LID(dof)
obj[point] = self.V_LHS[axis][dof]
def converge(self, stopping_div = 1.0e-8, max_iter = inf, use_biot = True, printing = 0):
'''Run the iterative version of the solver until the first of input stopping criteria are met.'''
# Make sure we are setup properly
self.force_la_setup()
self.force_bc_setup()
self.update_D()
# If the current solution is valid
# (and it isn't the first iteration), break out
if self.I != 0 and self.max_D < stopping_div:
return
# Start the timer
self.solve_start = time.time()
while True:
# Perform an iteration
self.iterate()
self.dbprint("\tUpdating Divergence" , level = 2)
# Update the divergence and max divergence
self.update_D()
# Update Bi
if use_biot: self.update_Bi()
# Print some status Crap
self.dbprint("Iteration:%i, Max Divergence:%e" % ( self.I, self.max_D ) )
# If it is time to break, do so
if self.max_D < stopping_div:
self.solve_time = time.time() - self.solve_start
break
# Alternative breaking criteria: exceeding iteration count
if self.I >= max_iter:
raise ValueError("Max iterations exceeded")
def getMetaDict(self):
'''This function returns a dictionary of metadata information about the solver.
It is only 'valid' after one or more solution methods have been used'''
ret_dict = {"Iteration_Count":self.I,
"P_and_V_DOF_Number":self.dof_number,
"Setup_Time":self.setup_time,
"Converge_Time":self.solve_time,
"Matrix_Solver":self.method,
"Total_Time":(self.setup_time + self.solve_time),
"Identity":gethostname()}
return ret_dict
def nuke_all_trilinos(self):
'''Trilinos and the iPython task-client dont clean up swig-attached memory correctly.
Why? Who knows? But call this to avoid unsightly memory leaks.'''
self.sync("Pre-Nuke Sync")
if not self.using_trilinos:
return
# Epetra Vectors
del self.PMap
del self.P_LHS
del self.P_RHS
del self.P_COR
del self.PTEMP
del self.Bi
del self.DIV_MULT
del self.last_abs_div
del self.D_LIN
del self.ABS_D_LIN
for x in range(3):
del self.VMaps[0]
del self.V_LHS[0]
del self.V_RHS[0]
del self.V_COR[0]
del self.VTEMP[0]
# Epetra Matrices
del self.PM
for x in range(3):
del self.VM[0]
del self.DM[0]
del self.ST[0]
del self.GM[0]
