def do_numeric_factorization(
self,
matrix: BlockMatrix,
raise_on_error: bool = True) -> LinearSolverResults:
"""
Parameters
----------
matrix: BlockMatrix
raise_on_error: bool
Returns
-------
res: LinearSolverResults
"""
self.block_matrix = block_matrix = matrix
res = LinearSolverResults()
res.status = LinearSolverStatus.successful
for ndx in range(self.block_dim - 1):
sub_res = self.subproblem_solvers[ndx].do_numeric_factorization(
matrix=block_matrix.get_block(ndx, ndx),
raise_on_error=raise_on_error)
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
break
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
return res
schur_complement = block_matrix.get_block(
self.block_dim - 1, self.block_dim - 1).toarray()
# in a scipy csr_matrix,
# data contains the values
# indices contains the column indices
# indptr contains the number of nonzeros in the row
for ndx in range(self.block_dim - 1):
A = block_matrix.get_block(self.block_dim - 1, ndx).tocsr()
for row_ndx in range(A.shape[0]):
row_nnz = A.indptr[row_ndx + 1] - A.indptr[row_ndx]
if row_nnz != 0:
_rhs = A[row_ndx, :].toarray()[0]
contribution = self.subproblem_solvers[ndx].do_back_solve(
_rhs)
schur_complement[:, row_ndx] -= A.dot(contribution)
schur_complement = coo_matrix(schur_complement)
sub_res = self.schur_complement_solver.do_symbolic_factorization(
schur_complement, raise_on_error=raise_on_error)
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
return res
sub_res = self.schur_complement_solver.do_numeric_factorization(
schur_complement, raise_on_error=raise_on_error)
_process_sub_results(res, sub_res)
return res
def _gather_results(res: LinearSolverResults) -> LinearSolverResults:
stat = res.status.value
stats = comm.allgather(stat)
sub_res = LinearSolverResults()
res = LinearSolverResults()
res.status = LinearSolverStatus.successful
for stat in stats:
sub_res.status = LinearSolverStatus(stat)
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
break
return res
def do_symbolic_factorization(self,
matrix: BlockMatrix,
raise_on_error: bool = True,
timer=None) -> LinearSolverResults:
"""
Parameters
----------
matrix: BlockMatrix
raise_on_error: bool
timer: HierarchicalTimer
Returns
-------
res: LinearSolverResults
"""
block_matrix = matrix
nbrows, nbcols = block_matrix.bshape
if nbrows != nbcols:
raise ValueError('The block matrix provided is not square.')
self.block_dim = nbrows
nrows, ncols = block_matrix.shape
if nrows != ncols:
raise ValueError('The block matrix provided is not square.')
self.dim = nrows
res = LinearSolverResults()
res.status = LinearSolverStatus.successful
for ndx in range(self.block_dim - 1):
sub_res = self.subproblem_solvers[ndx].do_symbolic_factorization(
matrix=block_matrix.get_block(ndx, ndx),
raise_on_error=raise_on_error)
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
break
return res
def do_numeric_factorization(
self,
matrix: MPIBlockMatrix,
raise_on_error: bool = True,
timer: Optional[HierarchicalTimer] = None) -> LinearSolverResults:
"""
Perform numeric factorization:
* perform numeric factorization on each diagonal block
* form and communicate the Schur-Complement
* factorize the schur-complement
This method should only be called after do_symbolic_factorization.
Parameters
----------
matrix: MPIBlockMatrix
A Pynumero MPIBlockMatrix. This is the A matrix in Ax=b
raise_on_error: bool
If False, an error will not be raised if an error occurs during symbolic factorization. Instead the
status attribute of the results object will indicate an error ocurred.
timer: HierarchicalTimer
A timer for profiling.
Returns
-------
res: LinearSolverResults
The results object
"""
if timer is None:
timer = HierarchicalTimer()
self.block_matrix = block_matrix = matrix
res = LinearSolverResults()
res.status = LinearSolverStatus.successful
timer.start('form SC')
for ndx in self.local_block_indices:
timer.start('factorize')
sub_res = self.subproblem_solvers[ndx].do_numeric_factorization(
matrix=block_matrix.get_block(ndx, ndx), raise_on_error=False)
timer.stop('factorize')
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
break
res = _gather_results(res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
if raise_on_error:
raise RuntimeError(
'Numeric factorization unsuccessful; status: ' +
str(res.status))
else:
timer.stop('form SC')
return res
# in a scipy csr_matrix,
# data contains the values
# indices contains the column indices
# indptr contains the number of nonzeros in the row
self.schur_complement.data = np.zeros(self.schur_complement.data.size,
dtype=np.double)
for ndx in self.local_block_indices:
border_matrix: _BorderMatrix = self.border_matrices[ndx]
A = border_matrix.csr
_rhs = np.zeros(A.shape[1], dtype=np.double)
solver = self.subproblem_solvers[ndx]
for row_ndx in border_matrix.nonzero_rows:
for indptr in range(A.indptr[row_ndx], A.indptr[row_ndx + 1]):
col = A.indices[indptr]
val = A.data[indptr]
_rhs[col] += val
timer.start('back solve')
contribution = solver.do_back_solve(_rhs)
timer.stop('back solve')
timer.start('dot product')
contribution = A.dot(contribution)
timer.stop('dot product')
self.schur_complement.data[self.sc_data_slices[ndx][
row_ndx]] -= contribution[border_matrix.nonzero_rows]
for indptr in range(A.indptr[row_ndx], A.indptr[row_ndx + 1]):
col = A.indices[indptr]
val = A.data[indptr]
_rhs[col] -= val
timer.start('communicate')
timer.start('zeros')
sc = np.zeros(self.schur_complement.data.size, dtype=np.double)
timer.stop('zeros')
timer.start('Barrier')
comm.Barrier()
timer.stop('Barrier')
timer.start('Allreduce')
comm.Allreduce(self.schur_complement.data, sc)
timer.stop('Allreduce')
self.schur_complement.data = sc
timer.start('add')
sc = self.schur_complement + block_matrix.get_block(
self.block_dim - 1, self.block_dim - 1).tocoo()
timer.stop('add')
timer.stop('communicate')
timer.stop('form SC')
timer.start('factor SC')
sub_res = self.schur_complement_solver.do_symbolic_factorization(
sc, raise_on_error=raise_on_error)
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
timer.stop('factor SC')
return res
sub_res = self.schur_complement_solver.do_numeric_factorization(sc)
_process_sub_results(res, sub_res)
timer.stop('factor SC')
return res
def do_symbolic_factorization(
self,
matrix: MPIBlockMatrix,
raise_on_error: bool = True,
timer: Optional[HierarchicalTimer] = None) -> LinearSolverResults:
"""
Perform symbolic factorization. This performs symbolic factorization for each diagonal block and
collects some information on the structure of the schur complement for sparse communication in
the numeric factorization phase.
Parameters
----------
matrix: MPIBlockMatrix
A Pynumero MPIBlockMatrix. This is the A matrix in Ax=b
raise_on_error: bool
If False, an error will not be raised if an error occurs during symbolic factorization. Instead the
status attribute of the results object will indicate an error ocurred.
timer: HierarchicalTimer
A timer for profiling.
Returns
-------
res: LinearSolverResults
The results object
"""
if timer is None:
timer = HierarchicalTimer()
block_matrix = matrix
nbrows, nbcols = block_matrix.bshape
if nbrows != nbcols:
raise ValueError('The block matrix provided is not square.')
self.block_dim = nbrows
# split up the blocks between ranks
self.local_block_indices = list()
for ndx in range(self.block_dim - 1):
if ((block_matrix.rank_ownership[ndx, ndx] == rank) or
(block_matrix.rank_ownership[ndx, ndx] == -1 and rank == 0)):
self.local_block_indices.append(ndx)
res = LinearSolverResults()
res.status = LinearSolverStatus.successful
timer.start('factorize')
for ndx in self.local_block_indices:
sub_res = self.subproblem_solvers[ndx].do_symbolic_factorization(
matrix=block_matrix.get_block(ndx, ndx), raise_on_error=False)
_process_sub_results(res, sub_res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
break
timer.stop('factorize')
res = _gather_results(res)
if res.status not in {
LinearSolverStatus.successful, LinearSolverStatus.warning
}:
if raise_on_error:
raise RuntimeError(
'Symbolic factorization unsuccessful; status: ' +
str(res.status))
else:
return res
timer.start('sc_structure')
self._get_sc_structure(block_matrix=block_matrix, timer=timer)
timer.stop('sc_structure')
return res