def __init__(self): self.N_p1 = 2 self.N_p2 = 3 self.N_p3 = 4 self.N_q1 = 5 self.N_q2 = 6 self.N_ghost = 1 self.p1_start = self.p2_start = self.p3_start = 0 self.dp1 = 2/self.N_p1 self.dp2 = 2/self.N_p2 self.dp3 = 2/self.N_p3 self.p1, self.p2, self.p3 = self._calculate_p_center() self.physical_system = type('obj', (object,), {'moment_exponents': moment_exponents, 'moment_coeffs': moment_coeffs } ) self.f = af.randu(self.N_p1 * self.N_p2 * self.N_p3, self.N_q1 + 2 * self.N_ghost, self.N_q2 + 2 * self.N_ghost, dtype = af.Dtype.f64 ) self._comm = PETSc.COMM_WORLD self._da_dump_f = PETSc.DMDA().create([self.N_q1, self.N_q2], dof=( self.N_p1 * self.N_p2 * self.N_p3 ), stencil_width = self.N_ghost ) self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2], dof=len(self.physical_system.\ moment_exponents ), stencil_width = self.N_ghost ) self._glob_f = self._da_dump_f.createGlobalVec() self._glob_f_array = self._glob_f.getArray() self._glob_moments = self._da_dump_moments.createGlobalVec() self._glob_moments_array =self._glob_moments.getArray() PETSc.Object.setName(self._glob_f, 'distribution_function') PETSc.Object.setName(self._glob_moments, 'moments')
def main(): import petsc4py from petsc4py import PETSc n_fine = 5 n_coarse = int((n_fine - 1) / 2) + 1 da_fine = PETSc.DMDA().create([n_fine, n_fine], stencil_width=1) da_coarse = PETSc.DMDA().create([n_coarse, n_coarse], stencil_width=1) x_fine = da_fine.createGlobalVec() xa = da_fine.getVecArray(x_fine) (xs, xe), (ys, ye) = da_fine.getRanges() nx, ny = da_fine.getSizes() for i in range(xs, xe): for j in range(ys, ye): # xa[i, j] = 1.0 # xa[i, j] = i / nx xa[i, j] = np.sin(2 * np.pi * i / (nx + 1)) * np.sin(2 * np.pi * j / (ny + 1)) da_coarse.setInterpolationType(PETSc.DMDA.InterpolationType.Q1) B, vec = da_coarse.createInterpolation(da_fine) # print(B, vec.getArray()) x_coarse = da_coarse.createGlobalVec() xa = da_coarse.getVecArray(x_coarse) (xs, xe), (ys, ye) = da_coarse.getRanges() nx, ny = da_coarse.getSizes() for i in range(xs, xe): for j in range(ys, ye): xa[i, j] = 1.0 # xa[i, j] = i / nx xa[i, j] = np.sin(2 * np.pi * i / (nx + 1)) * np.sin(2 * np.pi * j / (ny + 1)) y = da_fine.createGlobalVec() # x_coarse.pointwiseMult(x_coarse) # PETSc.Mat.Restrict(B, x_coarse, y) B.mult(x_coarse, y) # y.pointwiseMult(vec, y) # PETSc.VecPointwiseMult() # print(y.getArray()) # print(x_coarse.getArray()) print((y - x_fine).norm(PETSc.NormType.NORM_INFINITY)) y_coarse = da_coarse.createGlobalVec() B.multTranspose(x_fine, y_coarse) y_coarse.pointwiseMult(vec, y_coarse) print((y_coarse - x_coarse).norm(PETSc.NormType.NORM_INFINITY))
def __init__(self): self.N_q1 = 32 self.N_q2 = 32 self.N_p1 = 4 self.N_p2 = 5 self.N_p3 = 6 self.p1_start = self.p2_start = self.p3_start = 0 self.dp1 = 2 / self.N_p1 self.dp2 = 2 / self.N_p2 self.dp3 = 2 / self.N_p3 self.p1, self.p2, self.p3 = self._calculate_p_center() # Creating an object with required attributes for the test: self.physical_system = type('obj', (object, ), { 'moment_exponents': moment_exponents, 'moment_coeffs': moment_coeffs }) self.single_mode_evolution = False self.f = af.randu(self.N_q1, self.N_q2, self.N_p1 * self.N_p2 * self.N_p3, dtype=af.Dtype.f64) self.Y = 2 * af.fft2(self.f) / (self.N_q1 * self.N_q2) self._da_dump_f = PETSc.DMDA().create( [self.N_q1, self.N_q2], dof=(self.N_p1 * self.N_p2 * self.N_p3), ) self._da_dump_moments = PETSc.DMDA().create([self.N_q1, self.N_q2], dof = len(self.physical_system.\ moment_exponents) ) self._glob_f = self._da_dump_f.createGlobalVec() self._glob_f_value = self._da_dump_f.getVecArray(self._glob_f) self._glob_moments = self._da_dump_moments.createGlobalVec() self._glob_moments_value = self._da_dump_moments.\ getVecArray(self._glob_moments) PETSc.Object.setName(self._glob_f, 'distribution_function') PETSc.Object.setName(self._glob_moments, 'moments')
def setup(nonlinear_solver, residual_function, context): dmda = PETSc.DMDA().create([context.grid.n_x, context.grid.n_y], dof=1, stencil_width=1, stencil_type='star') nonlinear_solver.setDM(dmda) n_x = context.grid.n_x n_y = context.grid.n_y r = PETSc.Vec().createSeq(n_x * n_y) # residual vector def wrapped_residual_function(snes, X, f, t, grid, old_grid, timestep_properties): residual_function(snes, X.getArray(readonly=True).reshape((n_y, n_x)), f, t, grid, old_grid, timestep_properties) nonlinear_solver.setFunction(wrapped_residual_function, r, context) def wrapped_nonlinear_solver_callable(initial_guess): x = PETSc.Vec().createSeq(n_x * n_y) # solution vector b = PETSc.Vec().createSeq(n_x * n_y) # right-hand side x.setArray(initial_guess) b.set(0) nonlinear_solver.solve(b, x) solution = x[:].reshape(n_y, n_x) return solution return wrapped_nonlinear_solver_callable
def dump_coordinate_info(self, arrays, name, file_name): self._da_coord_arrays = PETSc.DMDA().create( [self.N_q1, self.N_q2], dof=len(arrays), proc_sizes=(self._nproc_in_q1, self._nproc_in_q2), ownership_ranges=self._ownership_ranges, comm=self._comm) self._glob_coord = self._da_coord_arrays.createGlobalVec() self._glob_coord_array = self._glob_coord.getArray() N_g = self.N_ghost for i in range(len(arrays)): if (i == 0): array_to_dump = arrays[0][:, :, N_g:-N_g, N_g:-N_g] else: array_to_dump = af.join(0, array_to_dump, arrays[i][:, :, N_g:-N_g, N_g:-N_g]) af.flat(array_to_dump).to_ndarray(self._glob_coord_array) PETSc.Object.setName(self._glob_coord, name) viewer = PETSc.Viewer().createBinary(file_name + '.bin', 'w', comm=self._comm) viewer(self._glob_coord)
def __init__(self): self.q1_start = np.random.randint(0, 5) self.q2_start = np.random.randint(0, 5) self.q1_end = np.random.randint(5, 10) self.q2_end = np.random.randint(5, 10) self.N_q1 = np.random.randint(16, 32) self.N_q2 = np.random.randint(16, 32) self.dq1 = (self.q1_end - self.q1_start) / self.N_q1 self.dq2 = (self.q2_end - self.q2_start) / self.N_q2 N_g = self.N_ghost = np.random.randint(1, 5) self.q1 = self.q1_start \ * (0.5 + np.arange(-self.N_ghost, self.N_q1 + self.N_ghost ) ) * self.dq1 self.q2 = self.q2_start \ * (0.5 + np.arange(-self.N_ghost, self.N_q2 + self.N_ghost ) ) * self.dq2 self.q2, self.q1 = np.meshgrid(self.q2, self.q1) self.q1, self.q2 = af.to_array(self.q1), af.to_array(self.q2) self.q1 = af.reorder(self.q1, 2, 0, 1) self.q2 = af.reorder(self.q2, 2, 0, 1) self.q1 = af.tile(self.q1, 6) self.q2 = af.tile(self.q2, 6) self._da_fields = PETSc.DMDA().create([self.N_q1, self.N_q2], dof=6, stencil_width=self.N_ghost, boundary_type=('periodic', 'periodic'), stencil_type=1, ) self._glob_fields = self._da_fields.createGlobalVec() self._local_fields = self._da_fields.createLocalVec() self._glob_fields_array = self._glob_fields.getArray() self._local_fields_array = self._local_fields.getArray() self.cell_centered_EM_fields = af.constant(0, 6, self.q1.shape[1], self.q1.shape[2], dtype=af.Dtype.f64 ) self.cell_centered_EM_fields[:, N_g:-N_g, N_g:-N_g] = \ af.sin(2 * np.pi * self.q1 + 4 * np.pi * self.q2)[:, N_g:-N_g,N_g:-N_g] self.performance_test_flag = False
def run_mgrit(nt, space_procs, coarsening, freq=1, a=1.0, t_start=0, t_stop=1, cycle='V'): size = MPI.COMM_WORLD.Get_size() comm_x, comm_t = split_communicator(MPI.COMM_WORLD, space_procs) nx = ny = 65 dmda_coarse = PETSc.DMDA().create([nx, ny], stencil_width=1, comm=comm_x) dmda_fine = dmda_coarse.refine() t_interval = np.linspace(t_start, t_stop, nt) time_procs = int(size / space_procs) problem = [HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=freq, a=a, t_interval=t_interval)] for i in range(0, len(coarsening)): problem.append(HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=freq, a=a, t_interval=t_interval[::np.prod(coarsening[:i + 1], dtype=int)])) nested_iteration = True if len(coarsening) > 0 else False if cycle == 'V': mgrit = Mgrit(problem=problem, comm_time=comm_t, comm_space=comm_x, nested_iteration=nested_iteration) else: mgrit = Mgrit(problem=problem, comm_time=comm_t, comm_space=comm_x, nested_iteration=nested_iteration, cycle_type='F', cf_iter=0) info = mgrit.solve() return info['time_setup'], info['time_solve'], info['time_setup'] + info['time_solve']
def main(): n = 4 da = PETSc.DMDA().create([n, n], stencil_width=1) rank = PETSc.COMM_WORLD.getRank() x = da.createGlobalVec() xa = da.getVecArray(x) (xs, xe), (ys, ye) = da.getRanges() for i in range(xs, xe): for j in range(ys, ye): xa[i, j] = j * n + i print('x=', rank, x.getArray(), xs, xe, ys, ye) A = da.createMatrix() A.setType('aij') # sparse A.setFromOptions() Istart, Iend = A.getOwnershipRange() for I in range(Istart, Iend): A[I, I] = 1.0 # communicate off-processor values # and setup internal data structures # for performing parallel operations A.assemblyBegin() A.assemblyEnd() res = da.createGlobalVec() A.mult(x, res) print(rank, res.getArray()) print((res - x).norm())
def __init__(self, N_q1, N_q2, N_ghost): self._da_f = \ PETSc.DMDA().create([N_q1, N_q2], stencil_width=N_ghost, boundary_type=('periodic', 'periodic') ) self.N_q1 = N_q1 self.N_q2 = N_q2 self.dq1 = 1 / N_q1 self.dq2 = 1 / N_q2 self.N_ghost = N_ghost self._A_q1 = 1 self._A_q2 = 1 self.q1_start = self.q2_start = 0 self.q1_center, self.q2_center = calculate_q_center(self) self.f = af.sin(2 * np.pi * self.q1_center + 4 * np.pi * self.q2_center) self.performance_test_flag = False
def setUp(self): self.da = PETSc.DMDA().create(dim=len(self.SIZES), dof=self.DOF, sizes=self.SIZES, boundary_type=self.BOUNDARY, stencil_type=self.STENCIL, stencil_width=self.SWIDTH, comm=self.COMM)
def help(args=None): import sys, shlex # program name try: prog = sys.argv[0] except Exception: prog = getattr(sys, 'executable', 'python') # arguments if args is None: args = sys.argv[1:] elif isinstance(args, str): args = shlex.split(args) else: args = [str(a) for a in args] # import and initialize import petsc4py petsc4py.init([prog, '-help'] + args) from petsc4py import PETSc # help dispatcher COMM = PETSc.COMM_SELF if 'vec' in args: vec = PETSc.Vec().create(comm=COMM) vec.setSizes(0) vec.setFromOptions() vec.destroy() if 'mat' in args: mat = PETSc.Mat().create(comm=COMM) mat.setSizes([0, 0]) mat.setFromOptions() mat.destroy() if 'pc' in args: pc = PETSc.PC().create(comm=COMM) pc.setFromOptions() pc.destroy() if 'ksp' in args: ksp = PETSc.KSP().create(comm=COMM) ksp.setFromOptions() ksp.destroy() if 'snes' in args: snes = PETSc.SNES().create(comm=COMM) snes.setFromOptions() snes.destroy() if 'ts' in args: ts = PETSc.TS().create(comm=COMM) ts.setFromOptions() ts.destroy() if 'tao' in args: tao = PETSc.TAO().create(comm=COMM) tao.setFromOptions() tao.destroy() if 'dmda' in args: dmda = PETSc.DMDA().create(comm=COMM) dmda.setFromOptions() dmda.destroy() if 'dmplex' in args: dmplex = PETSc.DMPlex().create(comm=COMM) dmplex.setFromOptions() dmplex.destroy()
def __init__(self, n, L, overlap, comm): # Build Grid self.dim = len(n) self.L = L self.comm = comm self.isBnd = lambda x: self.isBoundary(x) self.da = PETSc.DMDA().create(n, dof=1, stencil_width=overlap) self.da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2]) self.da.setMatType(PETSc.Mat.Type.AIJ) self.numSub = comm.Get_size() self.sub2proc = [[] for i in range(self.numSub)] for i in range(self.numSub): self.sub2proc[i] = [i, 0] self.M = 1 # Define Finite Element space self.fe = DarcyQ1(self.dim) # Setup global and local matrices + communicators self.A = self.da.createMatrix() r, _ = self.A.getLGMap() # Get local to global mapping self.is_A = PETSc.IS().createGeneral( r.indices) # Create Index Set for local indices A_local = self.A.createSubMatrices( self.is_A)[0] # Construct local submatrix on domain vglobal = self.da.createGlobalVec() vlocal = self.da.createLocalVec() self.scatter_l2g = PETSc.Scatter().create(vlocal, None, vglobal, self.is_A) self.A_local = A_local # Identify boundary nodes nnodes = int(self.da.getCoordinatesLocal()[:].size / self.dim) coords = np.transpose(self.da.getCoordinatesLocal()[:].reshape( (nnodes, self.dim))) Dirich, Neumann, P2P = checkFaces(self.da, self.isBnd, coords) # Construct Partition of Unity self.cS = coarseSpace(self.da, self.A, self.comm, self.scatter_l2g) self.cS.buildPOU(True)
def main(): import petsc4py from petsc4py import PETSc # set MPI communicator comm = MPI.COMM_WORLD world_rank = comm.Get_rank() world_size = comm.Get_size() if len(sys.argv) == 2: color = int(world_rank / int(sys.argv[1])) else: color = int(world_rank / 1) space_comm = comm.Split(color=color) space_rank = space_comm.Get_rank() space_size = space_comm.Get_size() if len(sys.argv) == 2: color = int(world_rank % int(sys.argv[1])) else: color = int(world_rank / world_size) time_comm = comm.Split(color=color) time_rank = time_comm.Get_rank() time_size = time_comm.Get_size() print( "IDs (world, space, time): %i / %i -- %i / %i -- %i / %i" % (world_rank, world_size, space_rank, space_size, time_rank, time_size)) n = 7 da = PETSc.DMDA().create([n, n], stencil_width=1, comm=space_comm) x = da.createGlobalVec() xa = da.getVecArray(x) (xs, xe), (ys, ye) = da.getRanges() for i in range(xs, xe): for j in range(ys, ye): xa[i, j] = np.sin(2 * np.pi * (i + 1) / (n + 1)) * np.sin(2 * np.pi * (j + 1) / (n + 1)) print('x=', x.getArray()) print('x:', x.getSizes(), da.getRanges()) print() if time_rank == 0: print('send', time_rank) time_comm.send(x.getArray(), dest=1, tag=0) else: print('recv', time_rank) y = da.createGlobalVec() y.setArray(time_comm.recv(source=0, tag=0)) print(type(y.getArray()))
def __init__(self, problem_params, dtype_u, dtype_f): """ Initialization routine Args: problem_params (dict): custom parameters for the example dtype_u: mesh data type (will be passed parent class) dtype_f: mesh data type (will be passed parent class) """ # these parameters will be used later, so assert their existence if 'comm' not in problem_params: problem_params['comm'] = PETSc.COMM_WORLD if 'sol_tol' not in problem_params: problem_params['sol_tol'] = 1E-10 if 'sol_maxiter' not in problem_params: problem_params['sol_maxiter'] = None essential_keys = ['nvars', 'nu', 'freq', 'comm'] for key in essential_keys: if key not in problem_params: msg = 'need %s to instantiate problem, only got %s' % (key, str(problem_params.keys())) raise ParameterError(msg) # make sure parameters have the correct form if len(problem_params['nvars']) != 2: raise ProblemError('this is a 2d example, got %s' % problem_params['nvars']) # create DMDA object which will be used for all grid operations da = PETSc.DMDA().create([problem_params['nvars'][0], problem_params['nvars'][1]], stencil_width=1, comm=problem_params['comm']) # invoke super init, passing number of dofs, dtype_u and dtype_f super(heat2d_petsc_forced, self).__init__(init=da, dtype_u=dtype_u, dtype_f=dtype_f, params=problem_params) # compute dx, dy and get local ranges self.dx = 1.0 / (self.params.nvars[0] - 1) self.dy = 1.0 / (self.params.nvars[1] - 1) (self.xs, self.xe), (self.ys, self.ye) = self.init.getRanges() # compute discretization matrix A and identity self.A = self.__get_A() self.Id = self.__get_Id() # setup solver self.ksp = PETSc.KSP() self.ksp.create(comm=self.params.comm) self.ksp.setType('cg') pc = self.ksp.getPC() pc.setType('none') self.ksp.setInitialGuessNonzero(True) self.ksp.setFromOptions() self.ksp.setTolerances(rtol=self.params.sol_tol, atol=self.params.sol_tol, max_it=self.params.sol_maxiter)
def __init__(self, grid, xmin, xmax, ymin, ymax, max_window): from petsc4py import PETSc from mpi4py import MPI comm = MPI.COMM_WORLD self.comm = comm self.MPI = MPI # super(CurieParallel, self).__init__(grid, xmin, xmax, ymin, ymax) # determine stencil width (should be window size/2) ny, nx = grid.shape dx = (xmax - xmin) / nx dy = (ymax - ymin) / ny nw = max_window / dx n2w = int(nw / 2) + 1 # add some buffer if abs(dx - dy) > 1.0: warnings.warn( "node spacing should be identical {}".format((dx, dy)), RuntimeWarning) dm = PETSc.DMDA().create(dim=2, sizes=(nx, ny), stencil_width=n2w, comm=comm) dm.setUniformCoordinates(xmin, xmax, ymin, ymax) self.dm = dm self.lgmap = dm.getLGMap() self.lvec = dm.createLocalVector() self.gvec = dm.createGlobalVector() coords = dm.getCoordinatesLocal().array.reshape(-1, 2) xmin, ymin = coords.min(axis=0) xmax, ymax = coords.max(axis=0) self.coords = coords self.max_window = max_window (imin, imax), (jmin, jmax) = dm.getGhostRanges() # now decompose grid for each processor grid_chunk = grid[jmin:jmax, imin:imax] super(CurieParallel, self).__init__(grid_chunk, xmin, xmax, ymin, ymax) reduce_methods = { 'sum': MPI.SUM, 'max': MPI.MAX, 'min': MPI.MIN, 'mean': MPI.SUM } self._reduce_methods = reduce_methods
def __init__(self, N): self.q1_start = 0 self.q2_start = 0 self.q1_end = 1 self.q2_end = 1 self.N_q1 = N self.N_q2 = N self.dq1 = (self.q1_end - self.q1_start) / self.N_q1 self.dq2 = (self.q2_end - self.q2_start) / self.N_q2 self.N_ghost = np.random.randint(3, 5) self.q1 = self.q1_start \ + (0.5 + np.arange(-self.N_ghost,self.N_q1 + self.N_ghost)) * self.dq1 self.q2 = self.q2_start \ + (0.5 + np.arange(-self.N_ghost,self.N_q2 + self.N_ghost)) * self.dq2 self.q2, self.q1 = np.meshgrid(self.q2, self.q1) self.q2, self.q1 = af.to_array(self.q2), af.to_array(self.q1) self.q1 = af.reorder(self.q1, 2, 0, 1) self.q2 = af.reorder(self.q2, 2, 0, 1) self.yee_grid_EM_fields = af.constant(0, 6, self.q1.shape[1], self.q1.shape[2], dtype=af.Dtype.f64) self._da_fields = PETSc.DMDA().create( [self.N_q1, self.N_q2], dof=6, stencil_width=self.N_ghost, boundary_type=('periodic', 'periodic'), stencil_type=1, ) self._glob_fields = self._da_fields.createGlobalVec() self._local_fields = self._da_fields.createLocalVec() self._glob_fields_array = self._glob_fields.getArray() self._local_fields_array = self._local_fields.getArray() self.boundary_conditions = type('obj', (object, ), { 'in_q1': 'periodic', 'in_q2': 'periodic' }) self.performance_test_flag = False
def testDuplicate(self, kargs=kargs): kargs = dict(kargs) dim = kargs.pop('dim') dof = kargs['dof'] boundary = kargs['boundary_type'] stencil = kargs['stencil_type'] width = kargs['stencil_width'] da = PETSc.DMDA().create([8*SCALE]*dim) newda = da.duplicate(**kargs) self.assertEqual(newda.dim, da.dim) self.assertEqual(newda.sizes, da.sizes) self.assertEqual(newda.proc_sizes, da.proc_sizes) self.assertEqual(newda.ranges, da.ranges) self.assertEqual(newda.corners, da.corners) if (newda.boundary_type == da.boundary_type and newda.stencil_width == da.stencil_width): self.assertEqual(newda.ghost_ranges, da.ghost_ranges) self.assertEqual(newda.ghost_corners, da.ghost_corners) if dof is None: dof = da.dof if boundary is None: boundary = da.boundary_type elif boundary == "none": boundary = (0,) * dim elif boundary == "mirror": boundary = (MIRROR,) * dim elif boundary == "ghosted": boundary = (GHOSTED,) * dim elif boundary == "periodic": boundary = (PERIODIC,) * dim elif boundary == "twist": boundary = (TWIST,) * dim elif isinstance(boundary, int): boundary = (boundary,) * dim if stencil is None: stencil = da.stencil[0] if width is None: width = da.stencil_width self.assertEqual(newda.dof, dof) self.assertEqual(newda.boundary_type, boundary) if dim == 1: self.assertEqual(newda.stencil, (stencil, width)) newda.destroy() da.destroy()
def main(): dim = 3 proc_sizes = tuple(MPI.Compute_dims(SIZE, dim)) # see lists.mcs.anl.gov/pipermail/petsc-dev/2016-April/018903.html # manually distribute the unkowns to the processors # mpiexec -n 3 if SIZE > 1: if 1: ownership_ranges = ( np.array([3, 4, 5], dtype='i4'), np.array([1], dtype='i4'), np.array([1], dtype='i4') ) else: ownership_ranges = ( # proc 0 (3, 4, 5), # proc 1 (1,), # proc 2 (1,) ) else: ownership_ranges = ((3,), (4,), (5,)) sizes = np.array(( sum(ownership_ranges[0]), sum(ownership_ranges[1]), sum(ownership_ranges[2])), dtype='i4' ) dm = PETSc.DMDA().create(dim=dim, sizes=sizes, proc_sizes=proc_sizes, ownership_ranges=ownership_ranges, boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.GHOSTED), stencil_type=PETSc.DMDA.StencilType.BOX, stencil_width=1, dof=1, comm=PETSc.COMM_WORLD, setup=True) assert dm.getComm().Get_rank() == RANK assert dm.getComm().Get_size() == SIZE print(f'create problem with {sizes.prod()} unknowns split into dim={dim} (sizes={sizes}) split accross {SIZE} procs with distribution {proc_sizes}') vec_l = dm.createLocalVector() vec_g = dm.createGlobalVector() initialise_values(dm, vec_l, vec_g) result_l, result_g = find_max_grad(dm, vec_l, vec_g) PETSc.Sys.syncPrint(f'Rank {RANK} had max gradient {result_l} while the global was {result_g}.') if RANK == 0: if testme(result_g, dm.getComm()): print('Result is correct.') else: print('Result is incorrect!')
def main(): def output_fcn(self): # Set path to solution path = 'results/petsc' # Create path if not existing pathlib.Path(path).mkdir(parents=True, exist_ok=True) # Save solution with corresponding time point to file np.save(path + '/petsc' + str(self.comm_time_rank) + str(self.comm_space_rank), [[[self.t[0][i], self.comm_space_rank, self.u[0][i].get_values().getArray()] for i in self.index_local[0]]]) # Split the communicator into space and time communicator comm_world = MPI.COMM_WORLD comm_x, comm_t = split_communicator(comm_world, 4) # Create PETSc DMDA grids nx = 129 ny = 129 dmda_coarse = PETSc.DMDA().create([nx, ny], stencil_width=1, comm=comm_x) dmda_fine = dmda_coarse.refine() # Set up the problem heat_petsc_0 = HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=1, a=1.0, t_start=0, t_stop=1, nt=33) heat_petsc_1 = HeatPetsc(dmda=dmda_coarse, comm_x=comm_x, freq=1, a=1.0, t_interval=heat_petsc_0.t[::2]) heat_petsc_2 = HeatPetsc(dmda=dmda_coarse, comm_x=comm_x, freq=1, a=1.0, t_interval=heat_petsc_1.t[::2]) # Setup three-level MGRIT solver with the space and time communicators and # solve the problem mgrit = Mgrit(problem=[heat_petsc_0, heat_petsc_1, heat_petsc_2], transfer=[GridTransferPetsc(fine_prob=dmda_fine, coarse_prob=dmda_coarse), GridTransferCopy()], comm_time=comm_t, comm_space=comm_x, output_fcn=output_fcn) info = mgrit.solve() import time if comm_t.Get_rank() == 0: time.sleep(1) sol = [] path = 'results/petsc/' for filename in os.listdir(path): data = np.load(path + filename, allow_pickle=True).tolist()[0] sol += data sol = [item for item in sol if item[1] == comm_x.Get_rank()] sol.sort(key=lambda tup: tup[0]) u_e = heat_petsc_0.u_exact(t=heat_petsc_0.t[-1]).get_values().getArray() diff = sol[-1][2] - u_e print('Difference at time point', heat_petsc_0.t[-1], ':', np.linalg.norm(diff, np.inf), '(space rank',comm_x.Get_rank() , ')')
def run_timestepping(nt, space_procs, freq=1, a=1.0, t_start=0, t_stop=1): comm_x, comm_t = split_communicator(MPI.COMM_WORLD, space_procs) nx = ny = 65 dmda_coarse = PETSc.DMDA().create([nx, ny], stencil_width=1, comm=comm_x) dmda_fine = dmda_coarse.refine() t_interval = np.linspace(t_start, t_stop, nt) heat_petsc_0 = HeatPetsc(dmda=dmda_fine, comm_x=comm_x, freq=freq, a=a, t_interval=t_interval) import time start = time.time() sol = heat_petsc_0.vector_t_start for i in range(1, len(heat_petsc_0.t)): sol = heat_petsc_0.step(u_start=sol, t_start=heat_petsc_0.t[i - 1], t_stop=heat_petsc_0.t[i]) info = {'time_setup': 0, 'time_solve': time.time() - start} return info['time_setup'], info['time_solve'], info['time_setup'] + info['time_solve']
def __init__(self): self.q1_start = np.random.randint(0, 5) self.q2_start = np.random.randint(0, 5) self.q1_end = np.random.randint(5, 10) self.q2_end = np.random.randint(5, 10) self.N_q1 = np.random.randint(16, 32) self.N_q2 = np.random.randint(16, 32) self.dq1 = (self.q1_end - self.q1_start) / self.N_q1 self.dq2 = (self.q2_end - self.q2_start) / self.N_q2 self.N_ghost = np.random.randint(1, 5) self._da_f = PETSc.DMDA().create([self.N_q1, self.N_q2], dof=1, stencil_width=self.N_ghost)
def __init__(self, minCoord, maxCoord, res): minX, minY, minZ = tuple(minCoord) maxX, maxY, maxZ = tuple(maxCoord) resI, resJ, resK = tuple(res) dm = PETSc.DMDA().create(dim=3, sizes=[resI, resJ, resK], stencil_width=1, comm=comm) dm.setUniformCoordinates(minX, maxX, minY, maxY, minZ, maxZ) self.dm = dm self.lvec = dm.createLocalVector() self.gvec = dm.createGlobalVector() self.rhs = dm.createGlobalVector() self.lgmap = dm.getLGMap() # Setup matrix sizes self.sizes = self.gvec.getSizes() Nx, Ny, Nz = dm.getSizes() N = Nx*Ny*Nz dx = (maxX - minX)/(Nx - 1) dy = (maxY - minY)/(Ny - 1) dz = (maxZ - minZ)/(Nz - 1) # include ghost nodes in local domain (minI, maxI), (minJ, maxJ), (minK, maxK) = dm.getGhostRanges() nx = maxI - minI ny = maxJ - minJ nz = maxK - minK self.dx, self.dy, self.dz = dx, dy, dz self.nx, self.ny, self.nz = nx, ny, nz # local numbering self.nodes = np.arange(0, nx*ny*nz, dtype=PETSc.IntType) self._initialise_mesh_variables() self._initialise_boundary_dictionary() self._initialise_matrix()
def cavity_flow2D(nx, ny, lidvelocity, grashof, prandtl): # create application context # and PETSc nonlinear solver snes = PETSc.SNES().create() da = PETSc.DMDA().create([nx,ny],dof=4, stencil_width=1, stencil_type='star') # set up solution vector F = da.createGlobalVec() snes.setFunction(CavityFlow2D.formFunction, F, args=(da, lidvelocity, prandtl, grashof)) x = da.createGlobalVec() CavityFlow2D.formInitGuess(x, da, lidvelocity, prandtl, grashof) snes.setDM(da) snes.setFromOptions() # solve the nonlinear problem snes.solve(None, x) return x
def __init__(self): self.q1_start = np.random.randint(0, 5) self.q2_start = np.random.randint(0, 5) self.q1_end = np.random.randint(5, 10) self.q2_end = np.random.randint(5, 10) self.N_q1 = np.random.randint(16, 32) self.N_q2 = np.random.randint(16, 32) self.N_ghost = np.random.randint(1, 5) self.N_p1 = np.random.randint(16, 32) self.N_p2 = np.random.randint(16, 32) self.N_p3 = np.random.randint(16, 32) self._da_f = PETSc.DMDA().create( [self.N_q1, self.N_q2], dof=(self.N_p1 * self.N_p2 * self.N_p3), stencil_width=self.N_ghost, )
def __init__(self, N, ncores_x, ncores_y): self.N = N self.da = PETSc.DMDA().create(sizes=[N, N, N], proc_sizes=[ncores_x, ncores_y, 1], stencil_width=2) self.comm = self.da.getComm() self.rank = self.comm.getRank() if self.rank == 0: print 'The 3D grid is tiled according to (nproc_x, nproc_y, nproc_z) : '\ +str(self.da.proc_sizes) # init useful vectors self.PSI = self.da.createGlobalVec() self.RHS = self.da.createGlobalVec() # fill psi self.fill_PSI() # create and fill the operator self.L = self.da.createMat() self.set_L()
def __init__(self, vs): if vs.enable_cyclic_x: boundary_type = ('periodic', 'ghosted') else: boundary_type = ('ghosted', 'ghosted') self._da = PETSc.DMDA().create( [vs.nx, vs.ny], stencil_width=1, stencil_type='star', comm=rs.mpi_comm, proc_sizes=rs.num_proc, boundary_type=boundary_type, ownership_ranges=[ (vs.nx // rs.num_proc[0], ) * rs.num_proc[0], (vs.ny // rs.num_proc[1], ) * rs.num_proc[1], ]) self._matrix, self._boundary_fac = self._assemble_poisson_matrix(vs) petsc_options = PETSc.Options() # setup krylov method self._ksp = PETSc.KSP() self._ksp.create(rs.mpi_comm) self._ksp.setOperators(self._matrix) self._ksp.setType('gmres') self._ksp.setTolerances(atol=0, rtol=vs.congr_epsilon, max_it=vs.congr_max_iterations) # preconditioner self._ksp.getPC().setType('hypre') petsc_options['pc_hypre_type'] = 'boomeramg' petsc_options['pc_hypre_boomeramg_relax_type_all'] = 'SOR/Jacobi' self._ksp.getPC().setFromOptions() self._rhs_petsc = self._da.createGlobalVec() self._sol_petsc = self._da.createGlobalVec()
def main(): dims=tuple(MPI.Compute_dims(MPI.COMM_WORLD.Get_size(),3)) dm = PETSc.DMDA().create(dim=len(dims), ownership_ranges=(numpy.array([3]), numpy.array([4]), numpy.array([5])), proc_sizes=dims, boundary_type=(PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.PERIODIC, PETSc.DMDA.BoundaryType.GHOSTED), stencil_type=PETSc.DMDA.StencilType.BOX, stencil_width = 1, dof = 1, comm = PETSc.COMM_WORLD, setup = True) vec_l=dm.createLocalVector() vec_g=dm.createGlobalVector() initialise_values(dm, vec_l, vec_g) result_l, result_g = find_max_grad(dm, vec_l, vec_g) PETSc.Sys.syncPrint("Rank {rank} had max gradient {maxgrad_l} while the global was {maxgrad_g}." .format( rank=dm.getComm().getRank(), maxgrad_l=result_l, maxgrad_g=result_g)) if (dm.getComm().getRank() == 0): if (testme(result_g, dm.getComm())): print("Result is correct.") else: print("Result is incorrect!")
def LayerCake(nx, ny, Lx, Ly, t, nel_per_layer, plotMesh = True, overlap = 1): n = [nx, ny, np.sum(nel_per_layer) + 1] # Number of Nodes in each direction. L = [Lx, Ly, np.sum(nel_per_layer)] # Dimension in x - y plane are set, z dimension will be adjusted according to stacking sequence da = PETSc.DMDA().create(n, dof=3, stencil_width=overlap) da.setUniformCoordinates(xmax=L[0], ymax=L[1], zmax=L[2]) da.setMatType(PETSc.Mat.Type.AIJ) elementCutOffs = [0.0] for i in range(nel_per_layer.size): hz = t[i] / nel_per_layer[i] for j in range(nel_per_layer[i]): elementCutOffs.append(elementCutOffs[-1] + hz) nnodes = int(da.getCoordinatesLocal()[:].size/3) c = da.getCoordinatesLocal()[:] cnew = da.getCoordinatesLocal().copy() for i in range(nnodes): cnew[3 * i + 2] = elementCutOffs[np.int(c[3 * i + 2])] da.setCoordinates(cnew) # Redefine coordinates in transformed state. if(plotMesh): x = da.createGlobalVec() viewer = PETSc.Viewer().createVTK('initial_Geometry.vts', 'w', comm = PETSc.COMM_WORLD) x.view(viewer) viewer.destroy() return da
def __init__(self, N, ncores_x, ncores_y): self.N = N self.da = PETSc.DMDA().create(sizes=[N, N, N], proc_sizes=[ncores_x, ncores_y, 1], stencil_width=2) self.comm = self.da.getComm() self.rank = self.comm.getRank() if self.rank == 0: print 'The 3D grid is tiled according to (nproc_x, nproc_y, nproc_z) : '\ +str(self.da.proc_sizes) # init useful vectors self.PSI = self.da.createGlobalVec() self.RHS = self.da.createGlobalVec() # fill right-hand-side self.fill_RHS() # create and fill the operator self.L = self.da.createMat() self.set_L() # create the solver self.ksp = PETSc.KSP() self.ksp.create(PETSc.COMM_WORLD) self.ksp.setOperators(self.L) self.ksp.setType('bicg') self.ksp.setInitialGuessNonzero(True) self.ksp.setTolerances(max_it=1000) # for opt in sys.argv[1:]: PETSc.Options().setValue(opt, None) self.ksp.setFromOptions() pass
def __init__(self, N): self.N_p1 = N self.N_p2 = N self.N_p3 = N self.N_q1 = 1 self.N_q2 = 1 self.N_ghost = 0 self.dp1 = 20 / self.N_p1 self.dp2 = 20 / self.N_p2 self.dp3 = 20 / self.N_p3 self.p1_start = self.p2_start = self.p3_start = -10 self.p1, self.p2, self.p3 = calculate_p_center(self) self.q1_center = self.q2_center = np.random.rand(1) # Creating Dummy Values: self.cell_centered_EM_fields = np.random.rand(6) self.B_fields_at_nth_timestep = np.random.rand(3) self.f = af.exp(-self.p1**2 - 2*self.p2**2 - 3*self.p3**2) self.physical_system = type('obj', (object, ), {'params': type('obj', (object, ),{'p_dim':3})} ) self._da_f = PETSc.DMDA().create([self.N_q1, self.N_q2], dof=( self.N_p1 * self.N_p2 * self.N_p3 ) ) self.performance_test_flag = False