Пример #1
0
def test_mixed_constant_bc(mesh_factory):
    """Test that setting a dirichletbc with on a component of a mixed
    function yields the same result as setting it with a function"""
    func, args = mesh_factory
    mesh = func(*args)
    tdim, gdim = mesh.topology.dim, mesh.geometry.dim
    boundary_facets = locate_entities_boundary(
        mesh, tdim - 1, lambda x: np.ones(x.shape[1], dtype=bool))
    TH = ufl.MixedElement([
        ufl.VectorElement("Lagrange", mesh.ufl_cell(), 2),
        ufl.FiniteElement("Lagrange", mesh.ufl_cell(), 1)
    ])
    W = FunctionSpace(mesh, TH)
    U = Function(W)

    # Apply BC to component of a mixed space using a Constant
    c = Constant(mesh, (PETSc.ScalarType(2), PETSc.ScalarType(2)))
    dofs0 = locate_dofs_topological(W.sub(0), tdim - 1, boundary_facets)
    bc0 = dirichletbc(c, dofs0, W.sub(0))
    u = U.sub(0)
    set_bc(u.vector, [bc0])

    # Apply BC to component of a mixed space using a Function
    ubc1 = u.collapse()
    ubc1.interpolate(lambda x: np.full((gdim, x.shape[1]), 2.0))
    dofs1 = locate_dofs_topological((W.sub(0), ubc1.function_space), tdim - 1,
                                    boundary_facets)
    bc1 = dirichletbc(ubc1, dofs1, W.sub(0))
    U1 = Function(W)
    u1 = U1.sub(0)
    set_bc(u1.vector, [bc1])

    # Check that both approaches yield the same vector
    assert np.allclose(u.x.array, u1.x.array)
Пример #2
0
    def monolithic_solve():
        """Monolithic (interleaved) solver"""
        P2_el = ufl.VectorElement("Lagrange", mesh.ufl_cell(), 2)
        P1_el = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), 1)
        TH = P2_el * P1_el
        W = FunctionSpace(mesh, TH)
        (u, p) = ufl.TrialFunctions(W)
        (v, q) = ufl.TestFunctions(W)
        a00 = ufl.inner(ufl.grad(u), ufl.grad(v)) * dx
        a01 = ufl.inner(p, ufl.div(v)) * dx
        a10 = ufl.inner(ufl.div(u), q) * dx
        a = a00 + a01 + a10

        p00 = ufl.inner(ufl.grad(u), ufl.grad(v)) * dx
        p11 = ufl.inner(p, q) * dx
        p_form = p00 + p11

        f = Function(W.sub(0).collapse()[0])
        p_zero = Function(W.sub(1).collapse()[0])
        L0 = inner(f, v) * dx
        L1 = inner(p_zero, q) * dx
        L = L0 + L1

        a, p_form, L = form(a), form(p_form), form(L)

        bdofsW0_P2_0 = locate_dofs_topological(W.sub(0), facetdim, bndry_facets0)
        bdofsW0_P2_1 = locate_dofs_topological(W.sub(0), facetdim, bndry_facets1)

        bc0 = dirichletbc(bc_value, bdofsW0_P2_0, W.sub(0))
        bc1 = dirichletbc(bc_value, bdofsW0_P2_1, W.sub(0))

        A = assemble_matrix(a, bcs=[bc0, bc1])
        A.assemble()
        P = assemble_matrix(p_form, bcs=[bc0, bc1])
        P.assemble()

        b = assemble_vector(L)
        apply_lifting(b, [a], bcs=[[bc0, bc1]])
        b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
        set_bc(b, [bc0, bc1])

        ksp = PETSc.KSP()
        ksp.create(mesh.comm)
        ksp.setOperators(A, P)
        ksp.setType("minres")
        pc = ksp.getPC()
        pc.setType('lu')

        def monitor(ksp, its, rnorm):
            # print("Num it, rnorm:", its, rnorm)
            pass

        ksp.setTolerances(rtol=1.0e-8, max_it=50)
        ksp.setMonitor(monitor)
        ksp.setFromOptions()
        x = A.createVecRight()
        ksp.solve(b, x)
        assert ksp.getConvergedReason() > 0
        return b.norm(), x.norm(), A.norm(), P.norm()
Пример #3
0
    def monolithic_solve():
        """Monolithic version"""
        E = P * P
        W = FunctionSpace(mesh, E)
        U = Function(W)
        dU = ufl.TrialFunction(W)
        u0, u1 = ufl.split(U)
        v0, v1 = ufl.TestFunctions(W)

        F = inner((u0**2 + 1) * ufl.grad(u0), ufl.grad(v0)) * dx \
            + inner((u1**2 + 1) * ufl.grad(u1), ufl.grad(v1)) * dx \
            - inner(f, v0) * ufl.dx - inner(g, v1) * dx
        J = derivative(F, U, dU)

        F, J = form(F), form(J)

        u0_bc = Function(V0)
        u0_bc.interpolate(bc_val_0)
        u1_bc = Function(V1)
        u1_bc.interpolate(bc_val_1)
        bdofsW0_V0 = locate_dofs_topological((W.sub(0), V0), facetdim,
                                             bndry_facets)
        bdofsW1_V1 = locate_dofs_topological((W.sub(1), V1), facetdim,
                                             bndry_facets)
        bcs = [
            dirichletbc(u0_bc, bdofsW0_V0, W.sub(0)),
            dirichletbc(u1_bc, bdofsW1_V1, W.sub(1))
        ]

        Jmat = create_matrix(J)
        Fvec = create_vector(F)

        snes = PETSc.SNES().create(MPI.COMM_WORLD)
        snes.setTolerances(rtol=1.0e-15, max_it=10)

        snes.getKSP().setType("preonly")
        snes.getKSP().getPC().setType("lu")

        problem = NonlinearPDE_SNESProblem(F, J, U, bcs)
        snes.setFunction(problem.F_mono, Fvec)
        snes.setJacobian(problem.J_mono, J=Jmat, P=None)

        U.sub(0).interpolate(initial_guess_u)
        U.sub(1).interpolate(initial_guess_p)

        x = create_vector(F)
        x.array = U.vector.array_r

        snes.solve(None, x)
        assert snes.getKSP().getConvergedReason() > 0
        assert snes.getConvergedReason() > 0
        return x.norm()
Пример #4
0
def rigid_motions_nullspace(V: _fem.FunctionSpace):
    """
    Function to build nullspace for 2D/3D elasticity.

    Parameters:
    ===========
    V
        The function space
    """
    _x = _fem.Function(V)
    # Get geometric dim
    gdim = V.mesh.geometry.dim
    assert gdim == 2 or gdim == 3

    # Set dimension of nullspace
    dim = 3 if gdim == 2 else 6

    # Create list of vectors for null space
    nullspace_basis = [_x.vector.copy() for _ in range(dim)]

    with ExitStack() as stack:
        vec_local = [stack.enter_context(x.localForm()) for x in nullspace_basis]
        basis = [np.asarray(x) for x in vec_local]

        dofs = [V.sub(i).dofmap.list.array for i in range(gdim)]

        # Build translational null space basis
        for i in range(gdim):
            basis[i][dofs[i]] = 1.0

        # Build rotational null space basis
        x = V.tabulate_dof_coordinates()
        dofs_block = V.dofmap.list.array
        x0, x1, x2 = x[dofs_block, 0], x[dofs_block, 1], x[dofs_block, 2]
        if gdim == 2:
            basis[2][dofs[0]] = -x1
            basis[2][dofs[1]] = x0
        elif gdim == 3:
            basis[3][dofs[0]] = -x1
            basis[3][dofs[1]] = x0

            basis[4][dofs[0]] = x2
            basis[4][dofs[2]] = -x0
            basis[5][dofs[2]] = x1
            basis[5][dofs[1]] = -x2

    _la.orthonormalize(nullspace_basis)
    assert _la.is_orthonormal(nullspace_basis)
    return PETSc.NullSpace().create(vectors=nullspace_basis)
Пример #5
0
def test_locate_dofs_geometrical():
    """Test that locate_dofs_geometrical, when passed two function
    spaces, returns the correct degrees of freedom in each space"""
    mesh = create_unit_square(MPI.COMM_WORLD, 4, 8)
    p0, p1 = 1, 2
    P0 = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), p0)
    P1 = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), p1)

    W = FunctionSpace(mesh, P0 * P1)
    V = W.sub(0).collapse()[0]

    with pytest.raises(RuntimeError):
        locate_dofs_geometrical(
            W, lambda x: np.isclose(x.T, [0, 0, 0]).all(axis=1))

    dofs = locate_dofs_geometrical(
        (W.sub(0), V), lambda x: np.isclose(x.T, [0, 0, 0]).all(axis=1))

    # Collect dofs (global indices) from all processes
    dofs0_global = W.sub(0).dofmap.index_map.local_to_global(dofs[0])
    dofs1_global = V.dofmap.index_map.local_to_global(dofs[1])
    all_dofs0 = set(np.concatenate(MPI.COMM_WORLD.allgather(dofs0_global)))
    all_dofs1 = set(np.concatenate(MPI.COMM_WORLD.allgather(dofs1_global)))

    # Check only one dof pair is found globally
    assert len(all_dofs0) == 1
    assert len(all_dofs1) == 1

    # On process with the dof pair
    if len(dofs) == 1:
        # Check correct dof returned in W
        coords_W = W.tabulate_dof_coordinates()
        assert np.isclose(coords_W[dofs[0][0]], [0, 0, 0]).all()
        # Check correct dof returned in V
        coords_V = V.tabulate_dof_coordinates()
        assert np.isclose(coords_V[dofs[0][1]], [0, 0, 0]).all()
Пример #6
0
def test_tabulate_dofs(mesh_factory):
    func, args = mesh_factory
    mesh = func(*args)
    W0 = FiniteElement("Lagrange", mesh.ufl_cell(), 1)
    W1 = VectorElement("Lagrange", mesh.ufl_cell(), 1)
    W = FunctionSpace(mesh, W0 * W1)

    L0 = W.sub(0)
    L1 = W.sub(1)
    L01 = L1.sub(0)
    L11 = L1.sub(1)

    map = mesh.topology.index_map(mesh.topology.dim)
    num_cells = map.size_local + map.num_ghosts
    for c in range(num_cells):
        dofs0 = L0.dofmap.cell_dofs(c)
        dofs1 = L01.dofmap.cell_dofs(c)
        dofs2 = L11.dofmap.cell_dofs(c)
        dofs3 = L1.dofmap.cell_dofs(c)
        assert len(np.intersect1d(dofs0, dofs1)) == 0
        assert len(np.intersect1d(dofs0, dofs2)) == 0
        assert len(np.intersect1d(dofs1, dofs2)) == 0
        assert np.array_equal(np.append(dofs1, dofs2), dofs3)
Пример #7
0
def test_assembly_solve_taylor_hood_nl(mesh):
    """Assemble Stokes problem with Taylor-Hood elements and solve."""
    gdim = mesh.geometry.dim
    P2 = VectorFunctionSpace(mesh, ("Lagrange", 2))
    P1 = FunctionSpace(mesh, ("Lagrange", 1))

    def boundary0(x):
        """Define boundary x = 0"""
        return np.isclose(x[0], 0.0)

    def boundary1(x):
        """Define boundary x = 1"""
        return np.isclose(x[0], 1.0)

    def initial_guess_u(x):
        u_init = np.row_stack(
            (np.sin(x[0]) * np.sin(x[1]), np.cos(x[0]) * np.cos(x[1])))
        if gdim == 3:
            u_init = np.row_stack((u_init, np.cos(x[2])))
        return u_init

    def initial_guess_p(x):
        return -x[0]**2 - x[1]**3

    u_bc_0 = Function(P2)
    u_bc_0.interpolate(
        lambda x: np.row_stack(tuple(x[j] + float(j) for j in range(gdim))))

    u_bc_1 = Function(P2)
    u_bc_1.interpolate(
        lambda x: np.row_stack(tuple(np.sin(x[j]) for j in range(gdim))))

    facetdim = mesh.topology.dim - 1
    bndry_facets0 = locate_entities_boundary(mesh, facetdim, boundary0)
    bndry_facets1 = locate_entities_boundary(mesh, facetdim, boundary1)

    bdofs0 = locate_dofs_topological(P2, facetdim, bndry_facets0)
    bdofs1 = locate_dofs_topological(P2, facetdim, bndry_facets1)

    bcs = [dirichletbc(u_bc_0, bdofs0), dirichletbc(u_bc_1, bdofs1)]

    u, p = Function(P2), Function(P1)
    du, dp = ufl.TrialFunction(P2), ufl.TrialFunction(P1)
    v, q = ufl.TestFunction(P2), ufl.TestFunction(P1)

    F = [
        inner(ufl.grad(u), ufl.grad(v)) * dx + inner(p, ufl.div(v)) * dx,
        inner(ufl.div(u), q) * dx
    ]
    J = [[derivative(F[0], u, du),
          derivative(F[0], p, dp)],
         [derivative(F[1], u, du),
          derivative(F[1], p, dp)]]
    P = [[J[0][0], None], [None, inner(dp, q) * dx]]

    F, J, P = form(F), form(J), form(P)

    # -- Blocked and monolithic

    Jmat0 = create_matrix_block(J)
    Pmat0 = create_matrix_block(P)
    Fvec0 = create_vector_block(F)

    snes = PETSc.SNES().create(MPI.COMM_WORLD)
    snes.setTolerances(rtol=1.0e-15, max_it=10)
    snes.getKSP().setType("minres")
    snes.getKSP().getPC().setType("lu")

    problem = NonlinearPDE_SNESProblem(F, J, [u, p], bcs, P=P)
    snes.setFunction(problem.F_block, Fvec0)
    snes.setJacobian(problem.J_block, J=Jmat0, P=Pmat0)

    u.interpolate(initial_guess_u)
    p.interpolate(initial_guess_p)

    x0 = create_vector_block(F)
    with u.vector.localForm() as _u, p.vector.localForm() as _p:
        scatter_local_vectors(x0, [_u.array_r, _p.array_r],
                              [(u.function_space.dofmap.index_map,
                                u.function_space.dofmap.index_map_bs),
                               (p.function_space.dofmap.index_map,
                                p.function_space.dofmap.index_map_bs)])
    x0.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                   mode=PETSc.ScatterMode.FORWARD)

    snes.solve(None, x0)

    assert snes.getConvergedReason() > 0

    # -- Blocked and nested

    Jmat1 = create_matrix_nest(J)
    Pmat1 = create_matrix_nest(P)
    Fvec1 = create_vector_nest(F)

    snes = PETSc.SNES().create(MPI.COMM_WORLD)
    snes.setTolerances(rtol=1.0e-15, max_it=10)

    nested_IS = Jmat1.getNestISs()

    snes.getKSP().setType("minres")
    snes.getKSP().setTolerances(rtol=1e-12)
    snes.getKSP().getPC().setType("fieldsplit")
    snes.getKSP().getPC().setFieldSplitIS(["u", nested_IS[0][0]],
                                          ["p", nested_IS[1][1]])

    ksp_u, ksp_p = snes.getKSP().getPC().getFieldSplitSubKSP()
    ksp_u.setType("preonly")
    ksp_u.getPC().setType('lu')
    ksp_p.setType("preonly")
    ksp_p.getPC().setType('lu')

    problem = NonlinearPDE_SNESProblem(F, J, [u, p], bcs, P=P)
    snes.setFunction(problem.F_nest, Fvec1)
    snes.setJacobian(problem.J_nest, J=Jmat1, P=Pmat1)

    u.interpolate(initial_guess_u)
    p.interpolate(initial_guess_p)

    x1 = create_vector_nest(F)
    for x1_soln_pair in zip(x1.getNestSubVecs(), (u, p)):
        x1_sub, soln_sub = x1_soln_pair
        soln_sub.vector.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                                    mode=PETSc.ScatterMode.FORWARD)
        soln_sub.vector.copy(result=x1_sub)
        x1_sub.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                           mode=PETSc.ScatterMode.FORWARD)

    x1.set(0.0)
    snes.solve(None, x1)

    assert snes.getConvergedReason() > 0
    assert nest_matrix_norm(Jmat1) == pytest.approx(Jmat0.norm(), 1.0e-12)
    assert Fvec1.norm() == pytest.approx(Fvec0.norm(), 1.0e-12)
    assert x1.norm() == pytest.approx(x0.norm(), 1.0e-12)

    # -- Monolithic

    P2_el = ufl.VectorElement("Lagrange", mesh.ufl_cell(), 2)
    P1_el = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), 1)
    TH = P2_el * P1_el
    W = FunctionSpace(mesh, TH)
    U = Function(W)
    dU = ufl.TrialFunction(W)
    u, p = ufl.split(U)
    du, dp = ufl.split(dU)
    v, q = ufl.TestFunctions(W)

    F = inner(ufl.grad(u), ufl.grad(v)) * dx + inner(p, ufl.div(v)) * dx \
        + inner(ufl.div(u), q) * dx
    J = derivative(F, U, dU)
    P = inner(ufl.grad(du), ufl.grad(v)) * dx + inner(dp, q) * dx

    F, J, P = form(F), form(J), form(P)

    bdofsW0_P2_0 = locate_dofs_topological((W.sub(0), P2), facetdim,
                                           bndry_facets0)
    bdofsW0_P2_1 = locate_dofs_topological((W.sub(0), P2), facetdim,
                                           bndry_facets1)

    bcs = [
        dirichletbc(u_bc_0, bdofsW0_P2_0, W.sub(0)),
        dirichletbc(u_bc_1, bdofsW0_P2_1, W.sub(0))
    ]

    Jmat2 = create_matrix(J)
    Pmat2 = create_matrix(P)
    Fvec2 = create_vector(F)

    snes = PETSc.SNES().create(MPI.COMM_WORLD)
    snes.setTolerances(rtol=1.0e-15, max_it=10)
    snes.getKSP().setType("minres")
    snes.getKSP().getPC().setType("lu")

    problem = NonlinearPDE_SNESProblem(F, J, U, bcs, P=P)
    snes.setFunction(problem.F_mono, Fvec2)
    snes.setJacobian(problem.J_mono, J=Jmat2, P=Pmat2)

    U.sub(0).interpolate(initial_guess_u)
    U.sub(1).interpolate(initial_guess_p)

    x2 = create_vector(F)
    x2.array = U.vector.array_r

    snes.solve(None, x2)

    assert snes.getConvergedReason() > 0
    assert Jmat2.norm() == pytest.approx(Jmat0.norm(), 1.0e-12)
    assert Fvec2.norm() == pytest.approx(Fvec0.norm(), 1.0e-12)
    assert x2.norm() == pytest.approx(x0.norm(), 1.0e-12)
Пример #8
0
def test_matrix_assembly_block_nl():
    """Test assembly of block matrices and vectors into (a) monolithic
    blocked structures, PETSc Nest structures, and monolithic structures
    in the nonlinear setting
    """
    mesh = create_unit_square(MPI.COMM_WORLD, 4, 8)
    p0, p1 = 1, 2
    P0 = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), p0)
    P1 = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), p1)
    V0 = FunctionSpace(mesh, P0)
    V1 = FunctionSpace(mesh, P1)

    def initial_guess_u(x):
        return np.sin(x[0]) * np.sin(x[1])

    def initial_guess_p(x):
        return -x[0]**2 - x[1]**3

    def bc_value(x):
        return np.cos(x[0]) * np.cos(x[1])

    facetdim = mesh.topology.dim - 1
    bndry_facets = locate_entities_boundary(
        mesh, facetdim,
        lambda x: np.logical_or(np.isclose(x[0], 0.0), np.isclose(x[0], 1.0)))

    u_bc = Function(V1)
    u_bc.interpolate(bc_value)
    bdofs = locate_dofs_topological(V1, facetdim, bndry_facets)
    bc = dirichletbc(u_bc, bdofs)

    # Define variational problem
    du, dp = ufl.TrialFunction(V0), ufl.TrialFunction(V1)
    u, p = Function(V0), Function(V1)
    v, q = ufl.TestFunction(V0), ufl.TestFunction(V1)

    u.interpolate(initial_guess_u)
    p.interpolate(initial_guess_p)

    f = 1.0
    g = -3.0

    F0 = inner(u, v) * dx + inner(p, v) * dx - inner(f, v) * dx
    F1 = inner(u, q) * dx + inner(p, q) * dx - inner(g, q) * dx

    a_block = form([[derivative(F0, u, du),
                     derivative(F0, p, dp)],
                    [derivative(F1, u, du),
                     derivative(F1, p, dp)]])
    L_block = form([F0, F1])

    # Monolithic blocked
    x0 = create_vector_block(L_block)
    scatter_local_vectors(x0, [u.vector.array_r, p.vector.array_r],
                          [(u.function_space.dofmap.index_map,
                            u.function_space.dofmap.index_map_bs),
                           (p.function_space.dofmap.index_map,
                            p.function_space.dofmap.index_map_bs)])
    x0.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                   mode=PETSc.ScatterMode.FORWARD)

    # Ghosts are updated inside assemble_vector_block
    A0 = assemble_matrix_block(a_block, bcs=[bc])
    b0 = assemble_vector_block(L_block, a_block, bcs=[bc], x0=x0, scale=-1.0)
    A0.assemble()
    assert A0.getType() != "nest"
    Anorm0 = A0.norm()
    bnorm0 = b0.norm()

    # Nested (MatNest)
    x1 = create_vector_nest(L_block)
    for x1_soln_pair in zip(x1.getNestSubVecs(), (u, p)):
        x1_sub, soln_sub = x1_soln_pair
        soln_sub.vector.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                                    mode=PETSc.ScatterMode.FORWARD)
        soln_sub.vector.copy(result=x1_sub)
        x1_sub.ghostUpdate(addv=PETSc.InsertMode.INSERT,
                           mode=PETSc.ScatterMode.FORWARD)

    A1 = assemble_matrix_nest(a_block, bcs=[bc])
    b1 = assemble_vector_nest(L_block)
    apply_lifting_nest(b1, a_block, bcs=[bc], x0=x1, scale=-1.0)
    for b_sub in b1.getNestSubVecs():
        b_sub.ghostUpdate(addv=PETSc.InsertMode.ADD,
                          mode=PETSc.ScatterMode.REVERSE)
    bcs0 = bcs_by_block([L.function_spaces[0] for L in L_block], [bc])

    set_bc_nest(b1, bcs0, x1, scale=-1.0)
    A1.assemble()

    assert A1.getType() == "nest"
    assert nest_matrix_norm(A1) == pytest.approx(Anorm0, 1.0e-12)
    assert b1.norm() == pytest.approx(bnorm0, 1.0e-12)

    # Monolithic version
    E = P0 * P1
    W = FunctionSpace(mesh, E)
    dU = ufl.TrialFunction(W)
    U = Function(W)
    u0, u1 = ufl.split(U)
    v0, v1 = ufl.TestFunctions(W)

    U.sub(0).interpolate(initial_guess_u)
    U.sub(1).interpolate(initial_guess_p)

    F = inner(u0, v0) * dx + inner(u1, v0) * dx + inner(u0, v1) * dx + inner(u1, v1) * dx \
        - inner(f, v0) * ufl.dx - inner(g, v1) * dx
    J = derivative(F, U, dU)
    F, J = form(F), form(J)

    bdofsW_V1 = locate_dofs_topological((W.sub(1), V1), facetdim, bndry_facets)

    bc = dirichletbc(u_bc, bdofsW_V1, W.sub(1))
    A2 = assemble_matrix(J, bcs=[bc])
    A2.assemble()
    b2 = assemble_vector(F)
    apply_lifting(b2, [J], bcs=[[bc]], x0=[U.vector], scale=-1.0)
    b2.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
    set_bc(b2, [bc], x0=U.vector, scale=-1.0)
    assert A2.getType() != "nest"
    assert A2.norm() == pytest.approx(Anorm0, 1.0e-12)
    assert b2.norm() == pytest.approx(bnorm0, 1.0e-12)
Пример #9
0
def test_biharmonic():
    """Manufactured biharmonic problem.

    Solved using rotated Regge mixed finite element method. This is equivalent
    to the Hellan-Herrmann-Johnson (HHJ) finite element method in
    two-dimensions."""
    mesh = create_rectangle(
        MPI.COMM_WORLD,
        [np.array([0.0, 0.0]), np.array([1.0, 1.0])], [32, 32],
        CellType.triangle)

    element = ufl.MixedElement([
        ufl.FiniteElement("Regge", ufl.triangle, 1),
        ufl.FiniteElement("Lagrange", ufl.triangle, 2)
    ])

    V = FunctionSpace(mesh, element)
    sigma, u = ufl.TrialFunctions(V)
    tau, v = ufl.TestFunctions(V)

    x = ufl.SpatialCoordinate(mesh)
    u_exact = ufl.sin(ufl.pi * x[0]) * ufl.sin(ufl.pi * x[0]) * ufl.sin(
        ufl.pi * x[1]) * ufl.sin(ufl.pi * x[1])
    f_exact = div(grad(div(grad(u_exact))))
    sigma_exact = grad(grad(u_exact))

    # sigma and tau are tangential-tangential continuous according to the
    # H(curl curl) continuity of the Regge space. However, for the biharmonic
    # problem we require normal-normal continuity H (div div). Theorem 4.2 of
    # Lizao Li's PhD thesis shows that the latter space can be constructed by
    # the former through the action of the operator S:
    def S(tau):
        return tau - ufl.Identity(2) * ufl.tr(tau)

    sigma_S = S(sigma)
    tau_S = S(tau)

    # Discrete duality inner product eq. 4.5 Lizao Li's PhD thesis
    def b(tau_S, v):
        n = FacetNormal(mesh)
        return inner(tau_S, grad(grad(v))) * dx \
            - ufl.dot(ufl.dot(tau_S('+'), n('+')), n('+')) * jump(grad(v), n) * dS \
            - ufl.dot(ufl.dot(tau_S, n), n) * ufl.dot(grad(v), n) * ds

    # Non-symmetric formulation
    a = form(inner(sigma_S, tau_S) * dx - b(tau_S, u) + b(sigma_S, v))
    L = form(inner(f_exact, v) * dx)

    V_1 = V.sub(1).collapse()[0]
    zero_u = Function(V_1)
    zero_u.x.array[:] = 0.0

    # Strong (Dirichlet) boundary condition
    boundary_facets = locate_entities_boundary(
        mesh, mesh.topology.dim - 1,
        lambda x: np.full(x.shape[1], True, dtype=bool))
    boundary_dofs = locate_dofs_topological(
        (V.sub(1), V_1), mesh.topology.dim - 1, boundary_facets)

    bcs = [dirichletbc(zero_u, boundary_dofs, V.sub(1))]

    A = assemble_matrix(a, bcs=bcs)
    A.assemble()
    b = assemble_vector(L)
    apply_lifting(b, [a], bcs=[bcs])
    b.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
    set_bc(b, bcs)

    # Solve
    solver = PETSc.KSP().create(MPI.COMM_WORLD)
    PETSc.Options()["ksp_type"] = "preonly"
    PETSc.Options()["pc_type"] = "lu"
    # PETSc.Options()["pc_factor_mat_solver_type"] = "mumps"
    solver.setFromOptions()
    solver.setOperators(A)

    x_h = Function(V)
    solver.solve(b, x_h.vector)
    x_h.x.scatter_forward()

    # Recall that x_h has flattened indices.
    u_error_numerator = np.sqrt(
        mesh.comm.allreduce(assemble_scalar(
            form(
                inner(u_exact - x_h[4], u_exact - x_h[4]) *
                dx(mesh, metadata={"quadrature_degree": 5}))),
                            op=MPI.SUM))
    u_error_denominator = np.sqrt(
        mesh.comm.allreduce(assemble_scalar(
            form(
                inner(u_exact, u_exact) *
                dx(mesh, metadata={"quadrature_degree": 5}))),
                            op=MPI.SUM))

    assert np.absolute(u_error_numerator / u_error_denominator) < 0.05

    # Reconstruct tensor from flattened indices.
    # Apply inverse transform. In 2D we have S^{-1} = S.
    sigma_h = S(ufl.as_tensor([[x_h[0], x_h[1]], [x_h[2], x_h[3]]]))
    sigma_error_numerator = np.sqrt(
        mesh.comm.allreduce(assemble_scalar(
            form(
                inner(sigma_exact - sigma_h, sigma_exact - sigma_h) *
                dx(mesh, metadata={"quadrature_degree": 5}))),
                            op=MPI.SUM))
    sigma_error_denominator = np.sqrt(
        mesh.comm.allreduce(assemble_scalar(
            form(
                inner(sigma_exact, sigma_exact) *
                dx(mesh, metadata={"quadrature_degree": 5}))),
                            op=MPI.SUM))

    assert np.absolute(sigma_error_numerator / sigma_error_denominator) < 0.005
Пример #10
0
    print(
        "(C) Norm of pressure coefficient vector (blocked, direct): {}".format(
            norm_p_2))
assert np.isclose(norm_u_2, norm_u_0)
assert np.isclose(norm_p_2, norm_p_0)

# ### Non-blocked direct solver
#
# Again, solve the same problem but this time with a non-blocked direct
# solver approach

# +
# Create the function space
TH = P2 * P1
W = FunctionSpace(msh, TH)
W0, _ = W.sub(0).collapse()

# No slip boundary condition
noslip = Function(V)
facets = locate_entities_boundary(msh, 1, noslip_boundary)
dofs = locate_dofs_topological((W.sub(0), V), 1, facets)
bc0 = dirichletbc(noslip, dofs, W.sub(0))

# Driving velocity condition u = (1, 0) on top boundary (y = 1)
lid_velocity = Function(W0)
lid_velocity.interpolate(lid_velocity_expression)
facets = locate_entities_boundary(msh, 1, lid)
dofs = locate_dofs_topological((W.sub(0), V), 1, facets)
bc1 = dirichletbc(lid_velocity, dofs, W.sub(0))

# Since for this problem the pressure is only determined up to a
Пример #11
0
def test_assembly_solve_block(mode):
    """Solve a two-field mass-matrix like problem with block matrix approaches
    and test that solution is the same"""
    mesh = create_unit_square(MPI.COMM_WORLD, 32, 31, ghost_mode=mode)
    P = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), 1)
    V0 = FunctionSpace(mesh, P)
    V1 = V0.clone()

    # Locate facets on boundary
    facetdim = mesh.topology.dim - 1
    bndry_facets = locate_entities_boundary(mesh, facetdim, lambda x: np.logical_or(np.isclose(x[0], 0.0),
                                                                                    np.isclose(x[0], 1.0)))

    bdofsV0 = locate_dofs_topological(V0, facetdim, bndry_facets)
    bdofsV1 = locate_dofs_topological(V1, facetdim, bndry_facets)

    u0_bc = PETSc.ScalarType(50.0)
    u1_bc = PETSc.ScalarType(20.0)
    bcs = [dirichletbc(u0_bc, bdofsV0, V0), dirichletbc(u1_bc, bdofsV1, V1)]

    # Variational problem
    u, p = ufl.TrialFunction(V0), ufl.TrialFunction(V1)
    v, q = ufl.TestFunction(V0), ufl.TestFunction(V1)
    f = 1.0
    g = -3.0
    zero = Function(V0)

    a00 = form(inner(u, v) * dx)
    a01 = form(zero * inner(p, v) * dx)
    a10 = form(zero * inner(u, q) * dx)
    a11 = form(inner(p, q) * dx)
    L0 = form(inner(f, v) * dx)
    L1 = form(inner(g, q) * dx)

    def monitor(ksp, its, rnorm):
        pass
        # print("Norm:", its, rnorm)

    A0 = assemble_matrix_block([[a00, a01], [a10, a11]], bcs=bcs)
    b0 = assemble_vector_block([L0, L1], [[a00, a01], [a10, a11]], bcs=bcs)
    A0.assemble()
    A0norm = A0.norm()
    b0norm = b0.norm()
    x0 = A0.createVecLeft()
    ksp = PETSc.KSP()
    ksp.create(mesh.comm)
    ksp.setOperators(A0)
    ksp.setMonitor(monitor)
    ksp.setType('cg')
    ksp.setTolerances(rtol=1.0e-14)
    ksp.setFromOptions()
    ksp.solve(b0, x0)
    x0norm = x0.norm()

    # Nested (MatNest)
    A1 = assemble_matrix_nest([[a00, a01], [a10, a11]], bcs=bcs, diagonal=1.0)
    A1.assemble()
    b1 = assemble_vector_nest([L0, L1])
    apply_lifting_nest(b1, [[a00, a01], [a10, a11]], bcs=bcs)
    for b_sub in b1.getNestSubVecs():
        b_sub.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
    bcs0 = bcs_by_block([L0.function_spaces[0], L1.function_spaces[0]], bcs)
    set_bc_nest(b1, bcs0)
    b1.assemble()

    b1norm = b1.norm()
    assert b1norm == pytest.approx(b0norm, 1.0e-12)
    A1norm = nest_matrix_norm(A1)
    assert A0norm == pytest.approx(A1norm, 1.0e-12)

    x1 = b1.copy()
    ksp = PETSc.KSP()
    ksp.create(mesh.comm)
    ksp.setMonitor(monitor)
    ksp.setOperators(A1)
    ksp.setType('cg')
    ksp.setTolerances(rtol=1.0e-12)
    ksp.setFromOptions()
    ksp.solve(b1, x1)
    x1norm = x1.norm()
    assert x1norm == pytest.approx(x0norm, rel=1.0e-12)

    # Monolithic version
    E = P * P
    W = FunctionSpace(mesh, E)
    u0, u1 = ufl.TrialFunctions(W)
    v0, v1 = ufl.TestFunctions(W)
    a = inner(u0, v0) * dx + inner(u1, v1) * dx
    L = inner(f, v0) * ufl.dx + inner(g, v1) * dx
    a, L = form(a), form(L)

    bdofsW0_V0 = locate_dofs_topological(W.sub(0), facetdim, bndry_facets)
    bdofsW1_V1 = locate_dofs_topological(W.sub(1), facetdim, bndry_facets)
    bcs = [dirichletbc(u0_bc, bdofsW0_V0, W.sub(0)), dirichletbc(u1_bc, bdofsW1_V1, W.sub(1))]

    A2 = assemble_matrix(a, bcs=bcs)
    A2.assemble()
    b2 = assemble_vector(L)
    apply_lifting(b2, [a], [bcs])
    b2.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
    set_bc(b2, bcs)
    A2norm = A2.norm()
    b2norm = b2.norm()
    assert A2norm == pytest.approx(A0norm, 1.0e-12)
    assert b2norm == pytest.approx(b0norm, 1.0e-12)

    x2 = b2.copy()
    ksp = PETSc.KSP()
    ksp.create(mesh.comm)
    ksp.setMonitor(monitor)
    ksp.setOperators(A2)
    ksp.setType('cg')
    ksp.getPC().setType('jacobi')
    ksp.setTolerances(rtol=1.0e-12)
    ksp.setFromOptions()
    ksp.solve(b2, x2)
    x2norm = x2.norm()
    assert x2norm == pytest.approx(x0norm, 1.0e-10)
Пример #12
0
def test_matrix_assembly_block(mode):
    """Test assembly of block matrices and vectors into (a) monolithic
    blocked structures, PETSc Nest structures, and monolithic
    structures"""
    mesh = create_unit_square(MPI.COMM_WORLD, 4, 8, ghost_mode=mode)
    p0, p1 = 1, 2
    P0 = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), p0)
    P1 = ufl.FiniteElement("Lagrange", mesh.ufl_cell(), p1)
    V0 = FunctionSpace(mesh, P0)
    V1 = FunctionSpace(mesh, P1)

    # Locate facets on boundary
    facetdim = mesh.topology.dim - 1
    bndry_facets = locate_entities_boundary(mesh, facetdim, lambda x: np.logical_or(np.isclose(x[0], 0.0),
                                                                                    np.isclose(x[0], 1.0)))
    bdofsV1 = locate_dofs_topological(V1, facetdim, bndry_facets)
    u_bc = PETSc.ScalarType(50.0)
    bc = dirichletbc(u_bc, bdofsV1, V1)

    # Define variational problem
    u, p = ufl.TrialFunction(V0), ufl.TrialFunction(V1)
    v, q = ufl.TestFunction(V0), ufl.TestFunction(V1)
    f = 1.0
    g = -3.0
    zero = Function(V0)

    a00 = inner(u, v) * dx
    a01 = inner(p, v) * dx
    a10 = inner(u, q) * dx
    a11 = inner(p, q) * dx

    L0 = zero * inner(f, v) * dx
    L1 = inner(g, q) * dx

    a_block = form([[a00, a01], [a10, a11]])
    L_block = form([L0, L1])

    # Monolithic blocked
    A0 = assemble_matrix_block(a_block, bcs=[bc])
    A0.assemble()
    b0 = assemble_vector_block(L_block, a_block, bcs=[bc])
    assert A0.getType() != "nest"
    Anorm0 = A0.norm()
    bnorm0 = b0.norm()

    # Nested (MatNest)
    A1 = assemble_matrix_nest(a_block, bcs=[bc], mat_types=[["baij", "aij"], ["aij", ""]])
    A1.assemble()
    Anorm1 = nest_matrix_norm(A1)
    assert Anorm0 == pytest.approx(Anorm1, 1.0e-12)

    b1 = assemble_vector_nest(L_block)
    apply_lifting_nest(b1, a_block, bcs=[bc])
    for b_sub in b1.getNestSubVecs():
        b_sub.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
    bcs0 = bcs_by_block([L.function_spaces[0] for L in L_block], [bc])
    set_bc_nest(b1, bcs0)
    b1.assemble()

    bnorm1 = math.sqrt(sum([x.norm()**2 for x in b1.getNestSubVecs()]))
    assert bnorm0 == pytest.approx(bnorm1, 1.0e-12)

    # Monolithic version
    E = P0 * P1
    W = FunctionSpace(mesh, E)
    u0, u1 = ufl.TrialFunctions(W)
    v0, v1 = ufl.TestFunctions(W)
    a = inner(u0, v0) * dx + inner(u1, v1) * dx + inner(u0, v1) * dx + inner(
        u1, v0) * dx
    L = zero * inner(f, v0) * ufl.dx + inner(g, v1) * dx
    a, L = form(a), form(L)

    bdofsW_V1 = locate_dofs_topological(W.sub(1), mesh.topology.dim - 1, bndry_facets)
    bc = dirichletbc(u_bc, bdofsW_V1, W.sub(1))
    A2 = assemble_matrix(a, bcs=[bc])
    A2.assemble()
    b2 = assemble_vector(L)
    apply_lifting(b2, [a], bcs=[[bc]])
    b2.ghostUpdate(addv=PETSc.InsertMode.ADD, mode=PETSc.ScatterMode.REVERSE)
    set_bc(b2, [bc])
    assert A2.getType() != "nest"
    assert A2.norm() == pytest.approx(Anorm0, 1.0e-9)
    assert b2.norm() == pytest.approx(bnorm0, 1.0e-9)
Пример #13
0
# Output file
file = XDMFFile(MPI.COMM_WORLD, "output.xdmf", "w")
file.write_mesh(mesh)

# Step in time
t = 0.0

#  Reduce run time if on test (CI) server
if "CI" in os.environ.keys() or "GITHUB_ACTIONS" in os.environ.keys():
    T = 3 * dt
else:
    T = 50 * dt

# Get the sub-space for c and the corresponding dofs in the mixed space
# vector
V0, dofs = ME.sub(0).collapse()

# Prepare viewer for plotting the solution during the computation
if have_pyvista:
    # Create a VTK 'mesh' with 'nodes' at the function dofs
    topology, cell_types, x = plot.create_vtk_mesh(V0)
    grid = pv.UnstructuredGrid(topology, cell_types, x)

    # Set output data
    grid.point_data["c"] = u.x.array[dofs].real
    grid.set_active_scalars("c")

    p = pvqt.BackgroundPlotter(title="concentration", auto_update=True)
    p.add_mesh(grid, clim=[0, 1])
    p.view_xy(True)
    p.add_text(f"time: {t}", font_size=12, name="timelabel")
Пример #14
0
r=1
# r=2
# r=3

# x_extents = mesh_bounding_box(mesh, 0)
# y_extents = mesh_bounding_box(mesh, 1)

# Measures
dx = ufl.Measure("dx", domain=mesh)
ds = ufl.Measure("ds", domain=mesh)

element = ufl.MixedElement([ufl.FiniteElement("Regge", ufl.triangle, r),
                            ufl.FiniteElement("Lagrange", ufl.triangle, r+1)])

V = FunctionSpace(mesh, element)
V_1 = V.sub(1).collapse()

sigma, u = ufl.TrialFunctions(V)
tau, v = ufl.TestFunctions(V)
def S(tau):
    return tau - ufl.Identity(2) * ufl.tr(tau)

# Discrete duality inner product 
# cf. eq. 4.5 Lizao Li's PhD thesis

def b(tau_S, v):
    n = FacetNormal(mesh)
    return inner(tau_S, grad(grad(v))) * dx \
        - ufl.dot(ufl.dot(tau_S('+'), n('+')), n('+')) * jump(grad(v), n) * dS \
        - ufl.dot(ufl.dot(tau_S, n), n) * ufl.dot(grad(v), n) * ds
Пример #15
0
def create_dictionary_constraint(
        V: fem.FunctionSpace,
        slave_master_dict: typing.Dict[bytes, typing.Dict[bytes, float]],
        subspace_slave: int = None,
        subspace_master: int = None):
    """
    Returns a multi point constraint for a given function space
    and dictionary constraint.
    Parameters
    ----------
        V
            The function space
        slave_master_dict
            The dictionary.
        subspace_slave
            If using mixed or vector space, and only want to use dofs from
            a sub space as slave add index here.
        subspace_master
            Subspace index for mixed or vector spaces

    Example
    -------
        If the dof D located at [d0,d1] should be constrained to the dofs E and
        F at [e0,e1] and [f0,f1] as
        D = alpha E + beta F
        the dictionary should be:
        {np.array([d0, d1], dtype=numpy.float64).tobytes():
            {numpy.array([e0, e1], dtype=numpy.float64).tobytes(): alpha,
             numpy.array([f0, f1], dtype=numpy.float64).tobytes(): beta}}
    """
    dfloat = np.float64
    comm = V.mesh.comm
    bs = V.dofmap.index_map_bs
    local_size = V.dofmap.index_map.size_local * bs
    index_map = V.dofmap.index_map
    owned_entities = {}
    ghosted_entities = {}
    non_local_entities = {}
    slaves_local = {}
    slaves_ghost = {}
    slave_point_nd = np.zeros((3, 1), dtype=dfloat)
    for i, slave_point in enumerate(slave_master_dict.keys()):
        num_masters = len(list(slave_master_dict[slave_point].keys()))
        # Status for current slave, -1 if not on proc, 0 if ghost, 1 if owned
        slave_status = -1
        # Wrap slave point as numpy array
        sp = np.frombuffer(slave_point, dtype=dfloat)
        for j, coord in enumerate(sp):
            slave_point_nd[j] = coord
        slave_point_nd[len(sp):] = 0

        if subspace_slave is None:
            slave_dofs = fem.locate_dofs_geometrical(V,
                                                     close_to(slave_point_nd))
        else:
            Vsub = V.sub(subspace_slave).collapse()[0]
            slave_dofs = fem.locate_dofs_geometrical(
                (V.sub(subspace_slave), Vsub), close_to(slave_point_nd))[0]
        if len(slave_dofs) == 1:
            # Decide if slave is ghost or not
            if slave_dofs[0] < local_size:
                slaves_local[i] = slave_dofs[0]
                owned_entities[i] = {
                    "masters": np.full(num_masters, -1, dtype=np.int64),
                    "coeffs": np.full(num_masters, -1, dtype=np.float64),
                    "owners": np.full(num_masters, -1, dtype=np.int32),
                    "master_count": 0,
                    "local_index": []
                }
                slave_status = 1
            else:
                slaves_ghost[i] = slave_dofs[0]
                ghosted_entities[i] = {
                    "masters": np.full(num_masters, -1, dtype=np.int64),
                    "coeffs": np.full(num_masters, -1, dtype=np.float64),
                    "owners": np.full(num_masters, -1, dtype=np.int32),
                    "master_count": 0,
                    "local_index": []
                }
                slave_status = 0
        elif len(slave_dofs) > 1:
            raise RuntimeError("Multiple slaves found at same point. " +
                               "You should use sub-space locators.")
        # Wrap as list to ensure order later
        master_points = list(slave_master_dict[slave_point].keys())
        master_points_nd = np.zeros((3, len(master_points)), dtype=dfloat)
        for (j, master_point) in enumerate(master_points):
            # Wrap bytes as numpy array
            for k, coord in enumerate(np.frombuffer(master_point,
                                                    dtype=dfloat)):
                master_points_nd[k, j] = coord
            if subspace_master is None:
                master_dofs = fem.locate_dofs_geometrical(
                    V, close_to(master_points_nd[:, j:j + 1]))
            else:
                Vsub = V.sub(subspace_master).collapse()[0]
                master_dofs = fem.locate_dofs_geometrical(
                    (V.sub(subspace_master), Vsub),
                    close_to(master_points_nd[:, j:j + 1]))[0]

            # Only add masters owned by this processor
            master_dofs = master_dofs[master_dofs < local_size]
            if len(master_dofs) == 1:
                master_block = master_dofs[0] // bs
                master_rem = master_dofs % bs
                glob_master = index_map.local_to_global([master_block])[0]
                if slave_status == -1:
                    if i in non_local_entities.keys():
                        non_local_entities[i]["masters"].append(glob_master *
                                                                bs +
                                                                master_rem)
                        non_local_entities[i]["coeffs"].append(
                            slave_master_dict[slave_point][master_point])
                        non_local_entities[i]["owners"].append(comm.rank),
                        non_local_entities[i]["local_index"].append(j)
                    else:
                        non_local_entities[i] = {
                            "masters": [glob_master * bs + master_rem],
                            "coeffs":
                            [slave_master_dict[slave_point][master_point]],
                            "owners": [comm.rank],
                            "local_index": [j]
                        }
                elif slave_status == 0:
                    ghosted_entities[i]["masters"][
                        j] = glob_master * bs + master_rem
                    ghosted_entities[i]["owners"][j] = comm.rank
                    ghosted_entities[i]["coeffs"][j] = slave_master_dict[
                        slave_point][master_point]
                    ghosted_entities[i]["local_index"].append(j)
                elif slave_status == 1:
                    owned_entities[i]["masters"][
                        j] = glob_master * bs + master_rem
                    owned_entities[i]["owners"][j] = comm.rank
                    owned_entities[i]["coeffs"][j] = slave_master_dict[
                        slave_point][master_point]
                    owned_entities[i]["local_index"].append(j)
                else:
                    raise RuntimeError(
                        "Invalid slave status: {0:d} (-1,0,1 are valid options)"
                        .format(slave_status))
            elif len(master_dofs) > 1:
                raise RuntimeError(
                    "Multiple masters found at same point. You should use sub-space locators."
                )

    # Send the ghost and owned entities to processor 0 to gather them
    data_to_send = [owned_entities, ghosted_entities, non_local_entities]
    if comm.rank != 0:
        comm.send(data_to_send, dest=0, tag=1)
    del owned_entities, ghosted_entities, non_local_entities
    # Gather all info on proc 0 and sort data
    owned_slaves, ghosted_slaves = None, None
    if comm.rank == 0:
        recv = {0: data_to_send}
        for proc in range(1, comm.size):
            recv[proc] = comm.recv(source=proc, tag=1)

        for proc in range(comm.size):
            # Loop through all masters
            other_procs = np.arange(comm.size)
            other_procs = other_procs[other_procs != proc]
            # Loop through all owned slaves and ghosts, and update
            # the master entries
            for pair in [[0, 1], [1, 0]]:
                i, j = pair
                for slave in recv[proc][i].keys():
                    for o_proc in other_procs:
                        # If slave is ghost on other proc add local masters
                        if slave in recv[o_proc][j].keys():
                            # Update master with possible entries from ghost
                            o_masters = recv[o_proc][j][slave]["local_index"]
                            for o_master in o_masters:
                                recv[proc][i][slave]["masters"][
                                    o_master] = recv[o_proc][j][slave][
                                        "masters"][o_master]
                                recv[proc][i][slave]["coeffs"][
                                    o_master] = recv[o_proc][j][slave][
                                        "coeffs"][o_master]
                                recv[proc][i][slave]["owners"][
                                    o_master] = recv[o_proc][j][slave][
                                        "owners"][o_master]
                        # If proc only has master, but not the slave
                        if slave in recv[o_proc][2].keys():
                            o_masters = recv[o_proc][2][slave]["local_index"]
                            # As non owned indices only store non-zero entries
                            for k, o_master in enumerate(o_masters):
                                recv[proc][i][slave]["masters"][
                                    o_master] = recv[o_proc][2][slave][
                                        "masters"][k]
                                recv[proc][i][slave]["coeffs"][
                                    o_master] = recv[o_proc][2][slave][
                                        "coeffs"][k]
                                recv[proc][i][slave]["owners"][
                                    o_master] = recv[o_proc][2][slave][
                                        "owners"][k]
            if proc == comm.rank:
                owned_slaves = recv[proc][0]
                ghosted_slaves = recv[proc][1]
            else:
                # If no owned masters, do not send masters
                if len(recv[proc][0].keys()) > 0:
                    comm.send(recv[proc][0], dest=proc, tag=55)
                if len(recv[proc][1].keys()) > 0:
                    comm.send(recv[proc][1], dest=proc, tag=66)
    else:
        if len(slaves_local.keys()) > 0:
            owned_slaves = comm.recv(source=0, tag=55)
        if len(slaves_ghost.keys()) > 0:
            ghosted_slaves = comm.recv(source=0, tag=66)

    # Flatten slaves (local)
    slaves, masters, coeffs, owners, offsets = [], [], [], [], [0]
    for slave_index in slaves_local.keys():
        slaves.append(slaves_local[slave_index])
        masters.extend(owned_slaves[slave_index]["masters"])  # type: ignore
        owners.extend(owned_slaves[slave_index]["owners"])  # type: ignore
        coeffs.extend(owned_slaves[slave_index]["coeffs"])  # type: ignore
        offsets.append(len(masters))

    for slave_index in slaves_ghost.keys():
        slaves.append(slaves_ghost[slave_index])
        masters.extend(ghosted_slaves[slave_index]["masters"])  # type: ignore
        owners.extend(ghosted_slaves[slave_index]["owners"])  # type: ignore
        coeffs.extend(ghosted_slaves[slave_index]["coeffs"])  # type: ignore
        offsets.append(len(masters))
    return (np.asarray(slaves,
                       dtype=np.int32), np.asarray(masters, dtype=np.int64),
            np.asarray(coeffs, dtype=PETSc.ScalarType),
            np.asarray(owners,
                       dtype=np.int32), np.asarray(offsets, dtype=np.int32))