def test_two_mats_on_same_sparsity_share_data(self, backend, skip_opencl,
m1, skip_sequential,
skip_openmp, ds2):
"""Sparsity data should be shared between Mat objects.
Even on the device."""
sp = op2.Sparsity((ds2, ds2), (m1, m1))
mat1 = op2.Mat(sp, 'float64')
mat2 = op2.Mat(sp, 'float64')
assert mat1._colidx is mat2._colidx
assert mat1._rowptr is mat2._rowptr
def mat(self, request, msparsity, non_nest_mixed_sparsity):
if request.param:
mat = op2.Mat(msparsity)
else:
mat = op2.Mat(non_nest_mixed_sparsity)
opt = mat.handle.Option.NEW_NONZERO_ALLOCATION_ERR
opt2 = mat.handle.Option.UNUSED_NONZERO_LOCATION_ERR
mat.handle.setOption(opt, False)
mat.handle.setOption(opt2, False)
for m in mat:
m.handle.setOption(opt, False)
m.handle.setOption(opt2, False)
return mat
def test_mat_set_diagonal(self, nodes, elem_node, n):
"Set the diagonal of the entire matrix to 1.0"
mat = op2.Mat(op2.Sparsity(nodes**n, elem_node), valuetype)
nrows = mat.sparsity.nrows
mat.set_local_diagonal_entries(list(range(nrows)))
mat.assemble()
assert (mat.values == np.identity(nrows * n)).all()
def make_interpolator(expr, V, subset, access):
assert isinstance(expr, ufl.classes.Expr)
if isinstance(expr, firedrake.Expression):
arguments = ()
else:
arguments = extract_arguments(expr)
if len(arguments) == 0:
if isinstance(V, firedrake.Function):
f = V
V = f.function_space()
else:
f = firedrake.Function(V)
tensor = f.dat
elif len(arguments) == 1:
if isinstance(V, firedrake.Function):
raise ValueError(
"Cannot interpolate an expression with an argument into a Function"
)
argfs = arguments[0].function_space()
sparsity = op2.Sparsity((V.dof_dset, argfs.dof_dset),
((V.cell_node_map(), argfs.cell_node_map()), ),
name="%s_%s_sparsity" % (V.name, argfs.name),
nest=False,
block_sparse=True)
tensor = op2.Mat(sparsity)
f = tensor
else:
raise ValueError("Cannot interpolate an expression with %d arguments" %
len(arguments))
# Make sure we have an expression of the right length i.e. a value for
# each component in the value shape of each function space
dims = [numpy.prod(fs.ufl_element().value_shape(), dtype=int) for fs in V]
loops = []
if numpy.prod(expr.ufl_shape, dtype=int) != sum(dims):
raise RuntimeError('Expression of length %d required, got length %d' %
(sum(dims), numpy.prod(expr.ufl_shape, dtype=int)))
if not isinstance(expr, firedrake.Expression):
if len(V) > 1:
raise NotImplementedError(
"UFL expressions for mixed functions are not yet supported.")
loops.extend(_interpolator(V, tensor, expr, subset, arguments, access))
elif hasattr(expr, 'eval'):
if len(V) > 1:
raise NotImplementedError(
"Python expressions for mixed functions are not yet supported."
)
loops.extend(_interpolator(V, tensor, expr, subset, arguments, access))
else:
raise ValueError("Don't know how to interpolate a %r" % expr)
def callable(loops, f):
for l in loops:
l()
return f
return partial(callable, loops, f), arguments
def __init__(self, a, bcs, mat_type, *args, **kwargs):
# sets self._a, self._bcs, and self._mat_type
super(Matrix, self).__init__(a, bcs, mat_type)
options_prefix = kwargs.pop("options_prefix")
self.M = op2.Mat(*args, mat_type=mat_type, **kwargs)
self.petscmat = self.M.handle
self.petscmat.setOptionsPrefix(options_prefix)
self.mat_type = mat_type
def __init__(self, a, bcs, *args, **kwargs):
# sets self._a and self._bcs
super(Matrix, self).__init__(a, bcs)
options_prefix = kwargs.pop("options_prefix")
self._M = op2.Mat(*args, **kwargs)
self.petscmat = self._M.handle
self.petscmat.setOptionsPrefix(options_prefix)
self._thunk = None
self.assembled = False
def __init__(self, a, bcs, *args, **kwargs):
# sets self._a and self._bcs
super(Matrix, self).__init__(a, bcs)
self._M = op2.Mat(*args, **kwargs)
self.petscmat = self._M.handle
self._thunk = None
self._assembled = False
self._bcs_at_point_of_assembly = []
def test_matrix(self, backend):
"""Test a indirect par_loop with a matrix argument"""
iterset = op2.Set(2)
idset = op2.Set(2)
ss01 = op2.Subset(iterset, [0, 1])
ss10 = op2.Subset(iterset, [1, 0])
indset = op2.Set(4)
dat = op2.Dat(idset ** 1, data=[0, 1], dtype=np.float)
map = op2.Map(iterset, indset, 4, [0, 1, 2, 3, 0, 1, 2, 3])
idmap = op2.Map(iterset, idset, 1, [0, 1])
sparsity = op2.Sparsity((indset, indset), (map, map))
mat = op2.Mat(sparsity, np.float64)
mat01 = op2.Mat(sparsity, np.float64)
mat10 = op2.Mat(sparsity, np.float64)
assembly = c_for("i", 4,
c_for("j", 4,
Incr(Symbol("mat", ("i", "j")), FlatBlock("(*dat)*16+i*4+j"))))
kernel_code = FunDecl("void", "unique_id",
[Decl("double*", c_sym("dat")),
Decl("double", Symbol("mat", (4, 4)))],
Block([assembly], open_scope=False))
k = op2.Kernel(kernel_code, "unique_id")
mat.zero()
mat01.zero()
mat10.zero()
op2.par_loop(k, iterset,
dat(op2.READ, idmap[0]),
mat(op2.INC, (map[op2.i[0]], map[op2.i[1]])))
mat.assemble()
op2.par_loop(k, ss01,
dat(op2.READ, idmap[0]),
mat01(op2.INC, (map[op2.i[0]], map[op2.i[1]])))
mat01.assemble()
op2.par_loop(k, ss10,
dat(op2.READ, idmap[0]),
mat10(op2.INC, (map[op2.i[0]], map[op2.i[1]])))
mat10.assemble()
assert (mat01.values == mat.values).all()
assert (mat10.values == mat.values).all()
def __init__(self, a, bcs, *args, **kwargs):
self._a = a
self._M = op2.Mat(*args, **kwargs)
self._thunk = None
self._assembled = False
# Iteration over bcs must be in a parallel consistent order
# (so we can't use a set, since the iteration order may differ
# on different processes)
self._bcs = [bc for bc in bcs] if bcs is not None else []
self._bcs_at_point_of_assembly = []
def mat(self, msparsity, mmap, mdat):
mat = op2.Mat(msparsity)
addone = """static void addone_mat(double v[9], double d[3]) {
for (int i = 0; i < 3; i++)
for (int j = 0; j < 3; j++)
v[i*3 + j] += d[i]*d[j];
}"""
addone = op2.Kernel(addone, "addone_mat")
op2.par_loop(addone, mmap.iterset,
mat(op2.INC, (mmap, mmap)),
mdat(op2.READ, mmap))
mat.assemble()
return mat
def restriction_matrix(Pk, P1, Pk_bcs, P1_bcs):
sp = op2.Sparsity((P1.dof_dset, Pk.dof_dset),
(P1.cell_node_map(), Pk.cell_node_map()))
mat = op2.Mat(sp, PETSc.ScalarType)
matarg = mat(op2.WRITE, (P1.cell_node_map(P1_bcs)[op2.i[0]],
Pk.cell_node_map(Pk_bcs)[op2.i[1]]))
# # HACK HACK HACK, this seems like it might be a pyop2 bug
# sh = matarg._block_shape
# assert len(sh) == 1 and len(sh[0]) == 1 and len(sh[0][0]) == 2
# a, b = sh[0][0]
# nsh = (((a*self.P1.dof_dset.cdim, b*self.V.dof_dset.cdim), ), )
# matarg._block_shape = nsh
mesh = Pk.ufl_domain()
op2.par_loop(transfer_kernel(Pk, P1), mesh.cell_set, matarg)
mat.assemble()
mat._force_evaluation()
return mat.handle
def restriction_matrix(Pk, P1, Pk_bcs, P1_bcs):
sp = op2.Sparsity((P1.dof_dset, Pk.dof_dset),
(P1.cell_node_map(), Pk.cell_node_map()))
mat = op2.Mat(sp, PETSc.ScalarType)
rlgmap, clgmap = mat.local_to_global_maps
rlgmap = P1.local_to_global_map(P1_bcs, lgmap=rlgmap)
clgmap = Pk.local_to_global_map(Pk_bcs, lgmap=clgmap)
unroll = any(bc.function_space().component is not None
for bc in chain(P1_bcs, Pk_bcs) if bc is not None)
matarg = mat(op2.WRITE, (P1.cell_node_map(), Pk.cell_node_map()),
lgmaps=(rlgmap, clgmap),
unroll_map=unroll)
mesh = Pk.ufl_domain()
op2.par_loop(transfer_kernel(Pk, P1), mesh.cell_set, matarg)
mat.assemble()
return mat.handle
def test_mat_always_has_diagonal_space(self):
# A sparsity should always have space for diagonal entries
s = op2.Set(1)
d = op2.Set(4)
m = op2.Map(s, d, 1, [2])
d2 = op2.Set(3)
m2 = op2.Map(s, d2, 1, [1])
sparsity = op2.Sparsity((d, d2), (m, m2))
from petsc4py import PETSc
# petsc4py default error handler swallows SETERRQ, so just
# install the abort handler to notice an error.
PETSc.Sys.pushErrorHandler("abort")
mat = op2.Mat(sparsity)
PETSc.Sys.popErrorHandler()
assert np.allclose(mat.handle.getDiagonal().array, 0.0)
def mat(self, msparsity, mmap, mdat):
mat = op2.Mat(msparsity)
code = c_for("i", 3,
c_for("j", 3,
Incr(Symbol("v", ("i", "j")), FlatBlock("d[i][0] * d[j][0]"))))
addone = FunDecl("void", "addone_mat",
[Decl("double", Symbol("v", (3, 3))),
Decl("double", c_sym("**d"))],
Block([code], open_scope=False))
addone = op2.Kernel(addone, "addone_mat")
op2.par_loop(addone, mmap.iterset,
mat(op2.INC, (mmap[op2.i[0]], mmap[op2.i[1]])),
mdat(op2.READ, mmap))
mat.assemble()
mat._force_evaluation()
return mat
def prolongation_matrix_aij(Pk, P1, Pk_bcs, P1_bcs):
sp = op2.Sparsity((Pk.dof_dset, P1.dof_dset),
(Pk.cell_node_map(), P1.cell_node_map()))
mat = op2.Mat(sp, PETSc.ScalarType)
mesh = Pk.ufl_domain()
fele = Pk.ufl_element()
if isinstance(fele, MixedElement) and not isinstance(
fele, (VectorElement, TensorElement)):
for i in range(fele.num_sub_elements()):
Pk_bcs_i = [bc for bc in Pk_bcs if bc.function_space().index == i]
P1_bcs_i = [bc for bc in P1_bcs if bc.function_space().index == i]
rlgmap, clgmap = mat[i, i].local_to_global_maps
rlgmap = Pk.sub(i).local_to_global_map(Pk_bcs_i, lgmap=rlgmap)
clgmap = P1.sub(i).local_to_global_map(P1_bcs_i, lgmap=clgmap)
unroll = any(bc.function_space().component is not None
for bc in chain(Pk_bcs_i, P1_bcs_i) if bc is not None)
matarg = mat[i, i](
op2.WRITE,
(Pk.sub(i).cell_node_map(), P1.sub(i).cell_node_map()),
lgmaps=((rlgmap, clgmap), ),
unroll_map=unroll)
op2.par_loop(
prolongation_transfer_kernel_aij(Pk.sub(i), P1.sub(i)),
mesh.cell_set, matarg)
else:
rlgmap, clgmap = mat.local_to_global_maps
rlgmap = Pk.local_to_global_map(Pk_bcs, lgmap=rlgmap)
clgmap = P1.local_to_global_map(P1_bcs, lgmap=clgmap)
unroll = any(bc.function_space().component is not None
for bc in chain(Pk_bcs, P1_bcs) if bc is not None)
matarg = mat(op2.WRITE, (Pk.cell_node_map(), P1.cell_node_map()),
lgmaps=((rlgmap, clgmap), ),
unroll_map=unroll)
op2.par_loop(prolongation_transfer_kernel_aij(Pk, P1), mesh.cell_set,
matarg)
mat.assemble()
return mat.handle
def test_minimal_zero_mat(self):
"""Assemble a matrix that is all zeros."""
code = c_for("i", 1,
c_for("j", 1,
Assign(Symbol("local_mat", ("i", "j")), c_sym("0.0"))))
zero_mat_code = FunDecl("void", "zero_mat",
[Decl("double", Symbol("local_mat", (1, 1)))],
Block([code], open_scope=False))
nelems = 128
set = op2.Set(nelems)
map = op2.Map(set, set, 1, np.array(list(range(nelems)), np.uint32))
sparsity = op2.Sparsity((set, set), (map, map))
mat = op2.Mat(sparsity, np.float64)
kernel = op2.Kernel(zero_mat_code, "zero_mat")
op2.par_loop(kernel, set,
mat(op2.WRITE, (map[op2.i[0]], map[op2.i[1]])))
mat.assemble()
expected_matrix = np.zeros((nelems, nelems), dtype=np.float64)
eps = 1.e-12
assert_allclose(mat.values, expected_matrix, eps)
def test_mat_set_diagonal(self, backend, nodes, elem_node, n, skip_cuda):
"Set the diagonal of the entire matrix to 1.0"
mat = op2.Mat(op2.Sparsity(nodes**n, elem_node), valuetype)
nrows = mat.sparsity.nrows
mat.inc_local_diagonal_entries(range(nrows))
assert (mat.values == np.identity(nrows * n)).all()
def mat(elem_node, dnodes):
sparsity = op2.Sparsity((dnodes, dnodes), (elem_node, elem_node), "sparsity")
return op2.Mat(sparsity, valuetype, "mat")
def make_interpolator(expr, V, subset, access):
assert isinstance(expr, ufl.classes.Expr)
arguments = extract_arguments(expr)
if len(arguments) == 0:
if isinstance(V, firedrake.Function):
f = V
V = f.function_space()
else:
f = firedrake.Function(V)
if access in {firedrake.MIN, firedrake.MAX}:
finfo = numpy.finfo(f.dat.dtype)
if access == firedrake.MIN:
val = firedrake.Constant(finfo.max)
else:
val = firedrake.Constant(finfo.min)
f.assign(val)
tensor = f.dat
elif len(arguments) == 1:
if isinstance(V, firedrake.Function):
raise ValueError(
"Cannot interpolate an expression with an argument into a Function"
)
argfs = arguments[0].function_space()
target_mesh = V.ufl_domain()
source_mesh = argfs.mesh()
argfs_map = argfs.cell_node_map()
if target_mesh is not source_mesh:
if not isinstance(target_mesh.topology,
firedrake.mesh.VertexOnlyMeshTopology):
raise NotImplementedError(
"Can only interpolate onto a Vertex Only Mesh")
if target_mesh.geometric_dimension(
) != source_mesh.geometric_dimension():
raise ValueError(
"Cannot interpolate onto a mesh of a different geometric dimension"
)
if not hasattr(target_mesh, "_parent_mesh"
) or target_mesh._parent_mesh is not source_mesh:
raise ValueError(
"Can only interpolate across meshes where the source mesh is the parent of the target"
)
if argfs_map:
# Since the par_loop is over the target mesh cells we need to
# compose a map that takes us from target mesh cells to the
# function space nodes on the source mesh. NOTE: argfs_map is
# allowed to be None when interpolating from a Real space, even
# in the trans-mesh case.
argfs_map = compose_map_and_cache(
target_mesh.cell_parent_cell_map, argfs_map)
sparsity = op2.Sparsity((V.dof_dset, argfs.dof_dset),
((V.cell_node_map(), argfs_map), ),
name="%s_%s_sparsity" % (V.name, argfs.name),
nest=False,
block_sparse=True)
tensor = op2.Mat(sparsity)
f = tensor
else:
raise ValueError("Cannot interpolate an expression with %d arguments" %
len(arguments))
# Make sure we have an expression of the right length i.e. a value for
# each component in the value shape of each function space
dims = [numpy.prod(fs.ufl_element().value_shape(), dtype=int) for fs in V]
loops = []
if numpy.prod(expr.ufl_shape, dtype=int) != sum(dims):
raise RuntimeError('Expression of length %d required, got length %d' %
(sum(dims), numpy.prod(expr.ufl_shape, dtype=int)))
if len(V) > 1:
raise NotImplementedError(
"UFL expressions for mixed functions are not yet supported.")
loops.extend(_interpolator(V, tensor, expr, subset, arguments, access))
def callable(loops, f):
for l in loops:
l()
return f
return partial(callable, loops, f), arguments
def mat(cls, iter2ind1):
sparsity = op2.Sparsity(iter2ind1.toset, iter2ind1, "sparsity")
return op2.Mat(sparsity, 'float64', "mat")
def mat(cls, iter2ind1, dindset):
sparsity = op2.Sparsity((dindset, dindset), (iter2ind1, iter2ind1),
"sparsity")
return op2.Mat(sparsity, 'float64', "mat")
def mat(s2, m2):
return op2.Mat(op2.Sparsity((s2, s2), (m2, m2)))
def xtr_mat(xtr_elem_node, xtr_dnodes):
sparsity = op2.Sparsity((xtr_dnodes, xtr_dnodes),
(xtr_elem_node, xtr_elem_node), "xtr_sparsity")
return op2.Mat(sparsity, valuetype, "xtr_mat")
def run(diffusivity, current_time, dt, endtime, **kwargs):
op2.init(**kwargs)
# Set up finite element problem
T = FiniteElement("Lagrange", "triangle", 1)
V = VectorElement("Lagrange", "triangle", 1)
p = TrialFunction(T)
q = TestFunction(T)
t = Coefficient(T)
u = Coefficient(V)
diffusivity = 0.1
M = p * q * dx
adv_rhs = (q * t + dt * dot(grad(q), u) * t) * dx
d = -dt * diffusivity * dot(grad(q), grad(p)) * dx
diff_matrix = M - 0.5 * d
diff_rhs = action(M + 0.5 * d, t)
# Generate code for mass and rhs assembly.
mass = compile_form(M, "mass")[0]
adv_rhs = compile_form(adv_rhs, "adv_rhs")[0]
diff_matrix = compile_form(diff_matrix, "diff_matrix")[0]
diff_rhs = compile_form(diff_rhs, "diff_rhs")[0]
# Set up simulation data structures
valuetype = np.float64
nodes, coords, elements, elem_node = read_triangle(kwargs['mesh'])
num_nodes = nodes.size
sparsity = op2.Sparsity((elem_node, elem_node), 1, "sparsity")
mat = op2.Mat(sparsity, valuetype, "mat")
tracer_vals = np.asarray([0.0] * num_nodes, dtype=valuetype)
tracer = op2.Dat(nodes, 1, tracer_vals, valuetype, "tracer")
b_vals = np.asarray([0.0] * num_nodes, dtype=valuetype)
b = op2.Dat(nodes, 1, b_vals, valuetype, "b")
velocity_vals = np.asarray([1.0, 0.0] * num_nodes, dtype=valuetype)
velocity = op2.Dat(nodes, 2, velocity_vals, valuetype, "velocity")
# Set initial condition
i_cond_code = """
void i_cond(double *c, double *t)
{
double i_t = 0.01; // Initial time
double A = 0.1; // Normalisation
double D = 0.1; // Diffusivity
double pi = 3.141459265358979;
double x = c[0]-0.5;
double y = c[1]-0.5;
double r = sqrt(x*x+y*y);
if (r<0.25)
*t = A*(exp((-(r*r))/(4*D*i_t))/(4*pi*D*i_t));
else
*t = 0.0;
}
"""
i_cond = op2.Kernel(i_cond_code, "i_cond")
op2.par_loop(i_cond, nodes, coords(op2.IdentityMap, op2.READ),
tracer(op2.IdentityMap, op2.WRITE))
zero_dat_code = """
void zero_dat(double *dat)
{
*dat = 0.0;
}
"""
zero_dat = op2.Kernel(zero_dat_code, "zero_dat")
# Assemble and solve
have_advection = True
have_diffusion = True
def timestep_iteration():
# Advection
if have_advection:
tic('advection')
tic('assembly')
mat.zero()
op2.par_loop(
mass, elements(3, 3),
mat((elem_node[op2.i[0]], elem_node[op2.i[1]]), op2.INC),
coords(elem_node, op2.READ))
op2.par_loop(zero_dat, nodes, b(op2.IdentityMap, op2.WRITE))
op2.par_loop(adv_rhs, elements(3), b(elem_node[op2.i[0]], op2.INC),
coords(elem_node, op2.READ),
tracer(elem_node, op2.READ),
velocity(elem_node, op2.READ))
toc('assembly')
tic('solve')
op2.solve(mat, tracer, b)
toc('solve')
toc('advection')
# Diffusion
if have_diffusion:
tic('diffusion')
tic('assembly')
mat.zero()
op2.par_loop(
diff_matrix, elements(3, 3),
mat((elem_node[op2.i[0]], elem_node[op2.i[1]]), op2.INC),
coords(elem_node, op2.READ))
op2.par_loop(zero_dat, nodes, b(op2.IdentityMap, op2.WRITE))
op2.par_loop(diff_rhs, elements(3), b(elem_node[op2.i[0]],
op2.INC),
coords(elem_node, op2.READ),
tracer(elem_node, op2.READ))
toc('assembly')
tic('solve')
op2.solve(mat, tracer, b)
toc('solve')
toc('diffusion')
# Perform 1 iteration to warm up plan cache then reset initial condition
timestep_iteration()
op2.par_loop(i_cond, nodes, coords(op2.IdentityMap, op2.READ),
tracer(op2.IdentityMap, op2.WRITE))
reset()
# Timed iteration
t1 = clock()
while current_time < endtime:
timestep_iteration()
current_time += dt
runtime = clock() - t1
print "/fluidity :: %f" % runtime
summary('profile_pyop2_%s_%s.csv' %
(opt['mesh'].split('/')[-1], opt['backend']))
