def get_restriction_kernel(fiat_element, unique_indices, dim=1, no_weights=False):
weights = get_restriction_weights(fiat_element)[unique_indices].T
ncdof = weights.shape[0]
nfdof = weights.shape[1]
arglist = [ast.Decl("double", ast.Symbol("coarse", (ncdof*dim, ))),
ast.Decl("double", ast.Symbol("*restrict *restrict fine", ()),
qualifiers=["const"])]
if not no_weights:
arglist.append(ast.Decl("double", ast.Symbol("*restrict *restrict count_weights", ()),
qualifiers=["const"]))
all_ones = np.allclose(weights, 1.0)
if all_ones:
w = []
else:
w_sym = ast.Symbol("weights", (ncdof, nfdof))
init = ast.ArrayInit(format_array_literal(weights))
w = [ast.Decl("double", w_sym, init,
qualifiers=["const"])]
i = ast.Symbol("i", ())
j = ast.Symbol("j", ())
k = ast.Symbol("k", ())
fine = ast.Symbol("fine", (j, k))
if no_weights:
if all_ones:
assign = fine
else:
assign = ast.Prod(fine, ast.Symbol("weights", (i, j)))
else:
if all_ones:
assign = ast.Prod(fine, ast.Symbol("count_weights", (j, 0)))
else:
assign = ast.Prod(fine,
ast.Prod(ast.Symbol("weights", (i, j)),
ast.Symbol("count_weights", (j, 0))))
assignment = ast.Incr(ast.Symbol("coarse", (ast.Sum(k, ast.Prod(i, ast.c_sym(dim))),)),
assign)
k_loop = ast.For(ast.Decl("int", k, ast.c_sym(0)),
ast.Less(k, ast.c_sym(dim)),
ast.Incr(k, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
j_loop = ast.For(ast.Decl("int", j, ast.c_sym(0)),
ast.Less(j, ast.c_sym(nfdof)),
ast.Incr(j, ast.c_sym(1)),
ast.Block([k_loop], open_scope=True))
i_loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(ncdof)),
ast.Incr(i, ast.c_sym(1)),
ast.Block([j_loop], open_scope=True))
k = ast.FunDecl("void", "restriction", arglist, ast.Block(w + [i_loop]),
pred=["static", "inline"])
return op2.Kernel(k, "restriction", opts=parameters["coffee"])
def get_restriction_kernel(fiat_element, unique_indices, dim=1, no_weights=False):
weights = restriction_weights(fiat_element)[unique_indices].T
ncdof = weights.shape[0]
nfdof = weights.shape[1]
arglist = [ast.Decl("double", ast.Symbol("coarse", (ncdof*dim, ))),
ast.Decl("double *restrict *restrict ", ast.Symbol("fine", ()),
qualifiers=["const"])]
if not no_weights:
arglist.append(ast.Decl("double *restrict *restrict", ast.Symbol("count_weights", ()),
qualifiers=["const"]))
all_ones = np.allclose(weights, 1.0)
if all_ones:
w = []
else:
w_sym = ast.Symbol("weights", (ncdof, nfdof))
init = ast.ArrayInit(format_array_literal(weights))
w = [ast.Decl("double", w_sym, init,
qualifiers=["const"])]
i = ast.Symbol("i", ())
j = ast.Symbol("j", ())
k = ast.Symbol("k", ())
fine = ast.Symbol("fine", (j, k))
if no_weights:
if all_ones:
assign = fine
else:
assign = ast.Prod(fine, ast.Symbol("weights", (i, j)))
else:
if all_ones:
assign = ast.Prod(fine, ast.Symbol("count_weights", (j, 0)))
else:
assign = ast.Prod(fine,
ast.Prod(ast.Symbol("weights", (i, j)),
ast.Symbol("count_weights", (j, 0))))
assignment = ast.Incr(ast.Symbol("coarse", (ast.Sum(k, ast.Prod(i, ast.c_sym(dim))),)),
assign)
k_loop = ast.For(ast.Decl("int", k, ast.c_sym(0)),
ast.Less(k, ast.c_sym(dim)),
ast.Incr(k, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
j_loop = ast.For(ast.Decl("int", j, ast.c_sym(0)),
ast.Less(j, ast.c_sym(nfdof)),
ast.Incr(j, ast.c_sym(1)),
ast.Block([k_loop], open_scope=True))
i_loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(ncdof)),
ast.Incr(i, ast.c_sym(1)),
ast.Block([j_loop], open_scope=True))
k = ast.FunDecl("void", "restriction", arglist, ast.Block(w + [i_loop]),
pred=["static", "inline"])
return op2.Kernel(k, "restriction", opts=parameters["coffee"])
def get_count_kernel(arity):
arglist = [ast.Decl("double", ast.Symbol("weight", (arity, )))]
i = ast.Symbol("i", ())
assignment = ast.Incr(ast.Symbol("weight", (i, )), ast.c_sym(1.0))
loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(arity)),
ast.Incr(i, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
k = ast.FunDecl("void", "count_weights", arglist,
ast.Block([loop]),
pred=["static", "inline"])
return op2.Kernel(k, "count_weights", opts=parameters["coffee"])
def get_count_kernel(arity):
arglist = [ast.Decl("double", ast.Symbol("weight", (arity, )))]
i = ast.Symbol("i", ())
assignment = ast.Incr(ast.Symbol("weight", (i, )), ast.c_sym(1.0))
loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(arity)),
ast.Incr(i, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
k = ast.FunDecl("void", "count_weights", arglist,
ast.Block([loop]),
pred=["static", "inline"])
return op2.Kernel(k, "count_weights", opts=parameters["coffee"])
def get_injection_kernel(fiat_element, unique_indices, dim=1):
weights = injection_weights(fiat_element)[unique_indices].T
ncdof = weights.shape[0]
nfdof = weights.shape[1]
# What if we have multiple nodes in same location (DG)? Divide by
# rowsum.
weights = weights / np.sum(weights, axis=1).reshape(-1, 1)
all_same = np.allclose(weights, weights[0, 0])
arglist = [
ast.Decl("double", ast.Symbol("coarse", (ncdof * dim, ))),
ast.Decl("double *restrict *restrict ",
ast.Symbol("fine", ()),
qualifiers=["const"])
]
if all_same:
w_sym = ast.Symbol("weights", ())
w = [ast.Decl("double", w_sym, weights[0, 0], qualifiers=["const"])]
else:
init = ast.ArrayInit(format_array_literal(weights))
w_sym = ast.Symbol("weights", (ncdof, nfdof))
w = [ast.Decl("double", w_sym, init, qualifiers=["const"])]
i = ast.Symbol("i", ())
j = ast.Symbol("j", ())
k = ast.Symbol("k", ())
if all_same:
assign = ast.Prod(ast.Symbol("fine", (j, k)), w_sym)
else:
assign = ast.Prod(ast.Symbol("fine", (j, k)),
ast.Symbol("weights", (i, j)))
assignment = ast.Incr(
ast.Symbol("coarse", (ast.Sum(k, ast.Prod(i, ast.c_sym(dim))), )),
assign)
k_loop = ast.For(ast.Decl("int", k, ast.c_sym(0)),
ast.Less(k, ast.c_sym(dim)), ast.Incr(k, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
j_loop = ast.For(ast.Decl("int", j, ast.c_sym(0)),
ast.Less(j, ast.c_sym(nfdof)), ast.Incr(j, ast.c_sym(1)),
ast.Block([k_loop], open_scope=True))
i_loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(ncdof)), ast.Incr(i, ast.c_sym(1)),
ast.Block([j_loop], open_scope=True))
k = ast.FunDecl("void",
"injection",
arglist,
ast.Block(w + [i_loop]),
pred=["static", "inline"])
return op2.Kernel(k, "injection", opts=parameters["coffee"])
def get_injection_kernel(fiat_element, unique_indices, dim=1):
weights = get_injection_weights(fiat_element)[unique_indices].T
ncdof = weights.shape[0]
nfdof = weights.shape[1]
# What if we have multiple nodes in same location (DG)? Divide by
# rowsum.
weights = weights / np.sum(weights, axis=1).reshape(-1, 1)
all_same = np.allclose(weights, weights[0, 0])
arglist = [ast.Decl("double", ast.Symbol("coarse", (ncdof*dim, ))),
ast.Decl("double", ast.Symbol("*restrict *restrict fine", ()),
qualifiers=["const"])]
if all_same:
w_sym = ast.Symbol("weights", ())
w = [ast.Decl("double", w_sym, weights[0, 0],
qualifiers=["const"])]
else:
init = ast.ArrayInit(format_array_literal(weights))
w_sym = ast.Symbol("weights", (ncdof, nfdof))
w = [ast.Decl("double", w_sym, init,
qualifiers=["const"])]
i = ast.Symbol("i", ())
j = ast.Symbol("j", ())
k = ast.Symbol("k", ())
if all_same:
assign = ast.Prod(ast.Symbol("fine", (j, k)),
w_sym)
else:
assign = ast.Prod(ast.Symbol("fine", (j, k)),
ast.Symbol("weights", (i, j)))
assignment = ast.Incr(ast.Symbol("coarse", (ast.Sum(k, ast.Prod(i, ast.c_sym(dim))),)),
assign)
k_loop = ast.For(ast.Decl("int", k, ast.c_sym(0)),
ast.Less(k, ast.c_sym(dim)),
ast.Incr(k, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
j_loop = ast.For(ast.Decl("int", j, ast.c_sym(0)),
ast.Less(j, ast.c_sym(nfdof)),
ast.Incr(j, ast.c_sym(1)),
ast.Block([k_loop], open_scope=True))
i_loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(ncdof)),
ast.Incr(i, ast.c_sym(1)),
ast.Block([j_loop], open_scope=True))
k = ast.FunDecl("void", "injection", arglist, ast.Block(w + [i_loop]),
pred=["static", "inline"])
return op2.Kernel(k, "injection", opts=parameters["coffee"])
def get_prolongation_kernel(fiat_element, unique_indices, dim=1):
weights = get_restriction_weights(fiat_element)[unique_indices]
nfdof = weights.shape[0]
ncdof = weights.shape[1]
arglist = [
ast.Decl("double", ast.Symbol("fine", (nfdof * dim, ))),
ast.Decl("double",
ast.Symbol("*restrict *restrict coarse", ()),
qualifiers=["const"])
]
all_same = np.allclose(weights, weights[0, 0])
if all_same:
w_sym = ast.Symbol("weights", ())
w = [ast.Decl("double", w_sym, weights[0, 0], qualifiers=["const"])]
else:
w_sym = ast.Symbol("weights", (nfdof, ncdof))
init = ast.ArrayInit(format_array_literal(weights))
w = [ast.Decl("double", w_sym, init, qualifiers=["const"])]
i = ast.Symbol("i", ())
j = ast.Symbol("j", ())
k = ast.Symbol("k", ())
if all_same:
assign = ast.Prod(ast.Symbol("coarse", (j, k)), w_sym)
else:
assign = ast.Prod(ast.Symbol("coarse", (j, k)),
ast.Symbol("weights", (i, j)))
assignment = ast.Incr(
ast.Symbol("fine", (ast.Sum(k, ast.Prod(i, ast.c_sym(dim))), )),
assign)
k_loop = ast.For(ast.Decl("int", k, ast.c_sym(0)),
ast.Less(k, ast.c_sym(dim)), ast.Incr(k, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
j_loop = ast.For(ast.Decl("int", j, ast.c_sym(0)),
ast.Less(j, ast.c_sym(ncdof)), ast.Incr(j, ast.c_sym(1)),
ast.Block([k_loop], open_scope=True))
i_loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(nfdof)), ast.Incr(i, ast.c_sym(1)),
ast.Block([j_loop], open_scope=True))
k = ast.FunDecl("void",
"prolongation",
arglist,
ast.Block(w + [i_loop]),
pred=["static", "inline"])
return op2.Kernel(k, "prolongation", opts=parameters["coffee"])
def get_prolongation_kernel(fiat_element, unique_indices, dim=1):
weights = restriction_weights(fiat_element)[unique_indices]
nfdof = weights.shape[0]
ncdof = weights.shape[1]
arglist = [ast.Decl("double", ast.Symbol("fine", (nfdof*dim, ))),
ast.Decl("double *restrict *restrict ", ast.Symbol("coarse", ()),
qualifiers=["const"])]
all_same = np.allclose(weights, weights[0, 0])
if all_same:
w_sym = ast.Symbol("weights", ())
w = [ast.Decl("double", w_sym, weights[0, 0],
qualifiers=["const"])]
else:
w_sym = ast.Symbol("weights", (nfdof, ncdof))
init = ast.ArrayInit(format_array_literal(weights))
w = [ast.Decl("double", w_sym, init,
qualifiers=["const"])]
i = ast.Symbol("i", ())
j = ast.Symbol("j", ())
k = ast.Symbol("k", ())
if all_same:
assign = ast.Prod(ast.Symbol("coarse", (j, k)),
w_sym)
else:
assign = ast.Prod(ast.Symbol("coarse", (j, k)),
ast.Symbol("weights", (i, j)))
assignment = ast.Incr(ast.Symbol("fine", (ast.Sum(k, ast.Prod(i, ast.c_sym(dim))),)),
assign)
k_loop = ast.For(ast.Decl("int", k, ast.c_sym(0)),
ast.Less(k, ast.c_sym(dim)),
ast.Incr(k, ast.c_sym(1)),
ast.Block([assignment], open_scope=True))
j_loop = ast.For(ast.Decl("int", j, ast.c_sym(0)),
ast.Less(j, ast.c_sym(ncdof)),
ast.Incr(j, ast.c_sym(1)),
ast.Block([k_loop], open_scope=True))
i_loop = ast.For(ast.Decl("int", i, ast.c_sym(0)),
ast.Less(i, ast.c_sym(nfdof)),
ast.Incr(i, ast.c_sym(1)),
ast.Block([j_loop], open_scope=True))
k = ast.FunDecl("void", "prolongation", arglist, ast.Block(w + [i_loop]),
pred=["static", "inline"])
return op2.Kernel(k, "prolongation", opts=parameters["coffee"])
def _generate_integral_ir(points, terms, sets, optimise_parameters, parameters):
"Generate code to evaluate the element tensor."
# For checking if the integral code is for a matrix
def is_matrix(loop):
loop_indices = [ l[0] for l in loop ]
return (format["first free index"] in loop_indices and \
format["second free index"] in loop_indices)
# Prefetch formats to speed up code generation.
p_format = parameters["format"]
f_comment = format["comment"]
f_mul = format["mul"]
f_scale_factor = format["scale factor"]
f_iadd = format["iadd"]
f_add = format["add"]
f_A = format["element tensor"][p_format]
f_j = format["first free index"]
f_k = format["second free index"]
f_loop = format["generate loop"]
f_B = format["basis constant"]
# Initialise return values.
code = []
num_ops = 0
loops = {}
# Extract sets.
used_weights, used_psi_tables, used_nzcs, trans_set = sets
nests = []
# Loop terms and create code.
for loop, (data, entry_vals) in terms.items():
# If we don't have any entry values, there's no need to generate the loop.
if not entry_vals:
continue
# Get data.
t_set, u_weights, u_psi_tables, u_nzcs, basis_consts = data
# If we have a value, then we also need to update the sets of used variables.
trans_set.update(t_set)
used_weights.update(u_weights)
used_psi_tables.update(u_psi_tables)
used_nzcs.update(u_nzcs)
# @@@: A[0][0] += FE0[ip][j]*FE0[ip][k]*W24[ip]*det;
entry_ir = []
for entry, value, ops in entry_vals:
# Left hand side
it_vars = entry if len(loop) > 0 else (0,)
rank = tuple(i.loop_index if hasattr(i, 'loop_index') else i for i in it_vars)
offset = tuple((1, int(i.offset)) if hasattr(i, 'offset') else (1, 0) for i in it_vars)
local_tensor = pyop2.Symbol(f_A(''), rank, offset)
# Right hand side
pyop2_rhs = visit_rhs(value)
entry_ir.append(pyop2.Incr(local_tensor, pyop2_rhs, "#pragma coffee expression"))
# Disgusting hack, ensure resulting IR comes out in same order
# on all processes.
entry_ir = sorted(entry_ir, key=lambda x: x.gencode())
if len(loop) == 0:
nest = pyop2.Block(entry_ir, open_scope=True)
elif len(loop) in [1, 2]:
it_var = c_sym(loop[0][0])
end = c_sym(loop[0][2])
nest = pyop2.For(pyop2.Decl("int", it_var, c_sym(0)), pyop2.Less(it_var, end), \
pyop2.Incr(it_var, c_sym(1)), pyop2.Block(entry_ir, open_scope=True), "#pragma coffee itspace")
if len(loop) == 2:
it_var = c_sym(loop[1][0])
end = c_sym(loop[1][2])
nest_k = pyop2.For(pyop2.Decl("int", it_var, c_sym(0)), pyop2.Less(it_var, end), \
pyop2.Incr(it_var, c_sym(1)), pyop2.Block(entry_ir, open_scope=True), "#pragma coffee itspace")
nest.children[0] = pyop2.Block([nest_k], open_scope=True)
nests.append(nest)
return nests, num_ops
def _generate_functions(functions, sets):
"Generate declarations for functions and code to compute values."
f_comment = format["comment"]
f_double = format["float declaration"]
f_F = format["function value"]
f_float = format["floating point"]
f_decl = format["declaration"]
f_r = format["free indices"]
f_iadd = format["iadd"]
f_loop = format["generate loop"]
# Create the function declarations -- only the (unique) variables we need
const_vardecls = set(fd.id for fd in functions.values() if fd.cellwise_constant)
const_ast_items = [pyop2.Decl(f_double, c_sym(f_F(n)), c_sym(f_float(0)))
for n in const_vardecls]
vardecls = set(fd.id for fd in functions.values() if not fd.cellwise_constant)
ast_items = [pyop2.Decl(f_double, c_sym(f_F(n)), c_sym(f_float(0)))
for n in vardecls]
# Get sets.
used_psi_tables = sets[1]
used_nzcs = sets[2]
# Sort functions after being cellwise constant and loop ranges.
function_groups = collections.defaultdict(list)
for f, fd in functions.items():
function_groups[(fd.cellwise_constant, fd.loop_range)].append(f)
total_ops = 0
# Loop ranges and get list of functions.
for (cellwise_constant, loop_range), function_list in function_groups.iteritems():
function_expr = []
# Loop functions.
func_ops = 0
for function in function_list:
data = functions[function]
range_i = data.loop_range
if not isinstance(range_i, tuple):
range_i = tuple([range_i])
# Add name to used psi names and non zeros name to used_nzcs.
used_psi_tables.add(data.psi_name)
used_nzcs.update(data.used_nzcs)
# # TODO: This check can be removed for speed later.
# REMOVED this, since we might need to increment into the same
# number more than once for mixed element + interior facets
# ffc_assert(data.id not in function_expr, "This is definitely not supposed to happen!")
# Convert function to COFFEE ast node, save string
# representation for sorting (such that we're reproducible
# in parallel).
function = visit_rhs(function)
key = str(function)
function_expr.append((data.id, function, key))
# Get number of operations to compute entry and add to function operations count.
func_ops += (data.ops + 1)*sum(range_i)
# Gather, sorted by string rep of function.
lines = [pyop2.Incr(c_sym(f_F(n)), fn) for n, fn, _ in sorted(function_expr, key=lambda x: x[2])]
if isinstance(loop_range, tuple):
if not all(map(lambda x: x==loop_range[0], loop_range)):
raise RuntimeError("General mixed elements not yet supported in PyOP2")
loop_vars = [ (f_r[0], 0, loop_range[0]), (f_r[1], 0, len(loop_range)) ]
else:
loop_vars = [(f_r[0], 0, loop_range)]
# TODO: If loop_range == 1, this loop may be unneccessary. Not sure if it's safe to just skip it.
it_var = c_sym(loop_vars[0][0])
loop_size = c_sym(loop_vars[0][2])
ast_item = pyop2.For(pyop2.Decl("int", it_var, c_sym(0)), pyop2.Less(it_var, loop_size),
pyop2.Incr(it_var, c_sym(1)), pyop2.Block(lines, open_scope=True))
if cellwise_constant:
const_ast_items.append(ast_item)
else:
ast_items.append(ast_item)
return const_ast_items, ast_items, total_ops
def _generate_element_tensor(integrals, sets, optimise_parameters, parameters):
"Construct quadrature code for element tensors."
# Prefetch formats to speed up code generation.
f_comment = format["comment"]
f_ip = format["integration points"]
f_I = format["ip constant"]
f_loop = format["generate loop"]
f_ip_coords = format["generate ip coordinates"]
f_coords = format["coordinate_dofs"]
f_double = format["float declaration"]
f_decl = format["declaration"]
f_X = format["ip coordinates"]
f_C = format["conditional"]
# Initialise return values.
tensor_ops_count = 0
ffc_assert(1 == len(integrals), "This function is not capable of handling multiple integrals.")
# We receive a dictionary {num_points: form,}.
# Loop points and forms.
for points, terms, functions, ip_consts, coordinate, conditionals in integrals:
nest_ir = []
ip_ir = []
num_ops = 0
# Generate code to compute coordinates if used.
if coordinate:
raise RuntimeError("Don't know how to compute coordinates")
# Left in place for posterity
name, gdim, ip, r = coordinate
element_code += ["", f_comment("Declare array to hold physical coordinate of quadrature point.")]
element_code += [f_decl(f_double, f_X(points, gdim))]
ops, coord_code = f_ip_coords(gdim, points, name, ip, r)
ip_code += ["", f_comment("Compute physical coordinate of quadrature point, operations: %d." % ops)]
ip_code += [coord_code]
num_ops += ops
# Update used psi tables and transformation set.
sets[1].add(name)
sets[3].add(f_coords(r))
# Generate code to compute function values.
if functions:
const_func_code, func_code, ops = _generate_functions(functions, sets)
nest_ir += const_func_code
ip_ir += func_code
num_ops += ops
# Generate code to compute conditionals (might depend on coordinates
# and function values so put here).
# TODO: Some conditionals might only depend on geometry so they
# should be moved outside if possible.
if conditionals:
ip_ir.append(pyop2.Decl(f_double, c_sym(f_C(len(conditionals)))))
# Sort conditionals (need to in case of nested conditionals).
reversed_conds = dict([(n, (o, e)) for e, (t, o, n) in conditionals.items()])
for num in range(len(conditionals)):
name = format["conditional"](num)
ops, expr = reversed_conds[num]
ip_ir.append(pyop2.Assign(c_sym(name), visit_rhs(expr)))
num_ops += ops
# Generate code for ip constant declarations.
# TODO: this code should be removable as only executed when ffc's optimisations are on
ip_const_ops, ip_const_code = generate_aux_constants(ip_consts, f_I,\
format["assign"], True)
if len(ip_const_code) > 0:
raise RuntimeError("IP Const code not supported")
num_ops += ip_const_ops
# Generate code to evaluate the element tensor.
code, ops = _generate_integral_ir(points, terms, sets, optimise_parameters, parameters)
num_ops += ops
tensor_ops_count += num_ops*points
ip_ir += code
# Loop code over all IPs.
# @@@: for (ip ...) { A[0][0] += ... }
if points > 1:
it_var = pyop2.Symbol(f_ip, ())
nest_ir += [pyop2.For(pyop2.Decl("int", it_var, c_sym(0)),
pyop2.Less(it_var, c_sym(points)),
pyop2.Incr(it_var, c_sym(1)),
pyop2.Block(ip_ir, open_scope=True))]
else:
nest_ir += ip_ir
return (nest_ir, tensor_ops_count)