def __init__(self, expressions, **kwargs):
expressions = as_tuple(expressions)
# Input check
if any(not isinstance(i, Eq) for i in expressions):
raise InvalidOperator("Only `devito.Eq` expressions are allowed.")
self.name = kwargs.get("name", "Kernel")
subs = kwargs.get("subs", {})
dse = kwargs.get("dse", configuration['dse'])
# Header files, etc.
self._headers = list(self._default_headers)
self._includes = list(self._default_includes)
self._globals = list(self._default_globals)
# Required for compilation
self._compiler = configuration['compiler']
self._lib = None
self._cfunction = None
# References to local or external routines
self._func_table = OrderedDict()
# Internal state. May be used to store information about previous runs,
# autotuning reports, etc
self._state = {}
# Expression lowering: indexification, substitution rules, specialization
expressions = [indexify(i) for i in expressions]
expressions = self._apply_substitutions(expressions, subs)
expressions = self._specialize_exprs(expressions)
# Expression analysis
self.input = filter_sorted(flatten(e.reads for e in expressions))
self.output = filter_sorted(flatten(e.writes for e in expressions))
self.dimensions = filter_sorted(
flatten(e.dimensions for e in expressions))
# Group expressions based on their iteration space and data dependences,
# and apply the Devito Symbolic Engine (DSE) for flop optimization
clusters = clusterize(expressions)
clusters = rewrite(clusters, mode=set_dse_mode(dse))
self._dtype, self._dspace = clusters.meta
# Lower Clusters to a Schedule tree
stree = st_build(clusters)
# Lower Schedule tree to an Iteration/Expression tree (IET)
iet = iet_build(stree)
iet, self._profiler = self._profile_sections(iet)
iet = self._specialize_iet(iet, **kwargs)
iet = iet_insert_C_decls(iet)
iet = self._build_casts(iet)
# Derive parameters as symbols not defined in the kernel itself
parameters = self._build_parameters(iet)
# Finish instantiation
super(Operator, self).__init__(self.name, iet, 'int', parameters, ())
def _make_wait(self, f, hse, key='', msg=None):
bufs = FieldFromPointer(msg._C_field_bufs, msg)
ofss = [Symbol(name='os%s' % d.root) for d in f.dimensions]
fromrank = Symbol(name='fromrank')
sizes = [
FieldFromPointer('%s[%d]' % (msg._C_field_sizes, i), msg)
for i in range(len(f._dist_dimensions))
]
scatter = Call('scatter%s' % key, [bufs] + sizes + [f] + ofss)
# The `scatter` must be guarded as we must not alter the halo values along
# the domain boundary, where the sender is actually MPI.PROC_NULL
scatter = Conditional(CondNe(fromrank, Macro('MPI_PROC_NULL')),
scatter)
rrecv = Byref(FieldFromPointer(msg._C_field_rrecv, msg))
waitrecv = Call('MPI_Wait', [rrecv, Macro('MPI_STATUS_IGNORE')])
rsend = Byref(FieldFromPointer(msg._C_field_rsend, msg))
waitsend = Call('MPI_Wait', [rsend, Macro('MPI_STATUS_IGNORE')])
iet = List(body=[waitsend, waitrecv, scatter])
iet = List(body=iet_insert_C_decls(iet))
parameters = ([f] + ofss + [fromrank, msg])
return Callable('wait%s' % key, iet, 'void', parameters, ('static', ))
def _make_sendrecv(self, f, hse, key='', **kwargs):
comm = f.grid.distributor._obj_comm
buf_dims = [
Dimension(name='buf_%s' % d.root) for d in f.dimensions
if d not in hse.loc_indices
]
bufg = Array(name='bufg',
dimensions=buf_dims,
dtype=f.dtype,
scope='heap')
bufs = Array(name='bufs',
dimensions=buf_dims,
dtype=f.dtype,
scope='heap')
ofsg = [Symbol(name='og%s' % d.root) for d in f.dimensions]
ofss = [Symbol(name='os%s' % d.root) for d in f.dimensions]
fromrank = Symbol(name='fromrank')
torank = Symbol(name='torank')
gather = Call('gather%s' % key, [bufg] + list(bufg.shape) + [f] + ofsg)
scatter = Call('scatter%s' % key,
[bufs] + list(bufs.shape) + [f] + ofss)
# The `gather` is unnecessary if sending to MPI.PROC_NULL
gather = Conditional(CondNe(torank, Macro('MPI_PROC_NULL')), gather)
# The `scatter` must be guarded as we must not alter the halo values along
# the domain boundary, where the sender is actually MPI.PROC_NULL
scatter = Conditional(CondNe(fromrank, Macro('MPI_PROC_NULL')),
scatter)
count = reduce(mul, bufs.shape, 1)
rrecv = MPIRequestObject(name='rrecv')
rsend = MPIRequestObject(name='rsend')
recv = Call('MPI_Irecv', [
bufs, count,
Macro(dtype_to_mpitype(f.dtype)), fromrank,
Integer(13), comm, rrecv
])
send = Call('MPI_Isend', [
bufg, count,
Macro(dtype_to_mpitype(f.dtype)), torank,
Integer(13), comm, rsend
])
waitrecv = Call('MPI_Wait', [rrecv, Macro('MPI_STATUS_IGNORE')])
waitsend = Call('MPI_Wait', [rsend, Macro('MPI_STATUS_IGNORE')])
iet = List(body=[recv, gather, send, waitsend, waitrecv, scatter])
iet = List(body=iet_insert_C_decls(iet))
parameters = ([f] + list(bufs.shape) + ofsg + ofss +
[fromrank, torank, comm])
return Callable('sendrecv%s' % key, iet, 'void', parameters,
('static', ))
def sendrecv(f, fixed):
"""Construct an IET performing a halo exchange along arbitrary
dimension and side."""
assert f.is_Function
assert f.grid is not None
comm = f.grid.distributor._C_comm
buf_dims = [Dimension(name='buf_%s' % d.root) for d in f.dimensions if d not in fixed]
bufg = Array(name='bufg', dimensions=buf_dims, dtype=f.dtype, scope='heap')
bufs = Array(name='bufs', dimensions=buf_dims, dtype=f.dtype, scope='heap')
dat_dims = [Dimension(name='dat_%s' % d.root) for d in f.dimensions]
dat = Array(name='dat', dimensions=dat_dims, dtype=f.dtype, scope='external')
ofsg = [Symbol(name='og%s' % d.root) for d in f.dimensions]
ofss = [Symbol(name='os%s' % d.root) for d in f.dimensions]
fromrank = Symbol(name='fromrank')
torank = Symbol(name='torank')
parameters = [bufg] + list(bufg.shape) + [dat] + list(dat.shape) + ofsg
gather = Call('gather_%s' % f.name, parameters)
parameters = [bufs] + list(bufs.shape) + [dat] + list(dat.shape) + ofss
scatter = Call('scatter_%s' % f.name, parameters)
# The scatter must be guarded as we must not alter the halo values along
# the domain boundary, where the sender is actually MPI.PROC_NULL
scatter = Conditional(CondNe(fromrank, Macro('MPI_PROC_NULL')), scatter)
srecv = MPIStatusObject(name='srecv')
rrecv = MPIRequestObject(name='rrecv')
rsend = MPIRequestObject(name='rsend')
count = reduce(mul, bufs.shape, 1)
recv = Call('MPI_Irecv', [bufs, count, Macro(numpy_to_mpitypes(f.dtype)),
fromrank, '13', comm, rrecv])
send = Call('MPI_Isend', [bufg, count, Macro(numpy_to_mpitypes(f.dtype)),
torank, '13', comm, rsend])
waitrecv = Call('MPI_Wait', [rrecv, srecv])
waitsend = Call('MPI_Wait', [rsend, Macro('MPI_STATUS_IGNORE')])
iet = List(body=[recv, gather, send, waitsend, waitrecv, scatter])
iet = List(body=[ArrayCast(dat), iet_insert_C_decls(iet)])
parameters = ([dat] + list(dat.shape) + list(bufs.shape) +
ofsg + ofss + [fromrank, torank, comm])
return Callable('sendrecv_%s' % f.name, iet, 'void', parameters, ('static',))
def _make_sendrecv(self, f, hse, key='', msg=None):
comm = f.grid.distributor._obj_comm
bufg = FieldFromPointer(msg._C_field_bufg, msg)
bufs = FieldFromPointer(msg._C_field_bufs, msg)
ofsg = [Symbol(name='og%s' % d.root) for d in f.dimensions]
fromrank = Symbol(name='fromrank')
torank = Symbol(name='torank')
sizes = [
FieldFromPointer('%s[%d]' % (msg._C_field_sizes, i), msg)
for i in range(len(f._dist_dimensions))
]
gather = Call('gather%s' % key, [bufg] + sizes + [f] + ofsg)
# The `gather` is unnecessary if sending to MPI.PROC_NULL
gather = Conditional(CondNe(torank, Macro('MPI_PROC_NULL')), gather)
count = reduce(mul, sizes, 1)
rrecv = Byref(FieldFromPointer(msg._C_field_rrecv, msg))
rsend = Byref(FieldFromPointer(msg._C_field_rsend, msg))
recv = Call('MPI_Irecv', [
bufs, count,
Macro(dtype_to_mpitype(f.dtype)), fromrank,
Integer(13), comm, rrecv
])
send = Call('MPI_Isend', [
bufg, count,
Macro(dtype_to_mpitype(f.dtype)), torank,
Integer(13), comm, rsend
])
iet = List(body=[recv, gather, send])
iet = List(body=iet_insert_C_decls(iet))
parameters = ([f] + ofsg + [fromrank, torank, comm, msg])
return Callable('sendrecv%s' % key, iet, 'void', parameters,
('static', ))
def __init__(self, expressions, **kwargs):
expressions = as_tuple(expressions)
# Input check
if any(not isinstance(i, sympy.Eq) for i in expressions):
raise InvalidOperator("Only SymPy expressions are allowed.")
self.name = kwargs.get("name", "Kernel")
subs = kwargs.get("subs", {})
time_axis = kwargs.get("time_axis", Forward)
dse = kwargs.get("dse", configuration['dse'])
dle = kwargs.get("dle", configuration['dle'])
# Header files, etc.
self._headers = list(self._default_headers)
self._includes = list(self._default_includes)
self._globals = list(self._default_globals)
# Required for compilation
self._compiler = configuration['compiler']
self._lib = None
self._cfunction = None
# References to local or external routines
self.func_table = OrderedDict()
# Expression lowering and analysis
expressions = [LoweredEq(e, subs=subs) for e in expressions]
self.dtype = retrieve_dtype(expressions)
self.input, self.output, self.dimensions = retrieve_symbols(
expressions)
# Set the direction of time acoording to the given TimeAxis
for time in [d for d in self.dimensions if d.is_Time]:
if not time.is_Stepping:
time.reverse = time_axis == Backward
# Parameters of the Operator (Dimensions necessary for data casts)
parameters = self.input + self.dimensions
# Group expressions based on their iteration space and data dependences,
# and apply the Devito Symbolic Engine (DSE) for flop optimization
clusters = clusterize(expressions)
clusters = rewrite(clusters, mode=set_dse_mode(dse))
# Lower Clusters to an Iteration/Expression tree (IET)
nodes = iet_build(clusters, self.dtype)
# Introduce C-level profiling infrastructure
nodes, self.profiler = self._profile_sections(nodes, parameters)
# Translate into backend-specific representation (e.g., GPU, Yask)
nodes = self._specialize(nodes, parameters)
# Apply the Devito Loop Engine (DLE) for loop optimization
dle_state = transform(nodes, *set_dle_mode(dle))
# Update the Operator state based on the DLE
self.dle_arguments = dle_state.arguments
self.dle_flags = dle_state.flags
self.func_table.update(
OrderedDict([(i.name, MetaCall(i, True))
for i in dle_state.elemental_functions]))
parameters.extend([i.argument for i in self.dle_arguments])
self.dimensions.extend([
i.argument for i in self.dle_arguments
if isinstance(i.argument, Dimension)
])
self._includes.extend(list(dle_state.includes))
# Introduce the required symbol declarations
nodes = iet_insert_C_decls(dle_state.nodes, self.func_table)
# Initialise ArgumentEngine
self.argument_engine = ArgumentEngine(clusters.ispace, parameters,
self.dle_arguments)
parameters = self.argument_engine.arguments
# Finish instantiation
super(Operator, self).__init__(self.name, nodes, 'int', parameters, ())
def __init__(self, expressions, **kwargs):
expressions = as_tuple(expressions)
# Input check
if any(not isinstance(i, sympy.Eq) for i in expressions):
raise InvalidOperator("Only SymPy expressions are allowed.")
self.name = kwargs.get("name", "Kernel")
subs = kwargs.get("subs", {})
dse = kwargs.get("dse", configuration['dse'])
# Header files, etc.
self._headers = list(self._default_headers)
self._includes = list(self._default_includes)
self._globals = list(self._default_globals)
# Required for compilation
self._compiler = configuration['compiler']
self._lib = None
self._cfunction = None
# References to local or external routines
self.func_table = OrderedDict()
# Expression lowering: indexification, substitution rules, specialization
expressions = [indexify(i) for i in expressions]
expressions = [i.xreplace(subs) for i in expressions]
expressions = self._specialize_exprs(expressions)
# Expression analysis
self.input = filter_sorted(flatten(e.reads for e in expressions))
self.output = filter_sorted(flatten(e.writes for e in expressions))
self.dimensions = filter_sorted(flatten(e.dimensions for e in expressions))
# Group expressions based on their iteration space and data dependences,
# and apply the Devito Symbolic Engine (DSE) for flop optimization
clusters = clusterize(expressions)
clusters = rewrite(clusters, mode=set_dse_mode(dse))
self._dtype, self._dspace = clusters.meta
# Lower Clusters to a Schedule tree
stree = schedule(clusters)
stree = section(stree)
# Lower Sections to an Iteration/Expression tree (IET)
iet = iet_build(stree)
# Insert code for C-level performance profiling
iet, self.profiler = self._profile_sections(iet)
# Translate into backend-specific representation
iet = self._specialize_iet(iet, **kwargs)
# Insert the required symbol declarations
iet = iet_insert_C_decls(iet, self.func_table)
# Insert data and pointer casts for array parameters and profiling structs
iet = self._build_casts(iet)
# Derive parameters as symbols not defined in the kernel itself
parameters = self._build_parameters(iet)
# Finish instantiation
super(Operator, self).__init__(self.name, iet, 'int', parameters, ())
def __init__(self, expressions, **kwargs):
expressions = as_tuple(expressions)
# Input check
if any(not isinstance(i, sympy.Eq) for i in expressions):
raise InvalidOperator("Only SymPy expressions are allowed.")
self.name = kwargs.get("name", "Kernel")
subs = kwargs.get("subs", {})
dse = kwargs.get("dse", configuration['dse'])
dle = kwargs.get("dle", configuration['dle'])
# Header files, etc.
self._headers = list(self._default_headers)
self._includes = list(self._default_includes)
self._globals = list(self._default_globals)
# Required for compilation
self._compiler = configuration['compiler']
self._lib = None
self._cfunction = None
# References to local or external routines
self.func_table = OrderedDict()
# Expression lowering: indexification, substitution rules, specialization
expressions = [indexify(i) for i in expressions]
expressions = [i.xreplace(subs) for i in expressions]
expressions = self._specialize_exprs(expressions)
# Expression analysis
self.input = filter_sorted(flatten(e.reads for e in expressions))
self.output = filter_sorted(flatten(e.writes for e in expressions))
self.dimensions = filter_sorted(flatten(e.dimensions for e in expressions))
# Group expressions based on their iteration space and data dependences,
# and apply the Devito Symbolic Engine (DSE) for flop optimization
clusters = clusterize(expressions)
clusters = rewrite(clusters, mode=set_dse_mode(dse))
self._dtype, self._dspace = clusters.meta
# Lower Clusters to an Iteration/Expression tree (IET)
nodes = iet_build(clusters)
# Introduce C-level profiling infrastructure
nodes, self.profiler = self._profile_sections(nodes)
# Translate into backend-specific representation (e.g., GPU, Yask)
nodes = self._specialize_iet(nodes)
# Apply the Devito Loop Engine (DLE) for loop optimization
dle_state = transform(nodes, *set_dle_mode(dle))
# Update the Operator state based on the DLE
self.dle_args = dle_state.arguments
self.dle_flags = dle_state.flags
self.func_table.update(OrderedDict([(i.name, MetaCall(i, True))
for i in dle_state.elemental_functions]))
self.dimensions.extend([i.argument for i in self.dle_args
if isinstance(i.argument, Dimension)])
self._includes.extend(list(dle_state.includes))
# Introduce the required symbol declarations
nodes = iet_insert_C_decls(dle_state.nodes, self.func_table)
# Insert data and pointer casts for array parameters and profiling structs
nodes = self._build_casts(nodes)
# Derive parameters as symbols not defined in the kernel itself
parameters = self._build_parameters(nodes)
# Finish instantiation
super(Operator, self).__init__(self.name, nodes, 'int', parameters, ())
def _create_efuncs(self, nodes, state):
"""
Extract Iteration sub-trees and turn them into Calls+Callables.
Currently, only tagged, elementizable Iteration objects are targeted.
"""
noinline = self._compiler_decoration('noinline',
c.Comment('noinline?'))
efuncs = OrderedDict()
mapper = {}
for tree in retrieve_iteration_tree(nodes, mode='superset'):
# Search an elementizable sub-tree (if any)
tagged = filter_iterations(tree, lambda i: i.tag is not None,
'asap')
if not tagged:
continue
root = tagged[0]
if not root.is_Elementizable:
continue
target = tree[tree.index(root):]
# Build a new Iteration/Expression tree with free bounds
free = []
defined_args = {} # Map of argument values defined by loop bounds
for i in target:
name, bounds = i.dim.name, i.symbolic_bounds
# Iteration bounds
_min = Scalar(name='%sf_m' % name,
dtype=np.int32,
is_const=True)
_max = Scalar(name='%sf_M' % name,
dtype=np.int32,
is_const=True)
defined_args[_min.name] = bounds[0]
defined_args[_max.name] = bounds[1]
# Iteration unbounded indices
ufunc = [
Scalar(name='%s_ub%d' % (name, j), dtype=np.int32)
for j in range(len(i.uindices))
]
defined_args.update({
uf.name: j.symbolic_min
for uf, j in zip(ufunc, i.uindices)
})
uindices = [
IncrDimension(j.parent, i.dim + as_symbol(k), 1, j.name)
for j, k in zip(i.uindices, ufunc)
]
free.append(
i._rebuild(limits=(_min, _max, 1),
offsets=None,
uindices=uindices))
# Construct elemental function body
free = Transformer(dict((zip(target, free))),
nested=True).visit(root)
items = FindSymbols().visit(free)
# Insert array casts
casts = [ArrayCast(i) for i in items if i.is_Tensor]
free = List(body=casts + [free])
# Insert declarations
external = [i for i in items if i.is_Array]
free = iet_insert_C_decls(free, external)
# Create the Callable
name = "f_%d" % root.tag
params = derive_parameters(free)
efuncs.setdefault(name,
Callable(name, free, 'void', params, 'static'))
# Create the Call
args = [defined_args.get(i.name, i) for i in params]
mapper[root] = List(header=noinline, body=Call(name, args))
# Transform the main tree
processed = Transformer(mapper).visit(nodes)
return processed, {'efuncs': efuncs.values()}
