def __new__(cls, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], LoweredEq):
# origin: LoweredEq(devito.LoweredEq, **kwargs)
input_expr = args[0]
expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i, kwargs.get(i) or getattr(input_expr, i))
return expr
elif len(args) == 1 and isinstance(args[0], Eq):
# origin: LoweredEq(sympy.Eq)
input_expr = expr = args[0]
elif len(args) == 2:
expr = Eq.__new__(cls, *args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i, kwargs.pop(i))
return expr
else:
raise ValueError("Cannot construct LoweredEq from args=%s "
"and kwargs=%s" % (str(args), str(kwargs)))
# Well-defined dimension ordering
ordering = dimension_sort(expr)
# Analyze the expression
mapper = detect_accesses(expr)
oobs = detect_oobs(mapper)
conditionals = [i for i in ordering if i.is_Conditional]
# The iteration space is constructed so that information always flows
# from an iteration to another (i.e., no anti-dependences are created)
directions, _ = force_directions(detect_flow_directions(expr), lambda i: Any)
iterators = build_iterators(mapper)
intervals = build_intervals(Stencil.union(*mapper.values()))
intervals = IntervalGroup(intervals, relations=ordering.relations)
ispace = IterationSpace(intervals.zero(), iterators, directions)
# The data space is relative to the computational domain. Note that we
# are deliberately dropping the intervals ordering (by turning `intervals`
# into a list), as this is irrelevant (even more: dangerous) for data spaces
intervals = [i if i.dim in oobs else i.zero() for i in intervals]
intervals += [Interval(i, 0, 0) for i in ordering
if i not in ispace.dimensions + conditionals]
parts = {k: IntervalGroup(build_intervals(v)) for k, v in mapper.items() if k}
dspace = DataSpace(intervals, parts)
# Finally create the LoweredEq with all metadata attached
expr = super(LoweredEq, cls).__new__(cls, expr.lhs, expr.rhs, evaluate=False)
expr._is_Increment = getattr(input_expr, 'is_Increment', False)
expr._dspace = dspace
expr._ispace = ispace
expr._conditionals = tuple(conditionals)
expr._reads, expr._writes = detect_io(expr)
return expr
def callback(self, clusters, prefix):
if not prefix:
return clusters
d = prefix[-1].dim
processed = []
for c in clusters:
if SKEWABLE not in c.properties[d]:
return clusters
skew_dims = {
i.dim
for i in c.ispace if SEQUENTIAL in c.properties[i.dim]
}
if len(skew_dims) > 1:
return clusters
skew_dim = skew_dims.pop()
# Since we are here, prefix is skewable and nested under a SEQUENTIAL loop
intervals = []
for i in c.ispace:
if i.dim is d and (not d.is_Block or d._depth == 1):
intervals.append(Interval(d, skew_dim, skew_dim))
else:
intervals.append(i)
intervals = IntervalGroup(intervals, relations=c.ispace.relations)
ispace = IterationSpace(intervals, c.ispace.sub_iterators,
c.ispace.directions)
exprs = xreplace_indices(c.exprs, {d: d - skew_dim})
processed.append(c.rebuild(exprs=exprs, ispace=ispace))
return processed
def decompose(ispace, d, block_dims):
"""
Create a new IterationSpace in which the `d` Interval is decomposed
into a hierarchy of Intervals over ``block_dims``.
"""
# Create the new Intervals
intervals = []
for i in ispace:
if i.dim is d:
intervals.append(i.switch(block_dims[0]))
intervals.extend([i.switch(bd).zero() for bd in block_dims[1:]])
else:
intervals.append(i)
# Create the relations.
# Example: consider the relation `(t, x, y)` and assume we decompose `x` over
# `xbb, xb, xi`; then we decompose the relation as two relations, `(t, xbb, y)`
# and `(xbb, xb, xi)`
relations = [block_dims]
for r in ispace.intervals.relations:
relations.append([block_dims[0] if i is d else i for i in r])
# The level of a given Dimension in the hierarchy of block Dimensions
level = lambda dim: len([i for i in dim._defines if i.is_Incr])
# Add more relations
for n, i in enumerate(ispace):
if i.dim is d:
continue
elif i.dim.is_Incr:
# Make sure IncrDimensions on the same level stick next to each other.
# For example, we want `(t, xbb, ybb, xb, yb, x, y)`, rather than say
# `(t, xbb, xb, x, ybb, ...)`
for bd in block_dims:
if level(i.dim) >= level(bd):
relations.append([bd, i.dim])
else:
relations.append([i.dim, bd])
elif n > ispace.intervals.index(d):
# The non-Incr subsequent Dimensions must follow the block Dimensions
for bd in block_dims:
relations.append([bd, i.dim])
else:
# All other Dimensions must precede the block Dimensions
for bd in block_dims:
relations.append([i.dim, bd])
intervals = IntervalGroup(intervals, relations=relations)
sub_iterators = dict(ispace.sub_iterators)
sub_iterators.pop(d, None)
sub_iterators.update({bd: ispace.sub_iterators.get(d, []) for bd in block_dims})
directions = dict(ispace.directions)
directions.pop(d)
directions.update({bd: ispace.directions[d] for bd in block_dims})
return IterationSpace(intervals, sub_iterators, directions)
def decompose(ispace, d, block_dims):
"""
Create a new IterationSpace in which the `d` Interval is decomposed
into a hierarchy of Intervals over ``block_dims``.
"""
# Create the new Intervals
intervals = []
for i in ispace:
if i.dim is d:
intervals.append(i.switch(block_dims[0]))
intervals.extend([i.switch(bd).zero() for bd in block_dims[1:]])
else:
intervals.append(i)
# Create the intervals relations
# 1: `bbd > bd > d`
relations = [tuple(block_dims)]
# 2: Suitably replace `d` with all `bd`'s
for r in ispace.relations:
if d not in r:
relations.append(r)
continue
for bd in block_dims:
# Avoid e.g. `x > yb`
if any(i._depth < bd._depth for i in r if i.is_Block):
continue
relations.append(tuple(bd if i is d else i for i in r))
# 3: Make sure BlockDimensions at same depth stick next to each other
# E.g., `(t, xbb, ybb, xb, yb, x, y)`, and NOT e.g. `(t, xbb, xb, x, ybb, ...)`
# NOTE: this is perfectly legal since:
# TILABLE => (perfect nest & PARALLEL) => interchangeable
for i in ispace.itdimensions:
if not i.is_Block:
continue
for bd in block_dims:
if i._depth < bd._depth:
relations.append((i, bd))
intervals = IntervalGroup(intervals, relations=relations)
sub_iterators = dict(ispace.sub_iterators)
sub_iterators.pop(d, None)
sub_iterators.update({bd: () for bd in block_dims[:-1]})
sub_iterators.update({block_dims[-1]: ispace.sub_iterators[d]})
directions = dict(ispace.directions)
directions.pop(d)
directions.update({bd: ispace.directions[d] for bd in block_dims})
return IterationSpace(intervals, sub_iterators, directions)
def decompose(ispace, d, block_dims):
"""
Create a new IterationSpace in which the `d` Interval is decomposed
into a hierarchy of Intervals over ``block_dims``.
"""
# Create the new Intervals
intervals = []
for i in ispace.intervals:
if i.dim is d:
intervals.append(i.switch(block_dims[0]))
intervals.extend([i.switch(bd).zero() for bd in block_dims[1:]])
else:
intervals.append(i)
# Create the new "decomposed" relations.
# Example: consider the relation `(t, x, y)` and assume we decompose `x` over
# `xbb, xb, xi`; then we decompose the relation as two relations, `(t, xbb, y)`
# and `(xbb, xb, xi)`
relations = [block_dims]
for r in ispace.intervals.relations:
relations.append([block_dims[0] if i is d else i for i in r])
# Further, if there are other IncrDimensions, add relations such that
# IncrDimensions at the same level stick together, thus we obtain for
# example `(t, xbb, ybb, xb, yb, x, y)` instead of `(t, xbb, xb, x, ybb, ...)`
for i in intervals:
if not isinstance(i.dim, IncrDimension):
continue
for bd in block_dims:
if bd._defines & i.dim._defines:
break
if len(i.dim._defines) > len(bd._defines):
relations.append([bd, i.dim])
intervals = IntervalGroup(intervals, relations=relations)
sub_iterators = dict(ispace.sub_iterators)
sub_iterators.pop(d, None)
sub_iterators.update(
{bd: ispace.sub_iterators.get(d, [])
for bd in block_dims})
directions = dict(ispace.directions)
directions.pop(d)
directions.update({bd: ispace.directions[d] for bd in block_dims})
return IterationSpace(intervals, sub_iterators, directions)
def callback(self, clusters, prefix):
if not prefix:
return clusters
d = prefix[-1].dim
processed = []
for c in clusters:
if SKEWABLE not in c.properties[d]:
return clusters
if d is c.ispace[-1].dim and not self.skewinner:
return clusters
skew_dims = {i.dim for i in c.ispace if SEQUENTIAL in c.properties[i.dim]}
if len(skew_dims) > 1:
return clusters
skew_dim = skew_dims.pop()
# The level of a given Dimension in the hierarchy of block Dimensions, used
# to skew over the outer level of loops.
level = lambda dim: len([i for i in dim._defines if i.is_Incr])
# Since we are here, prefix is skewable and nested under a
# SEQUENTIAL loop.
intervals = []
for i in c.ispace:
if i.dim is d and level(d) <= 1: # Skew only at level 0 or 1
intervals.append(Interval(d, skew_dim, skew_dim))
else:
intervals.append(i)
intervals = IntervalGroup(intervals, relations=c.ispace.relations)
ispace = IterationSpace(intervals, c.ispace.sub_iterators,
c.ispace.directions)
exprs = xreplace_indices(c.exprs, {d: d - skew_dim})
processed.append(c.rebuild(exprs=exprs, ispace=ispace,
properties=c.properties))
return processed
def __new__(cls, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], LoweredEq):
# origin: LoweredEq(devito.LoweredEq, **kwargs)
input_expr = args[0]
expr = sympy.Eq.__new__(cls, *input_expr.args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i,
kwargs.get(i) or getattr(input_expr, i))
return expr
elif len(args) == 1 and isinstance(args[0], Eq):
# origin: LoweredEq(devito.Eq)
input_expr = expr = args[0]
elif len(args) == 2:
expr = sympy.Eq.__new__(cls, *args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i, kwargs.pop(i))
return expr
else:
raise ValueError("Cannot construct LoweredEq from args=%s "
"and kwargs=%s" % (str(args), str(kwargs)))
# Well-defined dimension ordering
ordering = dimension_sort(expr)
# Analyze the expression
mapper = detect_accesses(expr)
oobs = detect_oobs(mapper)
conditionals = [i for i in ordering if i.is_Conditional]
# Construct Intervals for IterationSpace and DataSpace
intervals = build_intervals(Stencil.union(*mapper.values()))
iintervals = [] # iteration Intervals
dintervals = [] # data Intervals
for i in intervals:
d = i.dim
if d in oobs:
iintervals.append(i.zero())
dintervals.append(i)
else:
iintervals.append(i.zero())
dintervals.append(i.zero())
# Construct the IterationSpace
iintervals = IntervalGroup(iintervals, relations=ordering.relations)
iterators = build_iterators(mapper)
ispace = IterationSpace(iintervals, iterators)
# Construct the DataSpace
dintervals.extend([
Interval(i, 0, 0) for i in ordering
if i not in ispace.dimensions + conditionals
])
parts = {
k: IntervalGroup(build_intervals(v)).add(iintervals)
for k, v in mapper.items() if k
}
dspace = DataSpace(dintervals, parts)
# Lower all Differentiable operations into SymPy operations
rhs = diff2sympy(expr.rhs)
# Finally create the LoweredEq with all metadata attached
expr = super(LoweredEq, cls).__new__(cls,
expr.lhs,
rhs,
evaluate=False)
expr._dspace = dspace
expr._ispace = ispace
expr._conditionals = tuple(conditionals)
expr._reads, expr._writes = detect_io(expr)
expr._is_Increment = input_expr.is_Increment
expr._implicit_dims = input_expr.implicit_dims
return expr
def __new__(cls, *args, **kwargs):
if len(args) == 1:
# origin: LoweredEq(expr)
expr = input_expr = args[0]
assert not isinstance(expr, LoweredEq) and isinstance(expr, Eq)
elif len(args) == 2:
# origin: LoweredEq(lhs, rhs, stamp=...)
stamp = kwargs.pop('stamp')
expr = Eq.__new__(cls, *args, evaluate=False)
assert isinstance(stamp, Eq)
expr.is_Increment = stamp.is_Increment
expr._ispace, expr._dspace = stamp.ispace, stamp.dspace
expr.reads, expr.writes = stamp.reads, stamp.writes
return expr
elif len(args) == 5:
# origin: LoweredEq(expr, ispace, space)
input_expr, ispace, dspace, reads, writes = args
assert isinstance(ispace, IterationSpace) and isinstance(
dspace, DataSpace)
expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
expr.is_Increment = input_expr.is_Increment
expr._ispace, expr._dspace = ispace, dspace
expr.reads, expr.writes = reads, writes
return expr
else:
raise ValueError("Cannot construct LoweredEq from args=%s "
"and kwargs=%s" % (str(args), str(kwargs)))
# Well-defined dimension ordering
ordering = dimension_sort(expr, key=lambda i: not i.is_Time)
# Introduce space sub-dimensions if need to
region = getattr(input_expr, '_region', DOMAIN)
if region == INTERIOR:
mapper = {
i: SubDimension("%si" % i, i, 1, -1)
for i in ordering if i.is_Space
}
expr = expr.xreplace(mapper)
ordering = [mapper.get(i, i) for i in ordering]
# Analyze data accesses
mapper = detect_accesses(expr)
oobs = detect_oobs(mapper)
# The iteration space is constructed so that information always flows
# from an iteration to another (i.e., no anti-dependences are created)
directions, _ = force_directions(detect_flow_directions(expr),
lambda i: Any)
intervals, iterators = build_intervals(mapper)
intervals = sorted(intervals, key=lambda i: ordering.index(i.dim))
ispace = IterationSpace([i.zero() for i in intervals], iterators,
directions)
# The data space is relative to the computational domain
intervals = [i if i.dim in oobs else i.zero() for i in intervals]
intervals += [
Interval(i, 0, 0) for i in ordering if i not in ispace.dimensions
]
parts = {
k:
IntervalGroup(Interval(i, min(j), max(j)) for i, j in v.items())
for k, v in mapper.items()
}
dspace = DataSpace(intervals, parts)
# Finally create the LoweredEq with all metadata attached
expr = super(LoweredEq, cls).__new__(cls,
expr.lhs,
expr.rhs,
evaluate=False)
expr.is_Increment = getattr(input_expr, 'is_Increment', False)
expr._dspace = dspace
expr._ispace = ispace
expr.reads, expr.writes = detect_io(expr)
return expr
def __new__(cls, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], LoweredEq):
# origin: LoweredEq(devito.LoweredEq, **kwargs)
input_expr = args[0]
expr = sympy.Eq.__new__(cls, *input_expr.args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i,
kwargs.get(i) or getattr(input_expr, i))
return expr
elif len(args) == 1 and isinstance(args[0], Eq):
# origin: LoweredEq(devito.Eq)
input_expr = expr = args[0]
elif len(args) == 2:
expr = sympy.Eq.__new__(cls, *args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i, kwargs.pop(i))
return expr
else:
raise ValueError("Cannot construct LoweredEq from args=%s "
"and kwargs=%s" % (str(args), str(kwargs)))
# Well-defined dimension ordering
ordering = dimension_sort(expr)
# Analyze the expression
accesses = detect_accesses(expr)
dimensions = Stencil.union(*accesses.values())
# Separate out the SubIterators from the main iteration Dimensions, that
# is those which define an actual iteration space
iterators = {}
for d in dimensions:
if d.is_SubIterator:
iterators.setdefault(d.root, set()).add(d)
elif d.is_Conditional:
# Use `parent`, and `root`, because a ConditionalDimension may
# have a SubDimension as parent
iterators.setdefault(d.parent, set())
else:
iterators.setdefault(d, set())
# Construct the IterationSpace
intervals = IntervalGroup([Interval(d, 0, 0) for d in iterators],
relations=ordering.relations)
ispace = IterationSpace(intervals, iterators)
# Construct the conditionals and replace the ConditionalDimensions in `expr`
conditionals = {}
for d in ordering:
if not d.is_Conditional:
continue
if d.condition is None:
conditionals[d] = GuardFactor(d)
else:
conditionals[d] = diff2sympy(lower_exprs(d.condition))
if d.factor is not None:
expr = uxreplace(expr, {d: IntDiv(d.index, d.factor)})
conditionals = frozendict(conditionals)
# Lower all Differentiable operations into SymPy operations
rhs = diff2sympy(expr.rhs)
# Finally create the LoweredEq with all metadata attached
expr = super(LoweredEq, cls).__new__(cls,
expr.lhs,
rhs,
evaluate=False)
expr._ispace = ispace
expr._conditionals = conditionals
expr._reads, expr._writes = detect_io(expr)
expr._is_Increment = input_expr.is_Increment
expr._implicit_dims = input_expr.implicit_dims
return expr
def dspace(self):
"""
Derive the DataSpace of the Cluster from its expressions, IterationSpace,
and Guards.
"""
accesses = detect_accesses(self.exprs)
# Construct the `parts` of the DataSpace, that is a projection of the data
# space for each Function appearing in `self.exprs`
parts = {}
for f, v in accesses.items():
if f is None:
continue
intervals = [
Interval(d, min(offs), max(offs)) for d, offs in v.items()
]
intervals = IntervalGroup(intervals)
# Factor in the IterationSpace -- if the min/max points aren't zero,
# then the data intervals need to shrink/expand accordingly
intervals = intervals.promote(lambda d: d.is_Block)
shift = self.ispace.intervals.promote(lambda d: d.is_Block)
intervals = intervals.add(shift)
# Map SubIterators to the corresponding data space Dimension
# E.g., `xs -> x -> x0_blk0 -> x` or `t0 -> t -> time`
intervals = intervals.promote(lambda d: d.is_SubIterator)
# If the bound of a Dimension is explicitly guarded, then we should
# shrink the `parts` accordingly
for d, v in self.guards.items():
ret = v.find(BaseGuardBoundNext)
assert len(ret) <= 1
if len(ret) != 1:
continue
if ret.pop().direction is Forward:
intervals = intervals.translate(d, v1=-1)
else:
intervals = intervals.translate(d, 1)
# Special case: if the factor of a ConditionalDimension has value 1,
# then we can safely resort to the parent's Interval
intervals = intervals.promote(
lambda d: d.is_Conditional and d.factor == 1)
parts[f] = intervals
# Determine the Dimensions requiring shifted min/max points to avoid
# OOB accesses
oobs = set()
for f, v in parts.items():
for i in v:
if i.dim.is_Sub:
d = i.dim.parent
else:
d = i.dim
try:
if i.lower < 0 or \
i.upper > f._size_nodomain[d].left + f._size_halo[d].right:
# It'd mean trying to access a point before the
# left halo (test0) or after the right halo (test1)
oobs.update(d._defines)
except (KeyError, TypeError):
# Unable to detect presence of OOB accesses (e.g., `d` not in
# `f._size_halo`, that is typical of indirect accesses `A[B[i]]`)
pass
# Construct the `intervals` of the DataSpace, that is a global,
# Dimension-centric view of the data space
intervals = IntervalGroup.generate('union', *parts.values())
# E.g., `db0 -> time`, but `xi NOT-> x`
intervals = intervals.promote(lambda d: not d.is_Sub)
intervals = intervals.zero(set(intervals.dimensions) - oobs)
return DataSpace(intervals, parts)
def __new__(cls, *args, **kwargs):
if len(args) == 1 and isinstance(args[0], LoweredEq):
# origin: LoweredEq(devito.LoweredEq, **kwargs)
input_expr = args[0]
expr = Eq.__new__(cls, *input_expr.args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i,
kwargs.get(i) or getattr(input_expr, i))
return expr
elif len(args) == 1 and isinstance(args[0], Eq):
# origin: LoweredEq(sympy.Eq)
input_expr = expr = args[0]
elif len(args) == 2:
expr = Eq.__new__(cls, *args, evaluate=False)
for i in cls._state:
setattr(expr, '_%s' % i, kwargs.pop(i))
return expr
else:
raise ValueError("Cannot construct LoweredEq from args=%s "
"and kwargs=%s" % (str(args), str(kwargs)))
# Well-defined dimension ordering
ordering = dimension_sort(expr, key=lambda i: not i.is_Time)
# Introduce space sub-dimensions if need to
region = getattr(input_expr, '_region', DOMAIN)
if region == INTERIOR:
mapper = {
i: SubDimension.middle("%si" % i, i, 1, 1)
for i in ordering if i.is_Space
}
expr = expr.xreplace(mapper)
for k, v in mapper.items():
ordering.insert(ordering.index(k) + 1, v)
# Analyze the expression
mapper = detect_accesses(expr)
oobs = detect_oobs(mapper)
# The iteration space is constructed so that information always flows
# from an iteration to another (i.e., no anti-dependences are created)
directions, _ = force_directions(detect_flow_directions(expr),
lambda i: Any)
iterators = build_iterators(mapper)
intervals = build_intervals(Stencil.union(*mapper.values()))
intervals = sorted(intervals, key=lambda i: ordering.index(i.dim))
ispace = IterationSpace([i.zero() for i in intervals], iterators,
directions)
# The data space is relative to the computational domain
intervals = [i if i.dim in oobs else i.zero() for i in intervals]
intervals += [
Interval(i, 0, 0) for i in ordering if i not in ispace.dimensions
]
parts = {
k: IntervalGroup(build_intervals(v))
for k, v in mapper.items() if k
}
dspace = DataSpace(intervals, parts)
# Finally create the LoweredEq with all metadata attached
expr = super(LoweredEq, cls).__new__(cls,
expr.lhs,
expr.rhs,
evaluate=False)
expr._is_Increment = getattr(input_expr, 'is_Increment', False)
expr._dspace = dspace
expr._ispace = ispace
expr._reads, expr._writes = detect_io(expr)
return expr