def test_dynamic_nthreads(self):
grid = Grid(shape=(16, 16, 16))
f = TimeFunction(name='f', grid=grid)
sf = SparseTimeFunction(name='sf', grid=grid, npoint=1, nt=5)
eqns = [Eq(f.forward, f + 1)]
eqns += sf.interpolate(f)
op = Operator(eqns, opt='openmp')
parregions = FindNodes(OmpRegion).visit(op)
assert len(parregions) == 2
# Check suitable `num_threads` appear in the generated code
# Not very elegant, but it does the trick
assert 'num_threads(nthreads)' in str(parregions[0].header[0])
assert 'num_threads(nthreads_nonaffine)' in str(
parregions[1].header[0])
# Check `op` accepts the `nthreads*` kwargs
op.apply(time=0)
op.apply(time_m=1, time_M=1, nthreads=4)
op.apply(time_m=1, time_M=1, nthreads=4, nthreads_nonaffine=2)
op.apply(time_m=1, time_M=1, nthreads_nonaffine=2)
assert np.all(f.data[0] == 2.)
# Check the actual value assumed by `nthreads` and `nthreads_nonaffine`
assert op.arguments(time=0, nthreads=123)['nthreads'] == 123
assert op.arguments(
time=0, nthreads_nonaffine=100)['nthreads_nonaffine'] == 100
def test_basic(self):
grid = Grid(shape=(3, 3, 3))
u = TimeFunction(name='u', grid=grid)
op = Operator(Eq(u.forward, u + 1), dle=('blocking', 'openmp'))
# Does it compile? Honoring the OpenMP specification isn't trivial
assert op.cfunction
# Does it produce the right result
op.apply(t_M=9)
assert np.all(u.data[0] == 10)
# Try again but this time supplying specific values for the num_threads
u.data[:] = 0.
op.apply(t_M=9, nthreads=1, nthreads_nested=2)
assert np.all(u.data[0] == 10)
assert op.arguments(t_M=9, nthreads_nested=2)['nthreads_nested'] == 2
# Same as above, but with the alias
assert op.arguments(t_M=9, nthreads1=2)['nthreads_nested'] == 2
iterations = FindNodes(Iteration).visit(op._func_table['bf0'])
assert iterations[0].pragmas[
0].value == 'omp for collapse(1) schedule(dynamic,1)'
assert iterations[2].pragmas[0].value == (
'omp parallel for collapse(1) '
'schedule(dynamic,1) '
'num_threads(nthreads_nested)')
def test_argument_derivation_order(self, nt=100):
""" Ensure the precedence order of arguments is respected
Defaults < (overriden by) Tensor Arguments < Dimensions < Scalar Arguments
"""
i, j, k = dimify('i j k')
shape = (10, 10, 10)
grid = Grid(shape=shape, dimensions=(i, j, k))
a = Function(name='a', grid=grid)
b = TimeFunction(name='b', grid=grid, save=nt)
time = b.indices[0]
op = Operator(Eq(b, a))
# Simple case, same as that tested above.
# Repeated here for clarity of further tests.
op_arguments = op.arguments()
assert(op_arguments[time.min_name] == 0)
assert(op_arguments[time.max_name] == nt-1)
# Providing a tensor argument should infer the dimension size from its shape
b1 = TimeFunction(name='b1', grid=grid, save=nt+1)
op_arguments = op.arguments(b=b1)
assert(op_arguments[time.min_name] == 0)
assert(op_arguments[time.max_name] == nt)
# Providing a dimension size explicitly should override the automatically inferred
op_arguments = op.arguments(b=b1, time=nt - 1)
assert(op_arguments[time.min_name] == 0)
assert(op_arguments[time.max_name] == nt - 1)
# Providing a scalar argument explicitly should override the automatically
# inferred
op_arguments = op.arguments(b=b1, time_M=nt - 2)
assert(op_arguments[time.min_name] == 0)
assert(op_arguments[time.max_name] == nt - 2)
def check_deviceid(self):
grid = Grid(shape=(6, 6))
u = TimeFunction(name='u', grid=grid, space_order=2, save=10)
op = Operator(Eq(u.forward, u.dx + 1))
deviceid = self.get_param(op, DeviceID)
assert deviceid is not None
assert op.arguments()[deviceid.name] == -1
assert op.arguments(deviceid=0)[deviceid.name] == 0
def test_default_composite_functions(self):
"""
Test the default argument derivation for composite functions.
"""
grid = Grid(shape=(5, 6, 7))
f = TimeFunction(name='f', grid=grid)
s = SparseFunction(name='s', grid=grid, npoint=3, nt=4)
s.coordinates.data[:, 0] = np.arange(0., 3.)
s.coordinates.data[:, 1] = np.arange(1., 4.)
s.coordinates.data[:, 2] = np.arange(2., 5.)
op = Operator(s.interpolate(f))
expected = {
's': s.data,
's_coords': s.coordinates.data,
# Default dimensions of the sparse data
'p_size': 3,
'p_s': 0,
'p_e': 3,
'd_size': 3,
'p_s': 0,
'p_e': 3,
'time_size': 4,
'time_s': 0,
'time_e': 4,
}
self.verify_arguments(op.arguments(), expected)
def test_default_functions(self):
"""
Test the default argument derivation for functions.
"""
grid = Grid(shape=(5, 6, 7))
f = TimeFunction(name='f', grid=grid)
g = Function(name='g', grid=grid)
op = Operator(Eq(g, g + f))
expected = {
'x_size': 5,
'x_s': 0,
'x_e': 5,
'y_size': 6,
'y_s': 0,
'y_e': 6,
'z_size': 7,
'z_s': 0,
'z_e': 7,
'f': f.data,
'g': g.data,
}
self.verify_arguments(op.arguments(time=4), expected)
exp_parameters = [
'f', 'g', 'x_s', 'x_e', 'x_size', 'y_s', 'y_e', 'y_size', 'z_s',
'z_e', 'z_size', 'time_s', 'time_e'
]
self.verify_parameters(op.parameters, exp_parameters)
def test_basic(self):
grid = Grid(shape=(3, 3, 3))
u = TimeFunction(name='u', grid=grid)
op = Operator(Eq(u.forward, u + 1),
opt=('blocking', 'openmp', {
'par-nested': 0,
'par-collapse-ncores': 10000,
'par-dynamic-work': 0
}))
# Does it compile? Honoring the OpenMP specification isn't trivial
assert op.cfunction
# Does it produce the right result
op.apply(t_M=9)
assert np.all(u.data[0] == 10)
# Try again but this time supplying specific values for the num_threads
u.data[:] = 0.
op.apply(t_M=9, nthreads=1, nthreads_nested=2)
assert np.all(u.data[0] == 10)
assert op.arguments(t_M=9, nthreads_nested=2)['nthreads_nested'] == 2
bns, _ = assert_blocking(op, {'x0_blk0'})
iterations = FindNodes(Iteration).visit(bns['x0_blk0'])
assert iterations[0].pragmas[
0].value == 'omp for collapse(1) schedule(dynamic,1)'
assert iterations[2].pragmas[0].value == (
'omp parallel for collapse(1) '
'schedule(dynamic,1) '
'num_threads(nthreads_nested)')
def test_override_function_subrange(self):
"""
Test runtime start/end override for :class:`Function` dimensions.
"""
grid = Grid(shape=(5, 6, 7))
g = Function(name='g', grid=grid)
op = Operator(Eq(g, 1.))
args = {'x_s': 1, 'x_e': 3, 'y_s': 2, 'y_e': 4, 'z_s': 3, 'z_e': 5}
arguments = op.arguments(**args)
expected = {
'x_size': 5,
'x_s': 1,
'x_e': 3,
'y_size': 6,
'y_s': 2,
'y_e': 4,
'z_size': 7,
'z_s': 3,
'z_e': 5,
'g': g.data
}
self.verify_arguments(arguments, expected)
# Verify execution
op(**args)
mask = np.ones((5, 6, 7), dtype=np.bool)
mask[1:3, 2:4, 3:5] = False
assert (g.data[mask] == 0.).all()
assert (g.data[1:3, 2:4, 3:5] == 1.).all()
def test_override_composite_data(self):
i, j = dimify('i j')
grid = Grid(shape=(10, 10), dimensions=(i, j))
original_coords = (1., 1.)
new_coords = (2., 2.)
p_dim = Dimension('p_src')
u = TimeFunction(name='u', grid=grid, time_order=2, space_order=2)
src1 = SparseFunction(name='src1',
grid=grid,
dimensions=[time, p_dim],
npoint=1,
nt=10,
coordinates=original_coords)
src2 = SparseFunction(name='src1',
grid=grid,
dimensions=[time, p_dim],
npoint=1,
nt=10,
coordinates=new_coords)
op = Operator(src1.inject(u, src1))
# Move the source from the location where the setup put it so we can test
# whether the override picks up the original coordinates or the changed ones
# Operator.arguments() returns a tuple of (data, dimension_sizes)
args = op.arguments(src1=src2)[0]
arg_name = src1.name + "_coords"
assert (np.array_equal(args[arg_name], np.asarray((new_coords, ))))
def test_override_timefunction_subrange(self):
"""
Test runtime start/end overrides for :class:`TimeFunction` dimensions.
"""
grid = Grid(shape=(5, 6, 7))
f = TimeFunction(name='f', grid=grid, time_order=0)
# Suppress DLE to work around a know bug with GCC and OpenMP:
# https://github.com/opesci/devito/issues/320
op = Operator(Eq(f, 1.), dle=None)
# TODO: Currently we require the `time` subrange to be set
# explicitly. Ideally `t` would directly alias with `time`,
# but this seems broken currently.
args = {'x_m': 1, 'x_M': 3, 'y_m': 2, 'y_M': 4,
'z_m': 3, 'z_M': 5, 't_m': 1, 't_M': 4}
arguments = op.arguments(**args)
expected = {
'x_size': 5, 'x_m': 1, 'x_M': 3,
'y_size': 6, 'y_m': 2, 'y_M': 4,
'z_size': 7, 'z_m': 3, 'z_M': 5,
'time_m': 1, 'time_M': 4,
'f': f.data_allocated
}
self.verify_arguments(arguments, expected)
# Verify execution
op(**args)
mask = np.ones((1, 5, 6, 7), dtype=np.bool)
mask[:, 1:4, 2:5, 3:6] = False
assert (f.data[mask] == 0.).all()
assert (f.data[:, 1:4, 2:5, 3:6] == 1.).all()
def test_override_sparse_data_fix_dim(self):
"""
Ensure the arguments are derived correctly for an input SparseFunction.
The dimensions are forced to be the same in this case to verify
the aliasing on the SparseFunction name.
"""
grid = Grid(shape=(10, 10))
time = grid.time_dim
u = TimeFunction(name='u', grid=grid, time_order=2, space_order=2)
original_coords = (1., 1.)
new_coords = (2., 2.)
p_dim = Dimension(name='p_src')
src1 = SparseTimeFunction(name='src1', grid=grid, dimensions=[time, p_dim], nt=10,
npoint=1, coordinates=original_coords, time_order=2)
src2 = SparseTimeFunction(name='src2', grid=grid, dimensions=[time, p_dim],
npoint=1, nt=10, coordinates=new_coords, time_order=2)
op = Operator(src1.inject(u, src1))
# Move the source from the location where the setup put it so we can test
# whether the override picks up the original coordinates or the changed ones
args = op.arguments(src1=src2, time=0)
arg_name = src1.name + "_coords"
assert(np.array_equal(args[arg_name], np.asarray((new_coords,))))
def test_override_function_size(self):
"""
Test runtime size overrides for :class:`Function` dimensions.
Note: The current behaviour for size-only arguments seems
ambiguous (eg. op(x=3, y=4), as it sets `dim_size` as well as
`dim_end`. Since `dim_size` is used for the cast, we can get
garbage results if it does not agree with the shape of the
provided data. This should error out, or potentially we could
set the corresponding size, while aliasing `dim` to `dim_e`?
The same should be tested for :class:`TimeFunction` once fixed.
"""
grid = Grid(shape=(5, 6, 7))
g = Function(name='g', grid=grid)
op = Operator(Eq(g, 1.))
args = {'x': 3, 'y': 4, 'z': 5}
arguments = op.arguments(**args)
expected = {
'x_size': 5, 'x_m': 0, 'x_M': 3,
'y_size': 6, 'y_m': 0, 'y_M': 4,
'z_size': 7, 'z_m': 0, 'z_M': 5,
'g': g.data_allocated
}
self.verify_arguments(arguments, expected)
# Verify execution
op(**args)
assert (g.data[4:] == 0.).all()
assert (g.data[:, 5:] == 0.).all()
assert (g.data[:, :, 6:] == 0.).all()
assert (g.data[:4, :5, :6] == 1.).all()
def test_devicerm(self):
grid = Grid(shape=(6, 6))
u = TimeFunction(name='u', grid=grid, space_order=2)
f = Function(name='f', grid=grid)
op = Operator(Eq(u.forward, u.dx + f))
devicerm = self.get_param(op, DeviceRM)
assert devicerm is not None
assert devicerm.data == 1 # Always evict, by default
assert op.arguments(time_M=2)[devicerm.name] == 1
assert op.arguments(time_M=2, devicerm=0)[devicerm.name] == 0
assert op.arguments(time_M=2, devicerm=1)[devicerm.name] == 1
assert op.arguments(time_M=2, devicerm=224)[devicerm.name] == 1
assert op.arguments(time_M=2, devicerm=True)[devicerm.name] == 1
assert op.arguments(time_M=2, devicerm=False)[devicerm.name] == 0
def test_dynamic_nthreads(self):
grid = Grid(shape=(16, 16, 16))
f = TimeFunction(name='f', grid=grid)
op = Operator(Eq(f.forward, f + 1.), dle='openmp')
# Check num_threads appears in the generated code
# Not very elegant, but it does the trick
assert 'num_threads(nthreads)' in str(op)
# Check `op` accepts the `nthreads` kwarg
op.apply(time=0)
op.apply(time_m=1, time_M=1, nthreads=4)
assert np.all(f.data[0] == 2.)
# Check the actual value assumed by `nthreads`
assert op.arguments(time=0)['nthreads'] == NThreads.default_value()
assert op.arguments(time=0, nthreads=123)['nthreads'] == 123 # user supplied
def test_dynamic_nthreads(self):
grid = Grid(shape=(16, 16, 16))
f = TimeFunction(name='f', grid=grid)
op = Operator(Eq(f.forward, f + 1.), dle='openmp')
# Check num_threads appears in the generated code
# Not very elegant, but it does the trick
assert 'num_threads(nthreads)' in str(op)
# Check `op` accepts the `nthreads` kwarg
op.apply(time=0)
op.apply(time_m=1, time_M=1, nthreads=4)
assert np.all(f.data[0] == 2.)
# Check the actual value assumed by `nthreads`
assert op.arguments(time=0)['nthreads'] == NThreads.default_value()
assert op.arguments(time=0,
nthreads=123)['nthreads'] == 123 # user supplied
def test_dimension_size_infer(self, nt=100):
"""Test that the dimension sizes are being inferred correctly"""
grid = Grid(shape=(3, 5, 7))
a = Function(name='a', grid=grid)
b = TimeFunction(name='b', grid=grid, save=nt)
op = Operator(Eq(b, a))
time = b.indices[0]
op_arguments = op.arguments()
assert(op_arguments[time.min_name] == 0)
assert(op_arguments[time.max_name] == nt-1)
def test_dimension_size_infer(self, nt=100):
"""Test that the dimension sizes are being inferred correctly"""
i, j, k = dimify('i j k')
shape = tuple([d.size for d in [i, j, k]])
a = DenseData(name='a', shape=shape).indexed
b = TimeData(name='b', shape=shape, save=True, time_dim=nt).indexed
eqn = Eq(b[time, x, y, z], a[x, y, z])
op = Operator(eqn)
_, op_dim_sizes = op.arguments()
assert(op_dim_sizes[time.name] == nt)
class Processing(object):
def setup(self):
grid = Grid(shape=(5, 5, 5))
funcs = [Function(name='f%d' % n, grid=grid) for n in range(30)]
tfuncs = [TimeFunction(name='u%d' % n, grid=grid) for n in range(30)]
stfuncs = [SparseTimeFunction(name='su%d' % n, grid=grid, npoint=1, nt=100)
for n in range(30)]
v = TimeFunction(name='v', grid=grid, space_order=2)
eq = Eq(v.forward, v.laplace + sum(funcs) + sum(tfuncs) + sum(stfuncs),
subdomain=grid.interior)
self.op = Operator(eq, opt='noop')
# Allocate data, populate cached properties, etc.
self.op.arguments(time_M=98)
def time_processing(self):
self.op.arguments(time_M=98)
def test_npthreads(self):
nt = 10
async_degree = 5
grid = Grid(shape=(300, 300, 300))
u = TimeFunction(name='u', grid=grid)
usave = TimeFunction(name='usave', grid=grid, save=nt)
eqns = [Eq(u.forward, u + 1), Eq(usave, u.forward)]
op = Operator(eqns,
opt=('buffering', 'tasking', 'orchestrate', {
'buf-async-degree': async_degree
}))
npthreads0 = self.get_param(op, NPThreads)
assert op.arguments(time_M=2)[npthreads0.name] == 1
assert op.arguments(time_M=2, npthreads0=4)[npthreads0.name] == 4
# Cannot provide a value larger than the thread pool size
with pytest.raises(InvalidArgument):
assert op.arguments(time_M=2, npthreads0=5)
def test_dimension_offset_adjust(self, nt=100):
"""Test that the dimension sizes are being inferred correctly"""
i, j, k = dimify('i j k')
shape = (10, 10, 10)
grid = Grid(shape=shape, dimensions=(i, j, k))
a = Function(name='a', grid=grid).indexed
b = TimeFunction(name='b', grid=grid, save=nt)
time = b.indices[0]
eqn = Eq(b.indexed[time + 1, i, j, k], b.indexed[time - 1, i, j, k]
+ b.indexed[time, i, j, k] + a[i, j, k])
op = Operator(eqn)
op_arguments = op.arguments(time=nt-10)
assert(op_arguments[time.min_name] == 1)
assert(op_arguments[time.max_name] == nt - 10)
def test_argument_derivation_order(self, nt=100):
""" Ensure the precedence order of arguments is respected
Defaults < (overriden by) Tensor Arguments < Dimensions < Scalar Arguments
"""
i, j, k = dimify('i j k')
shape = (10, 10, 10)
grid = Grid(shape=shape, dimensions=(i, j, k))
a = Function(name='a', grid=grid).indexed
b_function = TimeFunction(name='b', grid=grid, save=True, time_dim=nt)
b = b_function.indexed
time = b_function.indices[0]
b1 = TimeFunction(name='b1', grid=grid, save=True,
time_dim=nt + 1).indexed
eqn = Eq(b[time, i, j, k], a[i, j, k])
op = Operator(eqn)
# Simple case, same as that tested above.
# Repeated here for clarity of further tests.
op_arguments, _ = op.arguments()
assert (op_arguments[time.start_name] == 0)
assert (op_arguments[time.end_name] == nt)
# Providing a tensor argument should infer the dimension size from its shape
op_arguments, _ = op.arguments(b=b1)
assert (op_arguments[time.start_name] == 0)
assert (op_arguments[time.end_name] == nt + 1)
# Providing a dimension size explicitly should override the automatically inferred
op_arguments, _ = op.arguments(b=b1, time=nt - 1)
assert (op_arguments[time.start_name] == 0)
assert (op_arguments[time.end_name] == nt - 1)
# Providing a scalar argument explicitly should override the automatically\
# inferred
op_arguments, _ = op.arguments(b=b1, time=nt - 1, time_e=nt - 2)
assert (op_arguments[time.start_name] == 0)
assert (op_arguments[time.end_name] == nt - 2)
def test_explicit_run(self):
time_dim = 6
grid = Grid(shape=(11, 11))
a = TimeFunction(name='a', grid=grid, time_order=1, save=time_dim)
eqn = Eq(a.forward, a + 1.)
op = Operator(eqn)
assert isinstance(op, OperatorForeign)
args = OrderedDict(op.arguments())
assert args['a'] is None
# Emulate data feeding from outside
array = np.ndarray(shape=a.shape, dtype=np.float32)
array.fill(0.0)
args['a'] = array
op.cfunction(*list(args.values()))
assert all(np.allclose(args['a'][i], i) for i in range(time_dim))
def test_arg_offset_adjust(self, nt=100):
"""Test that the dimension sizes are being inferred correctly"""
i, j, k = dimify('i j k')
shape = (10, 10, 10)
grid = Grid(shape=shape, dimensions=(i, j, k))
a = Function(name='a', grid=grid).indexed
b = TimeFunction(name='b', grid=grid, save=True, time_dim=nt)
time = b.indices[0]
eqn = Eq(
b.indexed[time + 1, i, j, k], b.indexed[time - 1, i, j, k] +
b.indexed[time, i, j, k] + a[i, j, k])
op = Operator(eqn)
args = {time.end_name: nt - 10}
op_arguments, _ = op.arguments(**args)
assert (op_arguments[time.start_name] == 0)
assert (op_arguments[time.end_name] == nt - 8)
def test_code(self):
shape = (11, 11)
a = TimeData(name='a',
shape=shape,
time_order=1,
time_dim=6,
save=True)
eqn = Eq(a.forward, a + 1.)
b = TimeData(name='a',
shape=shape,
time_order=1,
time_dim=6,
save=True)
eqn2 = Eq(b.forward, b + 1.)
op = Operator(eqn)
op()
op2 = Operator(eqn2, external=True)
arg, _ = op.arguments(a=b)
op2.cfunction(*list(arg.values()))
assert (np.allclose(a.data[:], b.data[:]))
def test_mpi(self):
# Shape chosen to get a source in multiple ranks
shape = (91, 91)
grid = Grid(shape=shape)
x, y = grid.dimensions
# because we interpolate across 2 neighbouring points in each dimension
r = 2
nt = 10
# NOTE: halo on function (space_order//2?) must be at least >= r
m = TimeFunction(name="m",
grid=grid,
space_order=4,
save=None,
time_order=1)
m.data[:] = 0.0
m.data[:, 40, 40] = 1.0
m.data[:, 50, 50] = 1.0
# only rank 0 is allowed to have points
if grid.distributor.myrank == 0:
# A single dipole source - so two rows, one column
matrix = scipy.sparse.coo_matrix(
np.array([[1], [-1]], dtype=np.float32))
else:
matrix = scipy.sparse.coo_matrix((0, 0), dtype=np.float32)
sf = MatrixSparseTimeFunction(name="s",
grid=grid,
r=r,
matrix=matrix,
nt=nt)
if grid.distributor.myrank == 0:
# First component of the dipole at 40, 40
sf.gridpoints.data[0, 0] = 40
sf.gridpoints.data[0, 1] = 40
sf.interpolation_coefficients[x].data[0, 0] = 1.0
sf.interpolation_coefficients[x].data[0, 1] = 2.0
sf.interpolation_coefficients[y].data[0, 0] = 1.0
sf.interpolation_coefficients[y].data[0, 1] = 2.0
sf.gridpoints.data[1, 0] = 50
sf.gridpoints.data[1, 1] = 50
sf.interpolation_coefficients[x].data[1, 0] = 2.0
sf.interpolation_coefficients[x].data[1, 1] = 2.0
sf.interpolation_coefficients[y].data[1, 0] = 2.0
sf.interpolation_coefficients[y].data[1, 1] = 2.0
op = Operator(sf.interpolate(m))
sf.manual_scatter()
args = op.arguments(time_m=0, time_M=9)
print("rank %d: %s" % (grid.distributor.myrank, str(args)))
op.apply(time_m=0, time_M=0)
sf.manual_gather()
for i in range(grid.distributor.nprocs):
print("==== from rank %d" % i)
if i == grid.distributor.myrank:
print(repr(sf.data))
grid.distributor.comm.Barrier()
if grid.distributor.myrank == 0:
assert sf.data[0,
0] == -3.0 # 1 * (1 * 1) * 1 + (-1) * (2 * 2) * 1
