def _initialize_bcs(self, bcs="damp"):
# Create dampening field as symbol `damp`
if self.nbl == 0:
self.damp = 1 if bcs == "mask" else 0
return
# First initialization
init = self.damp is None
# Get current Function if alread yinitialized
self.damp = self.damp or Function(name="damp", grid=self.grid)
if callable(bcs):
bcs(self.damp, self.nbl)
else:
re_init = ((bcs == "mask" and mmin(self.damp) == 0)
or (bcs == "damp" and mmax(self.damp) == 1))
if init or re_init:
if re_init and not init:
bcs_o = "damp" if bcs == "mask" else "mask"
warning("Re-initializing damp profile from %s to %s" %
(bcs_o, bcs))
warning("Model has to be created with `bcs=\"%s\"`"
"for this WaveSolver" % bcs)
initialize_damp(self.damp,
self.padsizes,
self.spacing,
abc_type=bcs,
fs=self.fs)
self._physical_parameters.update(['damp'])
def run(problem, **kwargs):
"""
A single run with a specific set of performance parameters.
"""
setup = model_type[problem]['setup']
options = {}
time_order = kwargs.pop('time_order')[0]
space_order = kwargs.pop('space_order')[0]
autotune = kwargs.pop('autotune')
block_shapes = as_tuple(kwargs.pop('block_shape'))
# Should a specific block-shape be used? Useful if one wants to skip
# the autotuning pass as a good block-shape is already known
if block_shapes:
if autotune:
warning("Skipping autotuning (using explicit block-shape `%s`)"
% str(block_shapes))
autotune = False
# This is horribly hacky, but it works for now
for i, bs in enumerate(block_shapes):
for d, s in zip(['x', 'y', 'z'], bs):
options['%s0_blk%d_size' % (d, i)] = s
solver = setup(space_order=space_order, time_order=time_order, **kwargs)
solver.forward(autotune=autotune, **options)
def run(problem, **kwargs):
"""
A single run with a specific set of performance parameters.
"""
setup = tti_setup if problem == 'tti' else acoustic_setup
options = {}
time_order = kwargs.pop('time_order')[0]
space_order = kwargs.pop('space_order')[0]
autotune = kwargs.pop('autotune')
# Should a specific block-shape be used? Useful if one wants to skip
# the autotuning pass as a good block-shape is already known
block_shape = as_tuple(kwargs.pop('block_shape'))
if all(block_shape):
if autotune:
warning("Skipping autotuning (using explicit block-shape `%s`)"
% str(block_shape))
autotune = False
# This is quite hacky, but it does the trick
for d, bs in zip(['x', 'y', 'z'], block_shape):
options['%s0_blk_size' % d] = bs
solver = setup(space_order=space_order, time_order=time_order, **kwargs)
solver.forward(autotune=autotune, **options)
def run(problem, **kwargs):
"""
A single run with a specific set of performance parameters.
"""
setup = tti_setup if problem == 'tti' else acoustic_setup
options = {}
time_order = kwargs.pop('time_order')[0]
space_order = kwargs.pop('space_order')[0]
autotune = kwargs.pop('autotune')
# Should a specific block-shape be used? Useful if one wants to skip
# the autotuning pass as a good block-shape is already known
block_shape = as_tuple(kwargs.pop('block_shape'))
if all(block_shape):
if autotune:
warning("Skipping autotuning (using explicit block-shape `%s`)" %
str(block_shape))
autotune = False
# This is quite hacky, but it does the trick
for d, bs in zip(['x', 'y', 'z'], block_shape):
options['%s0_blk_size' % d] = bs
solver = setup(space_order=space_order, time_order=time_order, **kwargs)
solver.forward(autotune=autotune, **options)
def convert(self, value, param, ctx):
n_value = len(value)
n_type = len(self.types)
if n_value <= n_type:
warning(f"Processing {n_value} out of expected up to {n_type}")
else:
super().convert(value, param, ctx)
return tuple(self.types[i](value[i], param, ctx)
for i in range(n_value))
def new_src(self, name='src', src_type='self'):
if self.src_type is None or src_type is None:
warning("No surce type defined, returning uninistiallized (zero) source")
return PointSource(name=name, grid=self.grid,
time_range=self.time_axis, npoint=self.nsrc,
coordinates=self.src_positions)
else:
return sources[self.src_type](name=name, grid=self.grid, f0=self.f0,
time_range=self.time_axis, npoint=self.nsrc,
coordinates=self.src_positions,
t0=self._t0w, a=self._a)
def adj_src(self):
if self.src_type is None:
warning("No source type defined, returning uninitiallized (zero) shot record")
return self.new_rec()
adj_src = sources[self.src_type](name='rec', grid=self.grid, f0=self.f0,
time_range=self.time_axis, npoint=self.nrec,
coordinates=self.rec_positions,
t0=self._t0w, a=self._a)
# Revert time axis to have a proper shot record and not compute on zeros
for i in range(self.nrec):
adj_src.data[:, i] = adj_src.wavelet[::-1]
return adj_src
def __init__(self, model, geometry, space_order=4, **kwargs):
self.model = model
self.geometry = geometry
if space_order % 2 != 0:
raise ValueError("space_order must be even but got %s" %
space_order)
if space_order % 4 != 0:
warning("It is recommended for space_order to be a multiple of 4" +
"but got %s" % space_order)
self.space_order = space_order
# Cache compiler options
self._kwargs = kwargs
def config_autotuning(ctx, param, value):
"""Setup auto-tuning to run in ``{basic,aggressive,...}+preemptive`` mode."""
if value != 'off':
# Sneak-peek at the `block-shape` -- if provided, keep auto-tuning off
if ctx.params['block_shape']:
warning("Skipping autotuning (using explicit block-shape `%s`)"
% str(ctx.params['block_shape']))
level = False
else:
# Make sure to always run in preemptive mode
configuration['autotuning'] = [value, 'preemptive']
# We apply blocking to all parallel loops, including the innermost ones
configuration['dle-options']['blockinner'] = True
level = value
else:
level = False
return level
def config_autotuning(ctx, param, value):
"""Setup auto-tuning to run in ``{basic,aggressive,...}+preemptive`` mode."""
if value != 'off':
# Sneak-peek at the `block-shape` -- if provided, keep auto-tuning off
if ctx.params['block_shape']:
warning("Skipping autotuning (using explicit block-shape `%s`)"
% str(ctx.params['block_shape']))
level = False
else:
# Make sure to always run in preemptive mode
configuration['autotuning'] = [value, 'preemptive']
# We apply blocking to all parallel loops, including the innermost ones
# Note: see https://github.com/devitocodes/devito/issues/320 for why
# we use blockinner=True only if the backend compiler is Intel
flag = isinstance(configuration['compiler'], IntelCompiler)
configuration['opt-options']['blockinner'] = flag
level = value
else:
level = False
return level
def __init__(self,
model,
geometry,
space_order=4,
kernel='centered',
**kwargs):
self.model = model
self.model._initialize_bcs(bcs="damp")
self.geometry = geometry
self.kernel = kernel
if space_order % 2 != 0:
raise ValueError("space_order must be even but got %s" %
space_order)
if space_order % 4 != 0:
warning("It is recommended for space_order to be a multiple of 4" +
"but got %s" % space_order)
self.space_order = space_order
# Cache compiler options
self._kwargs = kwargs
def model(self):
warning("Model is kept for backward compatibility but should not be"
"obtained from the geometry")
return self._model
from devito import warning
warning(
"""The location of Devito's checkpointing has changed. This location will be
deprecated soon. Please change your imports to 'from devito import
DevitoCheckpoint, CheckpointOperato'""")
from devito.checkpointing import * # noqa
def plot(problem, **kwargs):
"""
Plotting mode to generate plots for performance analysis.
"""
backend = kwargs.pop('backend')
resultsdir = kwargs.pop('resultsdir')
max_bw = kwargs.pop('max_bw')
flop_ceils = kwargs.pop('flop_ceil')
point_runtime = kwargs.pop('point_runtime')
arch = kwargs['arch']
space_order = "[%s]" % ",".join(str(i) for i in kwargs['space_order'])
time_order = kwargs['time_order']
shape = "[%s]" % ",".join(str(i) for i in kwargs['shape'])
RooflinePlotter = get_ob_plotter()
bench = get_ob_bench(problem, resultsdir, kwargs)
bench.load()
if not bench.loaded:
warning("Could not load any results, nothing to plot. Exiting...")
sys.exit(0)
gflopss = bench.lookup(params=kwargs, measure="gflopss", event='main')
oi = bench.lookup(params=kwargs, measure="oi", event='main')
time = bench.lookup(params=kwargs, measure="timings", event='main')
# What plot am I?
modes = [i for i in ['dse', 'dle', 'autotune']
if len(set(dict(j)[i] for j in gflopss)) > 1]
# Filename
figname = "%s_dim%s_so%s_to%s_arch[%s]_bkend[%s].pdf" % (
problem, shape, space_order, time_order, arch, backend
)
# Legend setup. Do not plot a legend if there's no variation in performance
# options (dse, dle, autotune)
if modes:
legend = {'loc': 'upper left', 'fontsize': 7, 'ncol': 4}
else:
legend = 'drop'
avail_colors = ['r', 'g', 'b', 'y', 'k', 'm']
avail_markers = ['o', 'x', '^', 'v', '<', '>']
used_colors = {}
used_markers = {}
# Find min and max runtimes for instances having the same OI
min_max = {v: [0, sys.maxsize] for v in oi.values()}
for k, v in time.items():
i = oi[k]
min_max[i][0] = v if min_max[i][0] == 0 else min(v, min_max[i][0])
min_max[i][1] = v if min_max[i][1] == sys.maxsize else max(v, min_max[i][1])
with RooflinePlotter(figname=figname, plotdir=resultsdir,
max_bw=max_bw, flop_ceils=flop_ceils,
fancycolor=True, legend=legend) as plot:
for k, v in gflopss.items():
so = dict(k)['space_order']
oi_value = oi[k]
time_value = time[k]
run = tuple(dict(k)[i] for i in modes)
label = ("<%s>" % ','.join(run)) if run else None
color = used_colors[run] if run in used_colors else avail_colors.pop(0)
used_colors.setdefault(run, color)
marker = used_markers[so] if so in used_markers else avail_markers.pop(0)
used_markers.setdefault(so, marker)
oi_loc = 0.076 if len(str(so)) == 1 else 0.09
oi_annotate = {'s': 'SO=%s' % so, 'size': 6, 'xy': (oi_value, oi_loc)}
if time_value in min_max[oi_value] and point_runtime:
# Only annotate min and max runtimes on each OI line, to avoid
# polluting the plot too much
point_annotate = {'s': "%.0fs" % time_value, 'xytext': (0.0, 5.5),
'size': 6, 'rotation': 0}
else:
point_annotate = None
oi_line = time_value == min_max[oi_value][0]
if oi_line:
perf_annotate = {'size': 6, 'xytext': (-4, 5)}
plot.add_point(gflops=v, oi=oi_value, marker=marker, color=color,
oi_line=oi_line, label=label, perf_annotate=perf_annotate,
oi_annotate=oi_annotate, point_annotate=point_annotate)
def __init__(self, origin, spacing, shape, space_order, vp, nbpml=20,
dtype=np.float32, epsilon=None, delta=None, theta=None, phi=None,
subdomains=(), **kwargs):
super(Model, self).__init__(origin, spacing, shape, space_order, nbpml, dtype,
subdomains)
# Create square slowness of the wave as symbol `m`
if isinstance(vp, np.ndarray):
self.m = Function(name="m", grid=self.grid, space_order=space_order)
else:
self.m = Constant(name="m", value=1/vp**2)
self._physical_parameters = ('m',)
# Set model velocity, which will also set `m`
self.vp = vp
# Create dampening field as symbol `damp`
self.damp = Function(name="damp", grid=self.grid)
initialize_damp(self.damp, self.nbpml, self.spacing)
# Additional parameter fields for TTI operators
self.scale = 1.
if epsilon is not None:
if isinstance(epsilon, np.ndarray):
self._physical_parameters += ('epsilon',)
self.epsilon = Function(name="epsilon", grid=self.grid)
initialize_function(self.epsilon, 1 + 2 * epsilon, self.nbpml)
# Maximum velocity is scale*max(vp) if epsilon > 0
if mmax(self.epsilon) > 0:
self.scale = np.sqrt(mmax(self.epsilon))
else:
self.epsilon = 1 + 2 * epsilon
self.scale = epsilon
else:
self.epsilon = 1
if delta is not None:
if isinstance(delta, np.ndarray):
self._physical_parameters += ('delta',)
self.delta = Function(name="delta", grid=self.grid)
initialize_function(self.delta, np.sqrt(1 + 2 * delta), self.nbpml)
else:
self.delta = delta
else:
self.delta = 1
if theta is not None:
if isinstance(theta, np.ndarray):
self._physical_parameters += ('theta',)
self.theta = Function(name="theta", grid=self.grid,
space_order=space_order)
initialize_function(self.theta, theta, self.nbpml)
else:
self.theta = theta
else:
self.theta = 0
if phi is not None:
if self.grid.dim < 3:
warning("2D TTI does not use an azimuth angle Phi, ignoring input")
self.phi = 0
elif isinstance(phi, np.ndarray):
self._physical_parameters += ('phi',)
self.phi = Function(name="phi", grid=self.grid, space_order=space_order)
initialize_function(self.phi, phi, self.nbpml)
else:
self.phi = phi
else:
self.phi = 0
gflopss, oi, timings, _ = self.func(*args, **kwargs)
for key in timings.keys():
self.register(gflopss[key], measure="gflopss", event=key.name)
self.register(oi[key], measure="oi", event=key.name)
self.register(timings[key], measure="timings", event=key.name)
return DevitoExecutor(func)
if __name__ == "__main__":
# If running with MPI, we emit logging messages from rank0 only
try:
MPI.Init() # Devito starts off with MPI disabled!
set_log_level('DEBUG', comm=MPI.COMM_WORLD)
if MPI.COMM_WORLD.size > 1 and not configuration['mpi']:
warning(
"It seems that you're running over MPI with %d processes, but "
"DEVITO_MPI is unset. Setting `DEVITO_MPI=basic`..." %
MPI.COMM_WORLD.size)
configuration['mpi'] = 'basic'
except TypeError:
# MPI not available
pass
# Profiling at max level
configuration['profiling'] = 'advanced'
benchmark()
def __init__(self,
origin,
spacing,
shape,
space_order,
vp,
nbpml=20,
dtype=np.float32,
epsilon=None,
delta=None,
theta=None,
phi=None,
subdomains=(),
**kwargs):
super(Model, self).__init__(origin, spacing, shape, space_order, nbpml,
dtype, subdomains)
# Create square slowness of the wave as symbol `m`
if isinstance(vp, np.ndarray):
self.m = Function(name="m",
grid=self.grid,
space_order=space_order)
else:
self.m = Constant(name="m", value=1 / vp**2)
self._physical_parameters = ('m', )
# Set model velocity, which will also set `m`
self.vp = vp
# Create dampening field as symbol `damp`
self.damp = Function(name="damp", grid=self.grid)
initialize_damp(self.damp, self.nbpml, self.spacing)
# Additional parameter fields for TTI operators
self.scale = 1.
if epsilon is not None:
if isinstance(epsilon, np.ndarray):
self._physical_parameters += ('epsilon', )
self.epsilon = Function(name="epsilon", grid=self.grid)
initialize_function(self.epsilon, 1 + 2 * epsilon, self.nbpml)
# Maximum velocity is scale*max(vp) if epsilon > 0
if np.max(self.epsilon.data_with_halo[:]) > 0:
self.scale = np.sqrt(np.max(
self.epsilon.data_with_halo[:]))
else:
self.epsilon = 1 + 2 * epsilon
self.scale = epsilon
else:
self.epsilon = 1
if delta is not None:
if isinstance(delta, np.ndarray):
self._physical_parameters += ('delta', )
self.delta = Function(name="delta", grid=self.grid)
initialize_function(self.delta, np.sqrt(1 + 2 * delta),
self.nbpml)
else:
self.delta = delta
else:
self.delta = 1
if theta is not None:
if isinstance(theta, np.ndarray):
self._physical_parameters += ('theta', )
self.theta = Function(name="theta",
grid=self.grid,
space_order=space_order)
initialize_function(self.theta, theta, self.nbpml)
else:
self.theta = theta
else:
self.theta = 0
if phi is not None:
if self.grid.dim < 3:
warning(
"2D TTI does not use an azimuth angle Phi, ignoring input")
self.phi = 0
elif isinstance(phi, np.ndarray):
self._physical_parameters += ('phi', )
self.phi = Function(name="phi",
grid=self.grid,
space_order=space_order)
initialize_function(self.phi, phi, self.nbpml)
else:
self.phi = phi
else:
self.phi = 0
def plot(problem, **kwargs):
"""
Plotting mode to generate plots for performance analysis.
"""
backend = kwargs.pop('backend')
resultsdir = kwargs.pop('resultsdir')
max_bw = kwargs.pop('max_bw')
flop_ceils = kwargs.pop('flop_ceil')
point_runtime = kwargs.pop('point_runtime')
autotune = kwargs['autotune']
arch = kwargs['arch']
space_order = "[%s]" % ",".join(str(i) for i in kwargs['space_order'])
time_order = kwargs['time_order']
shape = "[%s]" % ",".join(str(i) for i in kwargs['shape'])
section = kwargs.pop('section')
if not section:
warning("No `section` provided. Using `%s`'s default `%s`" %
(problem, model_type[problem]['default-section']))
section = model_type[problem]['default-section']
bench = get_ob_bench(problem, resultsdir, kwargs)
bench.load()
if not bench.loaded:
warning("Could not load any results, nothing to plot. Exiting...")
sys.exit(0)
gflopss = bench.lookup(params=kwargs, measure="gflopss", event=section)
oi = bench.lookup(params=kwargs, measure="oi", event=section)
time = bench.lookup(params=kwargs, measure="timings", event=section)
# What plot am I?
modes = [
i for i in ['dse', 'dle', 'autotune']
if len(set(dict(j)[i] for j in gflopss)) > 1
]
# Filename
figname = "%s_shape%s_so%s_to%s_arch[%s]_bkend[%s]_at[%s]" % (
problem, shape, space_order, time_order, arch, backend, autotune)
# Legend setup. Do not plot a legend if there's no variation in performance
# options (dse, dle, autotune)
if modes:
legend = {'loc': 'upper left', 'fontsize': 7, 'ncol': 4}
else:
legend = 'drop'
avail_colors = ['r', 'g', 'b', 'y', 'k', 'm']
avail_markers = ['o', 'x', '^', 'v', '<', '>']
used_colors = {}
used_markers = {}
# Find min and max runtimes for instances having the same OI
min_max = {v: [0, sys.maxsize] for v in oi.values()}
for k, v in time.items():
i = oi[k]
min_max[i][0] = v if min_max[i][0] == 0 else min(v, min_max[i][0])
min_max[i][1] = v if min_max[i][1] == sys.maxsize else max(
v, min_max[i][1])
with RooflinePlotter(figname=figname,
plotdir=resultsdir,
max_bw=max_bw,
flop_ceils=flop_ceils,
fancycolor=True,
legend=legend) as plot:
for k, v in gflopss.items():
so = dict(k)['space_order']
oi_value = oi[k]
time_value = time[k]
run = tuple(dict(k)[i] for i in modes)
label = ("<%s>" % ','.join(run)) if run else None
color = used_colors[
run] if run in used_colors else avail_colors.pop(0)
used_colors.setdefault(run, color)
marker = used_markers[
so] if so in used_markers else avail_markers.pop(0)
used_markers.setdefault(so, marker)
oi_loc = 0.076 if len(str(so)) == 1 else 0.09
oi_annotate = {
's': 'SO=%s' % so,
'size': 6,
'xy': (oi_value, oi_loc)
}
if time_value in min_max[oi_value] and point_runtime:
# Only annotate min and max runtimes on each OI line, to avoid
# polluting the plot too much
point_annotate = {
's': "%.0fs" % time_value,
'xytext': (0.0, 5.5),
'size': 6,
'rotation': 0
}
else:
point_annotate = None
oi_line = time_value == min_max[oi_value][0]
if oi_line:
perf_annotate = {'size': 6, 'xytext': (-4, 5)}
plot.add_point(gflops=v,
oi=oi_value,
marker=marker,
color=color,
oi_line=oi_line,
label=label,
perf_annotate=perf_annotate,
oi_annotate=oi_annotate,
point_annotate=point_annotate)
def wri_func(model,
src_coords,
wavelet,
rec_coords,
recin,
yin,
space_order=8,
isic=False,
ws=None,
t_sub=1,
grad="m",
grad_corr=False,
alpha_op=False,
w_fun=None,
eps=0,
freq_list=[],
wfilt=None):
"""
Time domain wavefield reconstruction inversion wrapper
"""
if freq_list is not None:
if grad_corr or grad in ["all", "y"]:
warning("On-the-fly DFT is not supported with gradient correction")
dft = True
else:
dft = False
freq_list = None
wfilt = wavelet
# F(m0) * q if y is not an input and compute y = r(m0)
if yin is None or grad_corr:
y, u0, _ = forward(model,
src_coords,
rec_coords,
wavelet,
save=grad_corr,
space_order=space_order,
ws=ws)
ydat = recin[:] - y.data[:]
else:
ydat = yin
# Compute wavefield vy = adjoint(F(m0))*y and norm on the fly
srca, v, norm_v, _ = adjoint(model,
ydat,
src_coords,
rec_coords,
norm_v=True,
w_fun=w_fun,
freq_list=freq_list,
save=not (grad is None or dft))
c1 = 1 / (recin.shape[1])
c2 = np.log(np.prod(model.shape))
# <PTy, d-F(m)*f> = <PTy, d>-<adjoint(F(m))*PTy, f>
ndt = np.sqrt(model.critical_dt)
PTy_dot_r = ndt**2 * (np.dot(ydat.reshape(-1), recin.reshape(-1)) -
np.dot(srca.data.reshape(-1), wavelet.reshape(-1)))
norm_y = ndt * np.linalg.norm(ydat)
# alpha
α = compute_optalpha(c2 * norm_y, c1 * norm_v, eps, comp_alpha=alpha_op)
# Lagrangian evaluation
fun = -.5 * c1 * α**2 * norm_v + c2 * α * PTy_dot_r - eps * np.abs(
α) * norm_y
gradm = grady = None
if grad is not None:
w = weight_fun(w_fun, model, src_coords)
w = c1 * α / w**2 if w is not None else c1 * α
Q = wf_as_src(v, w=w, freq_list=freq_list)
rcv, gradm, _ = forward_grad(model,
src_coords,
rec_coords,
c2 * wfilt,
freq=freq_list,
q=Q,
v=v)
# Compute gradient wrt y
if grad_corr or grad in ["all", "y"]:
grady = c2 * recin - rcv.data[:]
if norm_y != 0:
grady -= np.abs(eps) * ydat / norm_y
# Correcting for reduced gradient
if not grad_corr:
gradm = gradm.data
else:
gradm_corr, _ = gradient(model, grady, rec_coords, u0)
# Reduced gradient post-processing
gradm = gradm.data + gradm_corr.data
return fun, gradm if gradm is None else α * gradm, grady
