def test_gmem_access_counter_specialops():
knl = lp.make_kernel("{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}", [
"""
c[i, j, k] = (2*a[i,j,k])%(2+b[i,j,k]/3.0)
e[i, k] = (1+g[i,k])**(1+h[i,k+1])
"""
],
name="specialops",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(
knl, dict(a=np.float32, b=np.float32, g=np.float64, h=np.float64))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[(np.dtype(np.float32), 'uniform',
'load')].eval_with_dict(params)
f64 = poly[(np.dtype(np.float64), 'uniform',
'load')].eval_with_dict(params)
assert f32 == 2 * n * m * l
assert f64 == 2 * n * m
f32 = poly[(np.dtype(np.float32), 'uniform',
'store')].eval_with_dict(params)
f64 = poly[(np.dtype(np.float64), 'uniform',
'store')].eval_with_dict(params)
assert f32 == n * m * l
assert f64 == n * m
def test_gmem_access_counter_consec():
knl = lp.make_kernel("[n,m,l] -> {[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = a[i,j,k]*b[i,j,k]/3.0+a[i,j,k]
e[i, k] = g[i,k]*(2+h[i,k])
"""
],
name="consec",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(
knl, dict(a=np.float32, b=np.float32, g=np.float64, h=np.float64))
knl = lp.tag_inames(knl, {"k": "l.0", "i": "g.0", "j": "g.1"})
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f64consec = poly[(np.dtype(np.float64), 'consecutive',
'load')].eval_with_dict(params)
f32consec = poly[(np.dtype(np.float32), 'consecutive',
'load')].eval_with_dict(params)
assert f64consec == 2 * n * m
assert f32consec == 3 * n * m * l
f64consec = poly[(np.dtype(np.float64), 'consecutive',
'store')].eval_with_dict(params)
f32consec = poly[(np.dtype(np.float32), 'consecutive',
'store')].eval_with_dict(params)
assert f64consec == n * m
assert f32consec == n * m * l
def test_gmem_access_counter_bitwise():
knl = lp.make_kernel(
"{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = (a[i,j,k] | 1) + (b[i,j,k] & 1)
e[i, k] = (g[i,k] ^ k)*(~h[i,k+1]) + (g[i, k] << (h[i,k] >> k))
"""
],
name="bitwise", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(
knl, dict(
a=np.int32, b=np.int32,
g=np.int32, h=np.int32))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
i32 = poly[
(np.dtype(np.int32), 'uniform', 'load')
].eval_with_dict(params)
assert i32 == 4*n*m+2*n*m*l
i32 = poly[
(np.dtype(np.int32), 'uniform', 'store')
].eval_with_dict(params)
assert i32 == n*m+n*m*l
def test_gmem_access_counter_logic():
knl = lp.make_kernel("{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}", [
"""
e[i,k] = if(not(k<l-2) and k>6 or k/2==l, g[i,k]*2, g[i,k]+h[i,k]/2)
"""
],
name="logic",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(g=np.float32, h=np.float64))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[(np.dtype(np.float32), 'uniform',
'load')].eval_with_dict(params)
f64 = poly[(np.dtype(np.float64), 'uniform',
'load')].eval_with_dict(params)
assert f32 == 2 * n * m
assert f64 == n * m
f64 = poly[(np.dtype(np.float64), 'uniform',
'store')].eval_with_dict(params)
assert f64 == n * m
def test_gmem_access_counter_basic():
knl = lp.make_kernel("[n,m,l] -> {[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = a[i,j,k]*b[i,j,k]/3.0+a[i,j,k]
e[i, k] = g[i,k]*h[i,k+1]
"""
],
name="basic",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(
knl, dict(a=np.float32, b=np.float32, g=np.float64, h=np.float64))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[(np.dtype(np.float32), 'uniform',
'load')].eval_with_dict(params)
f64 = poly[(np.dtype(np.float64), 'uniform',
'load')].eval_with_dict(params)
assert f32 == 3 * n * m * l
assert f64 == 2 * n * m
f32 = poly[(np.dtype(np.float32), 'uniform',
'store')].eval_with_dict(params)
f64 = poly[(np.dtype(np.float64), 'uniform',
'store')].eval_with_dict(params)
assert f32 == n * m * l
assert f64 == n * m
def test_gmem_access_counter_logic():
knl = lp.make_kernel(
"{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
e[i,k] = if(not(k<l-2) and k>6 or k/2==l, g[i,k]*2, g[i,k]+h[i,k]/2)
"""
],
name="logic", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(g=np.float32, h=np.float64))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[
(np.dtype(np.float32), 'uniform', 'load')
].eval_with_dict(params)
f64 = poly[
(np.dtype(np.float64), 'uniform', 'load')
].eval_with_dict(params)
assert f32 == 2*n*m
assert f64 == n*m
f64 = poly[
(np.dtype(np.float64), 'uniform', 'store')
].eval_with_dict(params)
assert f64 == n*m
def test_all_counters_parallel_matmul():
knl = lp.make_kernel(
"{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"c[i, j] = sum(k, a[i, k]*b[k, j])"
],
name="matmul", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(a=np.float32, b=np.float32))
knl = lp.split_iname(knl, "i", 16, outer_tag="g.0", inner_tag="l.1")
knl = lp.split_iname(knl, "j", 16, outer_tag="g.1", inner_tag="l.0")
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
sync_poly = lp.get_synchronization_poly(knl)
assert len(sync_poly) == 1
assert sync_poly["kernel_launch"].eval_with_dict(params) == 1
op_map = lp.get_op_poly(knl)
f32mul = op_map[
(np.dtype(np.float32), 'mul')
].eval_with_dict(params)
f32add = op_map[
(np.dtype(np.float32), 'add')
].eval_with_dict(params)
i32ops = op_map[
(np.dtype(np.int32), 'add')
].eval_with_dict(params)
i32ops += op_map[
(np.dtype(np.int32), 'mul')
].eval_with_dict(params)
assert f32mul+f32add == n*m*l*2
assert i32ops == n*m*l*4 + l*n*4
subscript_map = lp.get_gmem_access_poly(knl)
f32uncoal = subscript_map[
(np.dtype(np.float32), 'nonconsecutive', 'load')
].eval_with_dict(params)
f32coal = subscript_map[
(np.dtype(np.float32), 'consecutive', 'load')
].eval_with_dict(params)
assert f32uncoal == n*m*l
assert f32coal == n*m*l
f32coal = subscript_map[
(np.dtype(np.float32), 'consecutive', 'store')
].eval_with_dict(params)
assert f32coal == n*l
def test_gmem_access_counter_mixed():
knl = lp.make_kernel(
"[n,m,l] -> {[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = a[i,j,k]*b[i,j,k]/3.0+a[i,j,k]+x[i,k]
e[i, k] = g[i,k]*(2+h[i,k])
"""
],
name="mixed", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(
a=np.float32, b=np.float32, g=np.float64, h=np.float64,
x=np.float32))
threads = 16
knl = lp.split_iname(knl, "j", threads)
knl = lp.tag_inames(knl, {"j_inner": "l.0", "j_outer": "g.0"})
poly = lp.get_gmem_access_poly(knl) # noqa
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f64uniform = poly[
(np.dtype(np.float64), 'uniform', 'load')
].eval_with_dict(params)
f32uniform = poly[
(np.dtype(np.float32), 'uniform', 'load')
].eval_with_dict(params)
f32nonconsec = poly[
(np.dtype(np.float32), 'nonconsecutive', 'load')
].eval_with_dict(params)
assert f64uniform == 2*n*m
assert f32uniform == n*m*l/threads
assert f32nonconsec == 3*n*m*l
f64uniform = poly[
(np.dtype(np.float64), 'uniform', 'store')
].eval_with_dict(params)
f32nonconsec = poly[
(np.dtype(np.float32), 'nonconsecutive', 'store')
].eval_with_dict(params)
assert f64uniform == n*m
assert f32nonconsec == n*m*l
def test_gmem_access_counter_mixed():
knl = lp.make_kernel("[n,m,l] -> {[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = a[i,j,k]*b[i,j,k]/3.0+a[i,j,k]+x[i,k]
e[i, k] = g[i,k]*(2+h[i,k])
"""
],
name="mixed",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(
knl,
dict(a=np.float32,
b=np.float32,
g=np.float64,
h=np.float64,
x=np.float32))
threads = 16
knl = lp.split_iname(knl, "j", threads)
knl = lp.tag_inames(knl, {"j_inner": "l.0", "j_outer": "g.0"})
poly = lp.get_gmem_access_poly(knl) # noqa
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f64uniform = poly[(np.dtype(np.float64), 'uniform',
'load')].eval_with_dict(params)
f32uniform = poly[(np.dtype(np.float32), 'uniform',
'load')].eval_with_dict(params)
f32nonconsec = poly[(np.dtype(np.float32), 'nonconsecutive',
'load')].eval_with_dict(params)
assert f64uniform == 2 * n * m
assert f32uniform == n * m * l / threads
assert f32nonconsec == 3 * n * m * l
f64uniform = poly[(np.dtype(np.float64), 'uniform',
'store')].eval_with_dict(params)
f32nonconsec = poly[(np.dtype(np.float32), 'nonconsecutive',
'store')].eval_with_dict(params)
assert f64uniform == n * m
assert f32nonconsec == n * m * l
def test_gmem_access_counter_reduction():
knl = lp.make_kernel("{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
["c[i, j] = sum(k, a[i, k]*b[k, j])"],
name="matmul",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(a=np.float32, b=np.float32))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[(np.dtype(np.float32), 'uniform',
'load')].eval_with_dict(params)
assert f32 == 2 * n * m * l
f32 = poly[(np.dtype(np.float32), 'uniform',
'store')].eval_with_dict(params)
assert f32 == n * l
def test_all_counters_parallel_matmul():
knl = lp.make_kernel("{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
["c[i, j] = sum(k, a[i, k]*b[k, j])"],
name="matmul",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(a=np.float32, b=np.float32))
knl = lp.split_iname(knl, "i", 16, outer_tag="g.0", inner_tag="l.1")
knl = lp.split_iname(knl, "j", 16, outer_tag="g.1", inner_tag="l.0")
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
sync_poly = lp.get_synchronization_poly(knl)
assert len(sync_poly) == 1
assert sync_poly["kernel_launch"].eval_with_dict(params) == 1
op_map = lp.get_op_poly(knl)
f32mul = op_map[(np.dtype(np.float32), 'mul')].eval_with_dict(params)
f32add = op_map[(np.dtype(np.float32), 'add')].eval_with_dict(params)
i32ops = op_map[(np.dtype(np.int32), 'add')].eval_with_dict(params)
i32ops += op_map[(np.dtype(np.int32), 'mul')].eval_with_dict(params)
assert f32mul + f32add == n * m * l * 2
assert i32ops == n * m * l * 4 + l * n * 4
subscript_map = lp.get_gmem_access_poly(knl)
f32uncoal = subscript_map[(np.dtype(np.float32), 'nonconsecutive',
'load')].eval_with_dict(params)
f32coal = subscript_map[(np.dtype(np.float32), 'consecutive',
'load')].eval_with_dict(params)
assert f32uncoal == n * m * l
assert f32coal == n * m * l
f32coal = subscript_map[(np.dtype(np.float32), 'consecutive',
'store')].eval_with_dict(params)
assert f32coal == n * l
def test_gmem_access_counter_consec():
knl = lp.make_kernel(
"[n,m,l] -> {[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = a[i,j,k]*b[i,j,k]/3.0+a[i,j,k]
e[i, k] = g[i,k]*(2+h[i,k])
"""
],
name="consec", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(
a=np.float32, b=np.float32, g=np.float64, h=np.float64))
knl = lp.tag_inames(knl, {"k": "l.0", "i": "g.0", "j": "g.1"})
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f64consec = poly[
(np.dtype(np.float64), 'consecutive', 'load')
].eval_with_dict(params)
f32consec = poly[
(np.dtype(np.float32), 'consecutive', 'load')
].eval_with_dict(params)
assert f64consec == 2*n*m
assert f32consec == 3*n*m*l
f64consec = poly[
(np.dtype(np.float64), 'consecutive', 'store')
].eval_with_dict(params)
f32consec = poly[
(np.dtype(np.float32), 'consecutive', 'store')
].eval_with_dict(params)
assert f64consec == n*m
assert f32consec == n*m*l
def test_gmem_access_counter_bitwise():
knl = lp.make_kernel("{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}", [
"""
c[i, j, k] = (a[i,j,k] | 1) + (b[i,j,k] & 1)
e[i, k] = (g[i,k] ^ k)*(~h[i,k+1]) + (g[i, k] << (h[i,k] >> k))
"""
],
name="bitwise",
assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(
knl, dict(a=np.int32, b=np.int32, g=np.int32, h=np.int32))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
i32 = poly[(np.dtype(np.int32), 'uniform', 'load')].eval_with_dict(params)
assert i32 == 4 * n * m + 2 * n * m * l
i32 = poly[(np.dtype(np.int32), 'uniform', 'store')].eval_with_dict(params)
assert i32 == n * m + n * m * l
def test_gmem_access_counter_reduction():
knl = lp.make_kernel(
"{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"c[i, j] = sum(k, a[i, k]*b[k, j])"
],
name="matmul", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl, dict(a=np.float32, b=np.float32))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[
(np.dtype(np.float32), 'uniform', 'load')
].eval_with_dict(params)
assert f32 == 2*n*m*l
f32 = poly[
(np.dtype(np.float32), 'uniform', 'store')
].eval_with_dict(params)
assert f32 == n*l
def test_gmem_access_counter_specialops():
knl = lp.make_kernel(
"{[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = (2*a[i,j,k])%(2+b[i,j,k]/3.0)
e[i, k] = (1+g[i,k])**(1+h[i,k+1])
"""
],
name="specialops", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl,
dict(a=np.float32, b=np.float32, g=np.float64, h=np.float64))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[
(np.dtype(np.float32), 'uniform', 'load')
].eval_with_dict(params)
f64 = poly[
(np.dtype(np.float64), 'uniform', 'load')
].eval_with_dict(params)
assert f32 == 2*n*m*l
assert f64 == 2*n*m
f32 = poly[
(np.dtype(np.float32), 'uniform', 'store')
].eval_with_dict(params)
f64 = poly[
(np.dtype(np.float64), 'uniform', 'store')
].eval_with_dict(params)
assert f32 == n*m*l
assert f64 == n*m
def test_gmem_access_counter_basic():
knl = lp.make_kernel(
"[n,m,l] -> {[i,k,j]: 0<=i<n and 0<=k<m and 0<=j<l}",
[
"""
c[i, j, k] = a[i,j,k]*b[i,j,k]/3.0+a[i,j,k]
e[i, k] = g[i,k]*h[i,k+1]
"""
],
name="basic", assumptions="n,m,l >= 1")
knl = lp.add_and_infer_dtypes(knl,
dict(a=np.float32, b=np.float32, g=np.float64, h=np.float64))
poly = lp.get_gmem_access_poly(knl)
n = 512
m = 256
l = 128
params = {'n': n, 'm': m, 'l': l}
f32 = poly[
(np.dtype(np.float32), 'uniform', 'load')
].eval_with_dict(params)
f64 = poly[
(np.dtype(np.float64), 'uniform', 'load')
].eval_with_dict(params)
assert f32 == 3*n*m*l
assert f64 == 2*n*m
f32 = poly[
(np.dtype(np.float32), 'uniform', 'store')
].eval_with_dict(params)
f64 = poly[
(np.dtype(np.float64), 'uniform', 'store')
].eval_with_dict(params)
assert f32 == n*m*l
assert f64 == n*m
def test_gnuma_horiz_kernel(ctx_factory, ilp_multiple, Nq, opt_level):
ctx = ctx_factory()
filename = "strongVolumeKernels.f90"
with open(filename, "r") as sourcef:
source = sourcef.read()
source = source.replace("datafloat", "real*4")
hsv_r, hsv_s = [
knl
for knl in lp.parse_fortran(source, filename, auto_dependencies=False)
if "KernelR" in knl.name or "KernelS" in knl.name
]
hsv_r = lp.tag_instructions(hsv_r, "rknl")
hsv_s = lp.tag_instructions(hsv_s, "sknl")
hsv = lp.fuse_kernels([hsv_r, hsv_s], ["_r", "_s"])
#hsv = hsv_s
from gnuma_loopy_transforms import (fix_euler_parameters,
set_q_storage_format,
set_D_storage_format)
hsv = lp.fix_parameters(hsv, Nq=Nq)
hsv = lp.set_loop_priority(hsv, "e,k,j,i")
hsv = lp.tag_inames(hsv, dict(e="g.0", j="l.1", i="l.0"))
hsv = lp.assume(hsv, "elements >= 1")
hsv = fix_euler_parameters(hsv, p_p0=1, p_Gamma=1.4, p_R=1)
for name in ["Q", "rhsQ"]:
hsv = set_q_storage_format(hsv, name)
hsv = set_D_storage_format(hsv)
#hsv = lp.add_prefetch(hsv, "volumeGeometricFactors")
ref_hsv = hsv
if opt_level == 0:
tap_hsv = hsv
hsv = lp.add_prefetch(hsv, "D[:,:]")
if opt_level == 1:
tap_hsv = hsv
# turn the first reads into subst rules
local_prep_var_names = set()
for insn in lp.find_instructions(hsv, "tag:local_prep"):
assignee, = insn.assignee_var_names()
local_prep_var_names.add(assignee)
hsv = lp.assignment_to_subst(hsv, assignee)
# precompute fluxes
hsv = lp.assignment_to_subst(hsv, "JinvD_r")
hsv = lp.assignment_to_subst(hsv, "JinvD_s")
r_fluxes = lp.find_instructions(hsv, "tag:compute_fluxes and tag:rknl")
s_fluxes = lp.find_instructions(hsv, "tag:compute_fluxes and tag:sknl")
if ilp_multiple > 1:
hsv = lp.split_iname(hsv, "k", 2, inner_tag="ilp")
ilp_inames = ("k_inner", )
flux_ilp_inames = ("kk", )
else:
ilp_inames = ()
flux_ilp_inames = ()
rtmps = []
stmps = []
flux_store_idx = 0
for rflux_insn, sflux_insn in zip(r_fluxes, s_fluxes):
for knl_tag, insn, flux_inames, tmps, flux_precomp_inames in [
("rknl", rflux_insn, (
"j",
"n",
), rtmps, (
"jj",
"ii",
)),
("sknl", sflux_insn, (
"i",
"n",
), stmps, (
"ii",
"jj",
)),
]:
flux_var, = insn.assignee_var_names()
print(insn)
reader, = lp.find_instructions(
hsv,
"tag:{knl_tag} and reads:{flux_var}".format(knl_tag=knl_tag,
flux_var=flux_var))
hsv = lp.assignment_to_subst(hsv, flux_var)
flux_store_name = "flux_store_%d" % flux_store_idx
flux_store_idx += 1
tmps.append(flux_store_name)
hsv = lp.precompute(hsv,
flux_var + "_subst",
flux_inames + ilp_inames,
temporary_name=flux_store_name,
precompute_inames=flux_precomp_inames +
flux_ilp_inames,
default_tag=None)
if flux_var.endswith("_s"):
hsv = lp.tag_array_axes(hsv, flux_store_name, "N0,N1,N2?")
else:
hsv = lp.tag_array_axes(hsv, flux_store_name, "N1,N0,N2?")
n_iname = "n_" + flux_var.replace("_r", "").replace("_s", "")
if n_iname.endswith("_0"):
n_iname = n_iname[:-2]
hsv = lp.rename_iname(hsv,
"n",
n_iname,
within="id:" + reader.id,
existing_ok=True)
hsv = lp.tag_inames(hsv, dict(ii="l.0", jj="l.1"))
for iname in flux_ilp_inames:
hsv = lp.tag_inames(hsv, {iname: "ilp"})
hsv = lp.alias_temporaries(hsv, rtmps)
hsv = lp.alias_temporaries(hsv, stmps)
if opt_level == 2:
tap_hsv = hsv
for prep_var_name in local_prep_var_names:
if prep_var_name.startswith("Jinv") or "_s" in prep_var_name:
continue
hsv = lp.precompute(
hsv, lp.find_one_rule_matching(hsv, prep_var_name + "_*subst*"))
if opt_level == 3:
tap_hsv = hsv
hsv = lp.add_prefetch(hsv, "Q[ii,jj,k,:,:,e]", sweep_inames=ilp_inames)
if opt_level == 4:
tap_hsv = hsv
tap_hsv = lp.tag_inames(
tap_hsv, dict(Q_dim_field_inner="unr", Q_dim_field_outer="unr"))
hsv = lp.buffer_array(hsv,
"rhsQ",
ilp_inames,
fetch_bounding_box=True,
default_tag="for",
init_expression="0",
store_expression="base + buffer")
if opt_level == 5:
tap_hsv = hsv
tap_hsv = lp.tag_inames(
tap_hsv,
dict(rhsQ_init_field_inner="unr",
rhsQ_store_field_inner="unr",
rhsQ_init_field_outer="unr",
rhsQ_store_field_outer="unr",
Q_dim_field_inner="unr",
Q_dim_field_outer="unr"))
# buffer axes need to be vectorized in order for this to work
hsv = lp.tag_array_axes(hsv, "rhsQ_buf", "c?,vec,c")
hsv = lp.tag_array_axes(hsv, "Q_fetch", "c?,vec,c")
hsv = lp.tag_array_axes(hsv, "D_fetch", "f,f")
hsv = lp.tag_inames(hsv, {
"Q_dim_k": "unr",
"rhsQ_init_k": "unr",
"rhsQ_store_k": "unr"
},
ignore_nonexistent=True)
if opt_level == 6:
tap_hsv = hsv
tap_hsv = lp.tag_inames(
tap_hsv,
dict(rhsQ_init_field_inner="unr",
rhsQ_store_field_inner="unr",
rhsQ_init_field_outer="unr",
rhsQ_store_field_outer="unr",
Q_dim_field_inner="unr",
Q_dim_field_outer="unr"))
hsv = lp.tag_inames(
hsv,
dict(rhsQ_init_field_inner="vec",
rhsQ_store_field_inner="vec",
rhsQ_init_field_outer="unr",
rhsQ_store_field_outer="unr",
Q_dim_field_inner="vec",
Q_dim_field_outer="unr"))
if opt_level == 7:
tap_hsv = hsv
hsv = lp.collect_common_factors_on_increment(
hsv, "rhsQ_buf", vary_by_axes=(0, ) if ilp_multiple > 1 else ())
if opt_level >= 8:
tap_hsv = hsv
hsv = tap_hsv
if 1:
print("OPS")
op_poly = lp.get_op_poly(hsv)
print(lp.stringify_stats_mapping(op_poly))
print("MEM")
gmem_poly = lp.sum_mem_access_to_bytes(lp.get_gmem_access_poly(hsv))
print(lp.stringify_stats_mapping(gmem_poly))
hsv = lp.set_options(hsv,
cl_build_options=[
"-cl-denorms-are-zero",
"-cl-fast-relaxed-math",
"-cl-finite-math-only",
"-cl-mad-enable",
"-cl-no-signed-zeros",
])
hsv = hsv.copy(name="horizontalStrongVolumeKernel")
results = lp.auto_test_vs_ref(ref_hsv,
ctx,
hsv,
parameters=dict(elements=300),
quiet=True)
elapsed = results["elapsed_wall"]
print("elapsed", elapsed)
def test_gnuma_horiz_kernel(ctx_factory, ilp_multiple, Nq, opt_level):
ctx = ctx_factory()
filename = "strongVolumeKernels.f90"
with open(filename, "r") as sourcef:
source = sourcef.read()
source = source.replace("datafloat", "real*4")
hsv_r, hsv_s = [
knl for knl in lp.parse_fortran(source, filename, auto_dependencies=False)
if "KernelR" in knl.name or "KernelS" in knl.name
]
hsv_r = lp.tag_instructions(hsv_r, "rknl")
hsv_s = lp.tag_instructions(hsv_s, "sknl")
hsv = lp.fuse_kernels([hsv_r, hsv_s], ["_r", "_s"])
#hsv = hsv_s
from gnuma_loopy_transforms import (
fix_euler_parameters,
set_q_storage_format, set_D_storage_format)
hsv = lp.fix_parameters(hsv, Nq=Nq)
hsv = lp.set_loop_priority(hsv, "e,k,j,i")
hsv = lp.tag_inames(hsv, dict(e="g.0", j="l.1", i="l.0"))
hsv = lp.assume(hsv, "elements >= 1")
hsv = fix_euler_parameters(hsv, p_p0=1, p_Gamma=1.4, p_R=1)
for name in ["Q", "rhsQ"]:
hsv = set_q_storage_format(hsv, name)
hsv = set_D_storage_format(hsv)
#hsv = lp.add_prefetch(hsv, "volumeGeometricFactors")
ref_hsv = hsv
if opt_level == 0:
tap_hsv = hsv
hsv = lp.add_prefetch(hsv, "D[:,:]")
if opt_level == 1:
tap_hsv = hsv
# turn the first reads into subst rules
local_prep_var_names = set()
for insn in lp.find_instructions(hsv, "tag:local_prep"):
assignee, = insn.assignee_var_names()
local_prep_var_names.add(assignee)
hsv = lp.assignment_to_subst(hsv, assignee)
# precompute fluxes
hsv = lp.assignment_to_subst(hsv, "JinvD_r")
hsv = lp.assignment_to_subst(hsv, "JinvD_s")
r_fluxes = lp.find_instructions(hsv, "tag:compute_fluxes and tag:rknl")
s_fluxes = lp.find_instructions(hsv, "tag:compute_fluxes and tag:sknl")
if ilp_multiple > 1:
hsv = lp.split_iname(hsv, "k", 2, inner_tag="ilp")
ilp_inames = ("k_inner",)
flux_ilp_inames = ("kk",)
else:
ilp_inames = ()
flux_ilp_inames = ()
rtmps = []
stmps = []
flux_store_idx = 0
for rflux_insn, sflux_insn in zip(r_fluxes, s_fluxes):
for knl_tag, insn, flux_inames, tmps, flux_precomp_inames in [
("rknl", rflux_insn, ("j", "n",), rtmps, ("jj", "ii",)),
("sknl", sflux_insn, ("i", "n",), stmps, ("ii", "jj",)),
]:
flux_var, = insn.assignee_var_names()
print(insn)
reader, = lp.find_instructions(hsv,
"tag:{knl_tag} and reads:{flux_var}"
.format(knl_tag=knl_tag, flux_var=flux_var))
hsv = lp.assignment_to_subst(hsv, flux_var)
flux_store_name = "flux_store_%d" % flux_store_idx
flux_store_idx += 1
tmps.append(flux_store_name)
hsv = lp.precompute(hsv, flux_var+"_subst", flux_inames + ilp_inames,
temporary_name=flux_store_name,
precompute_inames=flux_precomp_inames + flux_ilp_inames,
default_tag=None)
if flux_var.endswith("_s"):
hsv = lp.tag_data_axes(hsv, flux_store_name, "N0,N1,N2?")
else:
hsv = lp.tag_data_axes(hsv, flux_store_name, "N1,N0,N2?")
n_iname = "n_"+flux_var.replace("_r", "").replace("_s", "")
if n_iname.endswith("_0"):
n_iname = n_iname[:-2]
hsv = lp.rename_iname(hsv, "n", n_iname, within="id:"+reader.id,
existing_ok=True)
hsv = lp.tag_inames(hsv, dict(ii="l.0", jj="l.1"))
for iname in flux_ilp_inames:
hsv = lp.tag_inames(hsv, {iname: "ilp"})
hsv = lp.alias_temporaries(hsv, rtmps)
hsv = lp.alias_temporaries(hsv, stmps)
if opt_level == 2:
tap_hsv = hsv
for prep_var_name in local_prep_var_names:
if prep_var_name.startswith("Jinv") or "_s" in prep_var_name:
continue
hsv = lp.precompute(hsv,
lp.find_one_rule_matching(hsv, prep_var_name+"_*subst*"))
if opt_level == 3:
tap_hsv = hsv
hsv = lp.add_prefetch(hsv, "Q[ii,jj,k,:,:,e]", sweep_inames=ilp_inames)
if opt_level == 4:
tap_hsv = hsv
tap_hsv = lp.tag_inames(tap_hsv, dict(
Q_dim_field_inner="unr",
Q_dim_field_outer="unr"))
hsv = lp.buffer_array(hsv, "rhsQ", ilp_inames,
fetch_bounding_box=True, default_tag="for",
init_expression="0", store_expression="base + buffer")
if opt_level == 5:
tap_hsv = hsv
tap_hsv = lp.tag_inames(tap_hsv, dict(
rhsQ_init_field_inner="unr", rhsQ_store_field_inner="unr",
rhsQ_init_field_outer="unr", rhsQ_store_field_outer="unr",
Q_dim_field_inner="unr",
Q_dim_field_outer="unr"))
# buffer axes need to be vectorized in order for this to work
hsv = lp.tag_data_axes(hsv, "rhsQ_buf", "c?,vec,c")
hsv = lp.tag_data_axes(hsv, "Q_fetch", "c?,vec,c")
hsv = lp.tag_data_axes(hsv, "D_fetch", "f,f")
hsv = lp.tag_inames(hsv,
{"Q_dim_k": "unr", "rhsQ_init_k": "unr", "rhsQ_store_k": "unr"},
ignore_nonexistent=True)
if opt_level == 6:
tap_hsv = hsv
tap_hsv = lp.tag_inames(tap_hsv, dict(
rhsQ_init_field_inner="unr", rhsQ_store_field_inner="unr",
rhsQ_init_field_outer="unr", rhsQ_store_field_outer="unr",
Q_dim_field_inner="unr",
Q_dim_field_outer="unr"))
hsv = lp.tag_inames(hsv, dict(
rhsQ_init_field_inner="vec", rhsQ_store_field_inner="vec",
rhsQ_init_field_outer="unr", rhsQ_store_field_outer="unr",
Q_dim_field_inner="vec",
Q_dim_field_outer="unr"))
if opt_level == 7:
tap_hsv = hsv
hsv = lp.collect_common_factors_on_increment(hsv, "rhsQ_buf",
vary_by_axes=(0,) if ilp_multiple > 1 else ())
if opt_level >= 8:
tap_hsv = hsv
hsv = tap_hsv
if 1:
print("OPS")
op_poly = lp.get_op_poly(hsv)
print(lp.stringify_stats_mapping(op_poly))
print("MEM")
gmem_poly = lp.sum_mem_access_to_bytes(lp.get_gmem_access_poly(hsv))
print(lp.stringify_stats_mapping(gmem_poly))
hsv = lp.set_options(hsv, cl_build_options=[
"-cl-denorms-are-zero",
"-cl-fast-relaxed-math",
"-cl-finite-math-only",
"-cl-mad-enable",
"-cl-no-signed-zeros",
])
hsv = hsv.copy(name="horizontalStrongVolumeKernel")
results = lp.auto_test_vs_ref(ref_hsv, ctx, hsv, parameters=dict(elements=300),
quiet=True)
elapsed = results["elapsed_wall"]
print("elapsed", elapsed)