def inspecs_params():
inspecs = []
u = I.UniformInitializer((0.5, 1.0))
inspecs.append([Inspec((64, 1000), u)])
inspecs.append([Inspec((64, 32, 224, 224), u)])
inspecs.append([Inspec((64, 128, 56, 56), u)])
return inspecs
def pairwise_inspecs_params():
inspecs = []
u = I.UniformInitializer((0, 2))
inspecs.append(
[Inspec((64, 32, 224, 224), u),
Inspec((64, 32, 224, 224), u)])
return inspecs
def conv_params():
inputs = [Inspec((64, 3, 224, 224))]
func_kwargs = dict(
outmaps=64,
kernel=(7, 7),
pad=(3, 3),
stride=(2, 2),
with_bias=False)
list_params = [(inputs, func_kwargs)]
inputs = [Inspec((64, 64, 56, 56))]
func_kwargs = dict(
outmaps=64,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=False)
list_params.append((inputs, func_kwargs))
inputs = [Inspec((64, 512, 7, 7))]
func_kwargs = dict(
outmaps=512,
kernel=(3, 3),
pad=(1, 1),
stride=(1, 1),
with_bias=False)
list_params.append((inputs, func_kwargs))
return list_params
def pad_params():
inspecs = []
u = I.UniformInitializer((0.5, 1.0))
inspecs.append([Inspec((2, 2, 2, 2), u)])
inspecs.append([Inspec((2, 3, 2, 3), u)])
inspecs.append([Inspec((2, 20, 200, 200), u)])
return inspecs
def affine_params():
"""(inspecs, n_outmaps)"""
params = []
params.append(([Inspec((128, 1, 28, 28))], 64))
params.append(([Inspec((64, 512, 7, 7))], 4096))
params.append(([Inspec((64, 4096))], 1000))
params.append(([Inspec((64, 512))], 1000))
return params
def reduction_inspecs_params():
inspecs = []
inspecs.append([Inspec((2048, 8192 * 16))])
inspecs.append([Inspec((2048, 8192 * 2))])
inspecs.append([Inspec((64, 1000))])
inspecs.append([Inspec((64, 32, 224, 224))])
inspecs.append([Inspec((64, 512, 7, 7))])
return inspecs
def embed_params():
"""(inspecs, n_inputs, n_features)"""
params = []
params.append(([
Inspec((64, ), I.UniformIntInitializer((0, 40000)), need_grad=False)
], 40000, 256))
return params
def inspecs_params():
inspecs = []
inspecs.append(([Inspec((64, 80, 224, 224))], 1))
inspecs.append(([Inspec((64, 224, 224, 80))], 3))
inspecs.append(([Inspec((64, 20, 224, 224))], 1))
inspecs.append(([Inspec((64, 224, 224, 20))], 3))
inspecs.append(([Inspec((1, 1000))], 1))
inspecs.append(([Inspec((64, 1000))], 1))
inspecs.append(([Inspec((768, 50, 50))], 2))
return inspecs
def pairwise_inspecs_params(label_init=I.UniformIntInitializer((0, 2))):
inspecs = []
inspecs.append([Inspec((64, 1000)), Inspec((64, 1000), label_init, False)])
inspecs.append([
Inspec((64, 32, 224, 224)),
Inspec((64, 32, 224, 224), label_init, False)
])
inspecs.append([
Inspec((64, 128, 56, 56)),
Inspec((64, 128, 56, 56), label_init, False)
])
return inspecs
def classification_inspecs_params():
inspecs = []
ui = I.UniformIntInitializer
inspecs.append([Inspec((64, 1000)), Inspec((64, 1), ui((0, 1000)), False)])
inspecs.append([
Inspec((64, 32, 224, 224)),
Inspec((64, 1, 224, 224), ui((0, 32)), False)
])
inspecs.append([
Inspec((64, 128, 56, 56)),
Inspec((64, 1, 56, 56), ui((0, 128)), False)
])
return inspecs
def test_cumprod(seed, test_case, exclusive, reverse, with_mask, nnabla_opts):
x_shape = test_case.shape
axis = test_case.axis
def init(shape):
rng = np.random.RandomState(seed)
return create_cumprod_input(rng, shape, axis, with_mask)
need_grad = True
inputs = [Inspec(x_shape, init, need_grad)]
func_kwargs = dict(
axis=axis,
exclusive=exclusive,
reverse=reverse,
)
fb = FunctionBenchmark(F.cumprod, inputs, [], func_kwargs, nnabla_opts.ext,
nnabla_opts.ext_kwargs)
fb.benchmark()
fb.write(writer=nnabla_opts.function_benchmark_writer)
def large_reduction_cases_for_instance_norm(channel_last):
'''
Cases for checking large reduction size for instance normalization
'''
inspec_and_axis = []
batch = 1
ch = 256
res_mul = [1, 2, 4, 8]
base_res = 128
resolutions = [base_res * factor for factor in res_mul]
axis = 3 if channel_last else 1
# Create `inspec_and_axis`
for res in resolutions:
if channel_last:
shape = (batch, res, res, ch)
else:
shape = (batch, ch, res, res)
inspec_and_axis.append(([Inspec(shape)], axis))
return inspec_and_axis
def large_reduction_cases_for_group_norm(channel_last):
'''
Cases for checking large reduction size for group normalization
'''
inspec_and_axis = []
batch = 1
base_ch = 256
ch_mul = [1, 2, 4, 8, 16, 32]
channels = [base_ch * factor for factor in ch_mul]
res = 128
axis = 3 if channel_last else 1
# Create `inspec_and_axis`
for ch in channels:
if channel_last:
shape = (batch, res, res, ch)
else:
shape = (batch, ch, res, res)
inspec_and_axis.append(([Inspec(shape)], axis))
return inspec_and_axis
def general_cases(channel_last):
'''
Generate use cases
'''
inspec_and_axis = []
batch = 16
base_ch = 192
ch_mul = [1, 1, 2, 2, 4, 4]
channels = [base_ch * factor for factor in ch_mul]
resolutions = [256, 128, 64, 32, 16, 8]
axis = 3 if channel_last else 1
# Create `inspec_and_axis`
for ch, res in zip(channels, resolutions):
if channel_last:
shape = (batch, res, res, ch)
else:
shape = (batch, ch, res, res)
inspec_and_axis.append(([Inspec(shape)], axis))
return inspec_and_axis
def inspecs_params():
inspecs = []
inspecs.append([Inspec((64, 64, 224, 224))])
inspecs.append([Inspec((64, 128, 112, 112))])
inspecs.append([Inspec((64, 512, 14, 14))])
return inspecs
def inspecs_params():
inspecs = []
inspecs.append([Inspec((64, 64, 112, 112))])
inspecs.append([Inspec((64, 64, 56, 56))])
inspecs.append([Inspec((64, 512, 7, 7))])
return inspecs
def resnet50_inspecs_params_with_broadcast():
inspecs = []
u = I.UniformInitializer((0.5, 1.0))
inspecs.append([Inspec((5, 1024, 14, 14), u), Inspec((1, 1024, 1, 1), u)])
inspecs.append(
[Inspec((5, 1024, 14, 14), u),
Inspec((1, 1024, 14, 14), u)])
inspecs.append([Inspec((5, 112, 112, 64), u), Inspec((1, 1, 1, 64), u)])
inspecs.append(
[Inspec((5, 112, 112, 64), u),
Inspec((1, 112, 112, 64), u)])
inspecs.append([Inspec((5, 128, 28, 28), u), Inspec((1, 128, 1, 1), u)])
inspecs.append([Inspec((5, 128, 28, 28), u), Inspec((1, 128, 28, 28), u)])
inspecs.append([Inspec((5, 128, 56, 56), u), Inspec((1, 128, 1, 1), u)])
inspecs.append([Inspec((5, 128, 56, 56), u), Inspec((1, 128, 56, 56), u)])
inspecs.append([Inspec((5, 14, 14, 1024), u), Inspec((1, 1, 1, 1024), u)])
inspecs.append(
[Inspec((5, 14, 14, 1024), u),
Inspec((1, 14, 14, 1024), u)])
inspecs.append([Inspec((5, 14, 14, 256), u), Inspec((1, 1, 1, 256), u)])
inspecs.append([Inspec((5, 14, 14, 256), u), Inspec((1, 14, 14, 256), u)])
inspecs.append([Inspec((5, 14, 14, 512), u), Inspec((1, 1, 1, 512), u)])
inspecs.append([Inspec((5, 14, 14, 512), u), Inspec((1, 14, 14, 512), u)])
inspecs.append([Inspec((5, 2048, 7, 7), u), Inspec((1, 2048, 1, 1), u)])
inspecs.append([Inspec((5, 2048, 7, 7), u), Inspec((1, 2048, 7, 7), u)])
inspecs.append([Inspec((5, 256, 14, 14), u), Inspec((1, 256, 1, 1), u)])
inspecs.append([Inspec((5, 256, 14, 14), u), Inspec((1, 256, 14, 14), u)])
inspecs.append([Inspec((5, 256, 28, 28), u), Inspec((1, 256, 1, 1), u)])
inspecs.append([Inspec((5, 256, 28, 28), u), Inspec((1, 256, 28, 28), u)])
inspecs.append([Inspec((5, 256, 56, 56), u), Inspec((1, 256, 1, 1), u)])
inspecs.append([Inspec((5, 256, 56, 56), u), Inspec((1, 256, 56, 56), u)])
inspecs.append([Inspec((5, 28, 28, 128), u), Inspec((1, 1, 1, 128), u)])
inspecs.append([Inspec((5, 28, 28, 128), u), Inspec((1, 28, 28, 128), u)])
inspecs.append([Inspec((5, 28, 28, 256), u), Inspec((1, 1, 1, 256), u)])
inspecs.append([Inspec((5, 28, 28, 256), u), Inspec((1, 28, 28, 256), u)])
inspecs.append([Inspec((5, 28, 28, 512), u), Inspec((1, 1, 1, 512), u)])
inspecs.append([Inspec((5, 28, 28, 512), u), Inspec((1, 28, 28, 512), u)])
inspecs.append([Inspec((5, 512, 14, 14), u), Inspec((1, 512, 1, 1), u)])
inspecs.append([Inspec((5, 512, 14, 14), u), Inspec((1, 512, 14, 14), u)])
inspecs.append([Inspec((5, 512, 28, 28), u), Inspec((1, 512, 1, 1), u)])
inspecs.append([Inspec((5, 512, 28, 28), u), Inspec((1, 512, 28, 28), u)])
inspecs.append([Inspec((5, 512, 7, 7), u), Inspec((1, 512, 1, 1), u)])
inspecs.append([Inspec((5, 512, 7, 7), u), Inspec((1, 512, 7, 7), u)])
inspecs.append([Inspec((5, 56, 56, 128), u), Inspec((1, 1, 1, 128), u)])
inspecs.append([Inspec((5, 56, 56, 128), u), Inspec((1, 56, 56, 128), u)])
inspecs.append([Inspec((5, 56, 56, 256), u), Inspec((1, 1, 1, 256), u)])
inspecs.append([Inspec((5, 56, 56, 256), u), Inspec((1, 56, 56, 256), u)])
inspecs.append([Inspec((5, 56, 56, 64), u), Inspec((1, 1, 1, 64), u)])
inspecs.append([Inspec((5, 56, 56, 64), u), Inspec((1, 56, 56, 64), u)])
inspecs.append([Inspec((5, 64, 112, 112), u), Inspec((1, 64, 1, 1), u)])
inspecs.append(
[Inspec((5, 64, 112, 112), u),
Inspec((1, 64, 112, 112), u)])
inspecs.append([Inspec((5, 64, 56, 56), u), Inspec((1, 64, 1, 1), u)])
inspecs.append([Inspec((5, 64, 56, 56), u), Inspec((1, 64, 56, 56), u)])
inspecs.append([Inspec((5, 7, 7, 2048), u), Inspec((1, 1, 1, 2048), u)])
inspecs.append([Inspec((5, 7, 7, 2048), u), Inspec((1, 7, 7, 2048), u)])
inspecs.append([Inspec((5, 7, 7, 512), u), Inspec((1, 1, 1, 512), u)])
inspecs.append([Inspec((5, 7, 7, 512), u), Inspec((1, 7, 7, 512), u)])
return inspecs
def inspecs_params():
inspecs = []
# Reported bad performance cases
# These three cases were optimized well by cuTENSOR.
inspecs.append(([Inspec((32, 144, 28, 1))], (0, 1, 3, 2)))
inspecs.append(([Inspec((32, 144, 28, 3))], (0, 1, 3, 2)))
inspecs.append(([Inspec((768, 50, 50))], (0, 2, 1)))
# From ResNet-50
# Input side
inspecs.append(([Inspec((192, 224, 224, 3))], (0, 3, 1, 2)))
inspecs.append(([Inspec((192, 3, 224, 224))], (0, 2, 3, 1)))
# Output side
inspecs.append(([Inspec((192, 3, 3, 512))], (0, 3, 1, 2)))
inspecs.append(([Inspec((192, 512, 3, 3))], (0, 2, 3, 1)))
inspecs.append(([Inspec((192, 1, 1, 2048))], (0, 3, 1, 2)))
inspecs.append(([Inspec((192, 2048, 1, 1))], (0, 2, 3, 1)))
# Single input
inspecs.append(([Inspec((1, 224, 224, 3))], (0, 3, 1, 2)))
inspecs.append(([Inspec((1, 3, 224, 224))], (0, 2, 3, 1)))
inspecs.append(([Inspec((1, 3, 3, 512))], (0, 3, 1, 2)))
inspecs.append(([Inspec((1, 512, 3, 3))], (0, 2, 3, 1)))
inspecs.append(([Inspec((1, 1, 1, 2048))], (0, 3, 1, 2)))
inspecs.append(([Inspec((1, 2048, 1, 1))], (0, 2, 3, 1)))
# Other
# 2D
inspecs.append(([Inspec((64, 64))], (1, 0)))
inspecs.append(([Inspec((1024, 1024))], (1, 0)))
# 4D
inspecs.append(([Inspec((64, 64, 64, 64))], (0, 1, 2, 3)))
inspecs.append(([Inspec((64, 64, 64, 64))], (0, 1, 3, 2)))
inspecs.append(([Inspec((64, 64, 64, 64))], (0, 3, 2, 1)))
inspecs.append(([Inspec((64, 64, 64, 64))], (0, 2, 1, 3)))
inspecs.append(([Inspec((64, 64, 64, 64))], (0, 3, 1, 2)))
inspecs.append(([Inspec((64, 64, 64, 64))], (0, 2, 3, 1)))
# 4D misaligned
inspecs.append(([Inspec((65, 65, 65, 65))], (0, 1, 2, 3)))
inspecs.append(([Inspec((65, 65, 65, 65))], (0, 1, 3, 2)))
inspecs.append(([Inspec((65, 65, 65, 65))], (0, 3, 2, 1)))
inspecs.append(([Inspec((65, 65, 65, 65))], (0, 2, 1, 3)))
inspecs.append(([Inspec((65, 65, 65, 65))], (0, 3, 1, 2)))
inspecs.append(([Inspec((65, 65, 65, 65))], (0, 2, 3, 1)))
return inspecs