def test_nnabla_models_fixed_size(model_class, up_to_list, batch_size,
training, seed):
model_module = importlib.import_module("nnabla.models.imagenet")
nn.clear_parameters()
rng = np.random.RandomState(seed)
model = getattr(model_module, model_class)()
# Determine input shape and create input variable
input_shape = list(model.input_shape)
input_shape = tuple(input_shape)
x = nn.Variable.from_numpy_array(
rng.randint(0, 256, size=(batch_size, ) + input_shape))
# Test cases for all intermediate outputs
for use_up_to in up_to_list:
returns_net = False
def _execute():
y = model(x, training=training, use_up_to=use_up_to)
y.forward()
_execute()
net = model(x,
training=training,
use_up_to=use_up_to,
returns_net=True)
assert isinstance(net, NnpNetwork)
assert len(net.inputs.values()) == 1
assert len(net.outputs.values()) == 1
y = list(net.outputs.values())[0]
if training:
assert _check_trainable_parameters(y)
else:
assert not _check_trainable_parameters(y)
def __init__(self, args):
class Config:
pass
self._config = Config()
self._config.context = 'cpu'
self._config.device_id = 0
self._config.columns_size = 2
self._config.x_length = args.x_length
self._config.mlp_model_params_path = args.mlp_model_params_path
self._config.labels_path = args.labels_path
if not os.path.isfile(self._config.mlp_model_params_path):
logger.error("Model params path {} is not found.".format(self._config.mlp_model_params_path))
sys.exit(-1)
else:
logger.info("Path of the model parameters file is {}.".format(self._config.mlp_model_params_path))
if not os.path.isfile(self._config.labels_path):
logger.error("Labels path {} is not found.".format(self._config.labels_path))
sys.exit(-1)
else:
logger.info("Path of the labels file is {}.".format(self._config.labels_path))
seed(0)
logger.info("Running in %s" % self._config.context)
self._ctx = get_extension_context(self._config.context, device_id = self._config.device_id)
nn.set_default_context(self._ctx)
nn.clear_parameters()
self._mlp = MLP(self._config)
self._labels = None
with open(self._config.labels_path) as f:
self._labels = f.readlines()
self._points_buf = pointsbuffer.PointsBuffer()
def compute(grid_size, D, L, initial_sphere_radius):
nn.clear_parameters()
# 2D case
x = np.linspace(-1, 1, grid_size)
y = np.linspace(-1, 1, grid_size)
xx, yy = np.meshgrid(x, y)
xy = np.asarray([xx.flatten(), yy.flatten()]).T
x = nn.Variable.from_numpy_array(xy)
y = implicit_network(x,
D,
feature_size=256,
L=L,
initial_sphere_radius=initial_sphere_radius)
y = y[:, 0]
y.forward()
# Plot
plt.contourf(xx, yy, y.d.reshape(grid_size, grid_size))
plt.colorbar()
plt.contourf(xx,
yy,
y.d.reshape(grid_size, grid_size),
colors=('r', ),
levels=[-0.01, 0, 0.01])
plt.title(f"Contour of SDF(x, y) = 0 (D={D:03d} and L={L:03d})")
plt.savefig(f"sdf_contour_D{D:03d}_L{L:03d}.png")
plt.clf()
# error
gt = np.sqrt(np.sum(xy**2, axis=1)) - 1
mae = np.sum(np.abs(gt - y.d.flatten())) / len(gt)
return mae
def test_istft(window_size, stride, fft_size, window_type, center):
# clear all previous STFT conv/deconv kernels
nn.clear_parameters()
# Make sure that iSTFT(STFT(x)) = x
x = np.random.randn(1, window_size * 10)
nx = nn.Variable.from_numpy_array(x)
nyr, nyi = F.stft(nx,
window_size=window_size,
stride=stride,
fft_size=fft_size,
window_type=window_type,
center=center)
nz = F.istft(nyr,
nyi,
window_size=window_size,
stride=stride,
fft_size=fft_size,
window_type=window_type,
center=center)
nz.forward()
invalid = window_size - stride
assert (np.allclose(nx.d[:, invalid:-invalid],
nz.d[:, invalid:-invalid],
atol=1e-5,
rtol=1e-5))
def test_stft(window_size, stride, fft_size, window_type):
# clear all previous STFT conv/deconv kernels
nn.clear_parameters()
# Compare to `scipy.signal.stft` - only done if SciPy available
x = np.random.randn(1, window_size * 10)
nx = nn.Variable.from_numpy_array(x)
nyr, nyi = F.stft(nx,
window_size=window_size,
stride=stride,
fft_size=fft_size,
window_type=window_type,
center=False)
nn.forward_all([nyr, nyi])
stft_nnabla = nyr.d + 1j * nyi.d
_f, _t, stft_scipy = sig.stft(x,
window=window_type,
nperseg=window_size,
noverlap=window_size - stride,
nfft=fft_size,
boundary=None,
padded=False)
# scipy does a different scaling - take care here
stft_nnabla /= fft_size // 2
assert (np.allclose(stft_nnabla, stft_scipy, atol=1e-5, rtol=1e-5))
def __init__(self,
func,
inspecs,
func_args,
func_kwargs,
ext,
ext_kwargs,
min_run=1,
min_time=1.0):
nn.clear_parameters()
self.inputs = None
self.outputs = None
self.func_ins = None
self.setup_stat = None
self.forward_stat = None
self.backward_stat = None
self.func = func
self.module = func.__module__
self.inspecs = inspecs
self.inputs_f = create_inputs(inspecs)
self.func_args = func_args
self.func_kwargs = func_kwargs
self.ext = ext
self.ext_kwargs = ext_kwargs
self.mod_ext = importlib.import_module('.' + ext, 'nnabla_ext')
self.ctx = self.mod_ext.context(**ext_kwargs)
self.min_run = min_run
self.min_time = min_time
def main():
batch_size, m, h, w = 4, 3, 32, 32
extension_module = "cpu"
device_id = 0
ctx = extension_context(extension_module, device_id=device_id)
x_l_data = np.random.randn(batch_size, m, h, w)
y_l_data = (np.random.rand(batch_size, 1) * 10).astype(np.int32)
x_l = nn.Variable(x_l_data.shape)
y_l = nn.Variable(y_l_data.shape)
x_l.d = x_l_data
y_l.d = y_l_data
# CNN
print("# CNN")
pred = cnn_model_003(ctx, x_l)
s = 0
for n, v in nn.get_parameters().iteritems():
n_params = np.prod(v.shape)
print(n, n_params)
s += n_params
print("n_params={}".format(s))
nn.clear_parameters()
# Resnet
print("# Resnet")
inmaps = 256
pred = resnet_model(ctx, x_l, inmaps=inmaps)
s = 0
for n, v in nn.get_parameters().iteritems():
n_params = np.prod(v.shape)
print(n, n_params)
s += n_params
print("n_params={}".format(s))
nn.clear_parameters()
def main():
batch_size, m, h, w = 4, 3, 32, 32
extension_module = "cpu"
device_id = 0
ctx = extension_context(extension_module, device_id=device_id)
x_l_data = np.random.randn(batch_size, m, h, w)
y_l_data = (np.random.rand(batch_size, 1) * 10).astype(np.int32)
x_l = nn.Variable(x_l_data.shape)
y_l = nn.Variable(y_l_data.shape)
x_l.d = x_l_data
y_l.d = y_l_data
# CNN
print("# CNN")
pred = cnn_model_003(ctx, x_l)
s = 0
for n, v in nn.get_parameters().iteritems():
n_params = np.prod(v.shape)
print(n, n_params)
s += n_params
print("n_params={}".format(s))
nn.clear_parameters()
# Resnet
print("# Resnet")
inmaps = 256
pred = resnet_model(ctx, x_l, inmaps=inmaps)
s = 0
for n, v in nn.get_parameters().iteritems():
n_params = np.prod(v.shape)
print(n, n_params)
s += n_params
print("n_params={}".format(s))
nn.clear_parameters()
def test_nnabla_models_resnet(num_layers, image_size, batch_size, training, seed):
from nnabla.models.imagenet import ResNet
nn.clear_parameters()
rng = np.random.RandomState(seed)
model = ResNet(num_layers)
x = nn.Variable.from_numpy_array(rng.randint(
0, 256, size=(batch_size, 3, image_size, image_size)))
for use_up_to in ('classifier', 'pool', 'lastconv', 'lastconv+relu'):
check_global_pooling = True
force_global_pooling = False
returns_net = False
def _execute():
y = model(x, training=training, use_up_to=use_up_to, force_global_pooling=force_global_pooling,
check_global_pooling=check_global_pooling)
y.forward()
if image_size == 448 and use_up_to in ('classifier', 'pool'):
with pytest.raises(ValueError):
_execute()
if use_up_to == 'pool':
check_global_pooling = False
_execute()
force_global_pooling = True
_execute()
net = model(x, training=training, use_up_to=use_up_to, force_global_pooling=force_global_pooling,
check_global_pooling=check_global_pooling, returns_net=True)
assert isinstance(net, NnpNetwork)
assert len(net.inputs.values()) == 1
assert len(net.outputs.values()) == 1
y = list(net.outputs.values())[0]
if training:
assert _check_trainable_parameters(y)
else:
assert not _check_trainable_parameters(y)
def test_time_profiler(batch_size, n_class, ext_name, tmpdir):
nn.clear_parameters()
ctx = get_extension_context(ext_name)
nn.set_default_context(ctx)
x = nn.Variable.from_numpy_array(
np.random.normal(size=(batch_size, 3, 16, 16)))
t = nn.Variable.from_numpy_array(
np.random.randint(low=0, high=n_class, size=(batch_size, 1)))
y = simple_cnn(x, t, n_class)
tp = TimeProfiler(ext_name, device_id=ctx.device_id)
for i in range(5):
with tp.scope("forward"):
y.forward(clear_no_need_grad=True,
function_pre_hook=tp.pre_hook,
function_post_hook=tp.post_hook)
with tp.scope("backward"):
y.backward(clear_buffer=True,
function_pre_hook=tp.pre_hook,
function_post_hook=tp.post_hook)
tp.calc_elapsed_time(["forward", "backward", "summary"])
tp()
tp.to_csv(out_dir=str(tmpdir))
def test_grad_grad_resnet(seed, ctx, auto_forward, inplace, shared):
nn.clear_parameters()
# Settings
nn.set_default_context(ctx)
nn.set_auto_forward(auto_forward)
b, c, h, w = 4, 3, 32, 32
n_cls = 10
rng = np.random.RandomState(seed)
# Network
x = nn.Variable.from_numpy_array(rng.randn(b, c, h,
w)).apply(need_grad=True)
y = SmallResNet(x, inplace=inplace, shared=shared)
# Grad of grad
dx = nn.grad([y], [x])
ddx = nn.grad([dx[0]], [x])
ddx[0].forward() if not auto_forward else None
# Backward of grad
x.grad.zero()
dx[0].forward() if not auto_forward else None
dx[0].backward()
# Check between results of var.backward and nn.grad
backend = ctx.backend[0].split(":")[0]
if backend == 'cuda':
pytest.skip(
'CUDA Convolution N-D is only supported in CUDNN extension')
assert_allclose(x.g, ddx[0].d, atol=1e-6)
def test_generate_tmp_nnp():
nn.clear_parameters()
batch_size = 16
x0 = nn.Variable([batch_size, 100])
x1 = nn.Variable([batch_size, 100])
h1_0 = PF.affine(x0, 100, name='affine1_0')
h1_1 = PF.affine(x1, 100, name='affine1_0')
h1 = F.tanh(h1_0 + h1_1)
h2 = F.tanh(PF.affine(h1, 50, name='affine2'))
y0 = PF.affine(h2, 10, name='affiney_0')
y1 = PF.affine(h2, 10, name='affiney_1')
contents = {
'networks': [{
'name': 'net1',
'batch_size': batch_size,
'outputs': {
'y0': y0,
'y1': y1
},
'names': {
'x0': x0,
'x1': x1
}
}],
'executors': [{
'name': 'runtime',
'network': 'net1',
'data': ['x0', 'x1'],
'output': ['y0', 'y1']
}]
}
nnabla.utils.save.save('tmp.nnp', contents)
def test_istft(ctx, window_size, stride, fft_size, window_type, center):
backend = ctx.backend[0].split(":")[0]
if backend == 'cuda':
pytest.skip(
'CUDA Convolution N-D is only supported in CUDNN extension')
# clear all previous STFT conv/deconv kernels
nn.clear_parameters()
# Make sure that iSTFT(STFT(x)) = x
x = np.random.randn(1, window_size * 10)
nx = nn.Variable.from_numpy_array(x)
with nn.context_scope(ctx):
nyr, nyi = F._stft_v1(nx,
window_size=window_size,
stride=stride,
fft_size=fft_size,
window_type=window_type,
center=center)
nz = F._istft_v1(nyr,
nyi,
window_size=window_size,
stride=stride,
fft_size=fft_size,
window_type=window_type,
center=center)
nz.forward()
invalid = window_size - stride
assert (np.allclose(nx.d[:, invalid:-invalid],
nz.d[:, invalid:-invalid],
atol=1e-5,
rtol=1e-5))
def test_get_parameter_with_initializer():
"""Testing with initializer
"""
import nnabla as nn
from nnabla.parameter import get_parameter_or_create
nn.clear_parameters()
rng = np.random.RandomState(seed=313)
shape = (8, 8, 3, 3)
# Instnace inherited from BaseInitializer
initializer = UniformInitializer(lim=(-1, 1), rng=rng)
param1 = get_parameter_or_create('param1',
shape,
initializer=initializer,
need_grad=True)
assert np.min(param1.d > -1) and np.max(param1.d < 1)
# Numpy array
initializer = rng.randn(*shape)
param2 = get_parameter_or_create('param2',
initializer=initializer,
need_grad=True)
np.allclose(initializer, param2.d)
# Random
param3 = get_parameter_or_create('param3', shape, need_grad=True)
nn.clear_parameters()
def test_parametric_function_api():
"""
Testing :function:`nnabla.parametric_functions.parametric_function_api`.
"""
import nnabla as nn
import inspect
nn.clear_parameters()
shape = (2, 3, 4)
# Signature check
spec = inspect.getargspec(dummy_parametric_function)
assert spec.args == ['shape', 'f', 'i', 's', 'name']
assert spec.defaults == (10, 1, 'dummy', None)
assert dummy_parametric_function.__doc__.splitlines()[0] == 'Doc'
# Verify two different ways does the same thing.
# Using name argument
v = dummy_parametric_function(shape, name='group1')
# Using parameter_scope
with nn.parameter_scope('group1'):
v = dummy_parametric_function(shape)
params = nn.get_parameters()
assert len(params) == 2
assert list(iterkeys(params)) == ['group1/dummy/p1', 'group1/dummy/p2']
# No scope
v = dummy_parametric_function(shape)
params = nn.get_parameters()
len(params) == 4
assert list(iterkeys(params)) == ['group1/dummy/p1', 'group1/dummy/p2',
'dummy/p1', 'dummy/p2']
nn.clear_parameters()
def test_get_parameter_or_create_need_grad():
"""Testing if need_grad flag works not not.
"""
import nnabla as nn
from nnabla.parameter import get_parameter_or_create
nn.clear_parameters()
param1 = get_parameter_or_create('p/param1', (2, 3, 4, 5), need_grad=True)
p1d = np.random.randn(*param1.shape).astype(np.float32)
p1g = np.random.randn(*param1.shape).astype(np.float32)
param1.d = p1d
param1.g = p1g
param1_f = get_parameter_or_create('p/param1',
param1.shape,
need_grad=False)
assert not param1_f.need_grad
assert not param1.need_grad
assert np.all(param1.d == p1d)
assert np.all(param1.d == param1_f.d)
param1.d = 1
assert np.all(param1_f.d == 1)
param1_f2 = get_parameter_or_create('p/param1',
param1.shape,
need_grad=True,
as_need_grad=False)
assert param1.need_grad
assert param1_f.need_grad
assert not param1_f2.need_grad
nn.clear_parameters()
def test_parametric_function_api():
"""
Testing :function:`nnabla.parametric_functions.parametric_function_api`.
"""
import nnabla as nn
import inspect
nn.clear_parameters()
shape = (2, 3, 4)
# Signature check
spec = inspect.getargspec(dummy_parametric_function)
assert spec.args == ['shape', 'f', 'i', 's', 'name']
assert spec.defaults == (10, 1, 'dummy', None)
assert dummy_parametric_function.__doc__.splitlines()[0] == 'Doc'
# Verify two different ways does the same thing.
# Using name argument
v = dummy_parametric_function(shape, name='group1')
# Using parameter_scope
with nn.parameter_scope('group1'):
v = dummy_parametric_function(shape)
params = nn.get_parameters()
assert len(params) == 2
assert list(iterkeys(params)) == ['group1/dummy/p1', 'group1/dummy/p2']
# No scope
v = dummy_parametric_function(shape)
params = nn.get_parameters()
len(params) == 4
assert list(iterkeys(params)) == [
'group1/dummy/p1', 'group1/dummy/p2', 'dummy/p1', 'dummy/p2'
]
nn.clear_parameters()
def test_graph_more_than_2_outputs(seed, clear_buffer):
count = 0
def func_hook(f):
nonlocal count
if f.name == 'Split':
count += 1
nn.clear_parameters()
a = nn.Variable.from_numpy_array(np.ones((10, )))
b = nn.Variable.from_numpy_array(np.ones((10, )))
c = F.add2(a, b, inplace=True, outputs=[a.data])
y = F.split(c, axis=0)
nn.forward_all(y, function_pre_hook=func_hook)
assert count == 1
res = [x.d for x in y]
assert_allclose(res, [2.0] * 10)
a = nn.Variable.from_numpy_array(np.ones((10, )))
b = nn.Variable.from_numpy_array(np.ones((10, )))
c = F.add2(a, b, inplace=True, outputs=[a.data])
y = F.split(c, axis=0)
for yy in y:
yy.forward()
res = [x.d for x in y]
assert_allclose(res, [11.0] * 10)
def test_simple_loop():
nn.clear_parameters()
x = nn.Variable.from_numpy_array(np.random.randn(10, 3, 128, 128))
t = nn.Variable.from_numpy_array(np.random.randint(0, 100, (10, )))
unet = UNet(num_classes=1,
model_channels=128,
output_channels=3,
num_res_blocks=2,
attention_resolutions=(16, 8),
attention_num_heads=4,
channel_mult=(1, 1, 2, 2, 4, 4))
y = unet(x, t)
loss = F.mean(F.squared_error(y, x))
import nnabla.solvers as S
solver = S.Sgd()
solver.set_parameters(nn.get_parameters())
from tqdm import trange
tr = trange(100)
for i in tr:
loss.forward(clear_no_need_grad=True)
solver.zero_grad()
loss.backward(clear_buffer=True)
solver.update()
tr.set_description(f"diff: {loss.d.copy():.5f}")
def test_nnabla_models_vgg(num_layers, batch_size, training, seed):
from nnabla.models.imagenet import VGG
nn.clear_parameters()
rng = np.random.RandomState(seed)
model = VGG(num_layers)
# Determine input shape and create input variable
input_shape = list(model.input_shape)
input_shape = tuple(input_shape)
x = nn.Variable.from_numpy_array(
rng.randint(0, 256, size=(batch_size, ) + input_shape))
# Test cases for all intermediate outputs
for use_up_to in ('classifier', 'pool', 'lastconv', 'lastconv+relu'):
returns_net = False
def _execute():
y = model(x, training=training, use_up_to=use_up_to)
y.forward()
_execute()
net = model(x,
training=training,
use_up_to=use_up_to,
returns_net=True)
assert isinstance(net, NnpNetwork)
assert len(net.inputs.values()) == 1
assert len(net.outputs.values()) == 1
y = list(net.outputs.values())[0]
if training:
assert _check_trainable_parameters(y)
else:
assert not _check_trainable_parameters(y)
def test_no_grad(seed, graph):
from .graph_converter_test_utils import structure_tester, value_tester
nn.clear_parameters()
# Random number
np.random.seed(seed)
rng = np.random.RandomState(seed)
# Graph
x_data = rng.randn(4, 3, 32, 32)
x0 = nn.Variable.from_numpy_array(x_data)\
.apply(need_grad=False) \
.apply(persistent=True)
y0 = graph(x0)
y1 = y0.no_grad()
# Test
def assert_need_grad_flase(f):
for inp in f.inputs:
assert inp.need_grad == False, "need_grad must be false"
for out in f.outputs:
assert out.need_grad == False, "need_grad must be false"
y1.visit(assert_need_grad_flase)
structure_tester(y0, y1)
value_tester(y0, y1, clear_no_need_grad=True)
def test_batch_normalization_linear(seed, test, graph_ref, graph_act):
from graph_converter_test_utils import structure_tester, value_tester, print_params
# because linearized parameters (c0, c1) are to be shared among models
nn.clear_parameters()
# Random number
np.random.seed(seed)
rng = np.random.RandomState(seed)
# Graph
x_data = rng.randn(4, 3, 32, 32)
x = nn.Variable.from_numpy_array(x_data)
y_tgt = graph_act(x, test=test)
# Convert
name = "bn-linear-graph"
converter = GC.BatchNormalizationLinearConverter(name=name)
y_act = converter.convert(y_tgt, [x])
# Ref Graph
name = "bn-linear-graph-ref"
y_ref = graph_ref(x, name=name)
# Test
structure_tester(y_ref, y_act)
value_tester(y_tgt, y_act)
def test_nnabla_models_imagenet_etc(model_class, up_to_list, image_size_factor,
batch_size, training, seed):
nn.clear_parameters()
rng = np.random.RandomState(seed)
# Load model
from nnabla.models import imagenet
Model = getattr(imagenet, model_class)
model = Model()
# Determine input shape and create input variable
input_shape = list(model.input_shape)
input_shape[1] *= image_size_factor
input_shape[2] *= image_size_factor
input_shape = tuple(input_shape)
x = nn.Variable.from_numpy_array(
rng.randint(0, 256, size=(batch_size, ) + input_shape))
# Test cases for all intermediate outputs
for use_up_to in up_to_list:
check_global_pooling = True
force_global_pooling = False
returns_net = False
def _execute():
y = model(x,
training=training,
use_up_to=use_up_to,
force_global_pooling=force_global_pooling,
check_global_pooling=check_global_pooling)
y.forward()
# Need special care for SENet because it contains global average
# pooling in various points in a network.
if image_size_factor != 1 and (model_class == 'SENet'
or use_up_to in ('classifier', 'pool')):
with pytest.raises(ValueError):
_execute()
if use_up_to == 'pool' and model_class != 'SENet':
check_global_pooling = False
_execute()
force_global_pooling = True
_execute()
net = model(x,
training=training,
use_up_to=use_up_to,
force_global_pooling=force_global_pooling,
check_global_pooling=check_global_pooling,
returns_net=True)
assert isinstance(net, NnpNetwork)
assert len(net.inputs.values()) == 1
assert len(net.outputs.values()) == 1
y = list(net.outputs.values())[0]
if training:
assert _check_trainable_parameters(y)
else:
assert not _check_trainable_parameters(y)
def test_graph_model(model, seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4], need_grad=True)
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
nn.set_default_context(nn.Context())
# Forwardprop by definintion
nn.clear_parameters()
if model == "mlp":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
elif model == "recurrent":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
h = z2
for _ in range(2):
with nn.parameter_scope('fc2'):
h = PF.affine(h, 3)
h = F.relu(h, inplace=True)
with nn.parameter_scope('fc3'):
z3 = PF.affine(h, 5)
elif model == "convolution":
with nn.parameter_scope('conv1'):
z = PF.convolution(x, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
else:
raise ValueError()
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
L.forward(clear_no_need_grad=True)
# Backprop
# Diff should be initialized since they are always accumulated
x.grad.zero()
L.backward(clear_buffer=True)
x.g = rng.randn(*x.shape)
parameters = nn.get_parameters()
for param in parameters.values():
param.grad.zero()
inputs = [x] + list(parameters.values())
from nbla_test_utils import \
compute_analytical_and_numerical_grad_graph as grads
agrad, ngrad = grads(L, inputs, 1e-3)
assert np.allclose(ngrad, agrad, atol=1.05e-2)
def test_graph_model(model, seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4], need_grad=True)
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
nn.set_default_context(nn.Context())
# Forwardprop by definition
nn.clear_parameters()
if model == "mlp":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 3)
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
elif model == "recurrent":
with nn.parameter_scope('fc1'):
z = PF.affine(x, 8)
z2 = F.relu(z, inplace=True)
h = z2
for _ in range(2):
with nn.parameter_scope('fc2'):
h = PF.affine(h, 8)
h = F.relu(h, inplace=True)
with nn.parameter_scope('fc3'):
z3 = PF.affine(h, 5)
elif model == "convolution":
with nn.parameter_scope('conv1'):
z = PF.convolution(x, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
else:
raise ValueError()
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
L.forward(clear_no_need_grad=True)
# Backprop
# Diff should be initialized since they are always accumulated
x.grad.zero()
L.backward(clear_buffer=True)
x.g = rng.randn(*x.shape)
parameters = nn.get_parameters()
for param in parameters.values():
param.grad.zero()
inputs = [x] + list(parameters.values())
from nbla_test_utils import \
compute_analytical_and_numerical_grad_graph as grads
agrad, ngrad = grads(L, inputs, 1e-3)
assert_allclose(ngrad, agrad, atol=1.05e-2)
def get_saved_test_model(module):
module_func, module_input_shapes = module
with create_temp_with_dir(NNP_FILE) as nnp_file:
with nn.graph_def.graph() as g:
variables = [
nn.ProtoVariable(shape) for _, shape in module_input_shapes
]
outputs = module_func(*variables)
g.save(nnp_file)
nn.clear_parameters()
yield nnp_file
def test_parametric_function_1d(inshape, kernel, multiplier, outshape):
base_axis = len(inshape) - 2
sample_channels = inshape[base_axis]
outmap_channels = sample_channels * multiplier
x = nn.Variable(inshape)
y = PF.depthwise_convolution(x, kernel, multiplier=multiplier)
p = nn.get_parameters()
assert y.shape == outshape
assert p['depthwise_conv/W'].shape == (outmap_channels,) + kernel
assert p['depthwise_conv/b'].shape == (outmap_channels,)
nn.clear_parameters()
def test_parametric_function_2d(inshape, kernel, divisor, outshape):
base_axis = len(inshape) - 3
sample_channels = inshape[base_axis]
outmap_channels = sample_channels // divisor
x = nn.Variable(inshape)
y = PF.depthwise_deconvolution(x, kernel, divisor=divisor)
p = nn.get_parameters()
assert y.shape == outshape
assert p['depthwise_deconv/W'].shape == (sample_channels, ) + kernel
assert p['depthwise_deconv/b'].shape == (outmap_channels, )
nn.clear_parameters()
def test(args):
"""
Training
"""
## ~~~~~~~~~~~~~~~~~~~
## Initial settings
## ~~~~~~~~~~~~~~~~~~~
# Input Variable
nn.clear_parameters() # Clear
Input = nn.Variable([1, 3, 64, 64]) # Input
Trues = nn.Variable([1, 1]) # True Value
# Network Definition
Name = "CNN" # Name of scope which includes network models (arbitrary)
Output_test = network(Input, scope=Name, test=True) # Network & Output
Loss_test = F.mean(F.absolute_error(
Output_test, Trues)) # Loss Function (Squared Error)
# Load data
with nn.parameter_scope(Name):
nn.load_parameters(
os.path.join(args.model_save_path,
"network_param_{:04}.h5".format(args.epoch)))
# Training Data Setting
image_data, mos_data = dt.data_loader(test=True)
batches = dt.create_batch(image_data, mos_data, 1)
del image_data, mos_data
truth = []
result = []
for j in range(batches.iter_n):
Input.d, tures = next(batches)
Loss_test.forward(clear_no_need_grad=True)
result.append(Loss_test.d)
truth.append(tures)
result = np.array(result)
truth = np.squeeze(np.array(truth))
# Evaluation of performance
mae = np.average(np.abs(result - truth))
SRCC, p1 = stats.spearmanr(truth,
result) # Spearman's Correlation Coefficient
PLCC, p2 = stats.pearsonr(truth, result)
# Display
print("\n Model Parameter [epoch={0}]".format(args.epoch))
print(" Mean Absolute Error with Truth: {0:.4f}".format(mae))
print(" Speerman's Correlation Coefficient: {0:.3f}".format(SRCC))
print(" Pearson's Linear Correlation Coefficient: {0:.3f}".format(PLCC))
def __init__(self):
#パラメタの初期化
nn.clear_parameters()
#入力変数の準備
self.x = nn.Variable((1, 3, 256, 256)) #(枚数,色,高さ,幅)
#パラメタの読み込みparam
nn.load_parameters(os.path.join(current_dir, "parameters.h5"))
#推論ネットワークの構築
self.y = self.network(self.x, test=True)
def test_graph_rewire(seed, clear_buffer):
nn.clear_parameters()
# A. defining graph definition utility
def mlp2(x, scope):
with nn.parameter_scope(scope):
h = F.tanh(PF.affine(x, 10, name='a1'))
h = F.tanh(PF.affine(h, 10, name='a1'))
return h
# A. Create a graph A.
xa = nn.Variable((2, 10), need_grad=True)
ya = mlp2(xa, 'a')
# B. Create a graph B.
xb = nn.Variable((2, 10), need_grad=True)
yb = mlp2(xb, 'b')
# C. Create directly connected graph.
xc = nn.Variable((2, 10))
yc = mlp2(mlp2(xc, 'a'), 'b')
# D. Rewire the graphs A and B.
xb.rewire_on(ya)
# E. Check whether the results are the same.
rng = np.random.RandomState(seed)
data = rng.randn(*xa.shape)
xa.d = data
xc.d = data
params = nn.get_parameters()
def zero_grad():
for p in params.values():
p.grad.zero()
def backup_params():
return [p.g.copy() for p in params.values()]
# Checking forward
yb.forward(clear_no_need_grad=clear_buffer)
yc.forward(clear_no_need_grad=clear_buffer)
assert_allclose(yb.d, yc.d)
# Checking backward
zero_grad()
yb.backward(clear_buffer=clear_buffer)
gb = backup_params()
zero_grad()
yc.backward(clear_buffer=clear_buffer)
gc = backup_params()
assert_allclose(xa.d, xc.d)
for b, c in zip(gb, gc):
assert_allclose(b, c)
def test_all_reduce_callback(seed, pack_size, division, comm_nccl_opts):
if comm_nccl_opts is None:
pytest.skip(
"Communicator test is disabled. You can turn it on by an option `--test-communicator`."
)
if len(comm_nccl_opts.devices) < 2:
pytest.skip("Communicator test is disabled. Use more than 1 gpus.")
comm = comm_nccl_opts.comm
device_id = int(comm_nccl_opts.device_id)
nn.set_default_context(comm_nccl_opts.ctx)
nn.clear_parameters()
x_data_list = []
num_layers = 20
rng = np.random.RandomState(seed)
for l in range(num_layers):
x_data = rng.rand(3, 4)
x_data_list.append(x_data)
# all_reduce_callback
x_list1 = []
n1 = nn.Variable([3, 4])
n1.d = 0
for l in range(num_layers):
x = nn.Variable([3, 4], need_grad=True)
n1 = F.add2(n1, x)
x.d = x_data_list[l] * (device_id + 1)
x.g = 0
x_list1.append(x)
n1.backward(clear_buffer=True,
communicator_callbacks=comm.all_reduce_callback(
[v.grad for v in x_list1], pack_size, division))
# Ref AllReduce
x_list2 = []
n2 = nn.Variable([3, 4])
n2.d = 0
for l in range(num_layers):
x = nn.Variable([3, 4], need_grad=True)
n2 = F.add2(n2, x)
x.d = x_data_list[l] * (device_id + 1)
x.g = 0
x_list2.append(x)
n2.backward(clear_buffer=True)
comm.all_reduce([v.grad for v in x_list2],
inplace=False,
division=division)
# Check
for x, ref in zip(x_list1, x_list2):
assert np.allclose(x.g, ref.g)
def test_get_set_pop_parameter():
import nnabla as nn
from nnabla.parameter import set_parameter, pop_parameter, get_parameter
nn.clear_parameters()
x = nn.Variable((2, 3, 4, 5))
key = 'a/b/c'
set_parameter(key, x)
x2 = get_parameter(key)
assert x is x2
x3 = pop_parameter(key)
assert x is x3
x4 = get_parameter(key)
assert x4 is None
def test_parameter_scope_slash():
"""Testing if parameter_scope('aaa/bbb') works.
"""
import nnabla as nn
from nnabla.parameter import get_parameter_or_create
nn.clear_parameters()
with nn.parameter_scope('aaa/bbb'):
param = get_parameter_or_create('ccc', (2, 3, 4, 5))
ref = np.random.randn(*param.shape).astype(np.float32)
param.d = ref
with nn.parameter_scope('aaa'):
with nn.parameter_scope('bbb'):
param = get_parameter_or_create('ccc', (2, 3, 4, 5))
assert np.all(param.d == ref)
nn.clear_parameters()
def test_get_parameter_or_create_need_grad():
"""Testing if need_grad flag works not not.
"""
import nnabla as nn
from nnabla.parameter import get_parameter_or_create
nn.clear_parameters()
param1 = get_parameter_or_create('param1', (2, 3, 4, 5), need_grad=True)
p1d = np.random.randn(*param1.shape).astype(np.float32)
p1g = np.random.randn(*param1.shape).astype(np.float32)
param1.d = p1d
param1.g = p1g
param1_f = get_parameter_or_create('param1', param1.shape, need_grad=False)
assert not param1_f.need_grad
assert param1.need_grad
assert np.all(param1.d == p1d)
assert np.all(param1.d == param1_f.d)
param1.d = 1
assert np.all(param1_f.d == 1)
nn.clear_parameters()
def test_graph_clear_buffer(seed):
np.random.seed(313)
rng = np.random.RandomState(seed)
x = nn.Variable([2, 3, 4, 4])
t = nn.Variable([2, 1])
x.d = rng.randn(*x.shape)
t.d = rng.randint(0, 5, size=t.shape)
# Network definition
nn.set_default_context(nn.Context())
nn.clear_parameters()
x1 = x + 1
x2 = x1 - 1
with nn.parameter_scope('conv1'):
z = PF.convolution(x2, 3, (2, 2))
z2 = F.relu(z, inplace=True)
with nn.parameter_scope('fc2'):
z3 = PF.affine(z2, 5)
l = F.softmax_cross_entropy(z3, t, 1)
L = F.mean(l)
# Forwardprop
import tempfile
import os
tmpd = tempfile.mkdtemp()
nn.save_parameters(os.path.join(tmpd, 'parameter.h5'))
first = False
for cnng in [False, True]:
for cb in [False, True]:
_ = nn.load_parameters(os.path.join(tmpd, 'parameter.h5'))
for v in nn.get_parameters().values():
v.grad.zero()
L.forward(clear_no_need_grad=cnng)
L.backward(clear_buffer=cb)
if not first:
first = True
g = list(nn.get_parameters().values())[0].g.copy()
else:
g2 = list(nn.get_parameters().values())[0].g.copy()
assert np.all(g == g2)
def __init__(self, func, inspecs, func_args, func_kwargs,
ext, ext_kwargs, min_run=1, min_time=1.0):
nn.clear_parameters()
self.inputs = None
self.outputs = None
self.func_ins = None
self.setup_stat = None
self.forward_stat = None
self.backward_stat = None
self.func = func
self.module = func.__module__
self.inspecs = inspecs
self.inputs_f = create_inputs(inspecs)
self.func_args = func_args
self.func_kwargs = func_kwargs
self.ext = ext
self.ext_kwargs = ext_kwargs
self.mod_ext = importlib.import_module(
'.' + ext, 'nnabla.extensions')
self.ctx = self.mod_ext.context(**ext_kwargs)
self.min_run = min_run
self.min_time = min_time
