def test_share_params_linear(self):
# A's params are shared with B and C so that all the three share the
# same parameter arrays
model_a = L.Linear(2, 2)
arrays = async_.share_params_as_shared_arrays(model_a)
assert isinstance(arrays, dict)
assert set(arrays.keys()) == {'/W', '/b'}
model_b = L.Linear(2, 2)
model_c = L.Linear(2, 2)
async_.set_shared_params(model_b, arrays)
async_.set_shared_params(model_c, arrays)
# Pointers to parameters must be the same
_assert_same_pointers_to_param_data(model_a, model_b)
_assert_same_pointers_to_param_data(model_a, model_c)
# Pointers to gradients must be different
_assert_different_pointers_to_param_grad(model_a, model_b)
_assert_different_pointers_to_param_grad(model_a, model_c)
_assert_different_pointers_to_param_grad(model_b, model_c)
# Pointers to persistent values must be the same
_assert_same_pointers_to_persistent_values(model_a, model_b)
_assert_same_pointers_to_persistent_values(model_a, model_c)
def test_share_params_chain_list(self):
model_a = chainer.ChainList(
L.BatchNormalization(3),
chainer.ChainList(L.Linear(3, 5)),
)
arrays = async_.share_params_as_shared_arrays(model_a)
assert isinstance(arrays, dict)
assert set(arrays.keys()) == {
'/0/gamma', '/0/beta', '/0/avg_mean', '/0/avg_var', '/0/N',
'/1/0/W', '/1/0/b'}
model_b = chainer.ChainList(
L.BatchNormalization(3),
chainer.ChainList(L.Linear(3, 5)),
)
model_c = chainer.ChainList(
L.BatchNormalization(3),
chainer.ChainList(L.Linear(3, 5)),
)
async_.set_shared_params(model_b, arrays)
async_.set_shared_params(model_c, arrays)
# Pointers to parameters must be the same
_assert_same_pointers_to_param_data(model_a, model_b)
_assert_same_pointers_to_param_data(model_a, model_c)
# Pointers to gradients must be different
_assert_different_pointers_to_param_grad(model_a, model_b)
_assert_different_pointers_to_param_grad(model_a, model_c)
_assert_different_pointers_to_param_grad(model_b, model_c)
# Pointers to persistent values must be the same
_assert_same_pointers_to_persistent_values(model_a, model_b)
_assert_same_pointers_to_persistent_values(model_a, model_c)
def test_shared_link(self):
"""Check interprocess parameter sharing works if models share links"""
head = L.Linear(2, 2)
model_a = chainer.ChainList(head.copy(), L.Linear(2, 3))
model_b = chainer.ChainList(head.copy(), L.Linear(2, 4))
a_arrays = async_.extract_params_as_shared_arrays(
chainer.ChainList(model_a))
b_arrays = async_.extract_params_as_shared_arrays(
chainer.ChainList(model_b))
print(('model_a shared_arrays', a_arrays))
print(('model_b shared_arrays', b_arrays))
head = L.Linear(2, 2)
model_a = chainer.ChainList(head.copy(), L.Linear(2, 3))
model_b = chainer.ChainList(head.copy(), L.Linear(2, 4))
async_.set_shared_params(model_a, a_arrays)
async_.set_shared_params(model_b, b_arrays)
print('model_a replaced')
a_params = dict(model_a.namedparams())
for param_name, param in list(a_params.items()):
print((param_name, param.data.ctypes.data))
print('model_b replaced')
b_params = dict(model_b.namedparams())
for param_name, param in list(b_params.items()):
print((param_name, param.data.ctypes.data))
# Pointers to head parameters must be the same
self.assertEqual(a_params['/0/W'].data.ctypes.data,
b_params['/0/W'].data.ctypes.data)
self.assertEqual(a_params['/0/b'].data.ctypes.data,
b_params['/0/b'].data.ctypes.data)
# Pointers to tail parameters must be different
self.assertNotEqual(a_params['/1/W'].data.ctypes.data,
b_params['/1/W'].data.ctypes.data)
self.assertNotEqual(a_params['/1/b'].data.ctypes.data,
b_params['/1/b'].data.ctypes.data)
def test_share_params_batch_normalization(self):
# A's params and persistent values are all shared with B and C
model_a = L.BatchNormalization(3)
arrays = async_.share_params_as_shared_arrays(model_a)
assert isinstance(arrays, dict)
assert set(arrays.keys()) == {
'/gamma', '/beta', '/avg_mean', '/avg_var', '/N'
}
model_b = L.BatchNormalization(3)
model_c = L.BatchNormalization(3)
async_.set_shared_params(model_b, arrays)
async_.set_shared_params(model_c, arrays)
# Pointers to parameters must be the same
_assert_same_pointers_to_param_data(model_a, model_b)
_assert_same_pointers_to_param_data(model_a, model_c)
# Pointers to gradients must be different
_assert_different_pointers_to_param_grad(model_a, model_b)
_assert_different_pointers_to_param_grad(model_a, model_c)
_assert_different_pointers_to_param_grad(model_b, model_c)
# Pointers to persistent values must be the same
_assert_same_pointers_to_persistent_values(model_a, model_b)
_assert_same_pointers_to_persistent_values(model_a, model_c)
# Check if N is shared correctly among links
assert model_a.N == 0
assert model_b.N == 0
assert model_c.N == 0
test_input = np.random.normal(size=(2, 3)).astype(np.float32)
model_a(test_input, finetune=True)
assert model_a.N == 1
assert model_b.N == 1
assert model_c.N == 1
model_c(test_input, finetune=True)
assert model_a.N == 2
assert model_b.N == 2
assert model_c.N == 2
def test_share_params(self):
# A's params are shared with B and C so that all the three share the
# same parameter arrays
model_a = L.Linear(2, 2)
arrays = async_.share_params_as_shared_arrays(model_a)
model_b = L.Linear(2, 2)
model_c = L.Linear(2, 2)
async_.set_shared_params(model_b, arrays)
async_.set_shared_params(model_c, arrays)
a_params = dict(model_a.namedparams())
b_params = dict(model_b.namedparams())
c_params = dict(model_c.namedparams())
def assert_same_pointers_to_data(a, b):
self.assertEqual(a['/W'].data.ctypes.data,
b['/W'].data.ctypes.data)
self.assertEqual(a['/b'].data.ctypes.data,
b['/b'].data.ctypes.data)
def assert_different_pointers_to_grad(a, b):
self.assertNotEqual(a['/W'].grad.ctypes.data,
b['/W'].grad.ctypes.data)
self.assertNotEqual(a['/b'].grad.ctypes.data,
b['/b'].grad.ctypes.data)
# Pointers to parameters must be the same
assert_same_pointers_to_data(a_params, b_params)
assert_same_pointers_to_data(a_params, c_params)
# Pointers to gradients must be different
assert_different_pointers_to_grad(a_params, b_params)
assert_different_pointers_to_grad(a_params, c_params)