def test_gradient_bs5_normalised(self):
"""
Test the increment for online learning with a mini-batch of inputs.
"""
N = 1000
K = 3
M = 4
bs = 5 # batch size
w = 3 * rnd.randn(K, N)
B = 4 * rnd.randn(M, N)
xis = rnd.randn(bs, N)
g = scm.g_erf
dgdx = scm.dgdx_erf
gradients = np.zeros((bs, K, N))
# these are really the phis:
for b in range(bs):
phi_w = 1 / K * np.sum(
[g(w[k] @ xis[b] / np.sqrt(N)) for k in range(K)])
phi_B = 1 / M * np.sum(
[g(B[m] @ xis[b] / np.sqrt(N)) for m in range(M)])
for k in range(K):
gradients[b, k] = (phi_B - phi_w) * \
dgdx(w[k] @ xis[b] / np.sqrt(N)) * xis[b] / K
# average over the batch
gradient = np.mean(gradients, axis=0)
phi_B = scm.phi(B, xis, g, normalise=True)
self.assertTrue(
np.allclose(
gradient,
scm.gradient(w, xis, phi_B, g=g, dgdx=dgdx, normalise=True)))
def test_phi(self):
N = 5
K = 31
w = rnd.randn(K, N) # generate some weights
xi = rnd.randn(N) # an input
phi = 0
for k in range(K):
x = w[k] @ xi / np.sqrt(N)
phi += erf(x / np.sqrt(2))
self.assertTrue(np.allclose(phi, scm.phi(w, xi, scm.g_erf)),
msg='scm output wrong!')
def test_phi_normalised(self):
N = 5
K = 3
w = rnd.randn(K, N) # generate some weights
xi = rnd.randn(N) # an input
phi = 0
for k in range(K):
x = w[k] @ xi / np.sqrt(N)
phi += 1 / K * erf(x / np.sqrt(2))
self.assertTrue(np.allclose(phi,
scm.phi(w, xi, scm.g_erf, normalise=True)),
msg='scm output wrong when normalisation is on!')
def test_phi_batch(self):
N = 1000
K = 3
bs = 5
w = 3 * rnd.randn(K, N)
xis = rnd.randn(bs, N)
g = scm.g_erf
phis = np.zeros(bs)
for b in range(bs):
phis[b] = np.sum([g(w[k] @ xis[b] / np.sqrt(N)) for k in range(K)])
self.assertTrue(np.allclose(phis, scm.phi(w, xis, g)))
def test_discrimination_error_polar(self):
N = 1000
K = 3
num_samples = 10
xis = rnd.randn(num_samples, N)
ys = rnd.choice([-1., 1.], num_samples)
w = rnd.randn(K, N)
g = scm.g_erf
ys_guess = np.sign(scm.phi(w, xis, g))
error = 0
for i in range(num_samples):
if ys[i] != ys_guess[i]:
error += 1 / num_samples
self.assertAlmostEqual(error,
scm.e_dataset_discrimination(w, xis, ys, g))
def test_phi_batch_normalised(self):
N = 1000
K = 3
bs = 5
w = 3 * rnd.randn(K, N)
xis = rnd.randn(bs, N)
g = scm.g_erf
phis = np.zeros(bs)
for b in range(bs):
phis[b] = np.sum([g(w[k] @ xis[b] / np.sqrt(N)) for k in range(K)])
phis /= K
self.assertTrue(np.allclose(phis, scm.phi(w, xis, g, normalise=True)),
msg='scm output wrong when normalisation is on!')
def test_discrimination_error_binary(self):
N = 1000
K = 3
num_samples = 10
xis = rnd.randn(num_samples, N)
ys = rnd.choice([0., 1.], num_samples)
w = rnd.randn(K, N)
g = scm.g_relu
ys_guess = scm.phi(w, xis, g)
ys_guess[ys_guess > 0.5] = 1
ys_guess[ys_guess < 0.99] = 0
error = 0
for i in range(num_samples):
if ys[i] != ys_guess[i]:
error += 1 / num_samples
self.assertAlmostEqual(error,
scm.e_dataset_discrimination(w, xis, ys, g))
if rem > 0:
w0 = np.tile(B0, (quot + 1, 1))[:K]
scale = 1 / quot * np.ones(M)
scale[:rem] = 1 / (quot + 1)
scale = np.tile(scale, (quot + 1))[:K]
else:
w0 = np.tile(B0, (quot, 1))
scale = 1 / quot * np.ones(K)
w0 = np.diag(scale) @ w0 + 1e-9 * rnd.randn(*w0.shape)
else:
raise ValueError("init has to be between 1 and 4; see help!")
# if required, generate the training set
if args.ts > 0:
train_xis = rnd.randn(round(args.ts * N), N)
train_ys = scm.phi(B0, train_xis, g1, args.normalise)
if args.sigma > 0:
train_ys += sigma * rnd.randn(*train_ys.shape)
train_set = (train_xis, train_ys)
else:
train_set = None
# output file
log_fname = \
("scm_%s_%s_N%d_%sM%d_K%d_lr%g_wd%g_sigma%g_bs%d_%si%d%ssteps%g_s%d.dat" %
(utils.activation_name(args.g1), utils.activation_name(args.g2), N,
(('ts%g_' % args.ts) if args.ts > 0 else ''),
M, K, lr, wd, sigma, args.bs,
('normalised_' if args.normalise else ''), args.init,
('graded_' if args.graded else ''), args.steps, args.seed))
logfile = open(log_fname, "w", buffering=1)