def test_sgd():
results = []
for scale in scales:
A = cgt.shared(1.0)
B = cgt.shared(1.0)
updates = nn.sgd(f(A, scale) + f(B, scale), [A, B], learning_rate=0.1)
do_update = cgt.function([], [], updates=updates)
for _ in range(10):
do_update()
assert np.allclose(A.op.get_value(), B.op.get_value())
results.append(A.op.get_value().copy())
assert np.allclose(results, torch_values['sgd'])
def test_sgd():
results = []
for scale in scales:
A = cgt.shared(1.0)
B = cgt.shared(1.0)
updates = nn.sgd(f(A, scale) + f(B, scale), [A, B], learning_rate=0.1)
do_update = cgt.function([], [], updates=updates)
for _ in range(10):
do_update()
assert np.allclose(A.op.get_value(), B.op.get_value())
results.append(A.op.get_value().copy())
assert np.allclose(results, torch_values['sgd'])
conv2 = nn.rectify(
nn.SpatialConvolution(32, 32, kernelshape=(3,3), stride=(1,1), pad=(1,1), weight_init=nn.IIDGaussian(std=.1))(pool1)
)
pool2 = nn.max_pool_2d(conv2, kernelshape=(2,2), stride=(2,2))
d0, d1, d2, d3 = pool2.shape
flat = pool2.reshape([d0, d1*d2*d3])
nfeats = cgt.infer_shape(flat)[1]
probs = nn.softmax(nn.Affine(nfeats, 10)(flat))
cost = -categorical.loglik(y, probs).mean()
y_preds = cgt.argmax(probs, axis=1)
err = cgt.cast(cgt.not_equal(y, y_preds), cgt.floatX).mean()
params = nn.get_parameters(cost)
updates = nn.sgd(cost, params, 1e-3)
# training function
f = cgt.function(inputs=[X, y], outputs=[], updates=updates)
# compute the cost and error
cost_and_err = cgt.function(inputs=[X, y], outputs=[cost, err])
for i in xrange(epochs):
t0 = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = batch_size + start
f(Xtrainimg[start:end], ytrain[start:end])
elapsed = time.time() - t0
costval, errval = cost_and_err(Xtestimg, ytest)
print("Epoch {} took {}, test cost = {}, test error = {}".format(i, elapsed, costval, errval))
# followed by a logsoftmax.
X = cgt.matrix('X', fixed_shape=(None, 784))
y = cgt.vector('y', dtype='i8')
layer1 = nn.Affine(784, 400, weight_init=nn.XavierNormal())(X)
act1 = nn.rectify(layer1)
layer2 = nn.Affine(400, 400, weight_init=nn.XavierNormal())(act1)
act2 = nn.rectify(layer2)
probs = nn.softmax(nn.Affine(400, 10)(act2))
y_preds = cgt.argmax(probs, axis=1)
cost = -cgt.mean(categorical.loglik(y, probs))
err = cgt.cast(cgt.not_equal(y, y_preds), cgt.floatX).mean()
params = nn.get_parameters(cost)
updates = nn.sgd(cost, params, learning_rate) # train via sgd
# training function
f = cgt.function(inputs=[X, y], outputs=[], updates=updates)
# compute the cost and error
cost_and_err = cgt.function(inputs=[X, y], outputs=[cost, err])
for i in xrange(epochs):
t0 = time.time()
for start in xrange(0, Xtrain.shape[0], batch_size):
end = batch_size + start
f(Xtrain[start:end], ytrain[start:end])
elapsed = time.time() - t0
costval, errval = cost_and_err(Xtest, ytest)
print("Epoch {} took {}, test cost = {}, test error = {}".format(i+1, elapsed, costval, errval))
