def check_scalar_grads(precision, backend):
cgt.reset_config()
np.random.seed(0)
cgt.set_precision(precision)
cgt.core.update_config(backend=backend)
x = cgt.scalar('x')
y = cgt.scalar('y')
z = cgt.scalar('z')
vars = [x,y,z] #pylint: disable=W0622
vals = nr.rand(len(vars))+1
PROB2RESULT = {}
for ((key,_), cls) in it.chain(
it.izip(core.UNARY_INFO.items(),it.repeat(core.ElwiseUnary)),
it.izip(core.BINARY_INFO.items(),it.repeat(core.ElwiseBinary))
):
if key == "conj":
print "skipping conj"
continue
utils.colorprint(utils.Color.YELLOW, "Testing %s\n"%key)
if cls == core.ElwiseUnary:
n_in = 1
op = cls(key)
else:
n_in = 2
op = cls(key, (True,True))
inputvars = vars[0:n_in]
inputvals = vals[0:n_in]
out = core.Result(op, inputvars)
f = cgt.function(inputvars, out)
try:
grads = cgt.grad(out, inputvars)
except core.NonDifferentiable:
print "nondiff"
continue
if DISPLAY:
print "Function:"
cgt.print_tree(out)
print "Gradient original:"
cgt.print_tree(grads)
print "Gradient simplified:"
grads_simple = core.simplify(grads)
if DISPLAY: cgt.print_tree(grads_simple)
gradf = cgt.function(inputvars, grads)
eps = {"single":1e-4,"double":1e-9}[precision]
nugrad = numeric_grad(lambda li: f(*li), inputvals,eps=eps) #pylint: disable=W0640
cgtgrad = gradf(*inputvals)
np.testing.assert_almost_equal(nugrad,cgtgrad,decimal={"single":3,"double":6}[precision])
grad_count = core.count_nodes(grads_simple)
PROB2RESULT[key] = {}
PROB2RESULT[key]["grad"] = grad_count
if DISPLAY:
from thirdparty.tabulate import tabulate
print tabulate([[key,val["grad"]] for (key,val) in PROB2RESULT.iteritems()],headers=["funcname","gradcount"])
def test_scalars():
np.random.seed(0)
x = cgt.scalar('x')
y = cgt.scalar('y')
z = cgt.scalar('z')
vars = [x,y,z] #pylint: disable=W0622
vals = nr.rand(len(vars))+1
PROB2RESULT = {}
for ((key,_), cls) in it.chain(
it.izip(core.UNARY_INFO.items(),it.repeat(core.ElwiseUnary)),
it.izip(core.BINARY_INFO.items(),it.repeat(core.ElwiseBinary))
):
if key == "conj":
print "skipping conj"
continue
utils.colorprint(utils.Color.YELLOW, "Testing %s\n"%key)
if cls == core.ElwiseUnary:
n_in = 1
op = cls(key)
else:
n_in = 2
op = cls(key, (True,True))
inputvars = vars[0:n_in]
inputvals = vals[0:n_in]
out = core.Result(op, inputvars)
f = cgt.function(inputvars, out)
try:
grads = cgt.grad(out, inputvars)
except core.NonDifferentiable:
print "nondiff"
continue
if DISPLAY:
print "Function:"
cgt.print_tree(out)
print "Gradient original:"
cgt.print_tree(grads)
print "Gradient simplified:"
grads_simple = core.simplify(grads)
if DISPLAY: cgt.print_tree(grads_simple)
gradf = cgt.function(inputvars, grads)
eps = {"single":1e-4,"double":1e-9}[cgt.get_precision()]
nugrad = numeric_grad(lambda li: f(*li), inputvals,eps=eps) #pylint: disable=W0640
cgtgrad = gradf(*inputvals)
np.testing.assert_almost_equal(nugrad,cgtgrad,decimal={"single":3,"double":6}[cgt.get_precision()])
grad_count = core.count_nodes(grads_simple)
PROB2RESULT[key] = {}
PROB2RESULT[key]["grad"] = grad_count
if DISPLAY:
from thirdparty.tabulate import tabulate
print tabulate([[key,val["grad"]] for (key,val) in PROB2RESULT.iteritems()],headers=["funcname","gradcount"])
def __init__(self, n_actions):
Serializable.__init__(self, n_actions)
cgt.set_precision('double')
n_in = 128
o_no = cgt.matrix("o_no",fixed_shape=(None,n_in))
a_n = cgt.vector("a_n",dtype='i8')
q_n = cgt.vector("q_n")
oldpdist_np = cgt.matrix("oldpdists")
h0 = (o_no - 128.0)/128.0
nhid = 64
h1 = cgt.tanh(nn.Affine(128,nhid,weight_init=nn.IIDGaussian(std=.1))(h0))
probs_na = nn.softmax(nn.Affine(nhid,n_actions,weight_init=nn.IIDGaussian(std=0.01))(h1))
logprobs_na = cgt.log(probs_na)
b = cgt.size(o_no, 0)
logps_n = logprobs_na[cgt.arange(b), a_n]
surr = (logps_n*q_n).mean()
kl = (oldpdist_np * cgt.log(oldpdist_np/probs_na)).sum(axis=1).mean()
params = nn.get_parameters(surr)
gradsurr = cgt.grad(surr, params)
flatgrad = cgt.concatenate([p.flatten() for p in gradsurr])
lam = cgt.scalar()
penobj = surr - lam * kl
self._f_grad_lagrangian = cgt.function([lam, oldpdist_np, o_no, a_n, q_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj,params)]))
self.f_pdist = cgt.function([o_no], probs_na)
self.f_probs = cgt.function([o_no], probs_na)
self.f_surr_kl = cgt.function([oldpdist_np, o_no, a_n, q_n], [surr, kl])
self.f_gradlogp = cgt.function([oldpdist_np, o_no, a_n, q_n], flatgrad)
self.pc = ParamCollection(params)
def make_updater_fc():
X = cgt.matrix("X", fixed_shape=(None, 28 * 28))
y = cgt.vector("y", dtype="i8")
stepsize = cgt.scalar("stepsize")
loss = build_fc_return_loss(X, y)
params = nn.get_parameters(loss)
gparams = cgt.grad(loss, params)
updates = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X, y, stepsize], loss, updates=updates)
def make_updater_fc():
X = cgt.matrix("X", fixed_shape=(None, 28 * 28))
y = cgt.vector("y", dtype='i8')
stepsize = cgt.scalar("stepsize")
loss = build_fc_return_loss(X, y)
params = nn.get_parameters(loss)
gparams = cgt.grad(loss, params)
updates = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X, y, stepsize], loss, updates=updates)
def test_stack():
x = cgt.scalar()
y = cgt.scalar()
z = cgt.scalar()
s0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(s0, {x: 1, y: 2, z: 3}).shape == (3, )
x = cgt.vector()
y = cgt.vector()
z = cgt.vector()
v0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(v0, {
x: np.zeros(2),
y: np.zeros(2),
z: np.zeros(2)
}).shape == (3, 2)
v1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(v1, {
x: np.zeros(2),
y: np.ones(2),
z: np.zeros(2)
}).shape == (2, 3)
x = cgt.matrix()
y = cgt.matrix()
z = cgt.matrix()
m0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(m0, {
x: np.zeros((2, 4)),
y: np.zeros((2, 4)),
z: np.zeros((2, 4))
}).shape == (3, 2, 4)
m1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(m1, {
x: np.zeros((2, 4)),
y: np.zeros((2, 4)),
z: np.zeros((2, 4))
}).shape == (2, 3, 4)
m2 = cgt.stack([x, y, z], axis=2)
assert cgt.numeric_eval(m2, {
x: np.zeros((2, 4)),
y: np.zeros((2, 4)),
z: np.zeros((2, 4))
}).shape == (2, 4, 3)
def test_multi_output():
for x in (cgt.scalar('x'), cgt.vector('x'), cgt.matrix('x')):
for cls in (SinCos, SinCos2):
y,z = core.unpack(core.Result(cls(), [x]))
xnum = np.ones((3,)*x.ndim, cgt.floatX)
correct = (np.sin(xnum),np.cos(xnum))
yznum = cgt.numeric_eval([y,z], {x:xnum})
np.testing.assert_allclose(yznum, correct)
f = cgt.function([x],[y,z])
np.testing.assert_allclose(f(xnum), correct)
def test_incsubtensor1():
W = cgt.shared(np.zeros((5, 3)), name="W")
inc = cgt.matrix() # we'll increment W by this matrix
incval = np.arange(9).reshape(3, 3)
start = cgt.scalar(dtype='i8')
stop = cgt.scalar(dtype='i8')
updates = {W: cgt.inc_subtensor(W, slice(start, stop), inc)}
f = cgt.function([start, stop, inc], [], updates=updates)
f(0, 3, incval)
assert np.allclose(
W.op.get_value(),
np.array([
[0., 1., 2.],
[3., 4., 5.],
[6., 7., 8.],
[0., 0., 0.],
[0., 0., 0.],
]))
def make_updater_convnet():
X = cgt.tensor4("X", fixed_shape=(None, 1, 28, 28)) # so shapes can be inferred
y = cgt.vector("y", dtype="i8")
stepsize = cgt.scalar("stepsize")
loss = build_convnet_return_loss(X, y)
params = nn.get_parameters(loss)
gparams = cgt.grad(loss, params)
updates = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X, y, stepsize], loss, updates=updates)
def test_incsubtensor1():
W = cgt.shared(np.zeros((5,3)), name="W")
inc = cgt.matrix() # we'll increment W by this matrix
incval = np.arange(9).reshape(3,3)
start = cgt.scalar(dtype='i8')
stop = cgt.scalar(dtype='i8')
updates = {W : cgt.inc_subtensor(W, slice(start, stop), inc)}
f = cgt.function([start,stop,inc],[],updates=updates)
f(0,3,incval)
assert np.allclose(W.op.get_value(),
np.array(
[
[ 0., 1., 2.],
[ 3., 4., 5.],
[ 6., 7., 8.],
[ 0., 0., 0.],
[ 0., 0., 0.],
]))
def test_multi_output():
for x in (cgt.scalar('x'), cgt.vector('x'), cgt.matrix('x')):
for cls in (SinCos, SinCos2):
y, z = core.unpack(core.Result(cls(), [x]))
xnum = np.ones((3, ) * x.ndim, cgt.floatX)
correct = (np.sin(xnum), np.cos(xnum))
yznum = cgt.numeric_eval([y, z], {x: xnum})
np.testing.assert_allclose(yznum, correct)
f = cgt.function([x], [y, z])
np.testing.assert_allclose(f(xnum), correct)
def __init__(self, obs_dim, ctrl_dim):
cgt.set_precision('double')
Serializable.__init__(self, obs_dim, ctrl_dim)
self.obs_dim = obs_dim
self.ctrl_dim = ctrl_dim
o_no = cgt.matrix("o_no",fixed_shape=(None,obs_dim))
a_na = cgt.matrix("a_na",fixed_shape = (None, ctrl_dim))
adv_n = cgt.vector("adv_n")
oldpdist_np = cgt.matrix("oldpdist", fixed_shape=(None, 2*ctrl_dim))
self.logstd = logstd_1a = nn.parameter(np.zeros((1, self.ctrl_dim)), name="std_1a")
std_1a = cgt.exp(logstd_1a)
# Here's where we apply the network
h0 = o_no
nhid = 32
h1 = cgt.tanh(nn.Affine(obs_dim,nhid,weight_init=nn.IIDGaussian(std=0.1))(h0))
h2 = cgt.tanh(nn.Affine(nhid,nhid,weight_init=nn.IIDGaussian(std=0.1))(h1))
mean_na = nn.Affine(nhid,ctrl_dim,weight_init=nn.IIDGaussian(std=0.01))(h2)
b = cgt.size(o_no, 0)
std_na = cgt.repeat(std_1a, b, axis=0)
oldmean_na = oldpdist_np[:, 0:self.ctrl_dim]
oldstd_na = oldpdist_np[:, self.ctrl_dim:2*self.ctrl_dim]
logp_n = ((-.5) * cgt.square( (a_na - mean_na) / std_na ).sum(axis=1)) - logstd_1a.sum()
oldlogp_n = ((-.5) * cgt.square( (a_na - oldmean_na) / oldstd_na ).sum(axis=1)) - cgt.log(oldstd_na).sum(axis=1)
ratio_n = cgt.exp(logp_n - oldlogp_n)
surr = (ratio_n*adv_n).mean()
pdists_np = cgt.concatenate([mean_na, std_na], axis=1)
# kl = cgt.log(sigafter/)
params = nn.get_parameters(surr)
oldvar_na = cgt.square(oldstd_na)
var_na = cgt.square(std_na)
kl = (cgt.log(std_na / oldstd_na) + (oldvar_na + cgt.square(oldmean_na - mean_na)) / (2 * var_na) - .5).sum(axis=1).mean()
lam = cgt.scalar()
penobj = surr - lam * kl
self._compute_surr_kl = cgt.function([oldpdist_np, o_no, a_na, adv_n], [surr, kl])
self._compute_grad_lagrangian = cgt.function([lam, oldpdist_np, o_no, a_na, adv_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj,params)]))
self.f_pdist = cgt.function([o_no], pdists_np)
self.f_objs = cgt.function([oldpdist_np, o_no, a_na, adv_n], [surr, kl])
self.pc = ParamCollection(params)
def test_scalar_input():
x = cgt.scalar()
f = cgt.function([x], x**2)
xval = 2
yval = 4
assert np.allclose(f(2), 4)
assert np.allclose(f(2.0), 4)
assert np.allclose(f(np.array(2)), 4)
assert np.allclose(f(np.array(2.0)), 4)
assert np.allclose(f(np.array([2])[0]), 4)
assert np.allclose(f(np.array([2.0])[0]), 4)
def make_updater_convnet():
X = cgt.tensor4("X", fixed_shape=(None, 1, 28,
28)) # so shapes can be inferred
y = cgt.vector("y", dtype='i8')
stepsize = cgt.scalar("stepsize")
loss = build_convnet_return_loss(X, y)
params = nn.get_parameters(loss)
gparams = cgt.grad(loss, params)
updates = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X, y, stepsize], loss, updates=updates)
def test_sleeps():
with cgt.scoped_update_config(parallel=True):
x = cgt.scalar('x')
y1 = sleepfor(x, .1)
y2 = sleepfor(x, .1)
z = y1 + y2
fpar = cgt.function([x], z)
tstart = time.time()
fpar(0)
elapsed = time.time() - tstart
assert elapsed < .11
def test_sleeps():
with cgt.scoped_update_config(parallel = True, backend="native"):
x = cgt.scalar('x')
y1 = sleepfor(x, .1)
y2 = sleepfor(x, .1)
z=y1+y2
fpar = cgt.function([x],z)
tstart = time.time()
fpar(0)
elapsed = time.time() - tstart
assert elapsed < .11
def runTest(self):
if cgt.get_config()["backend"] != "python":
cgt.utils.warn("Skipping test -- only works for backend=python")
return
x = cgt.scalar()
with cgt.debug_context() as dbg:
cgt.assert_(cgt.equal(x, 1),"yoyoyo")
cgt.dbg_call(myfunc, x)
print "dbg",dbg.nodes
# cgt.assert_(cgt.equal(x, 2))
f = cgt.make_function([x],[x],dbg=dbg)
f(1)
with self.assertRaises(AssertionError):
f(2)
def runTest(self):
if cgt.get_config()["backend"] != "python":
cgt.utils.warn("Skipping test -- only works for backend=python")
return
x = cgt.scalar()
with cgt.debug_context() as dbg:
cgt.assert_(cgt.equal(x, 1), "yoyoyo")
cgt.dbg_call(myfunc, x)
print "dbg", dbg.nodes
# cgt.assert_(cgt.equal(x, 2))
f = cgt.make_function([x], [x], dbg=dbg)
f(1)
with self.assertRaises(AssertionError):
f(2)
def test_stack():
x = cgt.scalar()
y = cgt.scalar()
z = cgt.scalar()
s0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(s0, {x: 1, y: 2, z: 3}).shape == (3,)
x = cgt.vector()
y = cgt.vector()
z = cgt.vector()
v0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(v0, {x: np.zeros(2), y: np.zeros(2), z: np.zeros(2)}).shape == (3, 2)
v1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(v1, {x: np.zeros(2), y: np.ones(2), z: np.zeros(2)}).shape == (2, 3)
x = cgt.matrix()
y = cgt.matrix()
z = cgt.matrix()
m0 = cgt.stack([x, y, z], axis=0)
assert cgt.numeric_eval(m0, {x: np.zeros((2, 4)), y: np.zeros((2, 4)), z: np.zeros((2, 4))}).shape == (3, 2, 4)
m1 = cgt.stack([x, y, z], axis=1)
assert cgt.numeric_eval(m1, {x: np.zeros((2, 4)), y: np.zeros((2, 4)), z: np.zeros((2, 4))}).shape == (2, 3, 4)
m2 = cgt.stack([x, y, z], axis=2)
assert cgt.numeric_eval(m2, {x: np.zeros((2, 4)), y: np.zeros((2, 4)), z: np.zeros((2, 4))}).shape == (2, 4, 3)
def make_updater_fc_parallel():
X = cgt.matrix("X", fixed_shape=(None, 28 * 28))
y = cgt.vector("y", dtype='i8')
stepsize = cgt.scalar("stepsize")
loss = build_fc_return_loss(X, y)
params = nn.get_parameters(loss)
m = nn.Module([X, y], [loss])
split_loss = 0
for start in xrange(0, batch_size, batch_size // 4):
sli = slice(start, start + batch_size // 4)
split_loss += m([X[sli], y[sli]])[0]
split_loss /= 4
gparams = cgt.grad(split_loss, params)
updates2 = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X, y, stepsize], split_loss, updates=updates2)
def make_updater_convnet_parallel():
X = cgt.tensor4("X", fixed_shape=(None, 1, 28, 28)) # so shapes can be inferred
y = cgt.vector("y", dtype="i8")
stepsize = cgt.scalar("stepsize")
loss = build_convnet_return_loss(X, y)
m = nn.Module([X, y], [loss])
split_loss = 0
for start in xrange(0, batch_size, batch_size // 4):
sli = slice(start, start + batch_size // 4)
split_loss += m([X[sli], y[sli]])[0]
split_loss /= 4
params = nn.get_parameters(loss)
gparams = cgt.grad(split_loss, params)
updates2 = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X, y, stepsize], split_loss, updates=updates2)
def make_updater_fc_parallel():
X = cgt.matrix("X", fixed_shape=(None,28*28))
y = cgt.vector("y",dtype='i8')
stepsize = cgt.scalar("stepsize")
loss = build_fc_return_loss(X,y)
params = nn.get_parameters(loss)
m = nn.Module([X,y], [loss])
split_loss = 0
for start in xrange(0, batch_size, batch_size//4):
sli = slice(start, start+batch_size//4)
split_loss += m([X[sli], y[sli]])[0]
split_loss /= 4
gparams = cgt.grad(split_loss, params)
updates2 = [(p, p-stepsize*gp) for (p, gp) in zip(params, gparams)]
return cgt.function([X,y, stepsize], split_loss, updates=updates2)
def test_linreg():
cgt.reset_config()
cgt.set_precision('double')
N = 10
K = 3
Xval = np.random.randn(N, K)
wval = np.random.randn(K)
bval = np.random.randn()
yval = np.random.randn(N)
X_nk = cgt.matrix("X")
y_n = cgt.vector("y")
w_k = cgt.vector("w")
b = cgt.scalar(name="b")
ypred = cgt.dot(X_nk, w_k) + b
err = cgt.sum(cgt.square(ypred - y_n))
g = cgt.grad(err, [w_k, b])
g_simple, an, _ = cgt.core.simplify_and_analyze(g)
print "Loss function:"
cgt.print_tree([err])
print "Gradient:"
cgt.print_tree(g)
print "Gradient simplified"
cgt.print_tree(
g_simple,
nodefn=lambda node, o: o.write(" " + an["node2hash"][node][:5]))
print "-------"
d = {X_nk: Xval, w_k: wval, b: bval, y_n: yval}
np.testing.assert_allclose(cgt.numeric_eval(err, d),
np.linalg.norm(Xval.dot(wval) + bval - yval)**2)
np.testing.assert_allclose(cgt.numeric_eval(g[0], d),
2 * Xval.T.dot(Xval.dot(wval) + bval - yval))
np.testing.assert_allclose(cgt.numeric_eval(g[1], d),
2 * np.sum(Xval.dot(wval) + bval - yval, 0))
def __init__(self, num_features=None, num_hidden=100):
stepsize = 0.01
# with shape (batchsize, ncols)
X = cgt.matrix("X", fixed_shape=(1, num_features))
# y: a symbolic variable representing the rewards, which are integers
y = cgt.scalar("y", dtype='float64')
hid1 = nn.rectify(
nn.Affine(num_features, num_hidden, weight_init=nn.IIDGaussian(std=.1), bias_init=nn.Constant(1))(X)
)
# One final fully-connected layer, and then a linear activation output for reward
output = nn.Affine(num_hidden, 1, weight_init=nn.IIDGaussian(std=.1), bias_init=nn.Constant(1))(hid1)
abs_deviation = cgt.abs(output - y).mean()
params = nn.get_parameters(abs_deviation)
gparams = cgt.grad(abs_deviation, params)
updates = [(p, p-stepsize*gp) for (p, gp) in zip(params, gparams)]
self.predictor = cgt.function([X], output)
self.updater = cgt.function([X, y], abs_deviation, updates=updates)
def test_linreg():
N = 10
K = 3
Xval = np.random.randn(N,K)
wval = np.random.randn(K)
bval = np.random.randn()
yval = np.random.randn(N)
X_nk = cgt.matrix("X")
y_n = cgt.vector("y")
w_k = cgt.vector("w")
b = cgt.scalar(name="b")
ypred = cgt.dot(X_nk, w_k) + b
err = cgt.sum(cgt.square(ypred - y_n))
g = cgt.grad(err, [w_k, b])
g_simple,an,_ = cgt.core.simplify_and_analyze(g)
print "Loss function:"
cgt.print_tree([err])
print "Gradient:"
cgt.print_tree(g)
print "Gradient simplified"
cgt.print_tree(g_simple, nodefn=lambda node,o: o.write(" " + an["node2hash"][node][:5]))
print "-------"
d = {X_nk : Xval, w_k : wval, b : bval, y_n : yval}
np.testing.assert_allclose(cgt.numeric_eval(err,d), np.linalg.norm(Xval.dot(wval) + bval - yval)**2,
atol={"single":1e-3,"double":1e-6}[cgt.get_precision()])
np.testing.assert_allclose(cgt.numeric_eval(g[0],d), 2 * Xval.T.dot(Xval.dot(wval) + bval - yval),
atol={"single":1e-3,"double":1e-6}[cgt.get_precision()])
np.testing.assert_allclose(cgt.numeric_eval(g[1],d), 2 * np.sum(Xval.dot(wval) + bval - yval, 0),
atol={"single":1e-3,"double":1e-6}[cgt.get_precision()])
def __init__(self, n_actions):
Serializable.__init__(self, n_actions)
cgt.set_precision('double')
n_in = 128
o_no = cgt.matrix("o_no", fixed_shape=(None, n_in))
a_n = cgt.vector("a_n", dtype='i8')
q_n = cgt.vector("q_n")
oldpdist_np = cgt.matrix("oldpdists")
h0 = (o_no - 128.0) / 128.0
nhid = 64
h1 = cgt.tanh(
nn.Affine(128, nhid, weight_init=nn.IIDGaussian(std=.1))(h0))
probs_na = nn.softmax(
nn.Affine(nhid, n_actions,
weight_init=nn.IIDGaussian(std=0.01))(h1))
logprobs_na = cgt.log(probs_na)
b = cgt.size(o_no, 0)
logps_n = logprobs_na[cgt.arange(b), a_n]
surr = (logps_n * q_n).mean()
kl = (oldpdist_np * cgt.log(oldpdist_np / probs_na)).sum(axis=1).mean()
params = nn.get_parameters(surr)
gradsurr = cgt.grad(surr, params)
flatgrad = cgt.concatenate([p.flatten() for p in gradsurr])
lam = cgt.scalar()
penobj = surr - lam * kl
self._f_grad_lagrangian = cgt.function(
[lam, oldpdist_np, o_no, a_n, q_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj, params)]))
self.f_pdist = cgt.function([o_no], probs_na)
self.f_probs = cgt.function([o_no], probs_na)
self.f_surr_kl = cgt.function([oldpdist_np, o_no, a_n, q_n],
[surr, kl])
self.f_gradlogp = cgt.function([oldpdist_np, o_no, a_n, q_n], flatgrad)
self.pc = ParamCollection(params)
def __init__(self, num_features=None, num_hidden=100):
stepsize = 0.01
# with shape (batchsize, ncols)
X = cgt.matrix("X", fixed_shape=(1, num_features))
# y: a symbolic variable representing the rewards, which are integers
y = cgt.scalar("y", dtype='float64')
hid1 = nn.rectify(
nn.Affine(num_features,
num_hidden,
weight_init=nn.IIDGaussian(std=.1),
bias_init=nn.Constant(1))(X))
# One final fully-connected layer, and then a linear activation output for reward
output = nn.Affine(num_hidden,
1,
weight_init=nn.IIDGaussian(std=.1),
bias_init=nn.Constant(1))(hid1)
abs_deviation = cgt.abs(output - y).mean()
params = nn.get_parameters(abs_deviation)
gparams = cgt.grad(abs_deviation, params)
updates = [(p, p - stepsize * gp) for (p, gp) in zip(params, gparams)]
self.predictor = cgt.function([X], output)
self.updater = cgt.function([X, y], abs_deviation, updates=updates)
def scalar(name):
return cgt.scalar(name)
from __future__ import print_function
import cgt
a = cgt.scalar('a')
b = cgt.scalar('b')
y = a * b
multiply = cgt.function([a,b], y)
print(multiply(1, 2)) # 2
print(multiply(3, 3)) # 9
def scalar(name):
return cgt.scalar(name)
def __init__(self, obs_dim, ctrl_dim):
cgt.set_precision('double')
Serializable.__init__(self, obs_dim, ctrl_dim)
self.obs_dim = obs_dim
self.ctrl_dim = ctrl_dim
o_no = cgt.matrix("o_no", fixed_shape=(None, obs_dim))
a_na = cgt.matrix("a_na", fixed_shape=(None, ctrl_dim))
adv_n = cgt.vector("adv_n")
oldpdist_np = cgt.matrix("oldpdist", fixed_shape=(None, 2 * ctrl_dim))
self.logstd = logstd_1a = nn.parameter(np.zeros((1, self.ctrl_dim)),
name="std_1a")
std_1a = cgt.exp(logstd_1a)
# Here's where we apply the network
h0 = o_no
nhid = 32
h1 = cgt.tanh(
nn.Affine(obs_dim, nhid, weight_init=nn.IIDGaussian(std=0.1))(h0))
h2 = cgt.tanh(
nn.Affine(nhid, nhid, weight_init=nn.IIDGaussian(std=0.1))(h1))
mean_na = nn.Affine(nhid,
ctrl_dim,
weight_init=nn.IIDGaussian(std=0.01))(h2)
b = cgt.size(o_no, 0)
std_na = cgt.repeat(std_1a, b, axis=0)
oldmean_na = oldpdist_np[:, 0:self.ctrl_dim]
oldstd_na = oldpdist_np[:, self.ctrl_dim:2 * self.ctrl_dim]
logp_n = ((-.5) * cgt.square(
(a_na - mean_na) / std_na).sum(axis=1)) - logstd_1a.sum()
oldlogp_n = ((-.5) * cgt.square(
(a_na - oldmean_na) / oldstd_na).sum(axis=1)
) - cgt.log(oldstd_na).sum(axis=1)
ratio_n = cgt.exp(logp_n - oldlogp_n)
surr = (ratio_n * adv_n).mean()
pdists_np = cgt.concatenate([mean_na, std_na], axis=1)
# kl = cgt.log(sigafter/)
params = nn.get_parameters(surr)
oldvar_na = cgt.square(oldstd_na)
var_na = cgt.square(std_na)
kl = (cgt.log(std_na / oldstd_na) +
(oldvar_na + cgt.square(oldmean_na - mean_na)) / (2 * var_na) -
.5).sum(axis=1).mean()
lam = cgt.scalar()
penobj = surr - lam * kl
self._compute_surr_kl = cgt.function([oldpdist_np, o_no, a_na, adv_n],
[surr, kl])
self._compute_grad_lagrangian = cgt.function(
[lam, oldpdist_np, o_no, a_na, adv_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj, params)]))
self.f_pdist = cgt.function([o_no], pdists_np)
self.f_objs = cgt.function([oldpdist_np, o_no, a_na, adv_n],
[surr, kl])
self.pc = ParamCollection(params)
