def conv2d_fft(x_BKRC, f_LKrc, subsample, pad):
# TODO add shape assertion
f_LKrc = cgt.flip(f_LKrc, [2,3])
padnrows = size(x_BKRC, 2) + size(f_LKrc, 2) - 1
padncols = size(x_BKRC, 3) + size(f_LKrc, 3) - 1
tx = cgt.rfft(x_BKRC, (padnrows,padncols), (2,3))
tf = cgt.rfft(f_LKrc, (padnrows,padncols), (2,3))
out = cgt.irfft( cgt.einsum("BKrc,LKrc->BLrc",tx, tf), (2,3))
out = out[:,:,pad[0]:(padnrows-pad[0]):subsample[0],pad[1]:(padncols-pad[1]):subsample[1]] #pylint: disable=E1127
return out
def conv2d_fft(x_BKRC, f_LKrc, subsample, pad):
# TODO add shape assertion
f_LKrc = cgt.flip(f_LKrc, [2,3])
padnrows = size(x_BKRC, 2) + size(f_LKrc, 2) - 1
padncols = size(x_BKRC, 3) + size(f_LKrc, 3) - 1
tx = cgt.rfft(x_BKRC, (padnrows,padncols), (2,3))
tf = cgt.rfft(f_LKrc, (padnrows,padncols), (2,3))
out = cgt.irfft( cgt.einsum("BKrc,LKrc->BLrc",tx, tf), (2,3))
out = out[:,:,pad[0]:(padnrows-pad[0]):subsample[0],pad[1]:(padncols-pad[1]):subsample[1]] #pylint: disable=E1127
return out
def __init__(self, n_actions):
Serializable.__init__(self, n_actions)
cgt.set_precision('double')
n_in = 128
o_no = cgt.matrix("o_no",fixed_shape=(None,n_in))
a_n = cgt.vector("a_n",dtype='i8')
q_n = cgt.vector("q_n")
oldpdist_np = cgt.matrix("oldpdists")
h0 = (o_no - 128.0)/128.0
nhid = 64
h1 = cgt.tanh(nn.Affine(128,nhid,weight_init=nn.IIDGaussian(std=.1))(h0))
probs_na = nn.softmax(nn.Affine(nhid,n_actions,weight_init=nn.IIDGaussian(std=0.01))(h1))
logprobs_na = cgt.log(probs_na)
b = cgt.size(o_no, 0)
logps_n = logprobs_na[cgt.arange(b), a_n]
surr = (logps_n*q_n).mean()
kl = (oldpdist_np * cgt.log(oldpdist_np/probs_na)).sum(axis=1).mean()
params = nn.get_parameters(surr)
gradsurr = cgt.grad(surr, params)
flatgrad = cgt.concatenate([p.flatten() for p in gradsurr])
lam = cgt.scalar()
penobj = surr - lam * kl
self._f_grad_lagrangian = cgt.function([lam, oldpdist_np, o_no, a_n, q_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj,params)]))
self.f_pdist = cgt.function([o_no], probs_na)
self.f_probs = cgt.function([o_no], probs_na)
self.f_surr_kl = cgt.function([oldpdist_np, o_no, a_n, q_n], [surr, kl])
self.f_gradlogp = cgt.function([oldpdist_np, o_no, a_n, q_n], flatgrad)
self.pc = ParamCollection(params)
def circ_conv_1d(wg_bhn, s_bh3, axis=2):
"VERY inefficient way to implement circular convolution for the special case of filter size 3"
assert axis == 2
n = cgt.size(wg_bhn,2)
wback = cgt.concatenate([wg_bhn[:,:,n-1:n], wg_bhn[:,:,:n-1]], axis=2)
w = wg_bhn
wfwd = cgt.concatenate([wg_bhn[:,:,1:n], wg_bhn[:,:,0:1]], axis=2)
return cgt.broadcast("*", s_bh3[:,:,0:1] , wback, "xx1,xxx")\
+ cgt.broadcast("*", s_bh3[:,:,1:2] , w, "xx1,xxx")\
+ cgt.broadcast("*", s_bh3[:,:,2:3] , wfwd, "xx1,xxx")
def circ_conv_1d(wg_bhn, s_bh3, axis=2):
"VERY inefficient way to implement circular convolution for the special case of filter size 3"
assert axis == 2
n = cgt.size(wg_bhn,2)
wback = cgt.concatenate([wg_bhn[:,:,n-1:n], wg_bhn[:,:,:n-1]], axis=2)
w = wg_bhn
wfwd = cgt.concatenate([wg_bhn[:,:,1:n], wg_bhn[:,:,0:1]], axis=2)
return cgt.broadcast("*", s_bh3[:,:,0:1] , wback, "xx1,xxx")\
+ cgt.broadcast("*", s_bh3[:,:,1:2] , w, "xx1,xxx")\
+ cgt.broadcast("*", s_bh3[:,:,2:3] , wfwd, "xx1,xxx")
def __init__(self, obs_dim, ctrl_dim):
cgt.set_precision('double')
Serializable.__init__(self, obs_dim, ctrl_dim)
self.obs_dim = obs_dim
self.ctrl_dim = ctrl_dim
o_no = cgt.matrix("o_no",fixed_shape=(None,obs_dim))
a_na = cgt.matrix("a_na",fixed_shape = (None, ctrl_dim))
adv_n = cgt.vector("adv_n")
oldpdist_np = cgt.matrix("oldpdist", fixed_shape=(None, 2*ctrl_dim))
self.logstd = logstd_1a = nn.parameter(np.zeros((1, self.ctrl_dim)), name="std_1a")
std_1a = cgt.exp(logstd_1a)
# Here's where we apply the network
h0 = o_no
nhid = 32
h1 = cgt.tanh(nn.Affine(obs_dim,nhid,weight_init=nn.IIDGaussian(std=0.1))(h0))
h2 = cgt.tanh(nn.Affine(nhid,nhid,weight_init=nn.IIDGaussian(std=0.1))(h1))
mean_na = nn.Affine(nhid,ctrl_dim,weight_init=nn.IIDGaussian(std=0.01))(h2)
b = cgt.size(o_no, 0)
std_na = cgt.repeat(std_1a, b, axis=0)
oldmean_na = oldpdist_np[:, 0:self.ctrl_dim]
oldstd_na = oldpdist_np[:, self.ctrl_dim:2*self.ctrl_dim]
logp_n = ((-.5) * cgt.square( (a_na - mean_na) / std_na ).sum(axis=1)) - logstd_1a.sum()
oldlogp_n = ((-.5) * cgt.square( (a_na - oldmean_na) / oldstd_na ).sum(axis=1)) - cgt.log(oldstd_na).sum(axis=1)
ratio_n = cgt.exp(logp_n - oldlogp_n)
surr = (ratio_n*adv_n).mean()
pdists_np = cgt.concatenate([mean_na, std_na], axis=1)
# kl = cgt.log(sigafter/)
params = nn.get_parameters(surr)
oldvar_na = cgt.square(oldstd_na)
var_na = cgt.square(std_na)
kl = (cgt.log(std_na / oldstd_na) + (oldvar_na + cgt.square(oldmean_na - mean_na)) / (2 * var_na) - .5).sum(axis=1).mean()
lam = cgt.scalar()
penobj = surr - lam * kl
self._compute_surr_kl = cgt.function([oldpdist_np, o_no, a_na, adv_n], [surr, kl])
self._compute_grad_lagrangian = cgt.function([lam, oldpdist_np, o_no, a_na, adv_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj,params)]))
self.f_pdist = cgt.function([o_no], pdists_np)
self.f_objs = cgt.function([oldpdist_np, o_no, a_na, adv_n], [surr, kl])
self.pc = ParamCollection(params)
def shp_apply(self, inputs):
X, W, _b = inputs
h = cgt.ceil_divide(
cgt.size(X, 2) + self.ph * 2 - cgt.size(W, 2) + 1, self.sv)
w = cgt.ceil_divide(
cgt.size(X, 3) + self.pw * 2 - cgt.size(W, 3) + 1, self.sh)
return [cgt.size(X, 0), cgt.size(W, 0), h, w]
def shp_apply(self, inputs):
X, W, _b = inputs
h = cgt.ceil_divide(
cgt.size(X, 2) + self.ph * 2 - cgt.size(W, 2) + 1, self.sv)
w = cgt.ceil_divide(
cgt.size(X, 3) + self.pw * 2 - cgt.size(W, 3) + 1, self.sh)
return [cgt.size(X, 0), cgt.size(W, 0), h, w]
def __init__(self, n_actions):
Serializable.__init__(self, n_actions)
cgt.set_precision('double')
n_in = 128
o_no = cgt.matrix("o_no", fixed_shape=(None, n_in))
a_n = cgt.vector("a_n", dtype='i8')
q_n = cgt.vector("q_n")
oldpdist_np = cgt.matrix("oldpdists")
h0 = (o_no - 128.0) / 128.0
nhid = 64
h1 = cgt.tanh(
nn.Affine(128, nhid, weight_init=nn.IIDGaussian(std=.1))(h0))
probs_na = nn.softmax(
nn.Affine(nhid, n_actions,
weight_init=nn.IIDGaussian(std=0.01))(h1))
logprobs_na = cgt.log(probs_na)
b = cgt.size(o_no, 0)
logps_n = logprobs_na[cgt.arange(b), a_n]
surr = (logps_n * q_n).mean()
kl = (oldpdist_np * cgt.log(oldpdist_np / probs_na)).sum(axis=1).mean()
params = nn.get_parameters(surr)
gradsurr = cgt.grad(surr, params)
flatgrad = cgt.concatenate([p.flatten() for p in gradsurr])
lam = cgt.scalar()
penobj = surr - lam * kl
self._f_grad_lagrangian = cgt.function(
[lam, oldpdist_np, o_no, a_n, q_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj, params)]))
self.f_pdist = cgt.function([o_no], probs_na)
self.f_probs = cgt.function([o_no], probs_na)
self.f_surr_kl = cgt.function([oldpdist_np, o_no, a_n, q_n],
[surr, kl])
self.f_gradlogp = cgt.function([oldpdist_np, o_no, a_n, q_n], flatgrad)
self.pc = ParamCollection(params)
def loglik(self, labels, p):
return cgt.log(p[cgt.arange(cgt.size(labels, 0)), labels])
def __init__(self, obs_dim, ctrl_dim):
cgt.set_precision('double')
Serializable.__init__(self, obs_dim, ctrl_dim)
self.obs_dim = obs_dim
self.ctrl_dim = ctrl_dim
o_no = cgt.matrix("o_no", fixed_shape=(None, obs_dim))
a_na = cgt.matrix("a_na", fixed_shape=(None, ctrl_dim))
adv_n = cgt.vector("adv_n")
oldpdist_np = cgt.matrix("oldpdist", fixed_shape=(None, 2 * ctrl_dim))
self.logstd = logstd_1a = nn.parameter(np.zeros((1, self.ctrl_dim)),
name="std_1a")
std_1a = cgt.exp(logstd_1a)
# Here's where we apply the network
h0 = o_no
nhid = 32
h1 = cgt.tanh(
nn.Affine(obs_dim, nhid, weight_init=nn.IIDGaussian(std=0.1))(h0))
h2 = cgt.tanh(
nn.Affine(nhid, nhid, weight_init=nn.IIDGaussian(std=0.1))(h1))
mean_na = nn.Affine(nhid,
ctrl_dim,
weight_init=nn.IIDGaussian(std=0.01))(h2)
b = cgt.size(o_no, 0)
std_na = cgt.repeat(std_1a, b, axis=0)
oldmean_na = oldpdist_np[:, 0:self.ctrl_dim]
oldstd_na = oldpdist_np[:, self.ctrl_dim:2 * self.ctrl_dim]
logp_n = ((-.5) * cgt.square(
(a_na - mean_na) / std_na).sum(axis=1)) - logstd_1a.sum()
oldlogp_n = ((-.5) * cgt.square(
(a_na - oldmean_na) / oldstd_na).sum(axis=1)
) - cgt.log(oldstd_na).sum(axis=1)
ratio_n = cgt.exp(logp_n - oldlogp_n)
surr = (ratio_n * adv_n).mean()
pdists_np = cgt.concatenate([mean_na, std_na], axis=1)
# kl = cgt.log(sigafter/)
params = nn.get_parameters(surr)
oldvar_na = cgt.square(oldstd_na)
var_na = cgt.square(std_na)
kl = (cgt.log(std_na / oldstd_na) +
(oldvar_na + cgt.square(oldmean_na - mean_na)) / (2 * var_na) -
.5).sum(axis=1).mean()
lam = cgt.scalar()
penobj = surr - lam * kl
self._compute_surr_kl = cgt.function([oldpdist_np, o_no, a_na, adv_n],
[surr, kl])
self._compute_grad_lagrangian = cgt.function(
[lam, oldpdist_np, o_no, a_na, adv_n],
cgt.concatenate([p.flatten() for p in cgt.grad(penobj, params)]))
self.f_pdist = cgt.function([o_no], pdists_np)
self.f_objs = cgt.function([oldpdist_np, o_no, a_na, adv_n],
[surr, kl])
self.pc = ParamCollection(params)
def loglik(self, labels, p):
return cgt.log(p[cgt.arange(cgt.size(labels,0)),labels])
