class Agent:
def __init__(self,
environment,
x_dim,
y_dim,
state_dim,
action_dim,
observation_space_low,
observation_space_high,
action_space_low,
action_space_high,
unroll_steps,
no_samples,
discount_factor,
random_matrices,
biases,
basis_dims,
hidden_dim=32,
learn_reward=0,
use_mean_reward=0,
update_hyperstate=1,
policy_use_hyperstate=1,
learn_diff=0):
#assert environment in ['Pendulum-v0', 'MountainCarContinuous-v0']
assert x_dim == state_dim + action_dim
assert len(action_space_low.shape) == 1
np.testing.assert_equal(-action_space_low, action_space_high)
self.environment = environment
self.x_dim = x_dim
self.y_dim = y_dim
self.state_dim = state_dim
self.action_dim = action_dim
self.observation_space_low = observation_space_low
self.observation_space_high = observation_space_high
self.action_space_low = action_space_low
self.action_space_high = action_space_high
self.unroll_steps = unroll_steps
self.no_samples = no_samples
self.discount_factor = discount_factor
self.random_matrices = random_matrices
self.biases = biases
self.basis_dims = basis_dims
self.hidden_dim = hidden_dim
self.learn_reward = learn_reward
self.use_mean_reward = use_mean_reward
self.update_hyperstate = update_hyperstate
self.policy_use_hyperstate = policy_use_hyperstate
self.learn_diff = learn_diff
if self.environment == 'Pendulum-v0' and self.learn_reward == 0:
#self.reward_function = real_env_pendulum_reward()
self.reward_function = ANN(self.state_dim + self.action_dim, 1)
self.placeholders_reward = [
tf.placeholder(shape=v.shape, dtype=tf.float64)
for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
self.reward_function.scope)
]
self.assign_ops0 = [
v.assign(pl) for v, pl in zip(
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
self.reward_function.scope),
self.placeholders_reward)
]
elif self.environment == 'MountainCarContinuous-v0' and self.learn_reward == 0:
self.reward_function = mountain_car_continuous_reward_function()
#self.hyperstate_dim = sum([(basis_dim*(basis_dim+1))/2 + basis_dim for basis_dim in self.basis_dims])
self.hyperstate_dim = sum(
[basis_dim * (basis_dim + 1) for basis_dim in self.basis_dims])
self.random_projection_matrix = np.random.normal(
loc=0.,
scale=1. / np.sqrt(self.state_dim),
size=[self.hyperstate_dim, self.state_dim])
input_dim = self.state_dim
if self.policy_use_hyperstate == 1:
input_dim *= 2
self.w1 = np.concatenate([
np.random.normal(size=[input_dim, self.hidden_dim]),
np.random.uniform(-3e-3, 3e-3, size=[1, self.hidden_dim])
],
axis=0)
self.w2 = np.concatenate([
np.random.normal(size=[self.hidden_dim, self.hidden_dim]),
np.random.uniform(-3e-3, 3e-3, size=[1, self.hidden_dim])
],
axis=0)
self.w3 = np.concatenate([
np.random.normal(size=[self.hidden_dim, self.action_dim]),
np.random.uniform(-3e-3, 3e-3, size=[1, self.action_dim])
],
axis=0)
self.thetas = self._pack([self.w1, self.w2, self.w3])
self.sizes = [[input_dim + 1, self.hidden_dim],
[self.hidden_dim + 1, self.hidden_dim],
[self.hidden_dim + 1, self.action_dim]]
w1, w2, w3 = self._unpack(self.thetas, self.sizes)
np.testing.assert_equal(w1, self.w1)
np.testing.assert_equal(w2, self.w2)
np.testing.assert_equal(w3, self.w3)
def _pack(self, thetas):
return np.concatenate([theta.flatten() for theta in thetas])
def _unpack(self, thetas, sizes):
sidx = 0
weights = []
for size in sizes:
i, j = size
w = thetas[sidx:sidx + i * j].reshape([i, j])
sidx += i * j
weights.append(w)
return weights
def _forward(self, thetas, X, hyperstate):
#"Old" method of including hyperstate into policy network.
'''
w0, w1, w2, w3 = self._unpack(thetas, self.sizes)
XXtr, Xytr = hyperstate
A = [xx + noise for xx, noise in zip(XXtr, self.noises)]
wn = [solve(a, xy) for a, xy in zip(A, Xytr)]
indices = [np.triu_indices(basis_dim, 1) for basis_dim in self.basis_dims]
hyperstate = []
for i in range(len(X)):
tmp0 = []
for j in range(len(A)):
A[j][i][indices[j]] = np.nan
tmp1 = A[j][i]
tmp0.append(tmp1[~np.isnan(tmp1)])
tmp0.append(np.squeeze(wn[j][i]))
tmp0 = np.concatenate(tmp0)
hyperstate.append(tmp0)
hyperstate = np.stack(hyperstate, axis=0)
hyperstate = self._add_bias(hyperstate)
hyperstate_embedding = np.tanh(np.matmul(hyperstate, w0))
'''
w1, w2, w3 = self._unpack(thetas, self.sizes)
#Perform a simple random projection on the hyperstate.
if self.policy_use_hyperstate == 1:
hyperstate = np.concatenate([
np.concatenate([
np.reshape(XXtr, [len(XXtr), -1]),
np.reshape(Xytr, [len(Xytr), -1])
],
axis=-1) for XXtr, Xytr in zip(*hyperstate)
],
axis=-1)
hyperstate = np.tanh(hyperstate / 50000.)
hyperstate_embedding = np.matmul(hyperstate,
self.random_projection_matrix)
hyperstate_embedding = np.tanh(hyperstate_embedding)
state_hyperstate = np.concatenate([X, hyperstate_embedding],
axis=-1)
policy_net_input = self._add_bias(state_hyperstate)
else:
policy_net_input = self._add_bias(X)
h1 = np.tanh(np.matmul(policy_net_input, w1))
h1 = self._add_bias(h1)
h2 = np.tanh(np.matmul(h1, w2))
h2 = self._add_bias(h2)
out = np.tanh(np.matmul(h2, w3))
out = out * self.action_space_high #action bounds.
return out
def _add_bias(self, X):
assert len(X.shape) == 2
return np.concatenate([X, np.ones([len(X), 1])], axis=-1)
def _relu(self, X):
return np.maximum(X, 0.)
def _fit(self, cma_maxiter, X, XXtr, Xytr, hyperparameters, sess):
warnings.filterwarnings(
'ignore',
message=
'.*scipy.linalg.solve\nIll-conditioned matrix detected. Result is not guaranteed to be accurate.\nReciprocal.*'
)
assert len(XXtr) == self.state_dim + self.learn_reward
assert len(Xytr) == self.state_dim + self.learn_reward
assert len(hyperparameters) == self.state_dim + self.learn_reward
if self.use_mean_reward == 1:
print 'Warning: use_mean_reward is set to True but this flag is not used by this function.'
X = np.copy(X)
XXtr = [np.copy(ele) for ele in XXtr]
Xytr = [np.copy(ele) for ele in Xytr]
hyperparameters = [np.copy(ele) for ele in hyperparameters]
X = np.expand_dims(X, axis=1)
X = np.tile(X, [1, self.no_samples, 1])
X = np.reshape(X, [-1, self.state_dim])
Llowers = [
scipy.linalg.cholesky(
(hp[-2] / hp[-1])**2 * np.eye(basis_dim) + XX, lower=True) for
hp, basis_dim, XX in zip(hyperparameters, self.basis_dims, XXtr)
]
Llowers = [
np.tile(ele[np.newaxis, ...], [len(X), 1, 1]) for ele in Llowers
]
XXtr = [np.tile(ele[np.newaxis, ...], [len(X), 1, 1]) for ele in XXtr]
Xytr = [np.tile(ele[np.newaxis, ...], [len(X), 1, 1]) for ele in Xytr]
self.noises = [
(hp[2] / hp[3])**2 * np.eye(basis_dim)
for hp, basis_dim in zip(hyperparameters, self.basis_dims)
]
import cma
options = {'maxiter': cma_maxiter, 'verb_disp': 1, 'verb_log': 0}
print 'Before calling cma.fmin'
res = cma.fmin(self._loss,
self.thetas,
2.,
args=(np.copy(X), [np.copy(ele) for ele in Llowers
], [np.copy(ele) for ele in XXtr],
[np.copy(ele) for ele in Xytr], None,
[np.copy(ele) for ele in hyperparameters], sess),
options=options)
self.thetas = np.copy(res[0])
def _predict(self, Llower, Xytr, basis, noise_sd):
'''
Llower = Llower[0]
Xytr = Xytr[0]
basis = np.squeeze(basis, axis=1)
LinvXT = scipy.linalg.solve_triangular(Llower, basis.T, lower=True)
pred_sigma = np.sum(np.square(LinvXT), axis=0)*noise_sd**2+noise_sd**2
pred_sigma = pred_sigma[..., np.newaxis]
tmp0 = scipy.linalg.solve_triangular(Llower, basis.T, lower=True).T
tmp1 = scipy.linalg.solve_triangular(Llower, Xytr, lower=True)
pred_mu = np.matmul(tmp0, tmp1)
return pred_mu, pred_sigma
'''
#TODO:fix this.
LinvXT = solve_triangular(Llower, np.transpose(basis, [0, 2, 1]))
pred_sigma = np.sum(np.square(LinvXT),
axis=1) * noise_sd**2 + noise_sd**2
tmp0 = np.transpose(
solve_triangular(Llower, np.transpose(basis, [0, 2, 1])),
[0, 2, 1])
tmp1 = solve_triangular(Llower, Xytr)
pred_mu = np.matmul(tmp0, tmp1)
pred_mu = np.squeeze(pred_mu, axis=-1)
return pred_mu, pred_sigma
def _loss(self,
thetas,
X,
Llowers,
XXtr,
Xytr,
A=[],
hyperparameters=None,
sess=None):
rng_state = np.random.get_state()
X = np.copy(X)
Llowers = [np.copy(ele) for ele in Llowers]
XXtr = [np.copy(ele) for ele in XXtr]
Xytr = [np.copy(ele) for ele in Xytr]
hyperparameters = [np.copy(ele) for ele in hyperparameters]
try:
np.random.seed(2)
rewards = []
state = X
for unroll_step in xrange(self.unroll_steps):
action = self._forward(thetas,
state,
hyperstate=[Llowers, Xytr])
reward, basis_reward = self._reward(state, action, sess,
Llowers[-1], Xytr[-1],
hyperparameters[-1])
rewards.append((self.discount_factor**unroll_step) * reward)
state_action = np.concatenate([state, action], axis=-1)
means = []
covs = []
bases = []
for i in xrange(self.state_dim):
length_scale, signal_sd, noise_sd, prior_sd = hyperparameters[
i]
basis = _basis(state_action, self.random_matrices[i],
self.biases[i], self.basis_dims[i],
length_scale, signal_sd)
basis = np.expand_dims(basis, axis=1)
bases.append(basis)
pred_mu, pred_sigma = self._predict(
Llowers[i], Xytr[i], basis, noise_sd)
means.append(pred_mu)
covs.append(pred_sigma)
means = np.concatenate(means, axis=-1)
covs = np.concatenate(covs, axis=-1)
bases.append(basis_reward)
state_ = np.stack([
np.random.multivariate_normal(mean=mean, cov=np.diag(cov))
for mean, cov in zip(means, covs)
],
axis=0)
state = state + state_ if self.learn_diff else state_
if self.learn_diff == 0:
state_ = np.clip(state_, self.observation_space_low,
self.observation_space_high)
state = np.clip(state, self.observation_space_low,
self.observation_space_high)
# #Removable
# import copy
# Llowers2 = copy.deepcopy(Llowers)
# Xytr2 = copy.deepcopy(Xytr)
# XXtr2 = copy.deepcopy(XXtr)
# #Removable -END-
if self.update_hyperstate == 1 or self.policy_use_hyperstate == 1:
y = np.concatenate([state_, reward],
axis=-1)[..., :self.state_dim +
self.learn_reward]
y = y[..., np.newaxis, np.newaxis]
for i in xrange(self.state_dim + self.learn_reward):
Llowers[i] = Llowers[i].transpose([0, 2, 1])
for i in xrange(self.state_dim + self.learn_reward):
for j in xrange(len(Llowers[i])):
cholupdate(Llowers[i][j], bases[i][j, 0].copy())
Xytr[i] += np.matmul(bases[i].transpose([0, 2, 1]),
y[:, i, ...])
# #Removable
# _, _, noise_sd, prior_sd = hyperparameters[i]
# XXtr2[i], Xytr2[i], Llowers2[i] = self._update_hyperstate(XXtr2[i], XXtr2[i] + np.matmul(np.transpose(bases[i], [0, 2, 1]), bases[i]), Xytr2[i], Xytr2[i] + np.matmul(np.transpose(bases[i], [0, 2, 1]), y[:, i, ...]), Llowers2[i], (noise_sd/prior_sd)**2)
# print i
# print np.allclose(Llowers[i], Llowers2[i].transpose([0, 2, 1]))
# print np.allclose(Xytr[i], Xytr2[i])
# #Removable -END-
for i in xrange(self.state_dim + self.learn_reward):
Llowers[i] = Llowers[i].transpose([0, 2, 1])
rewards = np.concatenate(rewards, axis=-1)
rewards = np.sum(rewards, axis=-1)
loss = -np.mean(rewards)
np.random.set_state(rng_state)
return loss
except Exception as e:
np.random.set_state(rng_state)
print e, 'Returning 10e100'
return 10e100
def _update_hyperstate(self, XXold, XXnew, Xyold, Xynew, Llowerold,
var_ratio):
var_diag = var_ratio * np.eye(XXnew.shape[-1])
XX = []
Xy = []
Llower = []
for i in range(len(XXnew)):
try:
tmp = scipy.linalg.cholesky(XXnew[i] + var_diag, lower=True)
XX.append(XXnew[i].copy())
Xy.append(Xynew[i].copy())
Llower.append(tmp.copy())
except Exception as e:
XX.append(XXold[i].copy())
Xy.append(Xyold[i].copy())
Llower.append(Llowerold[i].copy())
XX = np.stack(XX, axis=0)
Xy = np.stack(Xy, axis=0)
Llower = np.stack(Llower, axis=0)
return XX, Xy, Llower
def _reward(self, state, action, sess, Llower, Xy, hyperparameters):
basis = None
if self.environment == 'Pendulum-v0' and self.learn_reward == 0:
reward = self.reward_function.build_np(sess, state, action)
elif self.environment == 'MountainCarContinuous-v0' and self.learn_reward == 0:
reward = self.reward_function.build_np(state, action)
else:
state_action = np.concatenate([state, action], axis=-1)
length_scale, signal_sd, noise_sd, prior_sd = hyperparameters
basis = _basis(state_action, self.random_matrices[-1],
self.biases[-1], self.basis_dims[-1], length_scale,
signal_sd)
basis = np.expand_dims(basis, axis=1)
pred_mu, pred_sigma = self._predict(Llower, Xy, basis, noise_sd)
if self.use_mean_reward == 1:
pred_sigma = np.zeros_like(pred_sigma)
reward = np.stack([
np.random.normal(loc=loc, scale=scale)
for loc, scale in zip(pred_mu, pred_sigma)
],
axis=0)
return reward, basis
class Agent:
def __init__(self,
environment,
x_dim,
y_dim,
state_dim,
action_dim,
observation_space_low,
observation_space_high,
action_space_low,
action_space_high,
unroll_steps,
no_samples,
discount_factor,
random_matrix_state,
bias_state,
basis_dim_state,
random_matrix_reward,
bias_reward,
basis_dim_reward,
hidden_dim=32,
learn_reward=0,
use_mean_reward=0,
update_hyperstate=1,
policy_use_hyperstate=1,
learn_diff=0,
dump_model=0):
#assert environment in ['Pendulum-v0', 'MountainCarContinuous-v0']
assert x_dim == state_dim + action_dim
assert len(action_space_low.shape) == 1
np.testing.assert_equal(-action_space_low, action_space_high)
self.environment = environment
self.x_dim = x_dim
self.y_dim = y_dim
self.state_dim = state_dim
self.action_dim = action_dim
self.observation_space_low = observation_space_low
self.observation_space_high = observation_space_high
self.action_space_low = action_space_low
self.action_space_high = action_space_high
self.unroll_steps = unroll_steps
self.no_samples = no_samples
self.discount_factor = discount_factor
self.random_matrix_state = random_matrix_state
self.bias_state = bias_state
self.basis_dim_state = basis_dim_state
self.random_matrix_reward = random_matrix_reward
self.bias_reward = bias_reward
self.basis_dim_reward = basis_dim_reward
self.hidden_dim = hidden_dim
self.learn_reward = learn_reward
self.use_mean_reward = use_mean_reward
self.update_hyperstate = update_hyperstate
self.policy_use_hyperstate = policy_use_hyperstate
self.learn_diff = learn_diff
self.dump_model = dump_model
self.uid = str(uuid.uuid4())
self.epoch = 0
if self.environment == 'Pendulum-v0' and self.learn_reward == 0:
#self.reward_function = real_env_pendulum_reward()
self.reward_function = ANN(self.state_dim + self.action_dim, 1)
self.placeholders_reward = [
tf.placeholder(shape=v.shape, dtype=tf.float64)
for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
self.reward_function.scope)
]
self.assign_ops0 = [
v.assign(pl) for v, pl in zip(
tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
self.reward_function.scope),
self.placeholders_reward)
]
elif self.environment == 'MountainCarContinuous-v0' and self.learn_reward == 0:
self.reward_function = mountain_car_continuous_reward_function()
self.hyperstate_dim = self.basis_dim_state * (self.basis_dim_state +
self.state_dim)
if self.learn_reward == 1:
self.hyperstate_dim += self.basis_dim_reward * (
self.basis_dim_reward + 1)
self.random_projection_matrix = np.random.normal(
loc=0.,
scale=1. / np.sqrt(self.state_dim),
size=[self.hyperstate_dim, self.state_dim])
input_dim = self.state_dim
if self.policy_use_hyperstate == 1:
input_dim *= 2
self.w1 = np.concatenate([
np.random.normal(size=[input_dim, self.hidden_dim]),
np.random.uniform(-3e-3, 3e-3, size=[1, self.hidden_dim])
],
axis=0)
self.w2 = np.concatenate([
np.random.normal(size=[self.hidden_dim, self.hidden_dim]),
np.random.uniform(-3e-3, 3e-3, size=[1, self.hidden_dim])
],
axis=0)
self.w3 = np.concatenate([
np.random.normal(size=[self.hidden_dim, self.action_dim]),
np.random.uniform(-3e-3, 3e-3, size=[1, self.action_dim])
],
axis=0)
self.thetas = self._pack([self.w1, self.w2, self.w3])
self.sizes = [[input_dim + 1, self.hidden_dim],
[self.hidden_dim + 1, self.hidden_dim],
[self.hidden_dim + 1, self.action_dim]]
w1, w2, w3 = self._unpack(self.thetas, self.sizes)
np.testing.assert_equal(w1, self.w1)
np.testing.assert_equal(w2, self.w2)
np.testing.assert_equal(w3, self.w3)
def _pack(self, thetas):
return np.concatenate([theta.flatten() for theta in thetas])
def _unpack(self, thetas, sizes):
sidx = 0
weights = []
for size in sizes:
i, j = size
w = thetas[sidx:sidx + i * j].reshape([i, j])
sidx += i * j
weights.append(w)
return weights
def _forward(self, thetas, X, hyperstate_params):
w1, w2, w3 = self._unpack(thetas, self.sizes)
#Perform a simple random projection on the hyperstate.
if self.policy_use_hyperstate == 1:
Llower_state, Xytr_state, Llower_reward, Xytr_reward = hyperstate_params
hyperstate = np.concatenate([
Llower_state.reshape([len(Llower_state), -1]),
Xytr_state.reshape([len(Xytr_state), -1]),
Llower_reward.reshape([len(Llower_reward), -1]),
Xytr_reward.reshape([len(Xytr_reward), -1])
],
axis=-1)
hyperstate = np.tanh(hyperstate / 50000.)
hyperstate_embedding = np.matmul(hyperstate,
self.random_projection_matrix)
hyperstate_embedding = np.tanh(hyperstate_embedding)
state_hyperstate = np.concatenate([X, hyperstate_embedding],
axis=-1)
policy_net_input = self._add_bias(state_hyperstate)
else:
policy_net_input = self._add_bias(X)
h1 = np.tanh(np.matmul(policy_net_input, w1))
h1 = self._add_bias(h1)
h2 = np.tanh(np.matmul(h1, w2))
h2 = self._add_bias(h2)
out = np.tanh(np.matmul(h2, w3))
out = out * self.action_space_high #action bounds.
return out
def _add_bias(self, X):
assert len(X.shape) == 2
return np.concatenate([X, np.ones([len(X), 1])], axis=-1)
def _relu(self, X):
return np.maximum(X, 0.)
def _fit(self, cma_maxiter, X, XXtr_state, Xytr_state,
hyperparameters_state, XXtr_reward, Xytr_reward,
hyperparameters_reward, sess):
warnings.filterwarnings(
'ignore',
message=
'.*scipy.linalg.solve\nIll-conditioned matrix detected. Result is not guaranteed to be accurate.\nReciprocal.*'
)
assert XXtr_state.shape == (self.basis_dim_state, self.basis_dim_state)
assert Xytr_state.shape == (self.basis_dim_state, self.state_dim)
assert XXtr_reward.shape == (self.basis_dim_reward,
self.basis_dim_reward)
assert Xytr_reward.shape == (self.basis_dim_reward, 1)
assert hyperparameters_state.shape == hyperparameters_reward.shape
if self.use_mean_reward == 1:
print(
'Warning: use_mean_reward is set to True but this flag is not used by this function.'
)
#Copy the arrays (just to be safe no overwriting occurs).
X = X.copy()
XXtr_state = XXtr_state.copy()
Xytr_state = Xytr_state.copy()
hyperparameters_state = hyperparameters_state.copy()
XXtr_reward = XXtr_reward.copy()
Xytr_reward = Xytr_reward.copy()
hyperparameters_reward = hyperparameters_reward.copy()
X = np.expand_dims(X, axis=1)
X = np.tile(X, [1, self.no_samples, 1])
X = np.reshape(X, [-1, self.state_dim])
#State
Llower_state = spla.cholesky(
(hyperparameters_state[-2] / hyperparameters_state[-1])**2 *
np.eye(self.basis_dim_state) + XXtr_state,
lower=True)
Llower_state = np.tile(Llower_state, [len(X), 1, 1])
XXtr_state = np.tile(XXtr_state, [len(X), 1, 1])
Xytr_state = np.tile(Xytr_state, [len(X), 1, 1])
#Reward
if self.learn_reward:
Llower_reward = spla.cholesky(
(hyperparameters_reward[-2] / hyperparameters_reward[-1])**2 *
np.eye(self.basis_dim_reward) + XXtr_reward,
lower=True)
Llower_reward = np.tile(Llower_reward, [len(X), 1, 1])
XXtr_reward = np.tile(XXtr_reward, [len(X), 1, 1])
Xytr_reward = np.tile(Xytr_reward, [len(X), 1, 1])
import cma
options = {'maxiter': cma_maxiter, 'verb_disp': 1, 'verb_log': 0}
print('Before calling cma.fmin')
res = cma.fmin(
self._loss,
self.thetas,
2.,
args=(X.copy(), Llower_state.copy(), XXtr_state.copy(),
Xytr_state.copy(), hyperparameters_state,
Llower_reward.copy() if self.learn_reward else None,
XXtr_reward.copy() if self.learn_reward else None,
Xytr_reward.copy() if self.learn_reward else None,
hyperparameters_reward if self.learn_reward else None, sess),
options=options)
self.thetas = np.copy(res[0])
if self.dump_model:
print('Unique identifier:', self.uid)
directory = './models/'
if not os.path.exists(directory):
os.makedirs(directory)
with open(
directory + self.uid + '_epoch:' + str(self.epoch) + '.p',
'wb') as fp:
pickle.dump(self.thetas, fp)
self.epoch += 1
def _predict(self, Llower, Xytr, basis, noise_sd):
#TODO: fix this.
LinvXT = solve_triangular(Llower, basis.transpose([0, 2, 1]))
sigma = np.sum(np.square(LinvXT), axis=1) * noise_sd**2 + noise_sd**2
tmp0 = solve_triangular(Llower,
basis.transpose([0, 2,
1])).transpose([0, 2, 1])
tmp1 = solve_triangular(Llower, Xytr)
mu = np.matmul(tmp0, tmp1).squeeze(axis=1)
return mu, sigma
def _loss(self,
thetas,
X,
Llower_state,
XXtr_state,
Xytr_state,
hyperparameters_state,
Llower_reward,
XXtr_reward,
Xytr_reward,
hyperparameters_reward,
sess=None):
X = X.copy()
Llower_state = Llower_state.copy()
XXtr_state = XXtr_state.copy()
Xytr_state = Xytr_state.copy()
hyperparameters_state = hyperparameters_state.copy()
if self.learn_reward:
Llower_reward = Llower_reward.copy()
XXtr_reward = XXtr_reward.copy()
Xytr_reward = Xytr_reward.copy()
hyperparameters_reward = hyperparameters_reward.copy()
rng_state = np.random.get_state()
#try:
np.random.seed(2)
rewards = []
state = X
for unroll_step in xrange(self.unroll_steps):
action = self._forward(thetas,
state,
hyperstate_params=[
Llower_state, Xytr_state, Llower_reward,
Xytr_reward
])
state_action = np.concatenate([state, action], axis=-1)
reward, basis_reward = self._reward(state, action, state_action,
sess, Llower_reward,
Xytr_reward,
hyperparameters_reward)
rewards.append((self.discount_factor**unroll_step) * reward)
length_scale, signal_sd, noise_sd, prior_sd = hyperparameters_state
basis_state = _basis(state_action, self.random_matrix_state,
self.bias_state, self.basis_dim_state,
length_scale, signal_sd)
basis_state = basis_state[:, None, ...]
mu, sigma = self._predict(Llower_state, Xytr_state, basis_state,
noise_sd)
state_ = mu + np.sqrt(sigma) * np.random.standard_normal(
size=mu.shape)
if self.learn_diff:
state_tmp = state.copy()
state = np.clip(state + state_, self.observation_space_low,
self.observation_space_high)
state_ = state - state_tmp
else:
state_ = np.clip(state_, self.observation_space_low,
self.observation_space_high)
state = state_.copy()
if self.update_hyperstate == 1 or self.policy_use_hyperstate == 1:
#Update state hyperstate
Llower_state = Llower_state.transpose([0, 2, 1])
for i in range(len(Llower_state)):
cholupdate(Llower_state[i], basis_state[i, 0].copy())
Llower_state = Llower_state.transpose([0, 2, 1])
Xytr_state += np.matmul(basis_state.transpose([0, 2, 1]),
state_[..., None, :])
#Update reward hyperstate
if self.learn_reward:
Llower_reward = Llower_reward.transpose([0, 2, 1])
for i in range(len(Llower_reward)):
cholupdate(Llower_reward[i], basis_reward[i, 0].copy())
Llower_reward = Llower_reward.transpose([0, 2, 1])
Xytr_reward += np.matmul(basis_reward.transpose([0, 2, 1]),
reward[..., None, :])
rewards = np.concatenate(rewards, axis=-1)
rewards = np.sum(rewards, axis=-1)
loss = -np.mean(rewards)
np.random.set_state(rng_state)
return loss
#except Exception as e:
#np.random.set_state(rng_state)
#print e, 'Returning 10e100'
#return 10e100
def _reward(self, state, action, state_action, sess, Llower, Xy,
hyperparameters):
basis = None
if self.environment == 'Pendulum-v0' and self.learn_reward == 0:
reward = self.reward_function.build_np(sess, state, action)
elif self.environment == 'MountainCarContinuous-v0' and self.learn_reward == 0:
reward = self.reward_function.build_np(state, action)
else:
#state_action = np.concatenate([state, action], axis=-1)
length_scale, signal_sd, noise_sd, prior_sd = hyperparameters
basis = _basis(state_action, self.random_matrix_reward,
self.bias_reward, self.basis_dim_reward,
length_scale, signal_sd)
basis = basis[:, None, ...]
mu, sigma = self._predict(Llower, Xy, basis, noise_sd)
if self.use_mean_reward == 1: sigma = np.zeros_like(sigma)
reward = mu + np.sqrt(sigma) * np.random.standard_normal(
size=mu.shape)
return reward, basis