def test_true_online_sarsa_lambda():
pi, _, mdp_continuous = initialize()
mdp_continuous.seed(1)
n_tilings = 1
tilings = Tiles.generate(n_tilings, [2, 2],
mdp_continuous.info.observation_space.low,
mdp_continuous.info.observation_space.high)
features = Features(tilings=tilings)
approximator_params = dict(
input_shape=(features.size, ),
output_shape=(mdp_continuous.info.action_space.n, ),
n_actions=mdp_continuous.info.action_space.n)
agent = TrueOnlineSARSALambda(mdp_continuous.info,
pi,
Parameter(.1),
.9,
features=features,
approximator_params=approximator_params)
core = Core(agent, mdp_continuous)
# Train
core.learn(n_steps=100, n_steps_per_fit=1, quiet=True)
test_w = np.array([
-17.30427303, 0., -13.54157504, 0., -16.82373134, 0., -10.29613337, 0.,
-14.79470382, 0., -10.50654665, 0.
])
assert np.allclose(agent.Q.get_weights(), test_w)
def test_sarsa_lambda_continuous_linear():
pi, _, mdp_continuous = initialize()
mdp_continuous.seed(1)
n_tilings = 1
tilings = Tiles.generate(n_tilings, [2, 2],
mdp_continuous.info.observation_space.low,
mdp_continuous.info.observation_space.high)
features = Features(tilings=tilings)
approximator_params = dict(
input_shape=(features.size,),
output_shape=(mdp_continuous.info.action_space.n,),
n_actions=mdp_continuous.info.action_space.n
)
agent = SARSALambdaContinuous(mdp_continuous.info, pi, LinearApproximator,
Parameter(.1), .9, features=features,
approximator_params=approximator_params)
core = Core(agent, mdp_continuous)
# Train
core.learn(n_steps=100, n_steps_per_fit=1, quiet=True)
test_w = np.array([-16.38428419, 0., -14.31250136, 0., -15.68571525, 0.,
-10.15663821, 0., -15.0545445, 0., -8.3683605, 0.])
assert np.allclose(agent.Q.get_weights(), test_w)
def test_sarsa_lambda_continuous_linear():
pi, _, mdp_continuous = initialize()
mdp_continuous.seed(1)
n_tilings = 1
tilings = Tiles.generate(n_tilings, [2, 2],
mdp_continuous.info.observation_space.low,
mdp_continuous.info.observation_space.high)
features = Features(tilings=tilings)
approximator_params = dict(
input_shape=(features.size, ),
output_shape=(mdp_continuous.info.action_space.n, ),
n_actions=mdp_continuous.info.action_space.n)
agent = SARSALambdaContinuous(mdp_continuous.info,
pi,
LinearApproximator,
Parameter(.1),
.9,
features=features,
approximator_params=approximator_params)
core = Core(agent, mdp_continuous)
# Train
core.learn(n_steps=100, n_steps_per_fit=1, quiet=True)
test_w = np.array([
-16.62627886, 0., -13.03033079, 0., -15.93237930, 0., -9.72299176, 0.,
-13.78884631, 0., -9.92157645, 0.
])
assert np.allclose(agent.Q.get_weights(), test_w)
def test_true_online_sarsa_lambda_save(tmpdir):
agent_path = tmpdir / 'agent_{}'.format(datetime.now().strftime("%H%M%S%f"))
pi, _, mdp_continuous = initialize()
mdp_continuous.seed(1)
n_tilings = 1
tilings = Tiles.generate(n_tilings, [2, 2],
mdp_continuous.info.observation_space.low,
mdp_continuous.info.observation_space.high)
features = Features(tilings=tilings)
approximator_params = dict(
input_shape=(features.size,),
output_shape=(mdp_continuous.info.action_space.n,),
n_actions=mdp_continuous.info.action_space.n
)
agent_save = TrueOnlineSARSALambda(mdp_continuous.info, pi,
Parameter(.1), .9, features=features,
approximator_params=approximator_params)
core = Core(agent_save, mdp_continuous)
# Train
core.learn(n_steps=100, n_steps_per_fit=1, quiet=True)
agent_save.save(agent_path)
agent_load = Agent.load(agent_path)
for att, method in vars(agent_save).items():
save_attr = getattr(agent_save, att)
load_attr = getattr(agent_load, att)
tu.assert_eq(save_attr, load_attr)
def test_true_online_sarsa_lambda():
pi, _, mdp_continuous = initialize()
mdp_continuous.seed(1)
n_tilings = 1
tilings = Tiles.generate(n_tilings, [2, 2],
mdp_continuous.info.observation_space.low,
mdp_continuous.info.observation_space.high)
features = Features(tilings=tilings)
approximator_params = dict(
input_shape=(features.size,),
output_shape=(mdp_continuous.info.action_space.n,),
n_actions=mdp_continuous.info.action_space.n
)
agent = TrueOnlineSARSALambda(mdp_continuous.info, pi,
Parameter(.1), .9, features=features,
approximator_params=approximator_params)
core = Core(agent, mdp_continuous)
# Train
core.learn(n_steps=100, n_steps_per_fit=1, quiet=True)
test_w = np.array([-17.27410736, 0., -15.04386343, 0., -16.6551805, 0.,
-11.31383707, 0., -16.11782002, 0., -9.6927357, 0.])
assert np.allclose(agent.Q.get_weights(), test_w)
def experiment(n_epochs, n_episodes):
np.random.seed()
logger = Logger(COPDAC_Q.__name__, results_dir=None)
logger.strong_line()
logger.info('Experiment Algorithm: ' + COPDAC_Q.__name__)
# MDP
n_steps = 5000
mdp = InvertedPendulum(horizon=n_steps)
# Agent
n_tilings = 10
alpha_theta = Parameter(5e-3 / n_tilings)
alpha_omega = Parameter(0.5 / n_tilings)
alpha_v = Parameter(0.5 / n_tilings)
tilings = Tiles.generate(n_tilings, [10, 10],
mdp.info.observation_space.low,
mdp.info.observation_space.high + 1e-3)
phi = Features(tilings=tilings)
input_shape = (phi.size, )
mu = Regressor(LinearApproximator,
input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
sigma = 1e-1 * np.eye(1)
policy = GaussianPolicy(mu, sigma)
agent = COPDAC_Q(mdp.info,
policy,
mu,
alpha_theta,
alpha_omega,
alpha_v,
value_function_features=phi,
policy_features=phi)
# Train
dataset_callback = CollectDataset()
visualization_callback = Display(agent._V, mu,
mdp.info.observation_space.low,
mdp.info.observation_space.high, phi, phi)
core = Core(agent, mdp, callbacks_fit=[dataset_callback])
for i in trange(n_epochs, leave=False):
core.learn(n_episodes=n_episodes, n_steps_per_fit=1, render=False)
J = compute_J(dataset_callback.get(), gamma=1.0)
dataset_callback.clean()
visualization_callback()
logger.epoch_info(i + 1, R_mean=np.sum(J) / n_steps / n_episodes)
logger.info('Press a button to visualize the pendulum...')
input()
sigma = 1e-8 * np.eye(1)
policy.set_sigma(sigma)
core.evaluate(n_steps=n_steps, render=True)
def experiment(alg, params, n_epochs, fit_per_epoch, ep_per_fit):
np.random.seed()
logger = Logger(alg.__name__, results_dir=None)
logger.strong_line()
logger.info('Experiment Algorithm: ' + alg.__name__)
# MDP
mdp = ShipSteering()
# Policy
high = [150, 150, np.pi]
low = [0, 0, -np.pi]
n_tiles = [5, 5, 6]
low = np.array(low, dtype=np.float)
high = np.array(high, dtype=np.float)
n_tilings = 1
tilings = Tiles.generate(n_tilings=n_tilings,
n_tiles=n_tiles,
low=low,
high=high)
phi = Features(tilings=tilings)
input_shape = (phi.size, )
approximator = Regressor(LinearApproximator,
input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
policy = DeterministicPolicy(approximator)
mu = np.zeros(policy.weights_size)
sigma = 4e-1 * np.ones(policy.weights_size)
distribution = GaussianDiagonalDistribution(mu, sigma)
# Agent
agent = alg(mdp.info, distribution, policy, features=phi, **params)
# Train
core = Core(agent, mdp)
dataset_eval = core.evaluate(n_episodes=ep_per_fit)
J = compute_J(dataset_eval, gamma=mdp.info.gamma)
logger.epoch_info(0, J=np.mean(J))
for i in range(n_epochs):
core.learn(n_episodes=fit_per_epoch * ep_per_fit,
n_episodes_per_fit=ep_per_fit)
dataset_eval = core.evaluate(n_episodes=ep_per_fit)
J = compute_J(dataset_eval, gamma=mdp.info.gamma)
logger.epoch_info(i + 1, J=np.mean(J))
def experiment(alg, params, n_epochs, n_iterations, ep_per_run):
np.random.seed()
# MDP
mdp = ShipSteering()
# Policy
high = [150, 150, np.pi]
low = [0, 0, -np.pi]
n_tiles = [5, 5, 6]
low = np.array(low, dtype=np.float)
high = np.array(high, dtype=np.float)
n_tilings = 1
tilings = Tiles.generate(n_tilings=n_tilings,
n_tiles=n_tiles,
low=low,
high=high)
phi = Features(tilings=tilings)
input_shape = (phi.size, )
approximator = Regressor(LinearApproximator,
input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
policy = DeterministicPolicy(approximator)
mu = np.zeros(policy.weights_size)
sigma = 4e-1 * np.ones(policy.weights_size)
distribution = GaussianDiagonalDistribution(mu, sigma)
# Agent
agent = alg(mdp.info, distribution, policy, features=phi, **params)
# Train
print(alg.__name__)
core = Core(agent, mdp)
dataset_eval = core.evaluate(n_episodes=ep_per_run)
J = compute_J(dataset_eval, gamma=mdp.info.gamma)
print('J at start : ' + str(np.mean(J)))
for i in range(n_epochs):
core.learn(n_episodes=n_iterations * ep_per_run,
n_episodes_per_fit=ep_per_run)
dataset_eval = core.evaluate(n_episodes=ep_per_run)
J = compute_J(dataset_eval, gamma=mdp.info.gamma)
print('J at iteration ' + str(i) + ': ' + str(np.mean(J)))
def test_copdac_q():
n_steps = 50
mdp = InvertedPendulum(horizon=n_steps)
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed(1)
# Agent
n_tilings = 1
alpha_theta = Parameter(5e-3 / n_tilings)
alpha_omega = Parameter(0.5 / n_tilings)
alpha_v = Parameter(0.5 / n_tilings)
tilings = Tiles.generate(n_tilings, [2, 2], mdp.info.observation_space.low,
mdp.info.observation_space.high + 1e-3)
phi = Features(tilings=tilings)
input_shape = (phi.size, )
mu = Regressor(LinearApproximator,
input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
sigma = 1e-1 * np.eye(1)
policy = GaussianPolicy(mu, sigma)
agent = COPDAC_Q(mdp.info,
policy,
mu,
alpha_theta,
alpha_omega,
alpha_v,
value_function_features=phi,
policy_features=phi)
# Train
core = Core(agent, mdp)
core.learn(n_episodes=2, n_episodes_per_fit=1)
w = agent.policy.get_weights()
w_test = np.array([0, -6.62180045e-7, 0, -4.23972882e-2])
assert np.allclose(w, w_test)
def test_tiles():
tilings = Tiles.generate(3, [3, 3], np.array([0., -.5]), np.array([1.,
.5]))
features = Features(tilings=tilings)
x = np.random.rand(10, 2) + [0., -.5]
y = features(x)
for i, x_i in enumerate(x):
assert np.all(features(x_i) == y[i])
x_1 = x[:, 0].reshape(-1, 1)
x_2 = x[:, 1].reshape(-1, 1)
assert np.all(features(x_1, x_2) == y)
for i, x_i in enumerate(zip(x_1, x_2)):
assert np.all(features(x_i[0], x_i[1]) == y[i])
def experiment(alpha):
np.random.seed()
# MDP
mdp = Gym(name='Acrobot-v1', horizon=np.inf, gamma=1.)
# Policy
epsilon = Parameter(value=1.)
pi = EpsGreedy(epsilon=epsilon)
# Agent
n_tilings = 10
tilings = Tiles.generate(n_tilings, [10, 10, 10, 10, 10, 10],
mdp.info.observation_space.low,
mdp.info.observation_space.high)
features = Features(tilings=tilings)
learning_rate = Parameter(alpha / n_tilings)
approximator_params = dict(input_shape=(features.size, ),
output_shape=(mdp.info.action_space.n, ),
n_actions=mdp.info.action_space.n)
algorithm_params = {'learning_rate': learning_rate, 'lambda_coeff': .9}
agent = TrueOnlineSARSALambda(mdp.info,
pi,
approximator_params=approximator_params,
features=features,
**algorithm_params)
#shape = agent.approximator.Q
#print(agent.Q)
# Algorithm
core = Core(agent, mdp)
# Train
core.learn(n_episodes=10, n_steps_per_fit=1, render=True)
dataset = core.evaluate(n_episodes=1, render=False)
#print(dataset)
print(episodes_length(dataset))
return np.mean(compute_J(dataset, .96))
def test_sarsa_lambda_continuous_linear_save():
pi, _, mdp_continuous = initialize()
mdp_continuous.seed(1)
n_tilings = 1
tilings = Tiles.generate(n_tilings, [2, 2],
mdp_continuous.info.observation_space.low,
mdp_continuous.info.observation_space.high)
features = Features(tilings=tilings)
approximator_params = dict(
input_shape=(features.size, ),
output_shape=(mdp_continuous.info.action_space.n, ),
n_actions=mdp_continuous.info.action_space.n)
agent_save = SARSALambdaContinuous(mdp_continuous.info,
pi,
LinearApproximator,
Parameter(.1),
.9,
features=features,
approximator_params=approximator_params)
core = Core(agent_save, mdp_continuous)
# Train
core.learn(n_steps=100, n_steps_per_fit=1, quiet=True)
agent_path = './agentdir{}/'.format(datetime.now().strftime("%H%M%S%f"))
agent_save.save(agent_path)
agent_load = Agent.load(agent_path)
shutil.rmtree(agent_path)
for att, method in agent_save.__dict__.items():
save_attr = getattr(agent_save, att)
load_attr = getattr(agent_load, att)
#print('{}: {}'.format(att, type(save_attr)))
tu.assert_eq(save_attr, load_attr)
def test_cmac_approximator():
np.random.seed(1)
# Generic regressor
x = np.random.rand(1000, 2)
k1 = np.random.rand(2)
k2 = np.random.rand(2)
y = np.array(
[np.sin(x.dot(k1) * 2 * np.pi),
np.sin(x.dot(k2) * 2 * np.pi)]).T
tilings = Tiles.generate(10, [10, 10], np.zeros(2), np.ones(2))
approximator = Regressor(CMAC,
tilings=tilings,
input_shape=(2, ),
output_shape=(2, ))
approximator.fit(x, y)
x = np.random.rand(2, 2)
y_hat = approximator.predict(x)
y_true = np.array(
[np.sin(x.dot(k1) * 2 * np.pi),
np.sin(x.dot(k2) * 2 * np.pi)]).T
y_test = np.array([[-0.73787754, 0.90673493], [-0.94972964, -0.72380013]])
assert np.allclose(y_hat, y_test)
point = np.random.rand(2)
derivative = approximator.diff(point)
assert np.array_equal(np.sum(derivative, axis=0), np.ones(2) * 10)
assert len(derivative) == approximator.weights_size
old_weights = approximator.get_weights()
approximator.set_weights(old_weights)
new_weights = approximator.get_weights()
assert np.array_equal(new_weights, old_weights)
random_weights = np.random.randn(*old_weights.shape).astype(np.float32)
approximator.set_weights(random_weights)
random_weight_new = approximator.get_weights()
assert np.array_equal(random_weights, random_weight_new)
assert not np.any(np.equal(random_weights, old_weights))
# Action regressor + Ensemble
n_actions = 2
s = np.random.rand(1000, 3)
a = np.random.randint(n_actions, size=(1000, 1))
q = np.random.rand(1000)
tilings = Tiles.generate(10, [10, 10, 10], np.zeros(3), np.ones(3))
approximator = Regressor(CMAC,
tilings=tilings,
input_shape=(3, ),
n_actions=n_actions,
n_models=5)
approximator.fit(s, a, q)
x_s = np.random.rand(2, 3)
x_a = np.random.randint(n_actions, size=(2, 1))
y = approximator.predict(x_s, x_a, prediction='mean')
y_test = np.array([[0.10921918, 0.09923379]])
assert np.allclose(y, y_test)
y = approximator.predict(x_s, x_a, prediction='sum')
y_test = np.array([0.54609592, 0.49616895])
assert np.allclose(y, y_test)
y = approximator.predict(x_s, x_a, prediction='min')
y_test = np.array([[0.10921918, 0.09923379]])
assert np.allclose(y, y_test)
y = approximator.predict(x_s)
y_test = np.array([[0.07606651, 0.10921918], [0.40698114, 0.09923379]])
assert np.allclose(y, y_test)
def experiment(n_epochs, n_episodes):
np.random.seed()
# MDP
n_steps = 5000
mdp = InvertedPendulum(horizon=n_steps)
# Agent
n_tilings = 11
alpha_r = Parameter(.0001)
alpha_theta = Parameter(.001 / n_tilings)
alpha_v = Parameter(.1 / n_tilings)
tilings = Tiles.generate(n_tilings-1, [10, 10],
mdp.info.observation_space.low,
mdp.info.observation_space.high + 1e-3)
phi = Features(tilings=tilings)
tilings_v = tilings + Tiles.generate(1, [1, 1],
mdp.info.observation_space.low,
mdp.info.observation_space.high + 1e-3)
psi = Features(tilings=tilings_v)
input_shape = (phi.size,)
mu = Regressor(LinearApproximator, input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
std = Regressor(LinearApproximator, input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
std_0 = np.sqrt(1.)
std.set_weights(np.log(std_0) / n_tilings * np.ones(std.weights_size))
policy = StateLogStdGaussianPolicy(mu, std)
agent = StochasticAC_AVG(mdp.info, policy,
alpha_theta, alpha_v, alpha_r,
lambda_par=.5,
value_function_features=psi,
policy_features=phi)
# Train
dataset_callback = CollectDataset()
display_callback = Display(agent._V, mu, std,
mdp.info.observation_space.low,
mdp.info.observation_space.high,
phi, psi)
core = Core(agent, mdp, callbacks_fit=[dataset_callback])
for i in range(n_epochs):
core.learn(n_episodes=n_episodes,
n_steps_per_fit=1, render=False)
J = compute_J(dataset_callback.get(), gamma=1.)
dataset_callback.clean()
display_callback()
print('Mean Reward at iteration ' + str(i) + ': ' +
str(np.sum(J) / n_steps/n_episodes))
print('Press a button to visualize the pendulum...')
input()
core.evaluate(n_steps=n_steps, render=True)
def learn(alg):
n_steps = 50
mdp = InvertedPendulum(horizon=n_steps)
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed(1)
# Agent
n_tilings = 2
alpha_r = Parameter(.0001)
alpha_theta = Parameter(.001 / n_tilings)
alpha_v = Parameter(.1 / n_tilings)
tilings = Tiles.generate(n_tilings - 1, [1, 1],
mdp.info.observation_space.low,
mdp.info.observation_space.high + 1e-3)
phi = Features(tilings=tilings)
tilings_v = tilings + Tiles.generate(
1, [1, 1], mdp.info.observation_space.low,
mdp.info.observation_space.high + 1e-3)
psi = Features(tilings=tilings_v)
input_shape = (phi.size, )
mu = Regressor(LinearApproximator,
input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
std = Regressor(LinearApproximator,
input_shape=input_shape,
output_shape=mdp.info.action_space.shape)
std_0 = np.sqrt(1.)
std.set_weights(np.log(std_0) / n_tilings * np.ones(std.weights_size))
policy = StateLogStdGaussianPolicy(mu, std)
if alg is StochasticAC:
agent = alg(mdp.info,
policy,
alpha_theta,
alpha_v,
lambda_par=.5,
value_function_features=psi,
policy_features=phi)
elif alg is StochasticAC_AVG:
agent = alg(mdp.info,
policy,
alpha_theta,
alpha_v,
alpha_r,
lambda_par=.5,
value_function_features=psi,
policy_features=phi)
core = Core(agent, mdp)
core.learn(n_episodes=2, n_episodes_per_fit=1)
return agent
from mushroom_rl.features import Features
from mushroom_rl.features.tiles import Tiles
from mushroom_rl.policy import EpsGreedy
from mushroom_rl.utils.callbacks import CollectDataset
from mushroom_rl.utils.parameters import Parameter
# MDP
mdp = Gym(name='MountainCar-v0', horizon=np.inf, gamma=1.)
# Policy
epsilon = Parameter(value=0.)
pi = EpsGreedy(epsilon=epsilon)
# Q-function approximator
n_tilings = 10
tilings = Tiles.generate(n_tilings, [10, 10], mdp.info.observation_space.low,
mdp.info.observation_space.high)
features = Features(tilings=tilings)
# Agent
learning_rate = Parameter(.1 / n_tilings)
approximator_params = dict(input_shape=(features.size, ),
output_shape=(mdp.info.action_space.n, ),
n_actions=mdp.info.action_space.n)
agent = SARSALambdaContinuous(mdp.info,
pi,
LinearApproximator,
approximator_params=approximator_params,
learning_rate=learning_rate,
lambda_coeff=.9,
features=features)
def test_cmac_approximator():
np.random.seed(1)
# Generic regressor
x = np.random.rand(1000, 2)
k1 = np.random.rand(2)
k2 = np.random.rand(2)
y = np.array(
[np.sin(x.dot(k1) * 2 * np.pi),
np.sin(x.dot(k2) * 2 * np.pi)]).T
tilings = Tiles.generate(10, [10, 10], np.zeros(2), np.ones(2))
approximator = Regressor(CMAC,
tilings=tilings,
input_shape=(2, ),
output_shape=(2, ))
approximator.fit(x, y)
x = np.random.rand(2, 2)
y_hat = approximator.predict(x)
y_true = np.array(
[np.sin(x.dot(k1) * 2 * np.pi),
np.sin(x.dot(k2) * 2 * np.pi)]).T
y_test = np.array([[-0.73581504, 0.90877225], [-0.95854488, -0.72429239]])
assert np.allclose(y_hat, y_test)
point = np.random.rand(2)
derivative = approximator.diff(point)
assert np.array_equal(np.sum(derivative, axis=0), np.ones(2) * 10)
assert len(derivative) == approximator.weights_size
old_weights = approximator.get_weights()
approximator.set_weights(old_weights)
new_weights = approximator.get_weights()
assert np.array_equal(new_weights, old_weights)
random_weights = np.random.randn(*old_weights.shape).astype(np.float32)
approximator.set_weights(random_weights)
random_weight_new = approximator.get_weights()
assert np.array_equal(random_weights, random_weight_new)
assert not np.any(np.equal(random_weights, old_weights))
# Action regressor + Ensemble
n_actions = 2
s = np.random.rand(1000, 3)
a = np.random.randint(n_actions, size=(1000, 1))
q = np.random.rand(1000)
tilings = Tiles.generate(10, [10, 10, 10], np.zeros(3), np.ones(3))
approximator = Regressor(CMAC,
tilings=tilings,
input_shape=(3, ),
n_actions=n_actions,
n_models=5)
approximator.fit(s, a, q)
np.random.seed(2)
x_s = np.random.rand(2, 3)
x_a = np.random.randint(n_actions, size=(2, 1))
y = approximator.predict(x_s, x_a, prediction='mean')
y_test = np.array([[0.56235045, 0.25080909]])
assert np.allclose(y, y_test)
y = approximator.predict(x_s, x_a, prediction='sum')
y_test = np.array([2.81175226, 1.25404543])
assert np.allclose(y, y_test)
y = approximator.predict(x_s, x_a, prediction='min')
y_test = np.array([0.56235045, 0.25080909])
assert np.allclose(y, y_test)
y = approximator.predict(x_s)
y_test = np.array([[0.10367145, 0.56235045], [0.05575822, 0.25080909]])
assert np.allclose(y, y_test)