def test_dqn_doesnt_store_invalid_transitions(self):
STEPS = 55
env = DummyEnv()
test_env = DummyEnv()
model = tf.keras.Sequential([tf.keras.layers.Dense(2, input_dim=2)])
model.compile(tf.keras.optimizers.SGD(learning_rate=0.), loss="mse")
agent = DQN.from_environment(env,
model,
discount_gamma=0.,
use_target_network=False)
rollout = Rolling(agent, env)
test_rollout = Trajectory(agent, test_env)
rollout.fit(epochs=10,
updates_per_epoch=12,
steps_per_update=STEPS,
update_batch_size=8,
testing_rollout=test_rollout,
buffer_warmup=False)
data = agent.memory_sampler.sample(-1)
np.testing.assert_array_less(data["state"], 10)
np.testing.assert_equal(
(data["state_next"] - data["state"]).sum(axis=1), 1)
def test_td3_doesnt_store_invalid_transitions(self):
STEPS = 55
env = DummyEnv(action_space="continuous")
test_env = DummyEnv(action_space="continuous")
actor_out = tf.convert_to_tensor([[0., 0.]])
critic_out = tf.convert_to_tensor([1.])
actor = arch.TestingModel(actor_out)
critic = arch.TestingModel(critic_out)
critic2 = arch.TestingModel(critic_out)
actor_target = arch.TestingModel(actor_out)
critic_target = arch.TestingModel(critic_out)
critic2_target = arch.TestingModel(critic_out)
actor.optimizer = tf.keras.optimizers.SGD(0)
critic.optimizer = tf.keras.optimizers.SGD(0)
critic2.optimizer = tf.keras.optimizers.SGD(0)
agent = TD3(actor,
actor_target,
critic,
critic_target,
critic2,
critic2_target,
discount_gamma=0.,
polyak_tau=0.,
action_minima=-1.,
action_maxima=1.,
update_actor_every=1)
rollout = Rolling(agent, env)
test_rollout = Trajectory(agent, test_env)
rollout.fit(epochs=10,
updates_per_epoch=12,
steps_per_update=STEPS,
update_batch_size=8,
testing_rollout=test_rollout,
buffer_warmup=False)
data = agent.memory_sampler.sample(-1)
np.testing.assert_array_less(data["state"], 10)
np.testing.assert_equal(
(data["state_next"] - data["state"]).sum(axis=1), 1)
Flatten(input_shape=test_env.shape),
Dense(256),
BatchNormalization(),
LeakyReLU(),
Dense(2, activation="softmax")
])
actor.compile(RMSprop(1e-4, rho=0.99), "categorical_crossentropy")
agent = REINFORCE(actor,
2,
Experience(),
discount_factor_gamma=0.99,
state_preprocessor=None)
rollout = MultiTrajectory([agent for _ in range(10)], envs)
test_rollout = Trajectory(agent, test_env)
rewards = deque(maxlen=100)
actor_loss = deque(maxlen=100)
actor_utility = deque(maxlen=100)
actor_entropy = deque(maxlen=100)
critic_loss = deque(maxlen=100)
episode = 0
while 1:
episode += 1
history = rollout.rollout(verbose=0, push_experience=True)
agent_history = agent.fit(batch_size=-1, verbose=0)
import gym
from trickster.agent import REINFORCE, A2C, PPO
from trickster.rollout import Trajectory
from trickster import callbacks
ENV_NAME = "LunarLanderContinuous-v2"
ALGO = "REINFORCE"
TRAJECTORY_MAX_STEPS = 100
EPOCHS = 1000
ROLLOUTS_PER_EPOCH = 4
env = gym.make(ENV_NAME)
algo = {"REINFORCE": REINFORCE, "A2C": A2C, "PPO": PPO}[ALGO]
agent = algo.from_environment(env)
rollout = Trajectory(agent, env, TRAJECTORY_MAX_STEPS)
cbs = [callbacks.ProgressPrinter(keys=rollout.progress_keys)]
rollout.fit(epochs=EPOCHS,
updates_per_epoch=1,
rollouts_per_update=4,
callbacks=cbs)
rollout.render(repeats=100)
ann = Sequential([
Dense(16, activation="relu", input_shape=input_shape),
Dense(16, activation="relu"),
Dense(num_actions, activation="linear")
])
ann.compile(loss="mse", optimizer=Adam(1e-3))
agent = DoubleDQN(ann,
action_space=2,
memory=Experience(max_length=10000),
epsilon=1.,
discount_factor_gamma=0.98)
rollout = Rolling(agent, env, config=RolloutConfig(max_steps=300))
test_rollout = Trajectory(agent, test_env)
rewards = []
losses = []
for episode in range(1, 501):
episode_losses = []
for update in range(32):
roll_history = rollout.roll(steps=4, verbose=0, push_experience=True)
agent_history = agent.fit(batch_size=32, verbose=0)
episode_losses.append(agent_history["loss"])
test_history = test_rollout.rollout(verbose=0,
push_experience=False,
render=False)
from trickster.agent import PPO
from trickster.rollout import MultiRolling, Trajectory
from trickster.utility import gymic
from trickster.model import mlp
envs = [gymic.rwd_scaled_env("LunarLander-v2") for _ in range(32)]
test_env = gymic.rwd_scaled_env("LunarLander-v2")
input_shape = envs[0].observation_space.shape
num_actions = envs[0].action_space.n
actor = mlp.wide_mlp_actor_categorical(input_shape, num_actions, adam_lr=1e-4)
critic = mlp.wide_mlp_critic_network(input_shape, output_dim=1, adam_lr=1e-4)
agent = PPO(actor,
critic,
action_space=num_actions,
discount_factor_gamma=0.99,
entropy_penalty_coef=0.05)
rollout = MultiRolling(agent, envs)
test_rollout = Trajectory(agent, test_env)
rollout.fit(episodes=1000,
updates_per_episode=1,
steps_per_update=32,
update_batch_size=32,
testing_rollout=test_rollout,
plot_curves=True)
test_rollout.render(repeats=10)
ALGO = "DQN"
NUM_ENVS = 4
TRAJECTORY_MAX_STEPS = 200
STEPS_PER_UPDATE = 1
UPDATES_PER_EPOCH = 64
EPOCHS = 200
UPDATE_BATCH_SIZE = 100
envs = [gym.make(ENV_NAME) for _ in range(NUM_ENVS)]
test_env = gym.make(ENV_NAME)
algo = {"DQN": DQN, "DoubleDQN": DoubleDQN}[ALGO]
agent = algo.from_environment(envs[0])
rollout = MultiRolling(agent, envs, TRAJECTORY_MAX_STEPS)
test_rollout = Trajectory(agent, test_env, TRAJECTORY_MAX_STEPS)
rollout.fit(epochs=EPOCHS,
updates_per_epoch=UPDATES_PER_EPOCH,
steps_per_update=STEPS_PER_UPDATE,
update_batch_size=UPDATE_BATCH_SIZE,
warmup_buffer=True,
callbacks=[
callbacks.TrajectoryEvaluator(testing_rollout=test_rollout,
repeats=4),
callbacks.ProgressPrinter(rollout.progress_keys)
])
test_rollout.render(repeats=10)
policy = Sequential([
Dense(24, activation="relu", input_shape=input_shape),
Dense(24, activation="relu"),
Dense(num_actions, activation="linear")
])
policy.compile(loss="mse", optimizer=Adam(1e-4))
agent = DQN(policy,
action_space=num_actions,
memory=Experience(max_length=10000),
epsilon=1.,
discount_factor_gamma=0.98,
use_target_network=True)
rollout = Trajectory(agent, env, RolloutConfig(max_steps=200))
rewards = []
losses = []
for warmup in range(1, 33):
rollout.rollout(verbose=0, learning_batch_size=0)
for episode in range(1, 501):
rollout._reset()
episode_rewards = []
episode_losses = []
while not rollout.finished:
roll_history = rollout.roll(steps=2, verbose=0, learning_batch_size=64)
episode_rewards.append(roll_history["reward_sum"])
episode_losses.append(roll_history["loss"])
from trickster.agent import A2C
from trickster.rollout import MultiRolling, Trajectory, RolloutConfig
from trickster.model import mlp
cfg = MatchConfig(canvas_size=(100, 100),
players_per_side=2,
learning_type=MatchConfig.LEARNING_TYPE_SINGLE_AGENT,
observation_type=MatchConfig.OBSERVATION_TYPE_VECTOR)
envs = [Match(cfg) for _ in range(8)]
test_env = Match(cfg)
actor, critic = mlp.wide_pg_actor_critic(envs[0].observation_space.shape,
envs[0].action_space.n,
actor_lr=1e-4,
critic_lr=1e-4)
agent = A2C(actor, critic, test_env.action_space, entropy_penalty_coef=0.1)
rcfg = RolloutConfig(max_steps=512, skipframes=2)
training_rollout = MultiRolling(agent, envs, rcfg)
testing_rollout = Trajectory(agent, test_env, rcfg)
training_rollout.fit(episodes=1000,
updates_per_episode=512,
steps_per_update=1,
testing_rollout=testing_rollout)
testing_rollout.render(repeats=10)
from trickster.agent import A2C
from trickster.rollout import Rolling, Trajectory, RolloutConfig
from trickster.model import mlp
from trickster.utility import gymic
env = gymic.rwd_scaled_env()
input_shape = env.observation_space.shape
num_actions = env.action_space.n
actor, critic = mlp.wide_pg_actor_critic(input_shape, num_actions)
agent = A2C(actor,
critic,
action_space=env.action_space,
discount_factor_gamma=0.98,
entropy_penalty_coef=0.01)
rollout = Rolling(agent, env, config=RolloutConfig(max_steps=300))
test_rollout = Trajectory(agent, gymic.rwd_scaled_env())
rollout.fit(episodes=1000, updates_per_episode=64, step_per_update=1, testing_rollout=test_rollout, plot_curves=True)
test_rollout.render(repeats=10)
critic = Sequential([
Dense(400, activation="relu", input_shape=input_shape),
Dense(300, activation="relu"),
Dense(1, activation="linear")
])
critic.compile(loss="mse", optimizer=Adam(1e-4))
agent = A2C(actor,
critic,
action_space=num_actions,
absolute_memory_limit=10000,
discount_factor_gamma=0.99,
entropy_penalty_coef=0.05)
rollout = Rolling(agent.create_workers(1)[0], env)
test_rollout = Trajectory(agent, Lunar())
def train():
hst = history.History("reward_sum", "actor_loss", "actor_utility",
"actor_utility_std", "actor_entropy", "values",
"advantages", "critic_loss")
for episode in range(1, 1001):
for update in range(1, 4):
rollout.roll(steps=128, verbose=0, push_experience=True)
agent_history = agent.fit(batch_size=-1,
verbose=0,
reset_memory=True)
hst.buffer(**agent_history)
envs = [gymic.rwd_scaled_env("CartPole-v1") for _ in range(8)]
test_env = gymic.rwd_scaled_env("CartPole-v1")
input_shape = envs[0].observation_space.shape
num_actions = envs[0].action_space.n
ann = mlp.wide_mlp_critic_network(input_shape, num_actions, adam_lr=1e-4)
agent = DQN(ann,
action_space=2,
memory=Experience(max_length=10000),
epsilon=1.,
epsilon_decay=0.99995,
epsilon_min=0.1,
discount_factor_gamma=0.98,
use_target_network=True)
rollout = MultiRolling(agent,
envs,
rollout_configs=RolloutConfig(max_steps=300))
test_rollout = Trajectory(agent, test_env)
rollout.fit(episodes=500,
updates_per_episode=128,
steps_per_update=1,
update_batch_size=32,
testing_rollout=test_rollout,
plot_curves=True)
test_rollout.render()
memory=Experience(max_length=10000),
discount_factor_gamma=0.995,
entropy_penalty_coef=0.0,
state_preprocessor=lambda state: state / 255.)
episode = 1
reward_memory = deque(maxlen=10)
step_lengths = deque(maxlen=10)
critic_losses = deque(maxlen=10)
actor_losses = deque(maxlen=10)
actor_utility = deque(maxlen=10)
actor_entropy = deque(maxlen=10)
rollout = MultiRolling(agent, envs, rollout_configs=RolloutConfig(max_steps=512, skipframes=2))
test_rollout = Trajectory(agent, test_env, config=RolloutConfig(max_steps=512, skipframes=2))
while 1:
episode_a_losses = []
episode_a_utility = []
episode_a_entropy = []
episode_c_losses = []
for update in range(32):
rollout.roll(steps=2, verbose=0, push_experience=True)
agent_history = agent.fit(batch_size=-1, verbose=0, reset_memory=True)
episode_a_losses.append(agent_history["actor_loss"])
episode_a_utility.append(agent_history["actor_utility"])
episode_a_entropy.append(agent_history["actor_entropy"])
episode_c_losses.append(agent_history["critic_loss"])
from trickster.agent import REINFORCE from trickster.rollout import Trajectory, RolloutConfig from trickster.utility import gymic from trickster.model import mlp env = gymic.rwd_scaled_env() input_shape = env.observation_space.shape num_actions = env.action_space.n policy = mlp.wide_mlp_actor_categorical(input_shape, num_actions, adam_lr=1e-4) agent = REINFORCE(policy, action_space=num_actions) rollout = Trajectory(agent, env, config=RolloutConfig(max_steps=300)) rollout.fit(episodes=500, rollouts_per_update=1, update_batch_size=-1) rollout.render(repeats=10)
input_shape=input_shape,
kernel_initializer="he_uniform"),
Dense(16, activation="relu", kernel_initializer="he_uniform"),
Dense(1, activation="linear", kernel_initializer="he_uniform")
])
critic.compile(loss="mse", optimizer=Adam(5e-4))
agent = A2C(actor,
critic,
action_space=env.action_space,
memory=Experience(max_length=10000),
discount_factor_gamma=0.98,
entropy_penalty_coef=0.01)
rollout = Rolling(agent, env, config=RolloutConfig(max_steps=300))
test_rollout = Trajectory(agent, gym.make("CartPole-v1"))
rewards = []
actor_loss = []
actor_utility = []
actor_entropy = []
critic_loss = []
for episode in range(1, 1001):
episode_actor_loss = []
episode_actor_utility = []
episode_actor_entropy = []
episode_critic_loss = []
for update in range(32):
rollout.roll(steps=2, verbose=0, push_experience=True)
num_actions = env.action_space.shape[0]
actor, critics = mlp.wide_ddpg_actor_critic(input_shape,
output_dim=num_actions,
action_range=2,
num_critics=2,
actor_lr=5e-4,
critic_lr=5e-4)
agent = TD3(actor,
critics,
action_space=spaces.CONTINUOUS,
memory=Experience(max_length=int(1e4)),
discount_factor_gamma=0.99,
action_noise_sigma=0.1,
action_noise_sigma_decay=1.,
action_minima=-2,
action_maxima=2,
target_noise_sigma=0.2,
target_noise_clip=0.5)
rollout = Rolling(agent, env)
test_rollout = Trajectory(agent, env, RolloutConfig(testing_rollout=True))
rollout.fit(episodes=1000,
updates_per_episode=64,
step_per_update=1,
update_batch_size=32,
testing_rollout=test_rollout)
test_rollout.render(repeats=10)
from trickster.utility import visual
env = gym.make("CartPole-v1")
input_shape = env.observation_space.shape
num_actions = env.action_space.n
policy = Sequential([
Dense(16, activation="relu", input_shape=input_shape),
Dense(16, activation="relu"),
Dense(num_actions, activation="softmax")
])
policy.compile(loss="categorical_crossentropy", optimizer=Adam(5e-3))
agent = REINFORCE(policy, action_space=num_actions)
rollout = Trajectory(agent, env, config=RolloutConfig(max_steps=300))
rewards = []
losses = []
for episode in range(1, 501):
rollout_history = rollout.rollout(verbose=0, push_experience=True)
agent_history = agent.fit(batch_size=-1, verbose=0, reset_memory=True)
rewards.append(rollout_history["reward_sum"])
losses.append(agent_history["loss"])
print("\rEpisode {:>4} RWD: {:>6.1f}, UTILITY: {: >8.4f}".format(
episode, np.mean(rewards[-10:]), np.mean(losses[-10:])),
end="")
import keras
from trickster.agent import REINFORCE
from trickster.rollout import Trajectory
from trickster.experience import Experience
from trickster.utility import gymic
from trickster.model import mlp
env = gymic.rwd_scaled_env("LunarLander-v2")
input_shape = env.observation_space.shape
num_actions = env.action_space.n
policy = mlp.wide_mlp_actor_categorical(input_shape, num_actions)
policy.compile(optimizer=keras.optimizers.SGD(lr=2e-4, momentum=0.9),
loss="categorical_crossentropy")
agent = REINFORCE(policy,
action_space=num_actions,
memory=Experience(max_length=10000),
discount_factor_gamma=0.99)
rollout = Trajectory(agent, env)
rollout.fit(episodes=1000, rollouts_per_update=16, update_batch_size=-1)
rollout.render(repeats=10)