def train_eval(
# tensorboard files
root_dir,
# environment
env_name="CartPole-v1",
random_seed=0,
# Params for collect
num_environment_steps=100000,
replay_buffer_capacity=1001, # Per-environment
# Params for eval
num_eval_episodes=30,
eval_interval=200,
# Params for summaries
summary_interval=50,
):
tf.compat.v1.set_random_seed(random_seed)
environment = TFPyEnvironment(suite_gym.load(env_name))
evaluation_environment = TFPyEnvironment(suite_gym.load(env_name))
actor_net = ActorDistributionNetwork(environment.observation_spec(),
environment.action_spec(),
fc_layer_params=(200, 100))
value_net = ValueNetwork(environment.observation_spec(),
fc_layer_params=(200, 100))
global_step = tf.compat.v1.train.get_or_create_global_step()
agent = PPOClipAgent( # should be closer to the paper than PPOAgent...
environment.time_step_spec(),
environment.action_spec(),
optimizer=tf.compat.v1.train.AdamOptimizer(
), # default None does not work
actor_net=actor_net,
value_net=value_net,
importance_ratio_clipping=0.2,
normalize_observations=False,
normalize_rewards=False,
use_gae=True,
lambda_value=0.5,
discount_factor=0.95,
train_step_counter=global_step,
)
agent_trainer = OnPolicyModelFreeAgentTrainer(400)
experiment_harness = ExperimentHarness(
root_dir,
environment,
evaluation_environment,
agent,
agent_trainer,
replay_buffer_capacity,
num_environment_steps,
summary_interval,
eval_interval,
num_eval_episodes,
number_of_initial_random_policy_steps=0,
use_tf_function=True,
)
experiment_harness.run()
def test_ppo(self):
env_class = PolicyUnittestEnv
learning_rate = 1e-1
iterations = 20
batch_size = 100
steps_per_episode = 13
env = env_class(batch_size, steps_per_episode)
env = TFPyEnvironment(env)
eval_env = env_class(batch_size, steps_per_episode)
eval_env = TFPyEnvironment(eval_env)
algorithm = create_algorithm(env, learning_rate=learning_rate)
driver = SyncOffPolicyDriver(env,
algorithm,
debug_summaries=DEBUGGING,
summarize_grads_and_vars=DEBUGGING)
replayer = driver.exp_replayer
eval_driver = OnPolicyDriver(eval_env,
algorithm,
training=False,
greedy_predict=True)
env.reset()
eval_env.reset()
time_step = driver.get_initial_time_step()
policy_state = driver.get_initial_policy_state()
for i in range(iterations):
time_step, policy_state = driver.run(max_num_steps=batch_size *
steps_per_episode,
time_step=time_step,
policy_state=policy_state)
experience = replayer.replay_all()
driver.train(experience, num_updates=4, mini_batch_size=25)
replayer.clear()
eval_env.reset()
eval_time_step, _ = eval_driver.run(
max_num_steps=(steps_per_episode - 1) * batch_size)
logging.info("%d reward=%f", i,
float(tf.reduce_mean(eval_time_step.reward)))
eval_env.reset()
eval_time_step, _ = eval_driver.run(
max_num_steps=(steps_per_episode - 1) * batch_size)
logging.info("reward=%f", float(tf.reduce_mean(eval_time_step.reward)))
self.assertAlmostEqual(1.0,
float(tf.reduce_mean(eval_time_step.reward)),
delta=1e-1)
def test_incorrect_termination_model():
"""
The generic model-based agent should only allow a ConstantFalseTermination model.
"""
# setup arguments for the model-based agent constructor
py_env = suite_gym.load("MountainCarContinuous-v0")
tf_env = TFPyEnvironment(py_env)
time_step_spec = tf_env.time_step_spec()
observation_spec = tf_env.observation_spec()
action_spec = tf_env.action_spec()
network = LinearTransitionNetwork(observation_spec)
transition_model = KerasTransitionModel([network], observation_spec, action_spec)
reward_model = MountainCarReward(observation_spec, action_spec)
initial_state_distribution_model = MountainCarInitialState(observation_spec)
termination_model = MountainCarTermination(observation_spec)
policy = RandomTFPolicy(time_step_spec, action_spec)
with pytest.raises(AssertionError) as excinfo:
ModelBasedAgent(
time_step_spec,
action_spec,
transition_model,
reward_model,
termination_model,
initial_state_distribution_model,
policy,
policy,
)
assert "Only constant false termination supported" in str(excinfo.value)
def test_tf_environment_with_random(n_episodes=20):
"""Test tf environment through random actions."""
print(f'Testing tf environment over {n_episodes} episodes.')
env = LakeMonsterEnvironment(**params)
env = TFPyEnvironment(env)
policy = RandomTFPolicy(time_step_spec=env.time_step_spec(),
action_spec=env.action_spec())
ts = env.reset()
rewards = []
n_steps = []
for _ in tqdm(range(n_episodes)):
n_step = 0
while not ts.is_last():
action = policy.action(ts).action
ts = env.step(action)
n_step += 1
reward = ts.reward
rewards.append(reward)
n_steps.append(n_step)
ts = env.reset()
# print results
print('average num of steps per episode:', np.mean(n_steps))
print('average reward per episode', np.mean(rewards))
def test_alf_metrics(self, num_envs, learn_queue_cap, unroll_length,
actor_queue_cap, num_actors, num_iterations):
episode_length = 5
env_f = lambda: TFPyEnvironment(
ValueUnittestEnv(batch_size=1, episode_length=episode_length))
envs = [env_f() for _ in range(num_envs)]
common.set_global_env(envs[0])
alg = _create_ac_algorithm()
driver = AsyncOffPolicyDriver(envs, alg, num_actors, unroll_length,
learn_queue_cap, actor_queue_cap)
driver.start()
total_num_steps_ = 0
for _ in range(num_iterations):
total_num_steps_ += driver.run_async()
driver.stop()
total_num_steps = int(driver.get_metrics()[1].result())
self.assertGreaterEqual(total_num_steps_, total_num_steps)
# An exp is only put in the log queue after it's put in the learning queue
# So when we stop the driver (which will force all queues to stop),
# some exps might be missing from the metric. Here we assert an arbitrary
# lower bound of 2/5. The upper bound is due to the fact that StepType.LAST
# is not recorded by the metric (episode_length==5).
self.assertLessEqual(total_num_steps, int(total_num_steps_ * 4 // 5))
self.assertGreaterEqual(total_num_steps,
int(total_num_steps_ * 2 // 5))
average_reward = int(driver.get_metrics()[2].result())
self.assertEqual(average_reward, episode_length - 1)
episode_length = int(driver.get_metrics()[3].result())
self.assertEqual(episode_length, episode_length)
def test_merlin_algorithm(self):
batch_size = 100
steps_per_episode = 15
gap = 10
env = RNNPolicyUnittestEnv(
batch_size, steps_per_episode, gap, obs_dim=3)
env = TFPyEnvironment(env)
common.set_global_env(env)
algorithm = _create_merlin_algorithm(
learning_rate=1e-3, debug_summaries=False)
driver = OnPolicyDriver(env, algorithm, train_interval=6)
eval_driver = OnPolicyDriver(env, algorithm, training=False)
proc = psutil.Process(os.getpid())
policy_state = driver.get_initial_policy_state()
time_step = driver.get_initial_time_step()
for i in range(100):
t0 = time.time()
time_step, policy_state, _ = driver.run(
max_num_steps=150 * batch_size,
time_step=time_step,
policy_state=policy_state)
mem = proc.memory_info().rss // 1e6
logging.info('%s time=%.3f mem=%s' % (i, time.time() - t0, mem))
env.reset()
time_step, _ = eval_driver.run(max_num_steps=14 * batch_size)
logging.info("eval reward=%.3f" % tf.reduce_mean(time_step.reward))
self.assertAlmostEqual(
1.0, float(tf.reduce_mean(time_step.reward)), delta=1e-2)
def test_actor_critic_continuous_policy(self):
batch_size = 100
steps_per_episode = 13
env = PolicyUnittestEnv(batch_size,
steps_per_episode,
action_type=ActionType.Continuous)
# We need to wrap env using TFPyEnvironment because the methods of env
# has side effects (e.g, env._current_time_step can be changed)
env = TFPyEnvironment(env)
action_spec = env.action_spec()
observation_spec = env.observation_spec()
algorithm = ActorCriticAlgorithm(
observation_spec=observation_spec,
action_spec=action_spec,
actor_network=ActorDistributionNetwork(observation_spec,
action_spec,
fc_layer_params=()),
value_network=ValueNetwork(observation_spec, fc_layer_params=()),
optimizer=tf.optimizers.Adam(learning_rate=1e-2))
driver = OnPolicyDriver(env, algorithm, train_interval=2)
eval_driver = OnPolicyDriver(env, algorithm, training=False)
driver.run = tf.function(driver.run)
t0 = time.time()
driver.run(max_num_steps=2600 * batch_size)
print("time=%s" % (time.time() - t0))
env.reset()
time_step, _ = eval_driver.run(max_num_steps=4 * batch_size)
print("reward=%s" % tf.reduce_mean(time_step.reward))
self.assertAlmostEqual(1.0,
float(tf.reduce_mean(time_step.reward)),
delta=5e-2)
def test_tf_environment_wrapping():
"""
Test wrapping the RL environment for use with TensorFlow Agents.
Use Simple Server Queue for simplicity
"""
# Set up single server queue.
cost_per_buffer = np.ones((1, 1))
initial_state = (0,)
capacity = np.ones((1, 1)) * np.inf
demand_rate_val = 0.7
job_conservation_flag = True
seed = 72
demand_rate = np.array([demand_rate_val])[:, None]
buffer_processing_matrix = - np.ones((1, 1))
constituency_matrix = np.ones((1, 1))
list_boundary_constraint_matrices = [constituency_matrix]
# Construct environment.
job_generator = ScaledBernoulliServicesPoissonArrivalsGenerator(
demand_rate, buffer_processing_matrix, job_gen_seed=seed)
assert job_generator.routes == {}
state_initialiser = stinit.DeterministicCRWStateInitialiser(initial_state)
env = RLControlledRandomWalk(cost_per_buffer, capacity, constituency_matrix, job_generator,
state_initialiser, job_conservation_flag,
list_boundary_constraint_matrices)
# Try wrapping environment for tf agents.
tf_env = TFPyEnvironment(GymWrapper(env))
del tf_env
def test_unknown_transition_model():
"""
Pets Agent has prespecified transition model, RuntimeError should raise on unknown model.
"""
# setup the environment and a prespecified model components
py_env = suite_gym.load("MountainCarContinuous-v0")
tf_env = TFPyEnvironment(py_env)
time_step_spec = tf_env.time_step_spec()
observation_spec = tf_env.observation_spec()
action_spec = tf_env.action_spec()
reward_model = MountainCarReward(observation_spec, action_spec)
initial_state_distribution_model = MountainCarInitialState(
observation_spec)
# trajectory optimiser
trajectory_optimiser_type = TrajectoryOptimizationType.CrossEntropyMethod
transition_model_type = "unknown_model"
trajectory_sampler_type = TrajectorySamplerType.TS1
# some parameters need to be set correctly
ensemble_size = 2
num_elites = 10
learning_rate = 0.9
max_iterations = 5
population_size = num_elites + 10
number_of_particles = 1
horizon = 1
with pytest.raises(RuntimeError) as excinfo:
PetsAgent(
time_step_spec,
action_spec,
transition_model_type,
1,
10,
tf.nn.relu,
ensemble_size,
False,
1,
1,
[tf.keras.callbacks.EarlyStopping(monitor="loss", patience=3)],
reward_model,
initial_state_distribution_model,
trajectory_sampler_type,
trajectory_optimiser_type,
horizon,
population_size,
number_of_particles,
num_elites,
learning_rate,
max_iterations,
)
assert "Unknown transition model" in str(excinfo.value)
def test_ensemble_size_set_correctly():
"""
For ensemble transition models ensemble size needs to be larger than 1.
"""
# setup the environment and a prespecified model components
py_env = suite_gym.load("MountainCarContinuous-v0")
tf_env = TFPyEnvironment(py_env)
time_step_spec = tf_env.time_step_spec()
observation_spec = tf_env.observation_spec()
action_spec = tf_env.action_spec()
reward_model = MountainCarReward(observation_spec, action_spec)
initial_state_distribution_model = MountainCarInitialState(observation_spec)
# transition model and model-free agent
transition_model_type = TransitionModelType.DeterministicEnsemble
trajectory_sampler_type = TrajectorySamplerType.TS1
model_free_agent_type = ModelFreeAgentType.Ppo
# some parameters need to be set correctly
ensemble_size = 1
population_size = 10
horizon = 1
# define agent, many transition model and trajectory optimiser parameters can
# be arbitrary
with pytest.raises(AssertionError) as excinfo:
MbpoAgent(
time_step_spec,
action_spec,
transition_model_type,
1,
10,
tf.nn.relu,
ensemble_size,
False,
1,
1,
[tf.keras.callbacks.EarlyStopping(monitor="loss", patience=3)],
reward_model,
initial_state_distribution_model,
trajectory_sampler_type,
horizon,
population_size,
model_free_agent_type,
1,
10,
tf.nn.relu,
2,
1,
)
assert "ensemble_size should be > 1" in str(excinfo.value)
def test_unknown_transition_model():
"""
Mepo Agent has prespecified transition model, RuntimeError should raise on unknown model.
"""
# setup the environment and a prespecified model components
py_env = suite_gym.load("MountainCarContinuous-v0")
tf_env = TFPyEnvironment(py_env)
time_step_spec = tf_env.time_step_spec()
observation_spec = tf_env.observation_spec()
action_spec = tf_env.action_spec()
reward_model = MountainCarReward(observation_spec, action_spec)
initial_state_distribution_model = MountainCarInitialState(observation_spec)
# transition model and model-free agent
transition_model_type = "unknown_model"
trajectory_sampler_type = TrajectorySamplerType.TS1
model_free_agent_type = ModelFreeAgentType.Ppo
# some parameters need to be set correctly
ensemble_size = 2
num_elites = 10
population_size = num_elites + 10
horizon = 1
with pytest.raises(RuntimeError) as excinfo:
MbpoAgent(
time_step_spec,
action_spec,
transition_model_type,
1,
10,
tf.nn.relu,
ensemble_size,
False,
1,
1,
[tf.keras.callbacks.EarlyStopping(monitor="loss", patience=3)],
reward_model,
initial_state_distribution_model,
trajectory_sampler_type,
horizon,
population_size,
model_free_agent_type,
1,
10,
tf.nn.relu,
2,
1,
)
assert "Unknown transition model" in str(excinfo.value)
def _create_environment_and_policy(batch_size):
tf_batched_environment = TFPyEnvironment(
BatchedPyEnvironment([
PyEnvironmentMock(final_state=TRAJECTORY_LENGTH)
for _ in range(batch_size)
]))
policy = TFPolicyMock(
tf_batched_environment.time_step_spec(),
tf_batched_environment.action_spec(),
batch_size=batch_size,
)
return tf_batched_environment, policy
def load_model_checkpoint(c):#returns the model at given chkpoint
dir_name = tf.train.latest_checkpoint(c.model_dir)
#if ver_name =='None':
# check_or_make_dir(dir_name)
#else:
# dir_name = os.path.join(dir_name,ver_name)
dummy_env= TFPyEnvironment(StockEnvBasic(**c.default_env))
time_step = dummy_env.reset()
temp = ValueNet(**c.model_vars)
#initialize model
temp(time_step.observation)
checkpoint2 = tf.train.Checkpoint(module=temp)
status=checkpoint2.restore(dir_name)
return temp,checkpoint2
def compile(self, X_train: np.ndarray, y_train: np.ndarray, lr: float, epsilon: float, gamma: float, imb_ratio: float,
replay_buffer_max_length: int, layers: dict) -> None:
"""
Create the Q-network, agent and policy
Args:
X_train: A np.ndarray for training samples.
y_train: A np.ndarray for the class labels of the training samples.
lr: learn rate for the optimizer (default Adam)
epsilon: Used for the default epsilon greedy policy for choosing a random action.
gamma: The discount factor for learning Q-values
imb_ratio: ratio of imbalance. Used to specifiy reward in the environment
replay_buffer_max_length: Maximum lenght of replay memory.
layers: A dict containing the layers of the Q-Network (eg, conv, dense, rnn, dropout).
"""
dense_layers = layers.get("dense")
conv_layers = layers.get("conv")
dropout_layers = layers.get("dropout")
self.train_env = TFPyEnvironment(ClassifyEnv(X_train, y_train, imb_ratio)) # create a custom environment
q_net = QNetwork(self.train_env.observation_spec(), self.train_env.action_spec(), conv_layer_params=conv_layers,
fc_layer_params=dense_layers, dropout_layer_params=dropout_layers)
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=lr)
train_step_counter = tf.Variable(0)
self.agent = DqnAgent(
self.train_env.time_step_spec(),
self.train_env.action_spec(),
q_network=q_net,
optimizer=optimizer,
td_errors_loss_fn=common.element_wise_squared_loss,
train_step_counter=train_step_counter,
gamma=gamma,
epsilon_greedy=epsilon,
)
self.agent.initialize()
self.replay_buffer = TFUniformReplayBuffer(
data_spec=self.agent.collect_data_spec,
batch_size=self.train_env.batch_size,
max_length=replay_buffer_max_length)
def test_planning_policy_batch_environment_model():
"""
Ensure that planning policy is operational.
"""
# number of trajectories for planning and planning horizon
population_size = 3
planner_horizon = 5
number_of_particles = 1
# setup the environment and a model of it
py_env = suite_gym.load("MountainCar-v0")
tf_env = TFPyEnvironment(py_env)
reward = MountainCarReward(tf_env.observation_spec(), tf_env.action_spec())
terminates = MountainCarTermination(tf_env.observation_spec())
network = LinearTransitionNetwork(tf_env.observation_spec())
transition_model = KerasTransitionModel(
[network],
tf_env.observation_spec(),
tf_env.action_spec(),
)
initial_state = MountainCarInitialState(tf_env.observation_spec())
environment_model = EnvironmentModel(
transition_model=transition_model,
reward_model=reward,
termination_model=terminates,
initial_state_distribution_model=initial_state,
)
# setup the trajectory optimiser
random_policy = RandomTFPolicy(tf_env.time_step_spec(),
tf_env.action_spec())
trajectory_optimiser = PolicyTrajectoryOptimiser(random_policy,
planner_horizon,
population_size,
number_of_particles)
planning_policy = PlanningPolicy(environment_model, trajectory_optimiser)
# test whether it runs
collect_driver_planning_policy = DynamicEpisodeDriver(tf_env,
planning_policy,
num_episodes=1)
time_step = tf_env.reset()
collect_driver_planning_policy.run(time_step)
def evaluate_episode(policy, env_params):
"""Use naive while loop to evaluate policy in single episode."""
if 'n_monsters' in env_params:
env = MultiMonsterEnvironment
elif 'is_jumping' in env_params:
env = JumpingEnvironment
else:
env = LakeMonsterEnvironment
py_env = env(**env_params)
tf_env = TFPyEnvironment(py_env)
ts = tf_env.reset()
n_steps = 0
while not ts.is_last():
action = policy.action(ts)
ts = tf_env.step(action.action)
n_steps += 1
reward = ts.reward.numpy().item()
return reward, n_steps * py_env.step_size
def collection_run(c, the_buffer, date_strs, cdict):
"""Fills replay buffer with experience from date_strs
"""
n_trades = 0
for date_str in date_strs:
print(date_str)
contract_str = cdict[date_str]
the_env = StockEnvBasic(date_str, contract_str, **c.stock_env)
the_env = TFPyEnvironment(the_env)
time_step = the_env.reset()
while not tf.reduce_all(time_step.is_last()):
action = 1
next_ts = the_env.step(action)
pol_step = policy_step.PolicyStep(action)
traj = trajectory.from_transition(time_step, pol_step, next_ts)
the_buffer.add_batch(traj)
time_step = next_ts
return the_buffer
def create_policy_eval_video(self,
env,
policy,
filename,
num_episodes=5,
fps=30):
filename = filename + ".mp4"
tf_env = TFPyEnvironment(env)
with imageio.get_writer(filename, fps=fps) as video:
for _ in range(num_episodes):
time_step = tf_env.reset()
tf_env.step(1)
video.append_data(env.render())
while not time_step.is_last():
action_step = policy.action(time_step)
time_step = tf_env.step(action_step.action)
video.append_data(env.render())
video.close()
return self.embed_mp4(filename)
def create_many_policy_gif(uid, file_path, monster_speed=4.0):
"""Create a gif superimposing the actions of many policies."""
n_steps = 300 # = timeout_factor / step_size
step_size = 0.01
fps = 10
p_paths = glob.glob(configs.POLICY_DIR + uid + '*')
all_positions = []
colors = []
for p_path in tqdm(p_paths):
color = (np.random.randint(256), np.random.randint(128), 0)
policy = tf.saved_model.load(p_path)
env_params = policy.get_metadata()
env_params = tf_to_py(env_params)
# overwriting parameters
env_params['step_size'] = step_size
env_params['monster_speed'] = monster_speed
py_env = LakeMonsterEnvironment(**env_params)
tf_env = TFPyEnvironment(py_env)
time_step = tf_env.reset()
agent_positions = {}
for step in range(n_steps):
if not time_step.is_last():
action = policy.action(time_step)
time_step = tf_env.step(action.action)
theta = py_env.total_monster_rotation - py_env.total_agent_rotation
c, s = np.cos(theta), np.sin(theta)
rot_matrix = np.array(((c, -s), (s, c)))
agent_positions[step] = np.dot(rot_matrix, np.array((py_env.r, 0)))
all_positions.append(agent_positions)
colors.append(color)
with imageio.get_writer(file_path, mode='I', fps=fps) as gif:
for step in range(n_steps):
positions = [item[step] for item in all_positions]
im = render_many_agents(positions, colors, step, step_size, 4,
monster_speed)
gif.append_data(np.array(im))
pygifsicle.optimize(file_path)
def episode_as_video(py_env, policy, filepath, fps=10):
"""Create mp4 video through py_environment render method."""
tf_env = TFPyEnvironment(py_env)
with imageio.get_writer('tmp.mp4', fps=fps) as video:
time_step = tf_env.reset()
video.append_data(py_env.render())
while not time_step.is_last():
action = policy.action(time_step).action
time_step = tf_env.step(action)
video.append_data(py_env.render())
for _ in range(3 * fps): # play for 3 more seconds
video.append_data(py_env.render())
# giving video file a more descriptive name
_, result = py_env.determine_reward()
assert filepath.split('.')[1] == 'mp4'
split = filepath.split('.')
split[0] += '-' + result
filepath = '.'.join(split)
os.rename('tmp.mp4', filepath)
def load_models():
"""Load existing saved models."""
if os.path.exists(configs.AGENT_ID_PATH):
deploy.save_model()
agent = train.restore_existing_agent()
model1 = verify.ModelWrapper(agent.q_net, agent.use_categorical)
p = agent.env_params
p['monster_speed'] = 3.5
p['step_size'] = 0.02
p['use_random_start'] = False
env = LakeMonsterEnvironment(**p)
step_increase = env.step_size / env.timeout_factor
env = TFPyEnvironment(env)
savepath = os.path.join(configs.TEMP_DIR, 'saved_model')
model2 = tf.keras.models.load_model(savepath)
return model1, model2, agent, env, step_increase
return None
def test_alf_metrics(self, num_envs, learn_queue_cap, unroll_length,
actor_queue_cap, num_actors, num_iterations):
episode_length = 5
env_f = lambda: TFPyEnvironment(
ValueUnittestEnv(batch_size=1, episode_length=episode_length))
alg = _create_ac_algorithm(env_f())
driver = AsyncOffPolicyDriver(env_f, alg, num_envs, num_actors,
unroll_length, learn_queue_cap,
actor_queue_cap)
driver.start()
total_num_steps_ = 0
for _ in range(num_iterations):
total_num_steps_ += driver.run_async()
driver.stop()
total_num_steps = int(driver.get_metrics()[1].result())
self.assertGreaterEqual(total_num_steps_, total_num_steps)
self.assertGreaterEqual(
total_num_steps, # multiply by 2/3 because 1/3 of steps are StepType.LAST
total_num_steps_ * 2 // 3)
average_reward = int(driver.get_metrics()[2].result())
self.assertEqual(average_reward, episode_length - 1)
episode_length = int(driver.get_metrics()[3].result())
self.assertEqual(episode_length, episode_length)
def episode_as_gif(py_env, policy, save_path, fps=10, show_path=True):
"""Create gif through py_environment render method."""
tf_env = TFPyEnvironment(py_env)
path = []
with imageio.get_writer(save_path, mode='I', fps=fps) as gif:
time_step = tf_env.reset()
# using the policy_state to deal with scripted_policy possibility
policy_state = policy.get_initial_state(batch_size=1)
gif.append_data(py_env.render())
while not time_step.is_last():
action = policy.action(time_step, policy_state)
time_step = tf_env.step(action.action)
im, real_position = py_env.render('return_real')
path.append(real_position)
if show_path:
im = render_agent_path(im, path)
policy_state = action.state
gif.append_data(np.array(im))
for _ in range(fps): # play for 1 more seconds
gif.append_data(py_env.render())
pygifsicle.optimize(save_path)
"""
# %matplotlib inline
# %% [markdown]
"""
This example is based on Mountain Car because it is straightforward to visualise the state space
and the linearised dynamics contain enough information to learn a controller using reinforcement
learning.
"""
# %%
global_step = tf.compat.v1.train.get_or_create_global_step()
tf_env = TFPyEnvironment(suite_gym.load("MountainCar-v0"))
# %% [markdown]
"""
## Dynamics model
We define the linear model that is going to be used to model the transition function of this
environment. We plot the predicted dynamics of the untrained model. The action space of the
mountain car environment consists of three discrete elements. These are represented in all
subsequent plots with three colours:
- left impulse: blue
- no impule: red
- right impulse: green
In these state-space plots, the x-axis is the agents position and the y-axis is the velocity.
"""
def get_env_specs(c):
dummy_env= TFPyEnvironment(StockEnvBasic(**c.default_env))
return dummy_env.observation_spec(),dummy_env.action_spec()
def train_eval(
# tensorboard files
root_dir,
# environment
env_name="Pendulum-v0",
random_seed=0,
# Params for collect
num_environment_steps=100000,
replay_buffer_capacity=1001, # Per-environment
# Params for eval
num_eval_episodes=30,
eval_interval=200,
# Params for summaries
summary_interval=50,
):
tf.compat.v1.set_random_seed(random_seed)
environment = TFPyEnvironment(suite_gym.load(env_name))
evaluation_environment = TFPyEnvironment(suite_gym.load(env_name))
critic_network = CriticNetwork(
input_tensor_spec=(environment.observation_spec(),
environment.action_spec()),
observation_fc_layer_params=None,
action_fc_layer_params=None,
joint_fc_layer_params=(200, 100),
)
actor_network = ActorNetwork(
input_tensor_spec=environment.observation_spec(),
output_tensor_spec=environment.action_spec(),
fc_layer_params=(200, 100),
)
global_step = tf.compat.v1.train.get_or_create_global_step()
agent = DdpgAgent(
time_step_spec=environment.time_step_spec(),
action_spec=environment.action_spec(),
critic_network=critic_network,
actor_network=actor_network,
actor_optimizer=tf.compat.v1.train.AdamOptimizer(),
critic_optimizer=tf.compat.v1.train.AdamOptimizer(),
train_step_counter=global_step,
)
agent_trainer = OffPolicyModelFreeAgentTrainer(1, 256)
experiment_harness = ExperimentHarness(
root_dir,
environment,
evaluation_environment,
agent,
agent_trainer,
replay_buffer_capacity,
num_environment_steps,
summary_interval,
eval_interval,
num_eval_episodes,
number_of_initial_random_policy_steps=0,
use_tf_function=True,
)
experiment_harness.run()
def create_environment(agent_mailbox, service_mailbox,
discount) -> TFPyEnvironment:
environment = PongEnvironment(agent_mailbox, service_mailbox, discount)
return TFPyEnvironment(environment)
from tensorflow_addons.optimizers.yogi import Yogi
from tensorflow.keras.optimizers.schedules import PolynomialDecay
from tensorflow.keras.losses import Huber
from tf_agents.environments.tf_py_environment import TFPyEnvironment
from tf_agents.networks.categorical_q_network import CategoricalQNetwork
from tf_agents.agents.categorical_dqn.categorical_dqn_agent import CategoricalDqnAgent
from tf_agents.utils.common import Checkpointer
from environment.carla_environment import CarlaEnvironment
# Allowing tensorflow to expand in physical memory, if it fails to allocate memory in GPU.
gpu = tf.config.experimental.list_physical_devices('GPU')[0]
tf.config.experimental.set_memory_growth(gpu, True)
# 1. Creating the tf-environment for validation.
carla_environment = CarlaEnvironment(log_dir='validation_log4/')
eval_env = TFPyEnvironment(environment=carla_environment)
# Allowing tensorflow to expand in physical memory, if it fails to allocate memory in GPU.
gpu = tf.config.experimental.list_physical_devices('GPU')[0]
tf.config.experimental.set_memory_growth(gpu, True)
# 2. Constructing the Categorical QNetworks: Online & Target.
# Default Activation Function: "Gelu".
# Default Weight Initialization: "He (Xavier) Initialization".
fc_layer_units = [128, 128]
conv_layer_units = [(4, 3, 1)]
num_atoms = 51
online_q_net = CategoricalQNetwork(
input_tensor_spec=eval_env.observation_spec(),
action_spec=eval_env.action_spec(),
def wrap_env(self):
tf_env = TFPyEnvironment(self.env)
return tf_env
losses[step, tr] = agent.train(experience=trajectories).loss
for met in range(len(train_metrics)):
train_metric_results[
met, step, tr] = train_metric_inst[met].result().numpy()
evaluations[step, :] = _evaluate_dyke_agent(eval_env, agent,
eval_episodes)
return {
'loss': losses,
'eval': evaluations,
'train-metrics': train_metric_results
}
if __name__ == '__main__':
train_py_env = DykeEnvironment(**dyke_environment_demo_params())
train_tf_env = TFPyEnvironment(train_py_env)
eval_py_env = DykeEnvironment(**dyke_environment_demo_params())
eval_tf_env = TFPyEnvironment(eval_py_env)
dqn_agent = dyke_dqn_agent(train_tf_env) # could also have been eval env
dqn_agent.initialize()
num_train_steps: int = int(1e2)
spe: int = int(np.ceil(train_py_env.timeout_time / train_py_env.delta_t))
# bat = _dyke_replay_buffer(train_tf_env, dqn_agent, int(1e4))
# print('Running the episode driver!')
# _, _ = DynamicStepDriver(train_tf_env, dqn_agent.policy, [bat.add_batch], num_steps=spe).run()
# print('Done!')
#
# it = iter(bat.as_dataset())