def load_act(path, scope):
with open(path, "rb") as f:
model_data, act_params = cloudpickle.load(f)
act = co_build_act(**act_params, scope=scope)
sess = tf.Session()
sess.__enter__()
with tempfile.TemporaryDirectory() as td:
arc_path = os.path.join(td, "packed.zip")
with open(arc_path, "wb") as f:
f.write(model_data)
zipfile.ZipFile(arc_path, 'r', zipfile.ZIP_DEFLATED).extractall(td)
load_variables(os.path.join(td, "model"))
return ActWrapper(act, act_params)
def __init__(
self,
env,
# observation_space,
# action_space,
network=None,
scope='deepq',
seed=None,
lr=None, # Was 5e-4
lr_mc=5e-4,
total_episodes=None,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=None, # was 0.02
train_freq=1,
train_log_freq=100,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
# checkpoint_path=None,
learning_starts=1000,
gamma=None,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
save_path=None,
load_path=None,
save_reward_threshold=None,
**network_kwargs):
super().__init__(env, seed)
if train_log_freq % train_freq != 0:
raise ValueError(
'Train log frequency should be a multiple of train frequency')
elif checkpoint_freq % train_log_freq != 0:
raise ValueError(
'Checkpoint freq should be a multiple of train log frequency, or model saving will not be logged properly'
)
print('init dqnlearningagent')
self.train_log_freq = train_log_freq
self.scope = scope
self.learning_starts = learning_starts
self.save_reward_threshold = save_reward_threshold
self.batch_size = batch_size
self.train_freq = train_freq
self.total_episodes = total_episodes
self.total_timesteps = total_timesteps
# TODO: scope not doing anything.
if network is None and 'lunar' in env.unwrapped.spec.id.lower():
if lr is None:
lr = 1e-3
if exploration_final_eps is None:
exploration_final_eps = 0.02
#exploration_fraction = 0.1
#exploration_final_eps = 0.02
target_network_update_freq = 1500
#print_freq = 100
# num_cpu = 5
if gamma is None:
gamma = 0.99
network = 'mlp'
network_kwargs = {
'num_layers': 2,
'num_hidden': 64,
}
self.target_network_update_freq = target_network_update_freq
self.gamma = gamma
get_session()
# set_global_seeds(seed)
# TODO: Check whether below is ok to substitue for set_global_seeds.
try:
import tensorflow as tf
tf.set_random_seed(seed)
except ImportError:
pass
self.q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
act, self.train, self.train_mc, self.update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=self.q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
optimizer_mc=tf.train.AdamOptimizer(learning_rate=lr_mc),
gamma=gamma,
grad_norm_clipping=10,
param_noise=False,
scope=scope,
# reuse=reuse,
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': self.q_func,
'num_actions': env.action_space.n,
}
self._act = ActWrapper(act, act_params)
self.print_freq = print_freq
self.checkpoint_freq = checkpoint_freq
# Create the replay buffer
self.prioritized_replay = prioritized_replay
self.prioritized_replay_eps = prioritized_replay_eps
if self.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(
buffer_size,
alpha=prioritized_replay_alpha,
)
if prioritized_replay_beta_iters is None:
if total_episodes is not None:
raise NotImplementedError(
'Need to check how to set exploration based on episodes'
)
prioritized_replay_beta_iters = total_timesteps
self.beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0,
)
else:
self.replay_buffer = ReplayBuffer(buffer_size)
self.replay_buffer_mc = ReplayBuffer(buffer_size)
self.beta_schedule = None
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(
schedule_timesteps=int(
exploration_fraction *
total_timesteps if total_episodes is None else total_episodes),
initial_p=1.0,
final_p=exploration_final_eps,
)
# Initialize the parameters and copy them to the target network.
U.initialize()
self.update_target()
self.episode_lengths = [0]
self.episode_rewards = [0.0]
self.discounted_episode_rewards = [0.0]
self.start_values = [None]
self.lunar_crashes = [0]
self.lunar_goals = [0]
self.saved_mean_reward = None
self.td = None
if save_path is None:
self.td = tempfile.mkdtemp()
outdir = self.td
self.model_file = os.path.join(outdir, "model")
else:
outdir = os.path.dirname(save_path)
os.makedirs(outdir, exist_ok=True)
self.model_file = save_path
print('DQN agent saving to:', self.model_file)
self.model_saved = False
if tf.train.latest_checkpoint(outdir) is not None:
# TODO: Check scope addition
load_variables(self.model_file, scope=self.scope)
# load_variables(self.model_file)
logger.log('Loaded model from {}'.format(self.model_file))
self.model_saved = True
raise Exception('Check that we want to load previous model')
elif load_path is not None:
# TODO: Check scope addition
load_variables(load_path, scope=self.scope)
# load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
self.train_log_file = None
if save_path and load_path is None:
self.train_log_file = self.model_file + '.log.csv'
with open(self.train_log_file, 'w') as f:
cols = [
'episode',
't',
'td_max',
'td_mean',
'100ep_r_mean',
'100ep_r_mean_discounted',
'100ep_v_mean',
'100ep_n_crashes_mean',
'100ep_n_goals_mean',
'saved_model',
'smoothing',
]
f.write(','.join(cols) + '\n')
self.training_episode = 0
self.t = 0
self.episode_t = 0
"""
n = observation_space.n
m = action_space.n
self.Q = np.zeros((n, m))
self._lr_schedule = lr_schedule
self._eps_schedule = eps_schedule
self._boltzmann_schedule = boltzmann_schedule
"""
# Make placeholder for Q values
self.q_values = debug['q_values']
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
initial_exploration_p=1.0,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=0,#,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
double_q=True,
num_heads=10,
**network_kwargs
):
"""Train a bootstrap-dqn model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
nenvs = env.num_envs
print("Bootstrap DQN with {} envs".format(nenvs))
q_func = build_q_func(network, num_heads, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = bootstrap_dqn.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
double_q=double_q,
num_heads=num_heads
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * total_timesteps),
initial_p=initial_exploration_p,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
episode_reward = np.zeros(nenvs, dtype = np.float32)
saved_mean_reward = None
reset = True
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_actions = []
epoch_episodes = 0
episode_rewards_history = deque(maxlen=100)
episode_step = np.zeros(nenvs, dtype = int)
episodes = 0 #scalar
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
print("Model will be saved at " , model_file)
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
print('Loaded model from {}'.format(load_path))
t = 0
while t < total_timesteps:
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
obs = env.reset()
head = np.random.randint(num_heads) # Head initialisation
for m in range(100):
action, q_values = act(np.array(obs)[None], head=head, update_eps=update_eps, **kwargs)
env_action = action
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, np.array([head]*nenvs), new_obs, done)
if np.random.rand() < 0.01:
print (head, obs[0], q_values[0])
obs = new_obs
episode_reward += rew
episode_step += 1
for d in range(len(done)):
if done[d]:
epoch_episode_rewards.append(episode_reward[d])
episode_rewards_history.append(episode_reward[d])
epoch_episode_steps.append(episode_step[d])
episode_reward[d] = 0.
episode_step[d] = 0
epoch_episodes += 1
episodes += 1
t += 100 * nenvs
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, head_t, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, head_t, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, head_t, head_t, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards_history), 2)
num_episodes = episodes
if print_freq is not None and num_episodes % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
print("episodes", num_episodes, "steps {}/{}".format(t, total_timesteps))
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
print("saving model")
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_variables(model_file)
return act
def __init__(self,
env,
network='mlp',
lr=5e-4,
buffer_size=50000,
exploration_epsilon=0.1,
train_freq=1,
batch_size=32,
learning_starts=1000,
target_network_update_freq=500,
**network_kwargs):
"""DQN wrapper to train option policies
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
lr: float
learning rate for adam optimizer
buffer_size: int
size of the replay buffer
exploration_epsilon: float
value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batch sampled from replay buffer for training
learning_starts: int
how many steps of the model to collect transitions for before learning starts
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
network_kwargs
additional keyword arguments to pass to the network builder.
"""
# Creating the network
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.controller_observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.controller_action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
grad_norm_clipping=10,
scope="controller")
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.controller_action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
replay_buffer = ReplayBuffer(buffer_size)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# Variables that are used during learning
self.act = act
self.train = train
self.update_target = update_target
self.replay_buffer = replay_buffer
self.exp_epsilon = exploration_epsilon
self.train_freq = train_freq
self.batch_size = batch_size
self.learning_starts = learning_starts
self.target_network_update_freq = target_network_update_freq
self.num_actions = env.controller_action_space.n
self.t = 0
def __init__(self,
env,
gamma,
total_timesteps,
network='mlp',
lr=5e-4,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
learning_starts=1000,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
**network_kwargs):
"""DQN wrapper to train option policies
Parameters
-------
env: gym.Env
environment to train on
gamma: float
discount factor
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
total_timesteps: int
number of env steps to optimizer for
lr: float
learning rate for adam optimizer
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batch sampled from replay buffer for training
learning_starts: int
how many steps of the model to collect transitions for before learning starts
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
**network_kwargs
additional keyword arguments to pass to the network builder.
"""
# Adjusting hyper-parameters by considering the number of options policies to learn
num_options = env.get_number_of_options()
buffer_size = num_options * buffer_size
batch_size = num_options * batch_size
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.option_observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
self.num_actions = env.option_action_space.n
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=self.num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
scope="options")
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': self.num_actions,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(
exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# Variables that are used during learning
self.act = act
self.train = train
self.update_target = update_target
self.replay_buffer = replay_buffer
self.beta_schedule = beta_schedule
self.exploration = exploration
self.param_noise = param_noise
self.train_freq = train_freq
self.batch_size = batch_size
self.learning_starts = learning_starts
self.target_network_update_freq = target_network_update_freq
self.prioritized_replay = prioritized_replay
self.prioritized_replay_alpha = prioritized_replay_alpha
self.prioritized_replay_beta0 = prioritized_replay_beta0
self.prioritized_replay_beta_iters = prioritized_replay_beta_iters
self.prioritized_replay_eps = prioritized_replay_eps
def main(args):
# configure logger, disable logging in child MPI processes (with rank > 0)
arg_parser = common_arg_parser()
args, unknown_args = arg_parser.parse_known_args(args)
extra_args = parse_cmdline_kwargs(unknown_args)
if MPI is None or MPI.COMM_WORLD.Get_rank() == 0:
rank = 0
configure_logger(args.log_path)
else:
rank = MPI.COMM_WORLD.Get_rank()
configure_logger(args.log_path, format_strs=[])
import json
with open(osp.join(logger.get_dir(), 'args.json'), 'w') as arg_record_file:
json.dump(args.__dict__, arg_record_file)
env, constraints = build_env(args)
hard_constraints = [c for c in constraints if c.is_hard]
from baselines.deepq.deepq import ActWrapper
model = ActWrapper.load_act(args.save_path)
q_vals = np.zeros((int(args.num_timesteps), env.action_space.n))
# if we have already collected trajectories
if 'experience_dir' in args.__dict__:
logger.log("Loading collected experiences")
# TODO: fix by adding loading of constraint state and finding the right files
states = np.load(osp.join(args.experience_dir, 'states'))
if file_exists:
constraint_states = np.load(
osp.join(args.experience_dir, 'constraint_states'))
else:
constraint_states = []
else:
print(extra_args)
if 'collect_states' in extra_args:
states = np.zeros((int(args.num_timesteps), ) +
env.observation_space.shape)
constraint_states = []
episode_rewards = []
logger.log("Running loaded model")
obs = env.reset()
state = model.initial_state if hasattr(model,
'initial_state') else None
dones = np.zeros((1, ))
episode_rew = np.zeros(env.num_envs) if isinstance(
env, VecEnv) else np.zeros(1)
timestep = 0
ready_to_exit = False
while True:
timestep += 1
if timestep >= args.num_timesteps:
ready_to_exit = True
if hard_constraints:
constraint_mask = reduce(lambda x, y: x + y, [
c.violating_mask(env.action_space.n)
for c in hard_constraints
])
else:
constraint_mask = None
if state is not None:
actions, _, state, _ = model.step(obs, S=state, M=dones, hard_constraint_mask=constraint_mask)
else:
actions, _, _, _ = model.step(obs, hard_constraint_mask=constraint_mask)
obs, rew, done, _ = env.step(actions)
if 'collect_states' in extra_args:
if type(obs) is tuple: # with augmentation
states[i] = obs[0]
constraint_states.append(obs[1])
else: # without aug
states[i] = obs
episode_rew += rew
done_any = done.any() if isinstance(done, np.ndarray) else done
if done_any:
for i in np.nonzero(done)[0]:
episode_rewards.append(episode_rew[0])
episode_rew[i] = 0
if ready_to_exit:
break
env.reset()
np.save(osp.join(logger.get_dir(), 'episode_rewards'), episode_rewards)
if 'collect_states' in extra_args:
np.save(osp.join(logger.get_dir(), 'states'), states)
if len(constraint_states) > 0:
np.save(osp.join(logger.get_dir(), 'constraint_states'),
np.array(constraint_states))
env.close()
# calculate q values
if 'collect_states' in extra_args:
for i, s in enumerate(states):
if len(constraint_states) > 0: # with augmentation
q_input = [(s, constraint_states[i])]
else:
q_input = s
q_vals[i] = model.q(q_input)
np.save(osp.join(logger.get_dir(), 'q_vals'), q_vals)
shutil.copyfile(osp.join(logger.get_dir(), 'log.txt'),
osp.join(logger.get_dir(), 'final_log.txt'))
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
stage1_total_timesteps=None,
stage2_total_timesteps=None,
buffer_size=50000,
exploration_fraction=0.3,
initial_exploration_p=1.0,
exploration_final_eps=0.0,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1,
gamma=1.0,
target_network_update_freq=100,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
double_q=True,
obs_dim=None,
qmdp_expert=None,
stage1_td_error_threshold=1e-3,
pretrain_experience=None,
flatten_belief=False,
num_experts=None,
**network_kwargs):
"""Train a bootstrap-dqn model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
qmdp_expert: takes obs, bel -> returns qmdp q-vals
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
nenvs = env.num_envs
print("{} envs".format(nenvs))
assert pretrain_experience is not None and qmdp_expert is not None and num_experts is not None
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
# import IPython; IPython.embed()
#assert isinstance(env.envs[0].env.env.env, ExplicitBayesEnv)
#belief_space = env.envs[0].env.env.env.belief_space
#observation_space = env.envs[0].env.env.env.internal_observation_space
obs_space = env.observation_space
assert obs_dim is not None
observation_space = Box(obs_space.low[:obs_dim],
obs_space.high[:obs_dim],
dtype=np.float32)
#belief_space = Box(obs_space.low[obs_dim:], obs_space.high[obs_dim:], dtype=np.float32)
observed_belief_space = Box(obs_space.low[obs_dim:],
obs_space.high[obs_dim:],
dtype=np.float32)
belief_space = Box(np.zeros(num_experts),
np.ones(num_experts),
dtype=np.float32) # rocksample
num_experts = belief_space.high.size
# print("Num experts", num_experts)
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
def make_bel_ph(name):
return ObservationInput(belief_space, name=name)
q_func = build_q_func(network, num_experts, **network_kwargs)
print('=============== got qfunc ============== ')
if stage1_total_timesteps is None and stage2_total_timesteps is None:
stage1_total_timesteps = total_timesteps // 2
stage2_total_timesteps = total_timesteps // 2
total_timesteps = stage1_total_timesteps + stage2_total_timesteps
act, train, update_target, debug = rbqnfe_staged.build_train(
make_obs_ph=make_obs_ph,
make_bel_ph=make_bel_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
double_q=double_q)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=initial_exploration_p,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_reward = np.zeros(nenvs, dtype=np.float32)
saved_mean_reward = None
reset = True
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_actions = []
epoch_episodes = 0
episode_rewards_history = deque(maxlen=1000)
episode_step = np.zeros(nenvs, dtype=int)
episodes = 0 #scalar
# Load model
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
print("Model will be saved at ", model_file)
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
print('Loaded model from {}'.format(load_path))
t = 0
accumulated_td_errors = deque(maxlen=100)
# copy all pre-experiences
for expert, experience in enumerate(pretrain_experience):
obs, val, action, rew, new_obs, done = experience
obs, bel = obs[:, :-observed_belief_space.
shape[0]], obs[:, -observed_belief_space.shape[0]:]
if flatten_belief:
bel = qmdp_expert.flatten_to_belief(bel,
approximate=True).transpose()
new_obs, new_bel = new_obs[:, :-observed_belief_space.
shape[0]], new_obs[:,
-observed_belief_space.
shape[0]:]
if flatten_belief:
new_bel = qmdp_expert.flatten_to_belief(
new_bel, approximate=True).transpose() # rocksample specific
new_expert_qval = qmdp_expert(new_obs, new_bel)
expert_qval = qmdp_expert(obs, bel)
obs = obs.astype(np.float32)
bel = bel.astype(np.float32)
expert_qval = expert_qval.astype(np.float32)
action = action.astype(np.float32)
rew = rew.astype(np.float32).ravel()
new_obs = new_obs.astype(np.float32)
new_bel = new_bel.astype(np.float32)
new_expert_qval = new_expert_qval.astype(np.float32)
replay_buffer.add(obs, bel, expert_qval, action, rew, new_obs, new_bel,
new_expert_qval, done)
print("Added {} samples to ReplayBuffer".format(
len(replay_buffer._storage)))
# Stage 1: Train Residual without exploration, just with batches from replay buffer
while t < stage1_total_timesteps:
if callback is not None:
if callback(locals(), globals()):
break
kwargs = {}
update_param_noise_threshold = 0.
obs = env.reset()
episode_reward = np.zeros(nenvs, dtype=np.float32)
episode_step[:] = 0
obs, bel = obs[:, :-observed_belief_space.
shape[0]], obs[:, -observed_belief_space.shape[0]:]
expert_qval = qmdp_expert(obs, bel)
t += 1
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size,
beta=beta_schedule.value(t))
obses_t, bels_t, expert_qvals, actions, rewards, obses_tp1, bels_tp1, expert_qvals_1, dones, weights, batch_idxes = experience
else:
experience = replay_buffer.sample(batch_size)
obses_t, bels_t, expert_qvals, actions, rewards, obses_tp1, bels_tp1, expert_qvals_1, dones = experience
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, bels_t, expert_qvals, actions, rewards,
obses_tp1, bels_tp1, expert_qvals_1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
accumulated_td_errors.append(np.mean(np.abs(td_errors)))
if np.random.rand() < 0.01:
print("Stage 1 TD error", np.around(td_errors, 1))
if t % target_network_update_freq == 0:
# Update target network periodically.
print("Update target")
update_target()
if len(accumulated_td_errors) == 100 and np.mean(
np.abs(accumulated_td_errors)) < stage1_td_error_threshold:
if saved_mean_reward is not None:
save_variables(model_file)
print("Breaking due to low td error",
np.mean(accumulated_td_errors))
break
if t % print_freq == 0:
# Just to get test rewards
obs = env.reset()
episode_reward = np.zeros(nenvs, dtype=np.float32)
episode_step[:] = 0
obs, bel = obs[:, :-observed_belief_space.
shape[0]], obs[:, -observed_belief_space.shape[0]:]
expert_qval = qmdp_expert(obs, bel)
episode_rewards_history = []
horizon = 100
while len(episode_rewards_history) < 1000:
action, q_values = act(np.array(obs)[None],
np.array(bel)[None],
np.array(expert_qval)[None],
update_eps=0,
**kwargs)
env_action = action
new_obs, rew, done, info = env.step(env_action)
new_obs, new_bel = new_obs[:, :-observed_belief_space.shape[
0]], new_obs[:, -observed_belief_space.shape[0]:]
new_expert_qval = qmdp_expert(new_obs, new_bel)
if flatten_belief:
new_bel = qmdp_expert.flatten_to_belief(new_bel)
obs = new_obs
bel = new_bel
expert_qval = new_expert_qval
episode_reward += 0.95**episode_step * rew
episode_step += 1
for d in range(len(done)):
if done[d]:
epoch_episode_rewards.append(episode_reward[d])
episode_rewards_history.append(episode_reward[d])
epoch_episode_steps.append(episode_step[d])
episode_reward[d] = 0.
episode_step[d] = 0
epoch_episodes += 1
episodes += 1
mean_100ep_reward = round(np.mean(episode_rewards_history), 2)
num_episodes = episodes
logger.record_tabular("stage", 1)
logger.record_tabular("steps", t)
logger.record_tabular("mean 1000 episode reward",
mean_100ep_reward)
logger.record_tabular("td errors", np.mean(accumulated_td_errors))
logger.dump_tabular()
print("episodes ", num_episodes,
"steps {}/{}".format(t, total_timesteps))
print("mean reward", mean_100ep_reward)
print("exploration", int(100 * exploration.value(t)))
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}".
format(saved_mean_reward, mean_100ep_reward))
print("saving model")
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
# Post stage1 saving
stage1_model_file = os.path.join(td, "stage1_model")
save_variables(stage1_model_file)
update_target()
print("===========================================")
print(" Stage 1 complete ")
print("===========================================")
stage1_total_timesteps = t
episode_rewards_history = deque(maxlen=1000)
# Stage 2: Train Resisual with explorationi
t = 0
while t < stage2_total_timesteps:
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
obs = env.reset()
episode_reward = np.zeros(nenvs, dtype=np.float32)
episode_step[:] = 0
obs, bel = obs[:, :-observed_belief_space.
shape[0]], obs[:, -observed_belief_space.shape[0]:]
expert_qval = qmdp_expert(obs, bel)
start_time = timer.time()
horizon = 100
for m in range(horizon):
action, q_values = act(np.array(obs)[None],
np.array(bel)[None],
np.array(expert_qval)[None],
update_eps=update_eps,
**kwargs)
env_action = action
new_obs, rew, done, info = env.step(env_action)
new_obs, new_bel = new_obs[:, :-observed_belief_space.shape[
0]], new_obs[:, -observed_belief_space.shape[0]:]
new_expert_qval = qmdp_expert(new_obs, new_bel)
if flatten_belief:
new_bel = qmdp_expert.flatten_to_belief(new_bel)
# Store transition in the replay buffer.
replay_buffer.add(obs, bel, expert_qval, action, rew, new_obs,
new_bel, new_expert_qval, done)
# if np.random.rand() < 0.05:
# # # write to file
# # with open('rbqn_fixed_expert.csv', 'a') as f:
# # out = ','.join(str(np.around(x,2)) for x in [bel[0], obs[0], q_values[0]])
# # f.write(out + "\n")
# print(np.around(bel[-1], 2), rew[-1], np.around(q_values[-1], 1), np.around(expert_qval[-1], 1))
obs = new_obs
bel = new_bel
expert_qval = new_expert_qval
episode_reward += 0.95**episode_step * rew
episode_step += 1
# print(action, done, obs)
for d in range(len(done)):
if done[d]:
epoch_episode_rewards.append(episode_reward[d])
episode_rewards_history.append(episode_reward[d])
epoch_episode_steps.append(episode_step[d])
episode_reward[d] = 0.
episode_step[d] = 0
epoch_episodes += 1
episodes += 1
print("Took {}".format(timer.time() - start_time))
t += 1
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size,
beta=beta_schedule.value(t))
if experience is None:
continue
obses_t, bels_t, expert_qvals, actions, rewards, obses_tp1, bels_tp1, expert_qvals_1, dones, weights, batch_idxes = experience
else:
experience = replay_buffer.sample(batch_size)
if experience is None:
continue
obses_t, bels_t, expert_qvals, actions, rewards, obses_tp1, bels_tp1, expert_qvals_1, dones = experience
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, bels_t, expert_qvals, actions, rewards,
obses_tp1, bels_tp1, expert_qvals_1, dones,
weights)
if np.random.rand() < 0.01:
print("TD error", np.around(td_errors, 1))
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
accumulated_td_errors.append(np.mean(td_errors))
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
print("Update target")
update_target()
mean_100ep_reward = round(np.mean(episode_rewards_history), 2)
num_episodes = episodes
if print_freq is not None and num_episodes % print_freq == 0:
logger.record_tabular("stage", 2)
logger.record_tabular("steps", t + stage1_total_timesteps)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 1000 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.record_tabular("td errors", np.mean(accumulated_td_errors))
logger.dump_tabular()
print("episodes ", num_episodes,
"steps {}/{}".format(t, total_timesteps))
print("mean reward", mean_100ep_reward)
print("exploration", int(100 * exploration.value(t)))
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}".
format(saved_mean_reward, mean_100ep_reward))
print("saving model")
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
initial_exploration_p=1.0,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=100,
prioritized_replay=True,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
pretraining_obs=None,
pretraining_targets=None,
pretrain_steps=1000,
pretrain_experience=None,
pretrain_num_episodes=0,
double_q=True,
expert_qfunc=None,
aggrevate_steps=0,
pretrain_lr=1e-4,
sampling_starts=0,
beb_agent=None,
qvalue_file="qvalue.csv",
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
beb_agent: takes Q values and suggests actions after adding beb bonus
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
nenvs = env.num_envs
print("Bayes-DeepQ:", env.num_envs)
print("Total timesteps", total_timesteps)
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, train_target, copy_target_to_q, debug = brl_deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
pretrain_optimizer=tf.train.AdamOptimizer(learning_rate=pretrain_lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
double_q=double_q)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=initial_exploration_p,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
print("Model will be saved at ", model_file)
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
print('Loaded model from {}'.format(load_path))
if pretraining_obs is not None:
# pretrain target and copy to qfunc
print("Pretrain steps ", pretrain_steps)
for i in range(pretrain_steps):
pretrain_errors = train_target(pretraining_obs,
pretraining_targets)
if i % 500 == 0:
print("Step {}".format(i), np.mean(pretrain_errors))
if np.mean(pretrain_errors) < 1e-5:
break
min_rew = 0
# copy all pre-experiences
if pretrain_experience is not None:
for obs, action, rew, new_obs, done in zip(*pretrain_experience):
replay_buffer.add(obs, action, rew, new_obs, float(done))
print("Added {} samples to ReplayBuffer".format(
len(replay_buffer._storage)))
min_rew = min(rew, min_rew)
print("Pretrain Error", np.mean(pretrain_errors))
else:
print("Skipping pretraining")
update_target()
print("Save the pretrained model", model_file)
save_variables(model_file)
episode_reward = np.zeros(nenvs, dtype=np.float32)
saved_mean_reward = None
obs = env.reset()
reset = True
epoch_episode_rewards = []
epoch_episode_steps = []
epoch_actions = []
epoch_episodes = 0
episode_rewards_history = deque(maxlen=100)
episode_step = np.zeros(nenvs, dtype=int)
episodes = 0 #scalar
start = 0
if expert_qfunc is None:
aggrevate_steps = 0
# if pretraining_obs is None or pretraining_obs.size == 0:
# episode_rewards = []
# else:
# episode_rewards = [[0.0]] * pretrain_num_episodes
# start = len(pretraining_obs)
# if print_freq is not None:
# for t in range(0, len(pretraining_obs), print_freq):
# logger.record_tabular("steps", t)
# logger.record_tabular("episodes", pretrain_num_episodes)
# logger.record_tabular("mean 100 episode reward", min_rew)
# logger.record_tabular("% time spent exploring", 0)
# logger.dump_tabular()
# print("pretraining episodes", pretrain_num_episodes, "steps {}/{}".format(t, total_timesteps))
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
print("Aggrevate: Model will be saved at ", model_file)
model_saved = False
for i in range(aggrevate_steps):
obses_t, values = [], []
for j in range(30):
# TODO: 30 should be changed to max horizon?
t = np.random.randint(50) + 1
obs = env.reset()
for k in range(t):
action, value = act(np.array(obs)[None],
update_eps=exploration.value(i))
obs, rew, done, _ = env.step(action)
obses_t.extend(obs)
# Roll out expert policy
episode_reward[:] = 0
dones = np.array([False] * obs.shape[0])
for k in range(51 - t):
obs, rew, done, _ = env.step(
[expert_qfunc.step(o) for o in obs])
dones[done] = True
rew[dones] = 0
episode_reward += 0.95**k * rew
# TODO: change this to exploration-savvy action
# action = np.random.randint(env.action_space.n, size=len(obs))
# Rocksample specific, take sensing actions
# prob = np.array([1] * 6 + [2] * (env.action_space.n - 6), dtype=np.float32)
# prob = prob / np.sum(prob)
# action = np.random.choice(env.action_space.n, p=prob, size=len(action))
# new_obs, rew, done, _ = env.step(action)
# value = rew.copy()
# value[np.logical_not(done)] += gamma * np.max(expert_qfunc.value(new_obs[np.logical_not(done)]), axis=1)
# current_value[tuple(np.array([np.arange(len(action)), action]))] = value
# episode reward
# episode_reward[np.logical_not(done)] += np.max(current_value[np.logical_not(done)], axis=1)
# episode_rewards_history.extend(np.max(current_value, axis=1))
value[tuple([np.arange(len(action)), action])] = episode_reward
values.extend(value)
print("Aggrevate got {} / {} new data".format(
obs.shape[0] * 30, len(obses_t)))
# print("Mean expected cost at the explored points", np.mean(np.max(values, axis=1)))
for j in range(1000):
obs, val = np.array(obses_t), np.array(values)
# indices = np.random.choice(len(obs), min(1000, len(obses_t)))
aggrevate_errors = train_target(obs, val)
if np.mean(aggrevate_errors) < 1e-5:
print("Aggrevate Step {}, {}".format(i, j),
np.mean(aggrevate_errors))
break
print("Aggrevate Step {}, {}".format(i, j),
np.mean(aggrevate_errors))
update_target()
print("Save the aggrevate model", i, model_file)
# Evaluate current policy
episode_reward[:] = 0
obs = env.reset()
num_episodes = 0
k = np.zeros(len(obs))
while num_episodes < 100:
action, _ = act(np.array(obs)[None], update_eps=0.0)
# print(action)
obs, rew, done, _ = env.step(action)
episode_reward += 0.95**k * rew
k += 1
for d in range(len(done)):
if done[d]:
episode_rewards_history.append(episode_reward[d])
episode_reward[d] = 0.
k[d] = 0
num_episodes += 1
mean_1000ep_reward = round(np.mean(episode_rewards_history), 2)
print("Mean discounted reward", mean_1000ep_reward)
logger.record_tabular("mean 100 episode reward",
mean_1000ep_reward)
logger.dump_tabular()
save_variables(model_file)
t = 0 # could start from pretrain-steps
epoch = 0
while True:
epoch += 1
if t >= total_timesteps:
break
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# no randomization
# update_eps = 0
print('update_eps', int(100 * exploration.value(t)))
qv_error = []
obs = env.reset()
for m in range(100):
action, q_values = act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)
if beb_agent is not None:
action = beb_agent.step(obs, action, q_values,
exploration.value(t))
# if expert_qfunc is not None:
# v = expert_qfunc.value(obs)
# qv_error += [v - q_values[0]]
env_action = action
reset = False
new_obs, rew, done, info = env.step(env_action)
if t >= sampling_starts:
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, done)
obs = new_obs
episode_reward += rew
episode_step += 1
for d in range(len(done)):
if done[d]:
# Episode done.
# discount(np.array(rewards), gamma) consider doing discount
epoch_episode_rewards.append(episode_reward[d])
episode_rewards_history.append(episode_reward[d])
epoch_episode_steps.append(episode_step[d])
episode_reward[d] = 0.
episode_step[d] = 0
epoch_episodes += 1
episodes += 1
t += 100 * nenvs
if t > learning_starts:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if target_network_update_freq is not None and t > sampling_starts \
and epoch % target_network_update_freq == 0:
# Update target network periodically.
print("Update target")
update_target()
mean_1000ep_reward = round(np.mean(episode_rewards_history), 2)
num_episodes = episodes
if print_freq is not None:
logger.record_tabular("steps", t)
logger.record_tabular("td errors", np.mean(td_errors))
logger.record_tabular("td errors std",
np.std(np.abs(td_errors)))
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 1000 episode reward",
mean_1000ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
print("episodes", num_episodes,
"steps {}/{}".format(t, total_timesteps))
if (checkpoint_freq is not None and t > learning_starts
and len(episode_rewards_history) >= 1000):
if saved_mean_reward is None or mean_1000ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_1000ep_reward))
print("saving model")
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_1000ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
initial_exploration_p=1.0,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
pretraining_obs=None,
pretraining_targets=None,
pretrain_steps=1000,
pretrain_experience=None,
pretrain_num_episodes=0,
double_q=True,
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, train_target, copy_target_to_q, debug = brl_deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
pretrain_optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
double_q=double_q)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=initial_exploration_p,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
if pretraining_obs is not None:
# pretrain target and copy to qfunc
for _ in range(pretrain_steps):
pretrain_errors = train_target(pretraining_obs,
pretraining_targets)
min_rew = 0
# copy all pre-experiences
if pretrain_experience is not None:
for obs, action, rew, new_obs, done in zip(*pretrain_experience):
replay_buffer.add(obs.reshape(1, -1), action, rew,
new_obs.reshape(1, -1), float(done))
print("Added {} samples to ReplayBuffer".format(
len(replay_buffer._storage)))
min_rew = min(rew, min_rew)
print("Pretrain Error", np.mean(pretrain_errors))
else:
print("Skipping pretraining")
update_target()
if pretraining_obs is None or pretraining_obs.size == 0:
episode_rewards = []
start = 0
else:
episode_rewards = [[0.0]] * pretrain_num_episodes
start = len(pretraining_obs)
if print_freq is not None:
for t in range(0, len(pretraining_obs), print_freq):
logger.record_tabular("steps", t)
logger.record_tabular("episodes", pretrain_num_episodes)
logger.record_tabular("mean 100 episode reward", min_rew)
logger.record_tabular("% time spent exploring", 0)
logger.dump_tabular()
print("pretraining episodes", pretrain_num_episodes,
"steps {}/{}".format(t, total_timesteps))
saved_mean_reward = None
obs = env.reset()
episode_rewards += [[0.0]]
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
print("Model will be saved at ", model_file)
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
print('Loaded model from {}'.format(load_path))
for t in range(start, total_timesteps, 1):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += [
rew if not isinstance(rew, np.ndarray) else rew[0]
]
if done:
obs = env.reset()
episode_rewards.append([0.0])
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
last_100ep_rewards = [
discount(np.array(rewards), gamma)[0]
for rewards in episode_rewards[-101:-1]
]
mean_100ep_reward = round(np.mean(last_100ep_rewards), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
print("episodes", num_episodes,
"steps {}/{}".format(t, total_timesteps))
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
print("saving model")
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act
def init_wrapper(env,
network_type,
lr=1e-4,
gamma=1.0,
param_noise=True,
buffer_size=int(1e5),
prioritized_replay_alpha=.6,
prioritized_replay=True,
prioritized_replay_beta_iters=None,
prioritized_replay_beta=.4,
exploration_fraction=.1,
grad_norm_clipping=10,
total_timesteps=int(1e6),
exploration_final_eps=0.02,
**network_kwargs):
# decomposes baseline deepq into initialize and inference components
# basically copied from deepqn repository
# see baselines repo for concise param documentation
q_func = build_q_func(network_type, **network_kwargs)
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=grad_norm_clipping,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
# WARNING: do not use internal replay buffer, use baselines only for
# stability reasons
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# return hashed objects
return {
'train_function': train,
'act_function': act,
'replay_buffer': replay_buffer,
'update_target_function': update_target,
'exploration_scheme': exploration,
'beta_schedule': beta_schedule
}
def learn(env,
network,
seed=None,
lr=1e-3,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
num_cpu=5,
callback=None,
scope='co_deepq',
pilot_tol=0,
pilot_is_human=False,
reuse=False,
load_path=None,
**network_kwargs):
# Create all the functions necessary to train the model
sess = get_session() #tf.Session(graph=tf.Graph())
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
using_control_sharing = True #pilot_tol > 0
if pilot_is_human:
utils.human_agent_action = init_human_action()
utils.human_agent_active = False
act, train, update_target, debug = co_build_train(
scope=scope,
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
reuse=tf.AUTO_REUSE if reuse else False,
using_control_sharing=using_control_sharing)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
episode_outcomes = []
saved_mean_reward = None
obs = env.reset()
reset = True
prev_t = 0
rollouts = []
if not using_control_sharing:
exploration = LinearSchedule(schedule_timesteps=int(
exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
for t in range(total_timesteps):
masked_obs = mask_helipad(obs)
act_kwargs = {}
if using_control_sharing:
if pilot_is_human:
act_kwargs['pilot_action'] = env.unwrapped.pilot_policy(
obs[None, :9])
else:
act_kwargs[
'pilot_action'] = env.unwrapped.pilot_policy.step(
obs[None, :9])
act_kwargs['pilot_tol'] = pilot_tol if not pilot_is_human or (
pilot_is_human and utils.human_agent_active) else 0
else:
act_kwargs['update_eps'] = exploration.value(t)
#action = act(masked_obs[None, :], **act_kwargs)[0][0]
action = act(np.array(masked_obs)[None], **act_kwargs)[0][0]
env_action = action
reset = False
new_obs, rew, done, info = env.step(env_action)
if pilot_is_human:
env.render()
# Store transition in the replay buffer.
masked_new_obs = mask_helipad(new_obs)
replay_buffer.add(masked_obs, action, rew, masked_new_obs,
float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if pilot_is_human:
utils.human_agent_action = init_human_action()
utils.human_agent_active = False
time.sleep(2)
if t > learning_starts and t % train_freq == 0:
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
episode_outcomes.append(rew)
episode_rewards.append(0.0)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
mean_100ep_succ = round(
np.mean(
[1 if x == 100 else 0 for x in episode_outcomes[-101:-1]]),
2)
mean_100ep_crash = round(
np.mean([
1 if x == -100 else 0 for x in episode_outcomes[-101:-1]
]), 2)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("mean 100 episode succ", mean_100ep_succ)
logger.record_tabular("mean 100 episode crash",
mean_100ep_crash)
logger.dump_tabular()
if checkpoint_freq is not None and t > learning_starts and num_episodes > 100 and t % checkpoint_freq == 0 and (
saved_mean_reward is None
or mean_100ep_reward > saved_mean_reward):
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}".
format(saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
reward_data = {'rewards': episode_rewards, 'outcomes': episode_outcomes}
return act, reward_data