def test_function():
with tf.Graph().as_default():
x = tf.compat.v1.placeholder(tf.int32, (), name="x")
y = tf.compat.v1.placeholder(tf.int32, (), name="y")
z = 3 * x + 2 * y
lin = function([x, y], z, givens={y: 0})
with single_threaded_session():
initialize()
assert lin(2) == 6
assert lin(2, 2) == 10
def test_multikwargs():
with tf.Graph().as_default():
x = tf.compat.v1.placeholder(tf.int32, (), name="x")
with tf.compat.v1.variable_scope("other"):
x2 = tf.compat.v1.placeholder(tf.int32, (), name="x")
z = 3 * x + 2 * x2
lin = function([x, x2], z, givens={x2: 0})
with single_threaded_session():
initialize()
assert lin(2) == 6
assert lin(2, 2) == 10
def test_runningmeanstd():
for (x1, x2, x3) in [
(np.random.randn(3), np.random.randn(4), np.random.randn(5)),
(np.random.randn(3, 2), np.random.randn(4, 2), np.random.randn(5, 2)),
]:
rms = RunningMeanStd(epsilon=0.0, shape=x1.shape[1:])
U.initialize()
x = np.concatenate([x1, x2, x3], axis=0)
ms1 = [x.mean(axis=0), x.std(axis=0)]
rms.update(x1)
rms.update(x2)
rms.update(x3)
ms2 = [rms.mean.eval(), rms.std.eval()]
assert np.allclose(ms1, ms2)
def test_dist():
np.random.seed(0)
p1, p2, p3 = (np.random.randn(3, 1), np.random.randn(4, 1),
np.random.randn(5, 1))
q1, q2, q3 = (np.random.randn(6, 1), np.random.randn(7, 1),
np.random.randn(8, 1))
# p1,p2,p3=(np.random.randn(3), np.random.randn(4), np.random.randn(5))
# q1,q2,q3=(np.random.randn(6), np.random.randn(7), np.random.randn(8))
comm = MPI.COMM_WORLD
assert comm.Get_size() == 2
if comm.Get_rank() == 0:
x1, x2, x3 = p1, p2, p3
elif comm.Get_rank() == 1:
x1, x2, x3 = q1, q2, q3
else:
assert False
rms = RunningMeanStd(epsilon=0.0, shape=(1, ))
U.initialize()
rms.update(x1)
rms.update(x2)
rms.update(x3)
bigvec = np.concatenate([p1, p2, p3, q1, q2, q3])
def checkallclose(x, y):
print(x, y)
return np.allclose(x, y)
assert checkallclose(
bigvec.mean(axis=0),
rms.mean.eval(),
)
assert checkallclose(
bigvec.std(axis=0),
rms.std.eval(),
)
def learn(env,
q_func,
lr=5e-4,
lr_decay_factor = 0.99,
lr_growth_factor = 1.01,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
epoch_steps=20000,
gpu_memory=1.0,
double_q=False,
scope="deepq",
save_dir='.',
nb_test_steps=10000,
test_eps=0.05,
render=False,
map_name=None,
num_targets=1
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines0/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.per_process_gpu_memory_fraction = gpu_memory
config.gpu_options.polling_inactive_delay_msecs = 25
sess = tf.Session(config=config)
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
act, act_greedy, q_values, train, update_target, lr_decay_op, lr_growth_op, _ = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer_f=tf.train.AdamOptimizer,
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
double_q = bool(double_q),
scope=scope,
test_eps=test_eps,
learning_rate = lr,
learning_rate_decay_factor = lr_decay_factor,
learning_rate_growth_factor = lr_growth_factor,
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
timelimit_env = env
while( not hasattr(timelimit_env, '_elapsed_steps')):
timelimit_env = timelimit_env.env
if timelimit_env.env.spec:
env_id = timelimit_env.env.spec.id
else:
env_id = timelimit_env.env.id
obs = env.reset()
reset = True
num_eps = 0
#recording
records = { 'loss':[], 'online_reward':[], 'test_reward':[],
'eval_value':[], 'learning_rate':[], 'q_vals':[], 'time':[]}
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
ep_losses, ep_qs, losses, qs = [], [], [], []
checkpt_loss = []
curr_lr = lr
s_time = time.time()
print("===== LEARNING STARTS =====")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, info = env.step(env_action)
# Store transition in the replay buffer.
if timelimit_env._elapsed_steps < timelimit_env._max_episode_steps:
replay_buffer.add(obs, action, rew, new_obs, float(done))
else:
replay_buffer.add(obs, action, rew, new_obs, float(not done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
reset = True
num_eps += 1
episode_rewards.append(0.0)
if losses:
ep_losses.append(np.mean(losses))
ep_qs.append(np.mean(qs))
losses = []
qs = []
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors, curr_lr, loss = train(obses_t, actions, rewards, obses_tp1, dones, weights)
q_values_t = q_values(obses_t)
losses.append(loss)
qs.append(np.mean(q_values_t))
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if render:
env.render(traj_num=num_eps)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if (t+1) % (epoch_steps/2) == 0 and (t+1) > learning_starts:
if ep_losses:
mean_loss = np.float16(np.mean(ep_losses))
if len(checkpt_loss) > 2 and mean_loss > np.float16(max(checkpt_loss[-3:])) and lr_decay_factor < 1.0:
sess.run(lr_decay_op)
print("Learning rate decayed due to an increase in loss: %.4f -> %.4f"%(np.float16(max(checkpt_loss[-3:])),mean_loss))
elif len(checkpt_loss) > 2 and mean_loss < np.float16(min(checkpt_loss[-3:])) and lr_growth_factor > 1.0:
sess.run(lr_growth_op)
print("Learning rate grown due to a decrease in loss: %.4f -> %.4f"%( np.float16(min(checkpt_loss[-3:])),mean_loss))
checkpt_loss.append(mean_loss)
if (t+1) % epoch_steps == 0 and (t+1) > learning_starts:
records['time'].append(time.time() - s_time)
test_reward, eval_value = test(env_id, act_greedy, nb_test_steps=nb_test_steps, map_name=map_name, num_targets=num_targets)
records['test_reward'].append(test_reward)
records['eval_value'].append(eval_value)
records['loss'].append(np.mean(ep_losses))
records['q_vals'].append(np.mean(ep_qs))
records['online_reward'].append(round(np.mean(episode_rewards[-101:-1]), 1))
records['learning_rate'].append(curr_lr)
pickle.dump(records, open(os.path.join(save_dir,"records.pkl"),"wb"))
print("==== EPOCH %d ==="%((t+1)/epoch_steps))
print(tabulate([[k,v[-1]] for (k,v) in records.items()]))
s_time = time.time()
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and (t+1) > learning_starts and
num_episodes > 100 and (t+1) % checkpoint_freq == 0):
print("Saving model to model_%d.pkl"%(t+1))
act.save(os.path.join(save_dir,"model_"+str(t+1)+".pkl"))
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_state(model_file)
return act, records
def learn(
env,
q_func,
lr=5e-4,
lr_decay_factor=0.99,
lr_growth_factor=1.01,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=0.9,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
callback=None,
varTH=1e-05,
noise=0.0,
epoch_steps=20000,
alg='adfq',
gpu_memory=1.0,
act_policy='egreedy',
save_dir='.',
nb_test_steps=10000,
scope='deepadfq',
test_eps=0.05,
init_t=0,
render=False,
map_name=None,
num_targets=1,
im_size=50,
):
"""Train a deepadfq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
varTH : variance threshold
noise : noise for stochastic cases
alg : 'adfq' or 'adfq-v2'
gpu_memory : a fraction of a gpu memory when running multiple programs in the same gpu
act_policy : action policy, 'egreedy' or 'bayesian'
save_dir : path for saving results
nb_test_steps : step bound in evaluation
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines0/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.per_process_gpu_memory_fraction = gpu_memory
config.gpu_options.polling_inactive_delay_msecs = 25
sess = tf.Session(config=config)
sess.__enter__()
num_actions = env.action_space.n
varTH = np.float32(varTH)
adfq_func = posterior_adfq if alg == 'adfq' else posterior_adfq_v2
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
# observation_space_shape = env.observation_space.shape
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return BatchInput((im_size * im_size + observation_space_shape[0], ),
name=name)
act, act_test, q_target_vals, train, update_target, lr_decay_op, lr_growth_op = build_graph.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer_f=tf.train.AdamOptimizer,
gamma=gamma,
grad_norm_clipping=10,
varTH=varTH,
act_policy=act_policy,
scope=scope,
test_eps=test_eps,
learning_rate=lr,
learning_rate_decay_factor=lr_decay_factor,
learning_rate_growth_factor=lr_growth_factor,
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
timelimit_env = env
while (not hasattr(timelimit_env, '_elapsed_steps')):
timelimit_env = timelimit_env.env
if timelimit_env.env.spec:
env_id = timelimit_env.env.spec.id
else:
env_id = timelimit_env.env.id
obs = env.reset()
reset = True
num_eps = 0
# recording
records = {
'q_mean': [],
'q_sd': [],
'loss': [],
'online_reward': [],
'test_reward': [],
'learning_rate': [],
'time': [],
'eval_value': []
}
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_saved = False
model_file = os.path.join(td, "model")
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
learning_starts += init_t
ep_losses, ep_means, ep_sds, losses, means, sds = [], [], [], [], [], []
ep_mean_err, ep_sd_err, mean_errs, sd_errs = [], [], [], []
checkpt_loss = []
curr_lr = lr
s_time = time.time()
print("===== LEARNING STARTS =====")
for t in range(init_t, max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
update_eps = exploration.value(t)
action = act(np.array(obs)[None], update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, info = env.step(env_action)
# Store transition in the replay buffer.
if timelimit_env._elapsed_steps < timelimit_env._max_episode_steps:
replay_buffer.add(obs, action, rew, new_obs, float(done))
else:
replay_buffer.add(obs, action, rew, new_obs, float(not done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
reset = True
num_eps += 1
episode_rewards.append(0.0)
if losses:
ep_losses.append(np.mean(losses))
ep_means.append(np.mean(means))
ep_sds.append(np.mean(sds))
losses, means, sds = [], [], []
ep_mean_err.append(np.mean(mean_errs))
ep_sd_err.append(np.mean(sd_errs))
mean_errs, sd_errs = [], []
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
stats_t = q_target_vals(obses_t)[0]
stats_tp1 = q_target_vals(obses_tp1)[0]
ind = np.arange(batch_size)
mean_t = stats_t[ind, actions.astype(int)]
sd_t = np.exp(-stats_t[ind, actions.astype(int) + num_actions])
mean_tp1 = stats_tp1[:, :num_actions]
sd_tp1 = np.exp(-stats_tp1[:, num_actions:])
var_t = np.maximum(varTH, np.square(sd_t))
var_tp1 = np.maximum(varTH, np.square(sd_tp1))
target_mean, target_var, _ = adfq_func(mean_tp1,
var_tp1,
mean_t,
var_t,
rewards,
gamma,
terminal=dones,
asymptotic=False,
batch=True,
noise=noise,
varTH=varTH)
target_mean = np.reshape(target_mean, (-1))
target_sd = np.reshape(np.sqrt(target_var), (-1))
loss, m_err, s_err, curr_lr = train(obses_t, actions,
target_mean, target_sd,
weights)
losses.append(loss)
means.append(np.mean(mean_tp1))
sds.append(np.mean(sd_tp1))
mean_errs.append(np.mean(np.abs(m_err)))
sd_errs.append(np.mean(np.abs(s_err)))
if prioritized_replay:
new_priorities = np.abs(m_err) + np.abs(
s_err) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if render:
env.render(traj_num=num_eps)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if (t + 1) % (epoch_steps / 2) == 0 and (t + 1) > learning_starts:
if ep_losses:
mean_loss = np.float16(np.mean(ep_losses))
if len(checkpt_loss) > 2 and mean_loss > np.float16(
max(checkpt_loss[-3:])) and lr_decay_factor < 1.0:
sess.run(lr_decay_op)
print(
"Learning rate decayed due to an increase in loss: %.4f -> %.4f"
% (np.float16(max(checkpt_loss[-3:])), mean_loss))
elif len(checkpt_loss) > 2 and mean_loss < np.float16(
min(checkpt_loss[-3:])) and lr_growth_factor > 1.0:
sess.run(lr_growth_op)
print(
"Learning rate grown due to a decrease in loss: %.4f -> %.4f"
% (np.float16(min(checkpt_loss[-3:])), mean_loss))
checkpt_loss.append(mean_loss)
if (t + 1) % epoch_steps == 0 and (t + 1) > learning_starts:
records['time'].append(time.time() - s_time)
test_reward, eval_value = test(env_id,
act_test,
nb_test_steps=nb_test_steps,
map_name=map_name,
num_targets=num_targets,
im_size=im_size)
records['test_reward'].append(test_reward)
records['eval_value'].append(eval_value)
records['q_mean'].append(np.mean(ep_means))
records['q_sd'].append(np.mean(ep_sds))
records['loss'].append(np.mean(ep_losses))
records['online_reward'].append(
round(np.mean(episode_rewards[-101:-1]), 1))
records['learning_rate'].append(curr_lr)
pickle.dump(records,
open(os.path.join(save_dir, "records.pkl"), "wb"))
print("==== EPOCH %d ===" % ((t + 1) / epoch_steps))
print(tabulate([[k, v[-1]] for (k, v) in records.items()]))
s_time = time.time()
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.record_tabular("averaged loss",
np.mean(ep_losses[-print_freq:]))
logger.record_tabular("averaged output mean",
np.mean(ep_means[-print_freq:]))
logger.record_tabular("averaged output sd",
np.mean(ep_sds[-print_freq:]))
logger.record_tabular("averaged error mean",
np.mean(ep_mean_err[-print_freq:]))
logger.record_tabular("averaged error sds",
np.mean(ep_sd_err[-print_freq:]))
logger.record_tabular("learning rate", curr_lr)
logger.dump_tabular()
if (checkpoint_freq is not None and (t + 1) > learning_starts and
(t + 1) % checkpoint_freq == 0): #num_episodes > 100 and
print("Saving model to model_%d.pkl" % (t + 1))
act.save(os.path.join(save_dir,
"model_" + str(t + 1) + ".pkl"))
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_state(model_file)
return act, records
def learn(
env,
q_func,
lr=5e-4,
lr_decay_factor=0.99,
lr_growth_factor=1.01,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=0.9,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
callback=None,
scope='deepadfq',
alg='adfq',
sdMin=1e-5,
sdMax=1e5,
noise=0.0,
act_policy='egreedy',
epoch_steps=20000,
eval_logger=None,
save_dir='.',
test_eps=0.05,
init_t=0,
gpu_memory=1.0,
render=False,
**kwargs,
):
"""Train a deepadfq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: float
update the target network every `target_network_update_freq` steps.
If it is less than 1, it performs the soft target network update with
the given rate.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
scope : str
scope of the network.
alg : str
'adfq' or 'adfq-v2'.
sdMin, sdMix : float
The minimum and maximum values for the standard deviations.
noise : flot
noise for stochastic cases.
act_policy : str
action policy, 'egreedy' or 'bayesian'.
epoch_step : int
the number of steps per epoch.
eval_logger : Logger()
the Logger() class object under deep_adfq folder.
save_dir : str
path for saving results.
test_eps : float
epsilon of the epsilon greedy action policy during testing.
init_t : int
an initial learning step if you start training from a pre-trained model.
gpu_memory : float
a fraction of a gpu memory when running multiple programs in the same gpu.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines0/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
config = tf.compat.v1.ConfigProto(allow_soft_placement=True)
config.gpu_options.per_process_gpu_memory_fraction = gpu_memory
config.gpu_options.polling_inactive_delay_msecs = 25
sess = tf.compat.v1.Session(config=config)
sess.__enter__()
num_actions = env.action_space.n
adfq_func = posterior_adfq if alg == 'adfq' else posterior_adfq_v2
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
# observation_space_shape = env.observation_space.shape
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
target_network_update_rate = np.minimum(target_network_update_freq, 1.0)
target_network_update_freq = np.maximum(target_network_update_freq, 1.0)
act, act_test, q_target_vals, train, update_target, lr_decay_op, lr_growth_op = build_graph.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer_f=tf.compat.v1.train.AdamOptimizer,
grad_norm_clipping=10,
sdMin=sdMin,
sdMax=sdMax,
act_policy=act_policy,
scope=scope,
test_eps=test_eps,
lr_init=lr,
lr_decay_factor=lr_decay_factor,
lr_growth_factor=lr_growth_factor,
reuse=tf.compat.v1.AUTO_REUSE,
tau=target_network_update_rate,
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
file_writer = tf.compat.v1.summary.FileWriter(save_dir, sess.graph)
U.initialize()
update_target()
saved_mean_reward = None
timelimit_env = env
while (not hasattr(timelimit_env, '_elapsed_steps')):
timelimit_env = timelimit_env.env
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_saved = False
model_file = os.path.join(td, "model")
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
learning_starts += init_t
checkpt_loss = []
eval_logger.log_epoch(act_test)
for t in range(init_t, max_timesteps):
if callback is not None and callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs_act = {}
update_eps = exploration.value(t)
action = act(np.array(obs)[None],
update_eps=update_eps,
**kwargs_act)[0]
env_action = action
reset = False
new_obs, rew, done, info = env.step(env_action)
# Store transition in the replay buffer.
if timelimit_env._elapsed_steps < timelimit_env._max_episode_steps:
replay_buffer.add(obs, action, rew, new_obs, float(done))
else:
replay_buffer.add(obs, action, rew, new_obs, float(not done))
obs = new_obs
eval_logger.log_reward(rew)
if done:
obs = env.reset()
reset = True
eval_logger.log_ep(info)
if t > learning_starts and (t + 1) % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
stats_t = q_target_vals(obses_t)[0]
stats_tp1 = q_target_vals(obses_tp1)[0]
ind = np.arange(batch_size)
mean_t = stats_t[ind, actions.astype(int)]
sd_t = np.exp(-np.clip(
stats_t[ind, actions.astype(int) +
num_actions], -np.log(sdMax), -np.log(sdMin)))
mean_tp1 = stats_tp1[:, :num_actions]
sd_tp1 = np.exp(-np.clip(stats_tp1[:, num_actions:],
-np.log(sdMax), -np.log(sdMin)))
target_mean, target_var, _ = adfq_func(mean_tp1,
np.square(sd_tp1),
mean_t,
np.square(sd_t),
rewards,
gamma,
terminal=dones,
asymptotic=False,
batch=True,
noise=noise,
varTH=sdMin * sdMin)
target_mean = np.reshape(target_mean, (-1))
target_sd = np.reshape(np.sqrt(target_var), (-1))
loss, m_err, s_err, summary = train(obses_t, actions,
target_mean, target_sd,
weights)
file_writer.add_summary(summary, t)
eval_logger.log_step(loss=loss)
if prioritized_replay:
new_priorities = np.abs(m_err) + np.abs(
s_err) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if render:
env.render()
if t > learning_starts and (t +
1) % target_network_update_freq == 0:
# Update target network periodically.
update_target()
if (t + 1) % epoch_steps == 0:
eval_logger.log_epoch(act_test)
if (checkpoint_freq is not None and t > learning_starts
and (t + 1) % checkpoint_freq == 0
and eval_logger.get_num_episode() > 10):
mean_loss = np.float16(np.mean(eval_logger.ep_history['loss']))
if len(checkpt_loss) > 2 and mean_loss > np.float16(
max(checkpt_loss[-3:])) and lr_decay_factor < 1.0:
sess.run(lr_decay_op)
print(
"Learning rate decayed due to an increase in loss: %.4f -> %.4f"
% (np.float16(max(checkpt_loss[-3:])), mean_loss))
elif len(checkpt_loss) > 2 and mean_loss < np.float16(
min(checkpt_loss[-3:])) and lr_growth_factor > 1.0:
sess.run(lr_growth_op)
print(
"Learning rate grown due to a decrease in loss: %.4f -> %.4f"
% (np.float16(min(checkpt_loss[-3:])), mean_loss))
checkpt_loss.append(mean_loss)
# print("Saving model to model_%d.pkl"%(t+1))
# act.save(os.path.join(save_dir,"model_"+str(t+1)+".pkl"))
mean_100ep_reward = eval_logger.get_100ep_reward()
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_state(model_file)
eval_logger.finish(max_timesteps, epoch_steps, learning_starts)
return act
def learn(env,
q_func,
lr=5e-4,
lr_period=0.6,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=0.05,
callback=None,
scope="setdeepq",
epoch_steps=20000,
eval_logger=None,
save_dir='.',
test_eps=0.05,
gpu_memory=1.0,
render=False,
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: float
update the target network every `target_network_update_freq` steps.
If it is less than 1, it performs the soft target network update with
the given rate.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
scope : str
scope of the network.
epoch_step : int
the number of steps per epoch.
eval_logger : Logger()
the Logger() class object under deep_adfq folder.
save_dir : str
path for saving results.
test_eps : float
epsilon of the epsilon greedy action policy during testing.
init_t : int
an initial learning step if you start training from a pre-trained model.
gpu_memory : float
a fraction of a gpu memory when running multiple programs in the same gpu.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines0/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
config = tf.compat.v1.ConfigProto(allow_soft_placement=True)
config.gpu_options.per_process_gpu_memory_fraction = gpu_memory
config.gpu_options.polling_inactive_delay_msecs = 25
config.intra_op_parallelism_threads = 4
config.inter_op_parallelism_threads = 4
sess = tf.compat.v1.Session(config=config)
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
# Make observation a Set with variable size [N,?,d_obs]
observation_space_shape = [None] + list(env.observation_space.shape)
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
# if target_network_update is < 1, update every step with polyak averaging
# if target_network_update is > 1, update periodically
target_network_update_rate = np.minimum(target_network_update_freq, 1.0)
target_network_update_freq = np.maximum(target_network_update_freq, 1.0)
act, act_test, q_values, train, update_target, _ = setdeepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer_f=tf.compat.v1.train.AdamOptimizer,
gamma=gamma,
grad_norm_clipping=5, #10,
scope=scope,
test_eps=test_eps,
lr_init=lr,
lr_period_steps=lr_period*max_timesteps,
reuse=tf.compat.v1.AUTO_REUSE,
tau=target_network_update_rate,
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
file_writer = tf.compat.v1.summary.FileWriter(save_dir, sess.graph)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
saved_mean_reward = None
timelimit_env = env
while(not hasattr(timelimit_env, '_elapsed_steps')):
timelimit_env = timelimit_env.env
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
checkpt_loss = []
eval_logger.log_epoch(act_test)
for t in range(max_timesteps):
if callback is not None and callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
update_eps = exploration.value(t)
action_dict = {}
for agent_id, a_obs in obs.items():
action_dict[agent_id] = act(np.array(a_obs)[None], update_eps=update_eps, **kwargs)[0]
reset = False
new_obs, rew, done, info = env.step(action_dict)
# Store transition in the replay buffer.
for agent_id, a_obs in obs.items():
if timelimit_env._elapsed_steps < timelimit_env._max_episode_steps:
replay_buffer.add(a_obs, action_dict[agent_id], rew['__all__'],
new_obs[agent_id], float(done['__all__']), env.nb_targets)
else:
replay_buffer.add(a_obs, action_dict[agent_id], rew['__all__'],
new_obs[agent_id], float(not done), env.nb_targets)
obs = new_obs
eval_logger.log_reward(rew['__all__'])
if type(done) is not dict:
obs = env.reset()
reset = True
eval_logger.log_ep(info['mean_nlogdetcov'])
if t > learning_starts and (t+1) % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size, env.num_targets)
weights, batch_idxes = np.ones_like(rewards), None
#Update with cosine learning rate
if t < max_timesteps*(1-lr_period):
lr_iter = 0
else:
lr_iter = t-max_timesteps*(1-lr_period)
td_errors, td_errors2, loss, summary = train(obses_t, actions, rewards, obses_tp1, dones, weights, lr_iter)
file_writer.add_summary(summary, t)
eval_logger.log_step(loss=loss)
if render:
env.render()
#Update target network
if t > learning_starts and (t+1) % target_network_update_freq == 0:
update_target()
if (t+1) % epoch_steps == 0:
eval_logger.log_epoch(act_test)
if (checkpoint_freq is not None and t > learning_starts and
(t+1) % checkpoint_freq == 0 and eval_logger.get_num_episode() > 10):
mean_loss = np.float16(np.mean(eval_logger.ep_history['loss']))
checkpt_loss.append(mean_loss)
# print("Saving current model to modellast.pkl")
act.save(os.path.join(save_dir,"modellast.pkl"))
# Save the best model based on evaluation sets
mean_eval_reward = eval_logger.get_eval_reward()
if saved_mean_reward is None or mean_eval_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_eval_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_eval_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_state(model_file)
eval_logger.finish(max_timesteps, epoch_steps, learning_starts)
return act
