def load_model(model_path):
if ScoutExploreTaskRL.act is not None:
return
class FakeEnv(object):
def __init__(self):
low = np.zeros(6)
high = np.ones(6)
self.observation_space = Box(low, high)
self.action_space = Discrete(8)
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
env = FakeEnv()
network = deepq.models.mlp([64, 32])
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': network,
'num_actions': env.action_space.n,
}
act = deepq.build_act(**act_params)
sess = tf.Session()
sess.__enter__()
print("load_model path=", model_path)
load_state(model_path)
ScoutExploreTaskRL.act = ActWrapper(act, act_params)
print("load_model ok")
def restore_act_and_value(env,
path,
num_cpu=4,
scope="saved/deepq",
reuse=None):
# pdb.set_trace()
qfunc_path = path + 'model.pkl'
with open(qfunc_path, "rb") as f:
q_func = dill.load(f)
def make_obs_ph(name):
return U.BatchInput(env.observation_space.shape, name=name)
act = build_act(make_obs_ph, q_func, env.action_space.n, scope, reuse)
value = build_value_function(make_obs_ph, q_func, env.action_space.n,
scope, True)
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
# for debugging
# for var in tf.global_variables():
# print(var.name)
U.load_state(tf.train.latest_checkpoint(path))
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n
}
return ActWrapper(act, act_params), value
def load_actor(sess, model_path):
model_file = osp.join(model_path, 'deepq1.pkl')
print("Loading act model from %s" % (model_file))
with open(model_file, "rb") as f:
model_data, act_params = cloudpickle.load(f)
act = deepq.build_act(**act_params)
with tempfile.TemporaryDirectory() as td:
arc_path = osp.join(td, "packed.zip")
with open(arc_path, "wb") as f:
f.write(model_data)
zipfile.ZipFile(arc_path, 'r', zipfile.ZIP_DEFLATED).extractall(td)
fname = osp.join(td, "model")
saver = tf.train.Saver()
saver.restore(sess, fname)
return ActWrapper(act, act_params)
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None):
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
if(env.is_single):
observation_space_shape = env.observation_space.shape
num_actions = env.action_space.n
else:
observation_space_shape = env.observation_space[0].shape
num_actions = env.action_space[0].n
num_agents=env.agentSize
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size*num_agents, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size*num_agents)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) + exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action=[]
qval=[]
for i in range(num_agents):
prediction=act(np.array(obs[i])[None], update_eps=update_eps, **kwargs)
#print(prediction[0],prediction[1][0])
action.append(prediction[0][0])
qval.append(prediction[1][0])
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action,qval)
# Store transition in the replay buffer.
for i in range(num_agents):
replay_buffer.add(obs[i], action[i], rew, new_obs[i], float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t*num_agents % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
#print(obses_t.shape,actions.shape,rewards.shape,obses_tp1.shape,dones.shape)
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_state(model_file)
return act,episode_rewards
def train(env,
eval_env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
my_skill_set=None,
log_dir = None,
num_eval_episodes=10,
render=False,
render_eval = False,
commit_for = 1
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
if my_skill_set: assert commit_for>=1, "commit_for >= 1"
save_idx = 0
with U.single_threaded_session() as sess:
## restore
if my_skill_set:
action_shape = my_skill_set.len
else:
action_shape = env.action_space.n
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return U.BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=action_shape,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': action_shape,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
# sess.run(tf.variables_initializer(new_variables))
# sess.run(tf.global_variables_initializer())
update_target()
if my_skill_set:
## restore skills
my_skill_set.restore_skillset(sess=sess)
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
model_saved = False
model_file = os.path.join(log_dir, "model", "deepq")
# save the initial act model
print("Saving the starting model")
os.makedirs(os.path.dirname(model_file), exist_ok=True)
act.save(model_file + '.pkl')
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
paction = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
if(my_skill_set):
skill_obs = obs.copy()
primitive_id = paction
rew = 0.
for _ in range(commit_for):
## break actions into primitives and their params
action = my_skill_set.pi(primitive_id=primitive_id, obs = skill_obs.copy(), primitive_params=None)
new_obs, skill_rew, done, _ = env.step(action)
if render:
# print(action)
env.render()
sleep(0.1)
rew += skill_rew
skill_obs = new_obs
terminate_skill = my_skill_set.termination(new_obs)
if done or terminate_skill:
break
else:
action= paction
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
if render:
env.render()
sleep(0.1)
# Store transition in the replay buffer for the outer env
replay_buffer.add(obs, paction, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
print("Time:%d, episodes:%d"%(t,len(episode_rewards)))
# add hindsight experience
if t > learning_starts and t % train_freq == 0:
# print('Training!')
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
# print(len(episode_rewards), episode_rewards[-11:-1])
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 50 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
act.save(model_file + '%d.pkl'%save_idx)
save_idx += 1
model_saved = True
saved_mean_reward = mean_100ep_reward
# else:
# print(saved_mean_reward, mean_100ep_reward)
if (eval_env is not None) and t > learning_starts and t % target_network_update_freq == 0:
# dumping other stats
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("%d time spent exploring", int(100 * exploration.value(t)))
print("Testing!")
eval_episode_rewards = []
eval_episode_successes = []
for i in range(num_eval_episodes):
eval_episode_reward = 0.
eval_obs = eval_env.reset()
eval_obs_start = eval_obs.copy()
eval_done = False
while(not eval_done):
eval_paction = act(np.array(eval_obs)[None])[0]
if(my_skill_set):
eval_skill_obs = eval_obs.copy()
eval_primitive_id = eval_paction
eval_r = 0.
for _ in range(commit_for):
## break actions into primitives and their params
eval_action, _ = my_skill_set.pi(primitive_id=eval_primitive_id, obs = eval_skill_obs.copy(), primitive_params=None)
eval_new_obs, eval_skill_rew, eval_done, eval_info = eval_env.step(eval_action)
# print('env reward:%f'%eval_skill_rew)
if render_eval:
print("Render!")
eval_env.render()
print("rendered!")
eval_r += eval_skill_rew
eval_skill_obs = eval_new_obs
eval_terminate_skill = my_skill_set.termination(eval_new_obs)
if eval_done or eval_terminate_skill:
break
else:
eval_action= eval_paction
env_action = eval_action
reset = False
eval_new_obs, eval_r, eval_done, eval_info = eval_env.step(env_action)
if render_eval:
# print("Render!")
eval_env.render()
# print("rendered!")
eval_episode_reward += eval_r
# print("eval_r:%f, eval_episode_reward:%f"%(eval_r, eval_episode_reward))
eval_obs = eval_new_obs
eval_episode_success = (eval_info["done"]=="goal reached")
if(eval_episode_success):
logger.info("success, training epoch:%d,starting config:"%t)
eval_episode_rewards.append(eval_episode_reward)
eval_episode_successes.append(eval_episode_success)
combined_stats = {}
# print(eval_episode_successes, np.mean(eval_episode_successes))
combined_stats['eval/return'] = normal_mean(eval_episode_rewards)
combined_stats['eval/success'] = normal_mean(eval_episode_successes)
combined_stats['eval/episodes'] = (len(eval_episode_rewards))
for key in sorted(combined_stats.keys()):
logger.record_tabular(key, combined_stats[key])
print("dumping the stats!")
logger.dump_tabular()
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
U.load_state(model_file)
def train(model_file, game="CartPole-v1"):
"""Train at a game."""
with tf_util.make_session(8):
env = gym.make(game)
def make_placeholder(name):
"""Make a placeholder input."""
return tf_util.BatchInput(env.observation_space.shape, name=name)
act_params = {
'make_obs_ph': make_placeholder,
'q_func': model,
'num_actions': env.action_space.n
}
act, train, update_target, debug = deepq.build_train(
**act_params,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4)
)
act = ActWrapper(act, act_params)
replay_buffer = ReplayBuffer(50000)
exploration = LinearSchedule(
schedule_timesteps=100000,
initial_p=1.0,
final_p=0.02
)
tf_util.initialize()
update_target()
episode_rewards = [0.0]
obs = env.reset()
for t in itertools.count():
action = act(obs[None], update_eps=exploration.value(t))[0]
new_obs, rew, done, _ = env.step(action)
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0)
if not len(episode_rewards) % 100:
env.render()
if t > 1000:
obses_t, actions, rewards, obses_tp1, dones = (
replay_buffer.sample(32)
)
train(
obses_t, actions, rewards, obses_tp1, dones,
np.ones_like(rewards)
)
if not t % 1000:
update_target()
if not t % 3000:
if model_file:
tf_util.save_state(model_file)
yield act
if done and len(episode_rewards) % 10 == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", len(episode_rewards))
logger.record_tabular(
"mean episode reward",
round(np.mean(episode_rewards[-101:-1]), 1)
)
logger.record_tabular(
"% time spent exploring",
int(100 * exploration.value(t))
)
logger.dump_tabular()
def learn_off_policy(env,
q_func,
sess,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
callback=None,
debug_info_freq=20,
train_reward_approximation=True,
train_rl_algo=True,
grad_norm_clipping=10):
""" This slight modification of OpenAI's baselines.deepq.simple.learn function allows you
to take control of an OpenAI Universe game manually and have the DQN learn off-policy.
"""
# Create all the functions necessary to train the model
def make_obs_ph(name):
return U.BatchInput(env.observation_space.shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=grad_norm_clipping)
# this is the only slight change that I made:
act = PygletController(act,
openai_env=env,
height=env.observation_space.shape[0],
width=env.observation_space.shape[1])
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
path = "./checkpoints/checkpoint_for_" + str(env.spec.id)
reward_func_path = path + "/reward_func/"
rl_algo_path = path + "/rl_algo/"
reward_func_saver, rl_algo_saver = init_tf_vars(
sess, reward_func_path, rl_algo_path, update_target,
train_reward_approximation, train_rl_algo)
reward_func_path += "model.cptk"
rl_algo_path += "model.cptk"
episode_rewards = [0.0]
#saved_mean_reward = None
obs = env.reset()
#with tempfile.TemporaryDirectory() as td:
model_saved = False
#model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
obs_t = np.array(obs)[None]
action = act(obs_t, update_eps=exploration.value(t))[0]
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
if train_rl_algo:
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
np.ones_like(rewards))
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
print("Updating target network.")
update_target()
if (checkpoint_freq is not None and t % checkpoint_freq == 0):
if train_reward_approximation:
saved_at = reward_func_saver.save(sess, reward_func_path)
print("Reward approximator model saved at %s" % saved_at)
if train_rl_algo:
saved_at = rl_algo_saver.save(sess, rl_algo_path)
print("RL model saved at %s" % saved_at)
model_saved = True
return ActWrapper(
act, act_params
), replay_buffer, beta_schedule, train, reward_func_saver, reward_func_path, rl_algo_saver, rl_algo_path
def actor_inter(
update_flag,
end_train_flag,
total_step,
net_list,
net_list_lock,
mem_queue,
actor_num,
env,
q_func,
actor_max_timesteps=100000,
# mem_buffer_size=5000, # 目前每次step都发送
exploration_fraction=0.1,
exploration_final_eps=0.02,
print_freq=100,
actor_network_update_freq=500,
update_starts=6000,
param_noise=False,
callback=None):
# actor_num # actor 的序号
# mem_buffer_size=50000, # 多久向trainer_buffer发送一次memory
# actor_network_update_freq=500, # actor_network 更新频率
# update_starts=6000, # 需要设置actor从多少步之后开始从trainer复制网络（trainer则需要在buffer一定数量之后才开始训练）
multi_step_num = 3 # multi step return 10, 5
gamma = 0.5 # 折扣率
# 使用gpu增长占用显存,无法正确启动程序
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# sess = tf.Session(config=config)
# sess = tf.Session(config=tf.ConfigProto(device_count={'gpu': 0}))
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.1 # 占用GPU 10%的显存
sess = tf.Session(config=config)
# sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
# 等待trainer完成网络初始化,该步骤应在actor_build()之前,因为要使用net_list来setup placeholder
while bool(update_flag.value) is not True:
time.sleep(0.01)
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
act, update_qfunc = actor_build(make_obs_ph=make_obs_ph,
q_func=q_func,
net_list=net_list,
num_actions=env.action_space.n,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params) # ActWrapper() 可能根本不需要
# 设置本actor的replay_buffer
# Create the replay buffer
# if prioritized_replay:
# replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
# if prioritized_replay_beta_iters is None:
# prioritized_replay_beta_iters = max_timesteps
# beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
# initial_p=prioritized_replay_beta0,
# final_p=1.0)
# else:
# replay_buffer = ReplayBuffer(buffer_size)
# beta_schedule = None
# Create the schedule for exploration starting from 1.(探索率,各robot不统一)
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
actor_max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy the qfunc network.
U.initialize() # 必要
# 此时,trainer的q network 是随机生成的,但为保证一致性,各actor仍要复制该网络
# update_qfunc(sess=sess, net_list=net_list, net_list_lock=net_list_lock)
update_qfunc(sess=sess, net_list_lock=net_list_lock)
episode_rewards = [0.0]
obs = env.reset()
reset = True
# 在最大步数内interact
for t in range(actor_max_timesteps):
# if callback is not None:
# if callback(locals(), globals()):
# break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
update_param_noise_threshold = -np.log(1. - exploration.value(t) +
exploration.value(t) /
float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# 这里选择动作
action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# 将交互的记忆添加至queue
single_mem = (obs, action, rew, new_obs, float(done))
if mem_queue.full() is not True:
mem_queue.put(single_mem) # trainer记得从元组中取出
obs = new_obs
total_step.value += 1 # 更新全局steps
# 将每个episode即时回报累加,在每个actor上单独计算（better）,
episode_rewards[-1] += rew
if done:
# # 类似倒垂摆问题,每个episode 的step数量较少,比较好的方法应该是done之后更新
if t > update_starts:
# update_qfunc(sess=sess, net_list=net_list, net_list_lock=net_list_lock)
update_qfunc(sess=sess, net_list_lock=net_list_lock)
obs = env.reset()
episode_rewards.append(0.0)
reset = True
# capital_t = 1000 + multi_step_num + 1
# 从trainer更新network的参数,由于使用多进程,q_func不能够去每个step都更新,那么这个频率就和环境关系比较大,像倒垂摆问题,每个
# episode 的step数量较少,比较好的方法应该是done之后更新,而robot navigation step比较多,更好是每隔一定步数更新
# if t > update_starts and t % actor_network_update_freq == 0:
# # update_qfunc(sess=sess, net_list=net_list, net_list_lock=net_list_lock)
# update_qfunc(sess=sess, net_list_index=net_list_index)
mean_100ep_reward = round(
np.mean(episode_rewards[-101:-1]),
1) # round(np.mean(episode_rewards[-21:-1]), 1)
num_episodes = len(episode_rewards)
# 下面关于输出训练信息
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("actor-", actor_num)
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.record_tabular("total_steps", total_step.value)
logger.dump_tabular()
if t == actor_max_timesteps - 1:
end_train_flag.value += 1
# for debug
# if t % 500 == 0:
# # print('net_list: ')
# # print(net_list)
# print('single_mem: ')
# print(single_mem)
# print('net_list size: ' + str(sys.getsizeof(net_list)))
# print('single_mem size: ' + str(sys.getsizeof(single_mem)))
# print('action size: ' + str(sys.getsizeof(action)))
# yx: 个人编写的multi step return version, 在经典RL环境下训练良好, 前期比原算法得分提升快得多
# 通常情况下达到预设分数也比单步回报要快, 但是似乎更不稳定, 有时分数波动非常剧烈
t = 0
# 由于gym环境实现的问题, i_episode, episode_t皆需要与具体环境相结合
# for i_episode in range(int(actor_max_timesteps/1000)):
for i_episode in range(int(actor_max_timesteps / 10)):
action_rewards_list = [0 for x in range(1500)]
obs_list = list()
obs_list.append(obs) # 将S(0)加入序列
action_list = []
capital_t = 1500 + multi_step_num + 1 # for multi step return, can be infinite
for episode_t in range(
1520): # 注意这里的episode_t 将不是正常的步数,done之后仍然要运行,并计算g_reward
g_reward = 0.0
if episode_t < capital_t: # 与环境交互的部分,done之后就不会运行到
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) +
exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# 这里选择动作
action = act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
# print("action: "+str(env_action))
new_obs, rew, done, _ = env.step(env_action)
action_rewards_list[episode_t] = rew # reward 存储入reward list
obs_list.append(new_obs) # obs 的list
action_list.append(action)
obs = new_obs
if done:
capital_t = episode_t + 1
# 将每个episode即时回报累加,在每个actor上单独计算（better）,
episode_rewards[-1] += rew
t += 1
total_step.value += 1 # 更新全局steps
# # 从trainer更新network的参数
# if t > update_starts and t % actor_network_update_freq == 0:
# # update_qfunc(sess=sess, net_list=net_list, net_list_lock=net_list_lock)
# update_qfunc(sess=sess, net_list_lock=net_list_lock)
tau = episode_t - multi_step_num + 1
if tau >= 0:
min_step = min(tau + multi_step_num, capital_t)
for i in range(tau + 1, min_step):
g_reward += (gamma**(i - tau - 1)) * action_rewards_list[i]
# mem 加入 S(tau+n),最后几步中done一直为True,那么trainer的td目标不会加maxQ(s,a),
if tau + multi_step_num < capital_t:
single_mem = (obs_list[tau], action_list[tau], g_reward,
obs_list[tau + multi_step_num], float(done))
else:
# 这里的S(tau+n)已经不重要了,trainer不会实际用到
done = 1
single_mem = (obs_list[tau], action_list[tau], g_reward,
obs_list[-1], float(done))
if mem_queue.full() is not True:
mem_queue.put(single_mem) # trainer记得从元组中取出
# print('actor -- ' + str(actor_num) + ' episode_t -- ' + str(episode_t))
# print('actor -- ' + str(actor_num) + ' capital_t -- ' + str(capital_t))
# print('actor -- ' + str(actor_num) + ' tau -- ' + str(tau))
if t == actor_max_timesteps - 1:
end_train_flag.value += 1
break
if tau == capital_t - 1:
break
# end for 一步之内
if t > update_starts:
# update_qfunc(sess=sess, net_list=net_list, net_list_lock=net_list_lock)
update_qfunc(sess=sess, net_list_lock=net_list_lock)
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t == actor_max_timesteps - 1:
break
# 平均100次情景的回报（）, 注意episode是轮数,不是步数
mean_100ep_reward = round(
np.mean(episode_rewards[-101:-1]),
1) # round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
# 下面关于输出训练信息
if print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("actor-", actor_num)
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward) # reward还是正常的,不是g_reward
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.record_tabular("total_steps", total_step.value)
logger.dump_tabular()