def __init__(self, sess, policy_name, learning_params, curriculum,
num_features, num_states, num_actions):
# initialize attributes
self.sess = sess
self.learning_params = learning_params
self.use_double_dqn = learning_params.use_double_dqn
self.use_priority = learning_params.prioritized_replay
self.policy_name = policy_name
self.tabular_case = learning_params.tabular_case
# This proxy adds the machine state representation to the MDP state
self.feature_proxy = FeatureProxy(num_features, num_states,
self.tabular_case)
self.num_actions = num_actions
self.num_features = self.feature_proxy.get_num_features()
# create dqn network
self._create_network(learning_params.lr, learning_params.gamma,
learning_params.num_neurons,
learning_params.num_hidden_layers)
# create experience replay buffer
if self.use_priority:
self.replay_buffer = PrioritizedReplayBuffer(
learning_params.buffer_size,
alpha=learning_params.prioritized_replay_alpha)
if learning_params.prioritized_replay_beta_iters is None:
learning_params.prioritized_replay_beta_iters = curriculum.total_steps
self.beta_schedule = LinearSchedule(
learning_params.prioritized_replay_beta_iters,
initial_p=learning_params.prioritized_replay_beta0,
final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(learning_params.buffer_size)
self.beta_schedule = None
# count of the number of environmental steps
self.step = 0
def __init__(self, config, env):
self.config = config
self.agent_ids = [a for a in range(config.num_agents)]
self.env = env
self.optimizer = tf.keras.optimizers.Adam(self.config.lr)
self.replay_memory, self.beta_schedule = init_replay_memory(config)
self.model = init_network(config)
self.target_model = init_network(config)
self.model.summary()
tf.keras.utils.plot_model(self.model, to_file='./model.png')
if self.config.dueling:
self.agent_heads = self.build_agent_heads_dueling()
self.target_agent_heads = self.build_agent_heads_dueling()
self.agent_heads[0].summary()
tf.keras.utils.plot_model(self.agent_heads[0], to_file='./agent_heads_model.png')
else:
self.agent_heads = self.build_agent_heads()
self.target_agent_heads = self.build_agent_heads()
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(schedule_timesteps=int(config.exploration_fraction * config.num_timesteps),
initial_p=1.0, final_p=config.exploration_final_eps)
if config.load_path is not None:
self.load_models(config.load_path)
self.loss = self.nstep_loss
self.eps = tf.Variable(0.0)
self.one_hot_agents = tf.expand_dims(tf.one_hot(self.agent_ids, len(self.agent_ids), dtype=tf.float32), axis=1)
print(f'self.onehot_agent.shape is {self.one_hot_agents.shape}')
def __init__(self, lp, num_features, num_actions, reward_machine):
super().__init__()
# learning parameters
self.lp = lp
self.policy_name = 'dqn_network'
# This proxy adds the machine state representation to the MDP state
num_states = 1 if reward_machine is None else len(
reward_machine.get_states())
self.feature_proxy = FeatureProxy(num_features, num_states)
self.num_actions = num_actions
self.num_features = self.feature_proxy.get_num_features()
# Creating the network
self.sess = tf.Session()
self._create_network()
# create experience replay buffer
if self.lp.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(
lp.buffer_size, alpha=lp.prioritized_replay_alpha)
if lp.prioritized_replay_beta_iters is None:
lp.prioritized_replay_beta_iters = lp.train_steps
self.beta_schedule = LinearSchedule(
lp.prioritized_replay_beta_iters,
initial_p=lp.prioritized_replay_beta0,
final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(lp.buffer_size)
self.beta_schedule = None
# count of the number of environmental steps
self.step = 0
def train_model(self,
max_iters=100000,
number_of_steps=10,
log_path='./log',
debug=False):
tb = TensorBoard(log_path)
tb.set_model(self.rl_model)
self.beta_schedule = LinearSchedule(100000, initial_p=0.4, final_p=1.0)
epsilon = 1.
for i in range(max_iters):
if epsilon > 0.1:
epsilon *= 0.98
with timer('all', i):
reward, step, loss = self.train_iterator(
debug, i, number_of_steps, epsilon)
if not debug:
summary = tf.Summary()
reward_value = summary.value.add()
reward_value.simple_value = reward
reward_value.tag = 'reward'
step_value = summary.value.add()
step_value.simple_value = step
step_value.tag = 'step'
if len(loss) > 0:
loss_value = summary.value.add()
loss_value.simple_value = sum(loss) / len(loss)
loss_value.tag = 'loss'
tb.writer.add_summary(summary, i)
tb.writer.flush()
#sys.stdout.write('\r'+str(i))
if (i + 1) % 10000 is 0:
self.save_model(i / 10000 + 1)
def __init__(self, lp, num_features, num_actions, reward_machine):
super().__init__()
# learning parameters
self.lp = lp
self.rm = reward_machine
self.num_features = num_features
self.num_actions = num_actions
# Creating the network
self.sess = tf.Session()
self._create_network()
# create experience replay buffer
if self.lp.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(lp.buffer_size, alpha=lp.prioritized_replay_alpha)
if lp.prioritized_replay_beta_iters is None:
lp.prioritized_replay_beta_iters = lp.train_steps
self.beta_schedule = LinearSchedule(lp.prioritized_replay_beta_iters, initial_p=lp.prioritized_replay_beta0, final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(lp.buffer_size)
self.beta_schedule = None
# count of the number of environmental steps
self.step = 0
def __init__(self, config, env):
# super().__init__()
self.config = config
self.env = env
self.agent_ids = self.get_agent_ids()
self.replay_memory, self.beta_schedule = self.init_replay_memory()
self.optimizer = tf.keras.optimizers.Adam(self.config.lr)
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(schedule_timesteps=int(
config.exploration_fraction * config.num_timesteps),
initial_p=1.0,
final_p=config.exploration_final_eps)
self.eps = tf.Variable(0.0)
self.models, self.target_models = self._init_networks()
self.agents = [
Agent(config, self.models[agent_id], self.target_models[agent_id],
agent_id) for agent_id in self.agent_ids
]
self.support_z = np.linspace(-5.0, 5.0, self.config.atoms)
self.fps_zeros = np.zeros(
(self.config.num_agents, self.config.fp_shape))
def create_experience_replay_buffer(buffer_size, use_prioritized_replay, prioritized_replay_alpha, prioritized_replay_beta0, prioritized_replay_beta_iters):
if use_prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
beta_schedule = LinearSchedule(prioritized_replay_beta_iters, initial_p=prioritized_replay_beta0, final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
return replay_buffer, beta_schedule
def init_replay_memory(self):
"""
:return: replay_buffer, beta_schedule
"""
if self.config.prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(self.config.buffer_size, alpha=self.config.prioritized_replay_alpha)
if self.config.prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = self.config.num_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=self.config.prioritized_replay_beta0, final_p=1.0)
else:
replay_buffer = ReplayBuffer(self.config.buffer_size)
beta_schedule = None
return replay_buffer, beta_schedule
def __init__(self, config):
self.writer = SummaryWriter()
self.device = 'cuda' if T.cuda.is_available() else 'cpu'
self.dqn_type = config["dqn-type"]
self.run_title = config["run-title"]
self.env = gym.make(config["environment"])
self.num_states = np.prod(self.env.observation_space.shape)
self.num_actions = self.env.action_space.n
layers = [
self.num_states,
*config["architecture"],
self.num_actions
]
self.policy_net = Q_Network(self.dqn_type, layers).to(self.device)
self.target_net = Q_Network(self.dqn_type, layers).to(self.device)
self.target_net.load_state_dict(self.policy_net.state_dict())
self.target_net.eval()
capacity = config["max-experiences"]
self.p_replay_eps = config["p-eps"]
self.prioritized_replay = config["prioritized-replay"]
self.replay_buffer = PrioritizedReplayBuffer(capacity, config["p-alpha"]) if self.prioritized_replay \
else ReplayBuffer(capacity)
self.beta_scheduler = LinearSchedule(config["episodes"], initial_p=config["p-beta-init"], final_p=1.0)
self.epsilon_decay = lambda e: max(config["epsilon-min"], e * config["epsilon-decay"])
self.train_freq = config["train-freq"]
self.use_soft_update = config["use-soft-update"]
self.target_update = config["target-update"]
self.tau = config["tau"]
self.gamma = config["gamma"]
self.batch_size = config["batch-size"]
self.time_step = 0
self.optim = T.optim.AdamW(self.policy_net.parameters(), lr=config["lr-init"], weight_decay=config["weight-decay"])
self.lr_scheduler = T.optim.lr_scheduler.StepLR(self.optim, step_size=config["lr-step"], gamma=config["lr-gamma"])
self.criterion = nn.SmoothL1Loss(reduction="none") # Huber Loss
self.min_experiences = max(config["min-experiences"], config["batch-size"])
self.save_path = config["save-path"]
def __init__(self, config, env):
self.config = config
self.agent_ids = [a for a in range(config.num_agents)]
self.env = env
self.optimizer = tf.keras.optimizers.Adam(self.config.lr)
self.replay_memory, self.beta_schedule = init_replay_memory(config)
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(schedule_timesteps=int(
config.exploration_fraction * config.num_timesteps),
initial_p=1.0,
final_p=config.exploration_final_eps)
self.loss = self.nstep_loss
self.eps = tf.Variable(0.0)
# init model
self.network = Network(config)
def create_players(env_id, model_dir, exploration_fraction, max_timesteps,
exploration_final_eps, param_noise, agents_count):
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
policy_path = os.path.join(model_dir, "Policy.pkl")
player_processes = list()
player_connections = list()
for i in range(agents_count):
process, connection = Player.player_process_factory(
env_id, policy_path, exploration, param_noise)
player_processes.append(process)
player_connections.append(connection)
time.sleep(1)
print("agent {} is created".format(i))
return player_processes, player_connections
def init_replay_memory(config):
"""
:param config: config object containing all parameters and hyper-parameters
:return: replay_buffer, beta_schedule
"""
if config.prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(
config.buffer_size, alpha=config.prioritized_replay_alpha)
if config.prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = config.num_timesteps
beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=config.prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(config.buffer_size)
beta_schedule = None
return replay_buffer, beta_schedule
demo_env = gym.wrappers.Monitor(make_test_env(env_name),
MONITOR_DIR,
resume=True,
mode="evaluation",
write_upon_reset=True)
steps, total_return = play_once(demo_env, 0.05, render=True)
print("Demo for %d steps, Return %d" % (steps, total_return))
summary = tf.Summary()
summary.value.add(tag="demo/return", simple_value=total_return)
summary.value.add(tag="demo/steps", simple_value=steps)
demo_env.close()
return summary
linear_schedule = LinearSchedule(int(EPSILON_STEPS),
final_p=EPSILON_MIN,
initial_p=EPSILON_MAX)
epsilon = linear_schedule.value(session.run(global_step))
# Populate replay buffer
print("Populating replay buffer with epsilon %f..." % epsilon)
while MINIMAL_SAMPLES > replay_buffer.number_of_samples():
steps, total_return = play_once(env, epsilon, render=False)
print("Played %d < %d steps" %
(replay_buffer.number_of_samples(), MINIMAL_SAMPLES))
# Main loop
print("Start Main Loop...")
for n in range(ITERATIONS):
gstep = tf.train.global_step(session, global_step)
epsilon = linear_schedule.value(gstep)
steps, total_return = play_once(env, epsilon)
if __name__ == '__main__':
with U.make_session(num_cpu=8):
# Create the environment
env = gym.make("CartPole-v0")
# Create all the functions necessary to train the model
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: ObservationInput(env.observation_space, name=name),
q_func=model,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
)
# Create the replay buffer
replay_buffer = ReplayBuffer(50000)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
exploration = LinearSchedule(schedule_timesteps=10000, initial_p=1.0, final_p=0.02)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
obs = env.reset()
# for t in itertools.count():
for t in range(100000):
# Take action and update exploration to the newest value
action = act(obs[None], update_eps=exploration.value(t))[0]
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
def learn(env,
seed=None,
num_agents = 2,
lr=0.00008,
total_timesteps=100000,
buffer_size=2000,
exploration_fraction=0.2,
exploration_final_eps=0.01,
train_freq=1,
batch_size=16,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=2000,
gamma=0.99,
target_network_update_freq=1000,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
set_global_seeds(seed)
double_q = True
grad_norm_clipping = True
shared_weights = True
play_test = 1000
nsteps = 16
agent_ids = env.agent_ids()
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
print(f'agent_ids {agent_ids}')
num_actions = env.action_space.n
print(f'num_actions {num_actions}')
dqn_agent = MAgent(env, agent_ids, nsteps, lr, replay_buffer, shared_weights, double_q, num_actions,
gamma, grad_norm_clipping, param_noise)
if load_path is not None:
load_path = osp.expanduser(load_path)
ckpt = tf.train.Checkpoint(model=dqn_agent.q_network)
manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None)
ckpt.restore(manager.latest_checkpoint)
print("Restoring from {}".format(manager.latest_checkpoint))
dqn_agent.update_target()
episode_rewards = [0.0 for i in range(101)]
saved_mean_reward = None
obs_all = env.reset()
obs_shape = obs_all
reset = True
done = False
# Start total timer
tstart = time.time()
for t in range(total_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
kwargs = {}
if not param_noise:
update_eps = tf.constant(exploration.value(t))
update_param_noise_threshold = 0.
else:
update_eps = tf.constant(0.)
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
if t % print_freq == 0:
time_1000_step = time.time()
nseconds = time_1000_step - tstart
tstart = time_1000_step
print(f'time spend to perform {t-print_freq} to {t} steps is {nseconds} ')
print('eps update', exploration.value(t))
mb_obs, mb_rewards, mb_actions, mb_values, mb_dones = [], [], [], [], []
# mb_states = states
epinfos = []
for _ in range(nsteps):
# Given observations, take action and value (V(s))
obs_ = tf.constant(obs_all)
# print(f'obs_.shape is {obs_.shape}')
# obs_ = tf.expand_dims(obs_, axis=1)
# print(f'obs_.shape is {obs_.shape}')
actions_list, fps_ = dqn_agent.choose_action(obs_, update_eps=update_eps, **kwargs)
fps = [[] for _ in agent_ids]
# print(f'fps_.shape is {np.asarray(fps_).shape}')
for a in agent_ids:
fps[a] = np.delete(fps_, a, axis=0)
# print(fps)
# print(f'actions_list is {actions_list}')
# print(f'values_list is {values_list}')
# Append the experiences
mb_obs.append(obs_all.copy())
mb_actions.append(actions_list)
mb_values.append(fps)
mb_dones.append([float(done) for _ in range(num_agents)])
# Take actions in env and look the results
obs1_all, rews, done, info = env.step(actions_list)
rews = [np.max(rews) for _ in range(len(rews))] # for cooperative purpose same reward for every one
# print(rews)
mb_rewards.append(rews)
obs_all = obs1_all
# print(rewards, done, info)
maybeepinfo = info[0].get('episode')
if maybeepinfo: epinfos.append(maybeepinfo)
episode_rewards[-1] += np.max(rews)
if done:
episode_rewards.append(0.0)
obs_all = env.reset()
reset = True
mb_dones.append([float(done) for _ in range(num_agents)])
# print(f'mb_actions is {mb_actions}')
# print(f'mb_rewards is {mb_rewards}')
# print(f'mb_values is {mb_values}')
# print(f'mb_dones is {mb_dones}')
mb_obs = np.asarray(mb_obs, dtype=obs_all[0].dtype)
mb_actions = np.asarray(mb_actions, dtype=actions_list[0].dtype)
mb_rewards = np.asarray(mb_rewards, dtype=np.float32)
mb_values = np.asarray(mb_values, dtype=np.float32)
# print(f'mb_values.shape is {mb_values.shape}')
mb_dones = np.asarray(mb_dones, dtype=np.bool)
mb_masks = mb_dones[:-1]
mb_dones = mb_dones[1:]
# print(f'mb_actions is {mb_actions}')
# print(f'mb_rewards is {mb_rewards}')
# print(f'mb_values is {mb_values}')
# print(f'mb_dones is {mb_dones}')
# print(f'mb_masks is {mb_masks}')
# print(f'mb_masks.shape is {mb_masks.shape}')
if gamma > 0.0:
# Discount/bootstrap off value fn
last_values = dqn_agent.value(tf.constant(obs_all))
# print(f'last_values is {last_values}')
if mb_dones[-1][0] == 0:
# print('================ hey ================ mb_dones[-1][0] == 0')
mb_rewards = discount_with_dones(np.concatenate((mb_rewards, [last_values])),
np.concatenate((mb_dones, [[float(False) for _ in range(num_agents)]]))
, gamma)[:-1]
else:
mb_rewards = discount_with_dones(mb_rewards, mb_dones, gamma)
# print(f'after discount mb_rewards is {mb_rewards}')
if replay_buffer is not None:
replay_buffer.add(mb_obs, mb_actions, mb_rewards, obs1_all, mb_masks[:,0],
mb_values, np.tile([exploration.value(t), t], (nsteps, num_agents, 1)))
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones, fps, extra_datas = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
obses_t, obses_tp1 = tf.constant(obses_t), None
actions, rewards, dones = tf.constant(actions), tf.constant(rewards, dtype=tf.float32), tf.constant(dones)
weights, fps, extra_datas = tf.constant(weights), tf.constant(fps), tf.constant(extra_datas)
s = obses_t.shape
# print(f'obses_t.shape is {s}')
obses_t = tf.reshape(obses_t, (s[0] * s[1], *s[2:]))
s = actions.shape
# print(f'actions.shape is {s}')
actions = tf.reshape(actions, (s[0] * s[1], *s[2:]))
s = rewards.shape
# print(f'rewards.shape is {s}')
rewards = tf.reshape(rewards, (s[0] * s[1], *s[2:]))
s = weights.shape
# print(f'weights.shape is {s}')
weights = tf.reshape(weights, (s[0] * s[1], *s[2:]))
s = fps.shape
# print(f'fps.shape is {s}')
fps = tf.reshape(fps, (s[0] * s[1], *s[2:]))
# print(f'fps.shape is {fps.shape}')
s = extra_datas.shape
# print(f'extra_datas.shape is {s}')
extra_datas = tf.reshape(extra_datas, (s[0] * s[1], *s[2:]))
s = dones.shape
# print(f'dones.shape is {s}')
dones = tf.reshape(dones, (s[0], s[1], *s[2:]))
# print(f'dones.shape is {s}')
td_errors = dqn_agent.nstep_train(obses_t, actions, rewards, obses_tp1, dones, weights, fps, extra_datas)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
dqn_agent.update_target()
if t % play_test == 0 and t != 0:
play_test_games(dqn_agent)
mean_100ep_reward = np.mean(episode_rewards[-101:-1])
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
print(f'last 100 episode mean reward {mean_100ep_reward} in {num_episodes} playing')
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
def learn(env_id,
q_func,
lr=5e-4,
max_timesteps=10000,
buffer_size=5000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
train_steps=10,
learning_starts=500,
batch_size=32,
print_freq=10,
checkpoint_freq=100,
model_dir=None,
gamma=1.0,
target_network_update_freq=50,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
player_processes=None,
player_connections=None):
env, _, _ = create_gvgai_environment(env_id)
# Create all the functions necessary to train the model
# expert_decision_maker = ExpertDecisionMaker(env=env)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise)
session = tf.Session()
session.__enter__()
policy_path = os.path.join(model_dir, "Policy.pkl")
model_path = os.path.join(model_dir, "model", "model")
if os.path.isdir(os.path.join(model_dir, "model")):
load_state(model_path)
else:
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
act.save(policy_path)
save_state(model_path)
env.close()
# Create the replay buffer
if prioritized_replay:
replay_buffer_path = os.path.join(model_dir, "Prioritized_replay.pkl")
if os.path.isfile(replay_buffer_path):
with open(replay_buffer_path, 'rb') as input_file:
replay_buffer = pickle.load(input_file)
else:
replay_buffer = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer_path = os.path.join(model_dir, "Normal_replay.pkl")
if os.path.isfile(replay_buffer_path):
with open(replay_buffer_path, 'rb') as input_file:
replay_buffer = pickle.load(input_file)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
episode_rewards = list()
saved_mean_reward = -999999999
signal.signal(signal.SIGQUIT, signal_handler)
global terminate_learning
total_timesteps = 0
for timestep in range(max_timesteps):
if terminate_learning:
break
for connection in player_connections:
experiences, reward = connection.recv()
episode_rewards.append(reward)
for experience in experiences:
replay_buffer.add(*experience)
total_timesteps += 1
if total_timesteps < learning_starts:
if timestep % 10 == 0:
print("not strated yet", flush=True)
continue
if timestep % train_freq == 0:
for i in range(train_steps):
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(total_timesteps))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if timestep % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if print_freq is not None and timestep % print_freq == 0:
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular(
"% time spent exploring",
int(100 * exploration.value(total_timesteps)))
logger.dump_tabular()
if timestep % checkpoint_freq == 0 and mean_100ep_reward > saved_mean_reward:
act.save(policy_path)
save_state(model_path)
saved_mean_reward = mean_100ep_reward
with open(replay_buffer_path, 'wb') as output_file:
pickle.dump(replay_buffer, output_file,
pickle.HIGHEST_PROTOCOL)
send_message_to_all(player_connections, Message.UPDATE)
send_message_to_all(player_connections, Message.TERMINATE)
if mean_100ep_reward > saved_mean_reward:
act.save(policy_path)
with open(replay_buffer_path, 'wb') as output_file:
pickle.dump(replay_buffer, output_file, pickle.HIGHEST_PROTOCOL)
for player_process in player_processes:
player_process.join()
# player_process.terminate()
return act.load(policy_path)
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
for t in range(total_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act
def learn_continuous_tasks(env,
q_func,
env_name,
dir_path,
time_stamp,
total_num_episodes,
num_actions_pad=33,
lr=1e-4,
grad_norm_clipping=10,
max_timesteps=int(1e8),
buffer_size=int(1e6),
train_freq=1,
batch_size=64,
print_freq=10,
learning_starts=1000,
gamma=0.99,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=int(1e8),
num_cpu=16,
epsilon_greedy=False,
timesteps_std=1e6,
initial_std=0.4,
final_std=0.05,
eval_freq=100,
n_eval_episodes=10,
eval_std=0.01,
log_index=0,
log_prefix='q',
loss_type="L2",
model_file='./',
callback=None):
"""Train a branching deepq model to solve continuous control tasks via discretization.
Current assumptions in the implementation:
- for solving continuous control domains via discretization (can be adjusted to be compatible with naturally disceret-action domains using 'env.action_space.n')
- uniform number of sub-actions per action dimension (can be generalized to heterogeneous number of sub-actions across branches)
Parameters
-------
env : gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
num_actions_pad: int
number of sub-actions per action dimension (= num of discretization grains/bars + 1)
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimize for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
0.1 for dqn-baselines
exploration_final_eps: float
final value of random action probability
0.02 for dqn-baselines
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
grad_norm_clipping: int
set None for no clipping
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the unified TD error for updating priorities.
Erratum: The camera-ready copy of this paper incorrectly reported 1e-8.
The value used to produece the results is 1e8.
num_cpu: int
number of cpus to use for training
dir_path: str
path for logs and results to be stored in
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
def make_obs_ph(name):
return U.BatchInput(env.observation_space.shape, name=name)
print('Observation shape:' + str(env.observation_space.shape))
num_action_grains = num_actions_pad - 1
num_action_dims = env.action_space.shape[0]
num_action_streams = num_action_dims
num_actions = num_actions_pad * num_action_streams # total numb network outputs for action branching with one action dimension per branch
print('Number of actions in total:' + str(num_actions))
act, q_val, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
num_action_streams=num_action_streams,
batch_size=batch_size,
optimizer_name="Adam",
learning_rate=lr,
grad_norm_clipping=grad_norm_clipping,
gamma=gamma,
double_q=True,
scope="deepq",
reuse=None,
loss_type="L2")
print('TRAIN VARS:')
print(tf.trainable_variables())
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
'num_action_streams': num_action_streams,
}
print('Create the log writer for TensorBoard visualizations.')
log_dir = "{}/tensorboard_logs/{}".format(dir_path, env_name)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
score_placeholder = tf.placeholder(tf.float32, [],
name='score_placeholder')
tf.summary.scalar('score', score_placeholder)
lr_constant = tf.constant(lr, name='lr_constant')
tf.summary.scalar('learning_rate', lr_constant)
eval_placeholder = tf.placeholder(tf.float32, [], name='eval_placeholder')
eval_summary = tf.summary.scalar('evaluation', eval_placeholder)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
if epsilon_greedy:
approximate_num_iters = 2e6 / 4
exploration = PiecewiseSchedule([(0, 1.0),
(approximate_num_iters / 50, 0.1),
(approximate_num_iters / 5, 0.01)],
outside_value=0.01)
else:
exploration = ConstantSchedule(value=0.0) # greedy policy
std_schedule = LinearSchedule(schedule_timesteps=timesteps_std,
initial_p=initial_std,
final_p=final_std)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# Initialize the parameters used for converting branching, discrete action indeces to continuous actions
low = env.action_space.low
high = env.action_space.high
actions_range = np.subtract(high, low)
print('###################################')
print(low)
print(high)
print('###################################')
episode_rewards = []
reward_sum = 0.0
time_steps = [0]
time_spent_exploring = [0]
prev_time = time.time()
n_trainings = 0
# Open a dircetory for recording results
results_dir = "{}/results/{}".format(dir_path, env_name)
if not os.path.exists(results_dir):
os.makedirs(results_dir)
displayed_mean_reward = None
score_timesteps = []
game_scores = []
def evaluate(step, episode_number):
global max_eval_reward_mean, model_saved
print('Evaluate...')
eval_reward_sum = 0.0
# Run evaluation episodes
for eval_episode in range(n_eval_episodes):
obs = env.reset()
done = False
while not done:
# Choose action
action_idxes = np.array(
act(np.array(obs)[None],
stochastic=False)) # deterministic
actions_greedy = action_idxes / num_action_grains * actions_range + low
if eval_std == 0.0:
action = actions_greedy
else:
action = []
for index in range(len(actions_greedy)):
a_greedy = actions_greedy[index]
out_of_range_action = True
while out_of_range_action:
a_stoch = np.random.normal(loc=a_greedy,
scale=eval_std)
a_idx_stoch = np.rint(
(a_stoch + high[index]) /
actions_range[index] * num_action_grains)
if a_idx_stoch >= 0 and a_idx_stoch < num_actions_pad:
action.append(a_stoch)
out_of_range_action = False
# Step
obs, rew, done, _ = env.step(action)
eval_reward_sum += rew
# Average the rewards and log
eval_reward_mean = eval_reward_sum / n_eval_episodes
print(eval_reward_mean, 'over', n_eval_episodes, 'episodes')
game_scores.append(eval_reward_mean)
score_timesteps.append(step)
if max_eval_reward_mean is None or eval_reward_mean > max_eval_reward_mean:
logger.log(
"Saving model due to mean eval increase: {} -> {}".format(
max_eval_reward_mean, eval_reward_mean))
U.save_state(model_file)
model_saved = True
max_eval_reward_mean = eval_reward_mean
intact = ActWrapper(act, act_params)
intact.save(model_file + "_" + str(episode_number) + "_" +
str(int(np.round(max_eval_reward_mean))))
print('Act saved to ' + model_file + "_" + str(episode_number) +
"_" + str(int(np.round(max_eval_reward_mean))))
with tempfile.TemporaryDirectory() as td:
td = './logs'
evaluate(0, 0)
obs = env.reset()
t = -1
all_means = []
q_stats = []
current_qs = []
training_game_scores = []
training_timesteps = []
while True:
t += 1
# Select action and update exploration probability
action_idxes = np.array(
act(np.array(obs)[None], update_eps=exploration.value(t)))
qs = np.array(q_val(np.array(obs)[None],
stochastic=False)) # deterministic
tt = []
for val in qs:
tt.append(np.std(val))
current_qs.append(tt)
# Convert sub-actions indexes (discrete sub-actions) to continuous controls
action = action_idxes / num_action_grains * actions_range + low
if not epsilon_greedy: # Gaussian noise
actions_greedy = action
action_idx_stoch = []
action = []
for index in range(len(actions_greedy)):
a_greedy = actions_greedy[index]
out_of_range_action = True
while out_of_range_action:
# Sample from a Gaussian with mean at the greedy action and a std following a schedule of choice
a_stoch = np.random.normal(loc=a_greedy,
scale=std_schedule.value(t))
# Convert sampled cont action to an action idx
a_idx_stoch = np.rint(
(a_stoch + high[index]) / actions_range[index] *
num_action_grains)
# Check if action is in range
if a_idx_stoch >= 0 and a_idx_stoch < num_actions_pad:
action_idx_stoch.append(a_idx_stoch)
action.append(a_stoch)
out_of_range_action = False
action_idxes = action_idx_stoch
new_obs, rew, done, _ = env.step(np.array(action))
# Store transition in the replay buffer
replay_buffer.add(obs, action_idxes, rew, new_obs, float(done))
obs = new_obs
reward_sum += rew
if done:
obs = env.reset()
time_spent_exploring[-1] = int(100 * exploration.value(t))
time_spent_exploring.append(0)
episode_rewards.append(reward_sum)
training_game_scores.append(reward_sum)
training_timesteps.append(t)
time_steps[-1] = t
reward_sum = 0.0
time_steps.append(0)
q_stats.append(np.mean(current_qs, 0))
current_qs = []
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(
obses_t, actions, rewards, obses_tp1, dones,
weights) # np.ones_like(rewards)) #TEMP AT NEW
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
n_trainings += 1
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically
update_target()
if len(episode_rewards) == 0:
mean_100ep_reward = 0
elif len(episode_rewards) < 100:
mean_100ep_reward = np.mean(episode_rewards)
else:
mean_100ep_reward = np.mean(episode_rewards[-100:])
all_means.append(mean_100ep_reward)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
current_time = time.time()
logger.record_tabular("trainings per second",
n_trainings / (current_time - prev_time))
logger.dump_tabular()
n_trainings = 0
prev_time = current_time
if t > learning_starts and num_episodes > 100:
if displayed_mean_reward is None or mean_100ep_reward > displayed_mean_reward:
if print_freq is not None:
logger.log("Mean reward increase: {} -> {}".format(
displayed_mean_reward, mean_100ep_reward))
displayed_mean_reward = mean_100ep_reward
# Performance evaluation with a greedy policy
if done and num_episodes % eval_freq == 0:
evaluate(t + 1, num_episodes)
obs = env.reset()
# STOP training
if num_episodes >= total_num_episodes:
break
pickle.dump(q_stats,
open(
str(log_index) + "q_stat_stds99_" + log_prefix +
".pkl", 'wb'),
protocol=pickle.HIGHEST_PROTOCOL)
pickle.dump(game_scores,
open(
str(log_index) + "q_stat_scores99_" + log_prefix +
".pkl", 'wb'),
protocol=pickle.HIGHEST_PROTOCOL)
return ActWrapper(act, act_params)
def learn_continuous_tasks(env,
q_func,
env_name,
time_stamp,
total_num_episodes,
num_actions_pad=33,
lr=1e-4,
grad_norm_clipping=10,
max_timesteps=int(1e8),
buffer_size=int(1e6),
train_freq=1,
batch_size=64,
print_freq=10,
learning_starts=1000,
gamma=0.99,
target_network_update_freq=500,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=2e6,
prioritized_replay_eps=int(1e8),
num_cpu=16,
timesteps_std=1e6,
initial_std=0.4,
final_std=0.05,
eval_freq=100,
n_eval_episodes=10,
eval_std=0.01,
callback=None):
"""Train a branching deepq model to solve continuous control tasks via discretization.
Current assumptions in the implementation:
- for solving continuous control domains via discretization (can be adjusted to be compatible with naturally disceret-action domains using 'env.action_space.n')
- uniform number of sub-actions per action dimension (can be generalized to heterogeneous number of sub-actions across branches)
Parameters
-------
env : gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
num_actions_pad: int
number of sub-actions per action dimension (= num of discretization grains/bars + 1)
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimize for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
0.1 for dqn-baselines
exploration_final_eps: float
final value of random action probability
0.02 for dqn-baselines
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
grad_norm_clipping: int
set None for no clipping
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the unified TD error for updating priorities.
Erratum: The camera-ready copy of this paper incorrectly reported 1e-8.
The value used to produece the results is 1e8.
num_cpu: int
number of cpus to use for training
losses_version: int
optimization version number
dir_path: str
path for logs and results to be stored in
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
def make_obs_ph(name):
return U.BatchInput(env.observation_space.shape, name=name)
num_action_grains = num_actions_pad - 1
num_action_dims = env.action_space.shape[0]
num_action_streams = num_action_dims
num_actions = num_actions_pad * num_action_streams # total numb network outputs for action branching with one action dimension per branch
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
num_action_streams=num_action_streams,
batch_size=batch_size,
learning_rate=lr,
grad_norm_clipping=grad_norm_clipping,
gamma=gamma,
scope="deepq",
reuse=None)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
'num_action_streams': num_action_streams,
}
# prioritized_replay: create the replay buffer
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
# epsilon_greedy = False: just greedy policy
exploration = ConstantSchedule(value=0.0) # greedy policy
std_schedule = LinearSchedule(schedule_timesteps=timesteps_std,
initial_p=initial_std,
final_p=final_std)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# Initialize the parameters used for converting branching, discrete action indeces to continuous actions
low = env.action_space.low
high = env.action_space.high
actions_range = np.subtract(high, low)
episode_rewards = []
reward_sum = 0.0
num_episodes = 0
time_steps = [0]
time_spent_exploring = [0]
prev_time = time.time()
n_trainings = 0
# Set up on-demand rendering of Gym environments using keyboard controls: 'r'ender or 's'top
import termios, fcntl, sys
fd = sys.stdin.fileno()
oldterm = termios.tcgetattr(fd)
newattr = termios.tcgetattr(fd)
newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO
render = False
displayed_mean_reward = None
def evaluate(step, episode_number):
global max_eval_reward_mean, model_saved
print('Evaluate...')
eval_reward_sum = 0.0
# Run evaluation episodes
for eval_episode in range(n_eval_episodes):
obs = env.reset()
done = False
while not done:
# Choose action
action_idxes = np.array(
act(np.array(obs)[None],
stochastic=False)) # deterministic
actions_greedy = action_idxes / num_action_grains * actions_range + low
if eval_std == 0.0:
action = actions_greedy
else:
action = []
for index in range(len(actions_greedy)):
a_greedy = actions_greedy[index]
out_of_range_action = True
while out_of_range_action:
a_stoch = np.random.normal(loc=a_greedy,
scale=eval_std)
a_idx_stoch = np.rint(
(a_stoch + high[index]) /
actions_range[index] * num_action_grains)
if a_idx_stoch >= 0 and a_idx_stoch < num_actions_pad:
action.append(a_stoch)
out_of_range_action = False
# Step
obs, rew, done, _ = env.step(action)
eval_reward_sum += rew
# Average the rewards and log
eval_reward_mean = eval_reward_sum / n_eval_episodes
print(eval_reward_mean, 'over', n_eval_episodes, 'episodes')
with open("results/{}_{}_eval.csv".format(time_stamp, env_name),
"a") as eval_fw:
eval_writer = csv.writer(
eval_fw,
delimiter="\t",
lineterminator="\n",
)
eval_writer.writerow([episode_number, step, eval_reward_mean])
if max_eval_reward_mean is None or eval_reward_mean > max_eval_reward_mean:
logger.log(
"Saving model due to mean eval increase: {} -> {}".format(
max_eval_reward_mean, eval_reward_mean))
U.save_state(model_file)
model_saved = True
max_eval_reward_mean = eval_reward_mean
with tempfile.TemporaryDirectory() as td:
model_file = os.path.join(td, "model")
evaluate(0, 0)
obs = env.reset()
with open("results/{}_{}.csv".format(time_stamp, env_name), "w") as fw:
writer = csv.writer(
fw,
delimiter="\t",
lineterminator="\n",
)
t = -1
while True:
t += 1
# Select action and update exploration probability
action_idxes = np.array(
act(np.array(obs)[None], update_eps=exploration.value(t)))
# Convert sub-actions indexes (discrete sub-actions) to continuous controls
action = action_idxes / num_action_grains * actions_range + low
# epsilon_greedy = False: use Gaussian noise
actions_greedy = action
action_idx_stoch = []
action = []
for index in range(len(actions_greedy)):
a_greedy = actions_greedy[index]
out_of_range_action = True
while out_of_range_action:
# Sample from a Gaussian with mean at the greedy action and a std following a schedule of choice
a_stoch = np.random.normal(loc=a_greedy,
scale=std_schedule.value(t))
# Convert sampled cont action to an action idx
a_idx_stoch = np.rint(
(a_stoch + high[index]) / actions_range[index] *
num_action_grains)
# Check if action is in range
if a_idx_stoch >= 0 and a_idx_stoch < num_actions_pad:
action_idx_stoch.append(a_idx_stoch)
action.append(a_stoch)
out_of_range_action = False
action_idxes = action_idx_stoch
new_obs, rew, done, _ = env.step(action)
# On-demand rendering
if (t + 1) % 100 == 0:
# TO DO better?
termios.tcsetattr(fd, termios.TCSANOW, newattr)
oldflags = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK)
try:
try:
c = sys.stdin.read(1)
if c == 'r':
print()
print('Rendering begins...')
render = True
elif c == 's':
print()
print('Stop rendering!')
render = False
env.render(close=True)
except IOError:
pass
finally:
termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm)
fcntl.fcntl(fd, fcntl.F_SETFL, oldflags)
# Visualize Gym environment on render
if render: env.render()
# Store transition in the replay buffer
replay_buffer.add(obs, action_idxes, rew, new_obs, float(done))
obs = new_obs
reward_sum += rew
if done:
obs = env.reset()
time_spent_exploring[-1] = int(100 * exploration.value(t))
time_spent_exploring.append(0)
episode_rewards.append(reward_sum)
time_steps[-1] = t
reward_sum = 0.0
time_steps.append(0)
# Frequently log to file
writer.writerow(
[len(episode_rewards), t, episode_rewards[-1]])
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer
# prioritized_replay
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
td_errors = train(
obses_t, actions, rewards, obses_tp1, dones,
weights) #np.ones_like(rewards)) #TEMP AT NEW
# prioritized_replay
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
n_trainings += 1
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically
update_target()
if len(episode_rewards) == 0: mean_100ep_reward = 0
elif len(episode_rewards) < 100:
mean_100ep_reward = np.mean(episode_rewards)
else:
mean_100ep_reward = np.mean(episode_rewards[-100:])
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
current_time = time.time()
logger.record_tabular(
"trainings per second",
n_trainings / (current_time - prev_time))
logger.dump_tabular()
n_trainings = 0
prev_time = current_time
if t > learning_starts and num_episodes > 100:
if displayed_mean_reward is None or mean_100ep_reward > displayed_mean_reward:
if print_freq is not None:
logger.log("Mean reward increase: {} -> {}".format(
displayed_mean_reward, mean_100ep_reward))
displayed_mean_reward = mean_100ep_reward
# Performance evaluation with a greedy policy
if done and num_episodes % eval_freq == 0:
evaluate(t + 1, num_episodes)
obs = env.reset()
# STOP training
if num_episodes >= total_num_episodes:
break
if model_saved:
logger.log("Restore model with mean eval: {}".format(
max_eval_reward_mean))
U.load_state(model_file)
data_to_log = {
'time_steps': time_steps,
'episode_rewards': episode_rewards,
'time_spent_exploring': time_spent_exploring
}
# Write to file the episodic rewards, number of steps, and the time spent exploring
with open("results/{}_{}.txt".format(time_stamp, env_name), 'wb') as fp:
pickle.dump(data_to_log, fp)
return ActWrapper(act, act_params)
def __init__(self,
states_n: tuple,
actions_n: int,
hidden_layers: list,
scope_name: str,
sess=None,
learning_rate=1e-4,
discount=0.98,
replay_memory_size=100000,
batch_size=32,
begin_train=1000,
targetnet_update_freq=1000,
epsilon_start=1.0,
epsilon_end=0.1,
epsilon_decay_step=50000,
seed=1,
logdir='logs',
savedir='save',
save_freq=10000,
use_tau=False,
tau=0.001):
"""
:param states_n: tuple
:param actions_n: int
:param hidden_layers: list
:param scope_name: str
:param sess: tf.Session
:param learning_rate: float
:param discount: float
:param replay_memory_size: int
:param batch_size: int
:param begin_train: int
:param targetnet_update_freq: int
:param epsilon_start: float
:param epsilon_end: float
:param epsilon_decay_step: int
:param seed: int
:param logdir: str
"""
self.states_n = states_n
self.actions_n = actions_n
self._hidden_layers = hidden_layers
self._scope_name = scope_name
self.lr = learning_rate
self._target_net_update_freq = targetnet_update_freq
self._current_time_step = 0
self._epsilon_schedule = LinearSchedule(epsilon_decay_step,
epsilon_end, epsilon_start)
self._train_batch_size = batch_size
self._begin_train = begin_train
self._gamma = discount
self._use_tau = use_tau
self._tau = tau
self.savedir = savedir
self.save_freq = save_freq
self.qnet_optimizer = tf.train.AdamOptimizer(self.lr)
self._replay_buffer = ReplayBuffer(replay_memory_size)
self._seed(seed)
with tf.Graph().as_default():
self._build_graph()
self._merged_summary = tf.summary.merge_all()
if sess is None:
self.sess = tf.Session()
else:
self.sess = sess
self.sess.run(tf.global_variables_initializer())
self._saver = tf.train.Saver()
self._summary_writer = tf.summary.FileWriter(logdir=logdir)
self._summary_writer.add_graph(tf.get_default_graph())
def main(env_name='KungFuMasterNoFrameskip-v0',
train_freq=4,
target_update_freq=10000,
checkpoint_freq=100000,
log_freq=1,
batch_size=32,
train_after=200000,
max_timesteps=5000000,
buffer_size=50000,
vmin=-10,
vmax=10,
n=51,
gamma=0.99,
final_eps=0.1,
final_eps_update=1000000,
learning_rate=0.00025,
momentum=0.95):
env = gym.make(env_name)
env = wrap_env(env)
state_dim = (4, 84, 84)
action_count = env.action_space.n
with C.default_options(activation=C.relu, init=C.he_uniform()):
model_func = Sequential([
Convolution2D((8, 8), 32, strides=4, name='conv1'),
Convolution2D((4, 4), 64, strides=2, name='conv2'),
Convolution2D((3, 3), 64, strides=1, name='conv3'),
Dense(512, name='dense1'),
Dense((action_count, n), activation=None, name='out')
])
agent = CategoricalAgent(state_dim, action_count, model_func, vmin, vmax, n, gamma,
lr=learning_rate, mm=momentum, use_tensorboard=True)
logger = agent.writer
epsilon_schedule = LinearSchedule(1.0, final_eps, final_eps_update)
replay_buffer = ReplayBuffer(buffer_size)
try:
obs = env.reset()
episode = 0
rewards = 0
steps = 0
for t in range(max_timesteps):
# Take action
if t > train_after:
action = agent.act(obs, epsilon=epsilon_schedule.value(t))
else:
action = np.random.choice(action_count)
obs_, reward, done, _ = env.step(action)
# Store transition in replay buffer
replay_buffer.add(obs, action, reward, obs_, float(done))
obs = obs_
rewards += reward
if t > train_after and (t % train_freq) == 0:
# Minimize error in projected Bellman update on a batch sampled from replay buffer
experience = replay_buffer.sample(batch_size)
agent.train(*experience) # experience is (s, a, r, s_, t) tuple
logger.write_value('loss', agent.trainer.previous_minibatch_loss_average, t)
if t > train_after and (t % target_update_freq) == 0:
agent.update_target()
if t > train_after and (t % checkpoint_freq) == 0:
agent.checkpoint('checkpoints/model_{}.chkpt'.format(t))
if done:
episode += 1
obs = env.reset()
if episode % log_freq == 0:
steps = t - steps + 1
logger.write_value('rewards', rewards, episode)
logger.write_value('steps', steps, episode)
logger.write_value('epsilon', epsilon_schedule.value(t), episode)
logger.flush()
rewards = 0
steps = t
finally:
agent.save_model('checkpoints/{}.cdqn'.format(env_name))
def learn(env,
network,
seed=None,
lr=5e-5,
total_timesteps=100000,
buffer_size=500000,
exploration_fraction=0.1,
exploration_final_eps=0.01,
train_freq=1,
batch_size=32,
print_freq=10,
checkpoint_freq=100000,
checkpoint_path=None,
learning_starts=0,
gamma=0.99,
target_network_update_freq=10000,
prioritized_replay=True,
prioritized_replay_alpha=0.4,
prioritized_replay_beta0=0.6,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-3,
param_noise=False,
callback=None,
load_path=None,
load_idx=None,
demo_path=None,
n_step=10,
demo_prioritized_replay_eps=1.0,
pre_train_timesteps=750000,
epsilon_schedule="constant",
**network_kwargs):
# Create all the functions necessary to train the model
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
with tf.device('/GPU:0'):
model = DQfD(q_func=q_func,
observation_shape=env.observation_space.shape,
num_actions=env.action_space.n,
lr=lr,
grad_norm_clipping=10,
gamma=gamma,
param_noise=param_noise)
# Load model from checkpoint
if load_path is not None:
load_path = osp.expanduser(load_path)
ckpt = tf.train.Checkpoint(model=model)
manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None)
if load_idx is None:
ckpt.restore(manager.latest_checkpoint)
print("Restoring from {}".format(manager.latest_checkpoint))
else:
ckpt.restore(manager.checkpoints[load_idx])
print("Restoring from {}".format(manager.checkpoints[load_idx]))
# Setup demo trajectory
assert demo_path is not None
with open(demo_path, "rb") as f:
trajectories = pickle.load(f)
# Create the replay buffer
replay_buffer = PrioritizedReplayBuffer(buffer_size,
prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
temp_buffer = deque(maxlen=n_step)
is_demo = True
for epi in trajectories:
for obs, action, rew, new_obs, done in epi:
obs, new_obs = np.expand_dims(
np.array(obs), axis=0), np.expand_dims(np.array(new_obs),
axis=0)
if n_step:
temp_buffer.append((obs, action, rew, new_obs, done, is_demo))
if len(temp_buffer) == n_step:
n_step_sample = get_n_step_sample(temp_buffer, gamma)
replay_buffer.demo_len += 1
replay_buffer.add(*n_step_sample)
else:
replay_buffer.demo_len += 1
replay_buffer.add(obs[0], action, rew, new_obs[0], float(done),
float(is_demo))
logger.log("trajectory length:", replay_buffer.demo_len)
# Create the schedule for exploration
if epsilon_schedule == "constant":
exploration = ConstantSchedule(exploration_final_eps)
else: # not used
exploration = LinearSchedule(schedule_timesteps=int(
exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
model.update_target()
# ============================================== pre-training ======================================================
start = time()
num_episodes = 0
temp_buffer = deque(maxlen=n_step)
for t in tqdm(range(pre_train_timesteps)):
# sample and train
experience = replay_buffer.sample(batch_size,
beta=prioritized_replay_beta0)
batch_idxes = experience[-1]
if experience[6] is None: # for n_step = 0
obses_t, actions, rewards, obses_tp1, dones, is_demos = tuple(
map(tf.constant, experience[:6]))
obses_tpn, rewards_n, dones_n = None, None, None
weights = tf.constant(experience[-2])
else:
obses_t, actions, rewards, obses_tp1, dones, is_demos, obses_tpn, rewards_n, dones_n, weights = tuple(
map(tf.constant, experience[:-1]))
td_errors, n_td_errors, loss_dq, loss_n, loss_E, loss_l2, weighted_error = model.train(
obses_t, actions, rewards, obses_tp1, dones, is_demos, weights,
obses_tpn, rewards_n, dones_n)
# Update priorities
new_priorities = np.abs(td_errors) + np.abs(
n_td_errors) + demo_prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
# Update target network periodically
if t > 0 and t % target_network_update_freq == 0:
model.update_target()
# Logging
elapsed_time = timedelta(time() - start)
if print_freq is not None and t % 10000 == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", 0)
logger.record_tabular("max 100 episode reward", 0)
logger.record_tabular("min 100 episode reward", 0)
logger.record_tabular("demo sample rate", 1)
logger.record_tabular("epsilon", 0)
logger.record_tabular("loss_td", np.mean(loss_dq.numpy()))
logger.record_tabular("loss_n_td", np.mean(loss_n.numpy()))
logger.record_tabular("loss_margin", np.mean(loss_E.numpy()))
logger.record_tabular("loss_l2", np.mean(loss_l2.numpy()))
logger.record_tabular("losses_all", weighted_error.numpy())
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.record_tabular("pre_train", True)
logger.record_tabular("elapsed time", elapsed_time)
logger.dump_tabular()
# ============================================== exploring =========================================================
sample_counts = 0
demo_used_counts = 0
episode_rewards = deque(maxlen=100)
this_episode_reward = 0.
best_score = 0.
saved_mean_reward = None
is_demo = False
obs = env.reset()
# Always mimic the vectorized env
obs = np.expand_dims(np.array(obs), axis=0)
reset = True
for t in tqdm(range(total_timesteps)):
if callback is not None:
if callback(locals(), globals()):
break
kwargs = {}
if not param_noise:
update_eps = tf.constant(exploration.value(t))
update_param_noise_threshold = 0.
else: # not used
update_eps = tf.constant(0.)
update_param_noise_threshold = -np.log(1. - exploration.value(t) +
exploration.value(t) /
float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action, epsilon, _, _ = model.step(tf.constant(obs),
update_eps=update_eps,
**kwargs)
action = action[0].numpy()
reset = False
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
new_obs = np.expand_dims(np.array(new_obs), axis=0)
if n_step:
temp_buffer.append((obs, action, rew, new_obs, done, is_demo))
if len(temp_buffer) == n_step:
n_step_sample = get_n_step_sample(temp_buffer, gamma)
replay_buffer.add(*n_step_sample)
else:
replay_buffer.add(obs[0], action, rew, new_obs[0], float(done), 0.)
obs = new_obs
# invert log scaled score for logging
this_episode_reward += np.sign(rew) * (np.exp(np.sign(rew) * rew) - 1.)
if done:
num_episodes += 1
obs = env.reset()
obs = np.expand_dims(np.array(obs), axis=0)
episode_rewards.append(this_episode_reward)
reset = True
if this_episode_reward > best_score:
best_score = this_episode_reward
ckpt = tf.train.Checkpoint(model=model)
manager = tf.train.CheckpointManager(ckpt,
'./best_model',
max_to_keep=1)
manager.save(t)
logger.log("saved best model")
this_episode_reward = 0.0
if t % train_freq == 0:
experience = replay_buffer.sample(batch_size,
beta=beta_schedule.value(t))
batch_idxes = experience[-1]
if experience[6] is None: # for n_step = 0
obses_t, actions, rewards, obses_tp1, dones, is_demos = tuple(
map(tf.constant, experience[:6]))
obses_tpn, rewards_n, dones_n = None, None, None
weights = tf.constant(experience[-2])
else:
obses_t, actions, rewards, obses_tp1, dones, is_demos, obses_tpn, rewards_n, dones_n, weights = tuple(
map(tf.constant, experience[:-1]))
td_errors, n_td_errors, loss_dq, loss_n, loss_E, loss_l2, weighted_error = model.train(
obses_t, actions, rewards, obses_tp1, dones, is_demos, weights,
obses_tpn, rewards_n, dones_n)
new_priorities = np.abs(td_errors) + np.abs(
n_td_errors
) + demo_prioritized_replay_eps * is_demos + prioritized_replay_eps * (
1. - is_demos)
replay_buffer.update_priorities(batch_idxes, new_priorities)
# for logging
sample_counts += batch_size
demo_used_counts += np.sum(is_demos)
if t % target_network_update_freq == 0:
# Update target network periodically.
model.update_target()
if t % checkpoint_freq == 0:
save_path = checkpoint_path
ckpt = tf.train.Checkpoint(model=model)
manager = tf.train.CheckpointManager(ckpt,
save_path,
max_to_keep=10)
manager.save(t)
logger.log("saved checkpoint")
elapsed_time = timedelta(time() - start)
if done and num_episodes > 0 and num_episodes % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
np.mean(episode_rewards))
logger.record_tabular("max 100 episode reward",
np.max(episode_rewards))
logger.record_tabular("min 100 episode reward",
np.min(episode_rewards))
logger.record_tabular("demo sample rate",
demo_used_counts / sample_counts)
logger.record_tabular("epsilon", epsilon.numpy())
logger.record_tabular("loss_td", np.mean(loss_dq.numpy()))
logger.record_tabular("loss_n_td", np.mean(loss_n.numpy()))
logger.record_tabular("loss_margin", np.mean(loss_E.numpy()))
logger.record_tabular("loss_l2", np.mean(loss_l2.numpy()))
logger.record_tabular("losses_all", weighted_error.numpy())
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.record_tabular("pre_train", False)
logger.record_tabular("elapsed time", elapsed_time)
logger.dump_tabular()
return model
def main():
with open('cartpole.json', encoding='utf-8') as config_file:
config = json.load(config_file)
env = gym.make('CartPole-v0')
state_shape = env.observation_space.shape
action_count = env.action_space.n
layers = []
for layer in config['layers']:
layers.append(Dense(layer, activation=C.relu))
layers.append(Dense((action_count, config['n']), activation=None))
model_func = Sequential(layers)
replay_buffer = ReplayBuffer(config['buffer_capacity'])
# Fill the buffer with randomly generated samples
state = env.reset()
for i in range(config['buffer_capacity']):
action = env.action_space.sample()
post_state, reward, done, _ = env.step(action)
replay_buffer.add(state.astype(np.float32), action, reward, post_state.astype(np.float32), float(done))
if done:
state = env.reset()
reward_buffer = np.zeros(config['max_episodes'], dtype=np.float32)
losses = []
epsilon_schedule = LinearSchedule(1, 0.01, config['max_episodes'])
agent = CategoricalAgent(state_shape, action_count, model_func, config['vmin'], config['vmax'], config['n'],
lr=config['lr'], gamma=config['gamma'])
log_freq = config['log_freq']
for episode in range(1, config['max_episodes'] + 1):
state = env.reset().astype(np.float32)
done = False
while not done:
action = agent.act(state, epsilon_schedule.value(episode))
post_state, reward, done, _ = env.step(action)
post_state = post_state.astype(np.float32)
replay_buffer.add(state, action, reward, post_state, float(done))
reward_buffer[episode - 1] += reward
state = post_state
minibatch = replay_buffer.sample(config['minibatch_size'])
agent.train(*minibatch)
loss = agent.trainer.previous_minibatch_loss_average
losses.append(loss)
if episode % config['target_update_freq'] == 0:
agent.update_target()
if episode % log_freq == 0:
average = np.sum(reward_buffer[episode - log_freq: episode]) / log_freq
print('Episode {:4d} | Loss: {:6.4f} | Reward: {}'.format(episode, loss, average))
agent.model.save('cartpole.cdqn')
sns.set_style('dark')
pd.Series(reward_buffer).rolling(window=log_freq).mean().plot()
plt.xlabel('Episode')
plt.ylabel('Reward')
plt.title('CartPole - Reward with Time')
plt.show()
plt.plot(np.arange(len(losses)), losses)
plt.xlabel('Episode')
plt.ylabel('Loss')
plt.title('CartPole - Loss with Time')
plt.show()
def main(env_name,
train_freq=1,
target_update_freq=1000,
batch_size=32,
train_after=64,
final_gamma=0.02,
max_timesteps=2000000,
buffer_size=10000,
prioritized_replay_alpha=0.6,
prioritized_replay_beta=0.4,
prioritized_replay_eps=1e-6,
log_freq=1,
checkpoint_freq=10000):
env = gym.make(env_name)
env = wrap_env(env)
state_dim = (4, 84, 84)
action_dim = env.action_space.n
agent = LearningAgent(state_dim, action_dim)
logger = agent.writer
eps_sched = LinearSchedule(1.0, final_gamma, max_timesteps)
beta_sched = LinearSchedule(prioritized_replay_beta, 1.0, max_timesteps)
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
try:
obs = env.reset()
episode = 0
rewards = 0
steps = 0
for t in range(max_timesteps):
# Take action and update exploration to newest value
action = agent.act(obs, epsilon=eps_sched.value(t))
obs_, reward, done, _ = env.step(action)
# Store transition in replay buffer
replay_buffer.add(obs, action, reward, obs_, float(done))
obs = obs_
rewards += reward
if done:
steps = t - steps
episode += 1
obs = env.reset()
if t > train_after and (t % train_freq) == 0:
print('Training...')
# Minimize the error in Bellman's equation on a batch sampled from replay buffer
experience = replay_buffer.sample(batch_size, beta=beta_sched.value(t))
(s, a, r, s_, t, weights, batch_idxes) = experience
td_errors = agent.train(s, a, r, s_, t, weights)
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > train_after and (t % target_update_freq) == 0:
agent.update_target()
if done and (episode % log_freq) == 0:
logger.write_value('rewards', rewards, episode)
logger.write_value('steps', steps, episode)
logger.write_value('epsilon', eps_sched.value(t), episode)
agent.trainer.summarize_training_progress()
logger.flush()
rewards = 0
steps = t
if t > train_after and (t % checkpoint_freq) == 0:
agent.checkpoint('model_{}.chkpt'.format(t))
finally:
agent.save_model('model.dnn')
def learn_att(env,
q_func,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs
):
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
# q_func = build_q_func(network, **network_kwargs) since no network setting
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = build_train_att(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
#add a mask function for the choice of actions
mask_func=
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
for t in range(total_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_variables(model_file)
return act
def learn(env,
num_actions=3,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16):
torch.set_num_threads(num_cpu)
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
exploration = LinearSchedule(
schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
screen = player_relative
obs, xy_per_marine = common.init(env, obs)
group_id = 0
reset = True
dqn = DQN(num_actions, lr, cuda)
print('\nCollecting experience...')
checkpoint_path = 'models/deepq/checkpoint.pth.tar'
if os.path.exists(checkpoint_path):
dqn, saved_mean_reward = load_checkpoint(dqn, cuda, filename=checkpoint_path)
for t in range(max_timesteps):
# Take action and update exploration to the newest value
# custom process for DefeatZerglingsAndBanelings
obs, screen, player = common.select_marine(env, obs)
# action = act(
# np.array(screen)[None], update_eps=update_eps, **kwargs)[0]
action = dqn.choose_action(np.array(screen)[None])
reset = False
rew = 0
new_action = None
obs, new_action = common.marine_action(env, obs, player, action)
army_count = env._obs[0].observation.player_common.army_count
try:
if army_count > 0 and _ATTACK_SCREEN in obs[0].observation["available_actions"]:
obs = env.step(actions=new_action)
else:
new_action = [sc2_actions.FunctionCall(_NO_OP, [])]
obs = env.step(actions=new_action)
except Exception as e:
# print(e)
1 # Do nothing
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
new_screen = player_relative
rew += obs[0].reward
done = obs[0].step_type == environment.StepType.LAST
selected = obs[0].observation["screen"][_SELECTED]
player_y, player_x = (selected == _PLAYER_FRIENDLY).nonzero()
if len(player_y) > 0:
player = [int(player_x.mean()), int(player_y.mean())]
if len(player) == 2:
if player[0] > 32:
new_screen = common.shift(LEFT, player[0] - 32, new_screen)
elif player[0] < 32:
new_screen = common.shift(RIGHT, 32 - player[0],
new_screen)
if player[1] > 32:
new_screen = common.shift(UP, player[1] - 32, new_screen)
elif player[1] < 32:
new_screen = common.shift(DOWN, 32 - player[1], new_screen)
# Store transition in the replay buffer.
replay_buffer.add(screen, action, rew, new_screen, float(done))
screen = new_screen
episode_rewards[-1] += rew
reward = episode_rewards[-1]
if done:
print("Episode Reward : %s" % episode_rewards[-1])
obs = env.reset()
player_relative = obs[0].observation["screen"][
_PLAYER_RELATIVE]
screen = player_relative
group_list = common.init(env, obs)
# Select all marines first
# env.step(actions=[sc2_actions.FunctionCall(_SELECT_UNIT, [_SELECT_ALL])])
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = dqn.learn(obses_t, actions, rewards, obses_tp1, gamma, batch_size)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
dqn.update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("reward", reward)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward,
mean_100ep_reward))
save_checkpoint({
'epoch': t + 1,
'state_dict': dqn.save_state_dict(),
'best_accuracy': mean_100ep_reward
}, checkpoint_path)
saved_mean_reward = mean_100ep_reward
def __init__(self, args, env, writer=None):
"""
init the agent here
"""
self.eval_env = copy.deepcopy(env)
self.args = args
self.state_dim = env.reset().shape
self.action_dim = env.action_space.n
self.device = torch.device("cuda" if (
torch.cuda.is_available() and self.args.gpu) else "cpu")
# set the same random seed in the main launcher
random.seed(self.args.seed)
torch.manual_seed(self.args.seed)
np.random.seed(self.args.seed)
if self.args.gpu:
torch.cuda.manual_seed(self.args.seed)
self.writer = writer
if self.args.env_name == "grid":
self.dqn = OneHotDQN(self.state_dim,
self.action_dim).to(self.device)
self.dqn_target = OneHotDQN(self.state_dim,
self.action_dim).to(self.device)
else:
raise Exception("not implemented yet!")
# copy parameters
self.dqn_target.load_state_dict(self.dqn.state_dict())
self.optimizer = torch.optim.Adam(self.dqn.parameters(),
lr=self.args.lr)
# for actors
def make_env():
def _thunk():
env = create_env(args)
return env
return _thunk
envs = [make_env() for i in range(self.args.num_envs)]
self.envs = SubprocVecEnv(envs)
# create epsilon and beta schedule
# NOTE: hardcoded for now
self.eps_decay = LinearSchedule(50000 * 200, 0.01, 1.0)
# self.eps_decay = LinearSchedule(self.args.num_episodes * 200, 0.01, 1.0)
self.total_steps = 0
self.num_episodes = 0
# for storing resutls
self.results_dict = {
"train_rewards": [],
"train_constraints": [],
"eval_rewards": [],
"eval_constraints": [],
}
self.cost_indicator = "none"
if "grid" in self.args.env_name:
self.cost_indicator = 'pit'
else:
raise Exception("not implemented yet")
self.eps = self.eps_decay.value(self.total_steps)