def _build_train(self):
env = self.env
act, train, update_target, debug = deepq.build_train(
make_obs_ph=self._observation_ph_generator,
q_func=model_fn,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(
learning_rate=self._learning_rate),
gamma=self._gamma)
self.act = act
self.train = train
self.update_target = update_target
self.debug = debug
def __init__(self,
observation_space_shape,
num_actions,
q_func,
replay_buffer,
exploration,
lr=5e-4,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
param_noise=False
):
self.num_actions = num_actions
self.replay_buffer = replay_buffer
self.exploration = exploration
self.train_freq = train_freq
self.batch_size = batch_size
self.print_freq = print_freq
self.learning_starts = learning_starts
self.target_network_update_freq = target_network_update_freq
self.param_noise = param_noise
def make_obs_ph(name):
return ObservationInput(observation_space_shape, name=name)
self.act, self.train, self.update_target, _ = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=self.param_noise
)
U.initialize()
self.update_target()
self.t = 0
self.episode_rewards = []
def __init__(self, identifier, actions, observation_shape, num_steps, x=0.0, y=0.0):
self.id = identifier
self.actions = actions
self.x = x
self.y = y
self.yellow_steps = 0
self.postponed_action = None
self.obs = None
self.current_action = None
self.weights = np.ones(32)
self.td_errors = np.ones(32)
self.pre_train = 2500
self.prioritized = False
self.prioritized_eps = 1e-4
self.batch_size = 32
self.buffer_size = 30000
self.learning_freq = 500
self.target_update = 5000
# Create all the functions necessary to train the model
self.act, self.train, self.update_target, self.debug = deepq.build_train(
make_obs_ph=lambda name: TrafficTfInput(observation_shape, name=name),
q_func=dueling_model,
num_actions=len(actions),
optimizer=tf.train.AdamOptimizer(learning_rate=1e-4, epsilon=1e-4),
gamma=0.99,
double_q=True,
scope="deepq" + identifier
)
# Create the replay buffer
if self.prioritized:
self.replay_buffer = PrioritizedReplayBuffer(size=self.buffer_size, alpha=0.6)
self.beta_schedule = LinearSchedule(num_steps // 4, initial_p=0.4, final_p=1.0)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
self.exploration = LinearSchedule(schedule_timesteps=int(num_steps * 0.1), initial_p=1.0, final_p=0.01)
# Initialize the parameters and copy them to the target network.
U.initialize()
self.update_target()
def _build_model(self):
sess = U.get_session()
if sess is None:
sess = U.make_session(8)
sess.__enter__()
self.act, self.train, self.update_target, self.debug = deepq.build_train(
make_obs_ph=lambda name: U.BatchInput(shape=[2, self.state_size],
name=name),
q_func=self.model2,
num_actions=self.action_size,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
scope=self.scope,
double_q=True,
param_noise=True)
# 初始化tf环境
U.initialize()
self.update_target()
def __init__(self, model, observation_space, n_actions, gamma, lr,
replay_buffer, batch_size, optim_iters, grad_norm_clip,
double_q):
self.replay_buffer = replay_buffer
self.batch_size = batch_size
self.optim_iters = optim_iters
optimizer = tf.train.AdamOptimizer(learning_rate=lr)
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
self.act, self.train, self.update_target, _ = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=model,
num_actions=n_actions,
optimizer=optimizer,
gamma=gamma,
grad_norm_clipping=grad_norm_clip,
double_q=double_q)
def make_build_train(self):
# Build act and train networks
self.act, self.train, self.update_target, self.debug = deepq.build_train(
make_obs_ph=self.make_obs_ph,
q_func=self.q_func,
num_actions=self.env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=self.lr),
gamma=self.gamma,
grad_norm_clipping=10,
param_noise=self.param_noise)
self.act_params = {
'make_obs_ph': self.make_obs_ph,
'q_func': self.q_func,
'num_actions': self.env.action_space.n,
}
self.act = ActWrapper(self.act, self.act_params)
return 'make_build_train() complete'
def __init__(self,
env,
gamma,
total_timesteps,
network='mlp',
lr=5e-4,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
learning_starts=1000,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
**network_kwargs):
"""DQN wrapper to train option policies
Parameters
-------
env: gym.Env
environment to train on
gamma: float
discount factor
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
total_timesteps: int
number of env steps to optimizer for
lr: float
learning rate for adam optimizer
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batch sampled from replay buffer for training
learning_starts: int
how many steps of the model to collect transitions for before learning starts
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
**network_kwargs
additional keyword arguments to pass to the network builder.
"""
# Adjusting hyper-parameters by considering the number of options policies to learn
num_options = env.get_number_of_options()
buffer_size = num_options * buffer_size
batch_size = num_options * batch_size
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.option_observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
self.num_actions = env.option_action_space.n
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=self.num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
scope="options")
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': self.num_actions,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(
exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# Variables that are used during learning
self.act = act
self.train = train
self.update_target = update_target
self.replay_buffer = replay_buffer
self.beta_schedule = beta_schedule
self.exploration = exploration
self.param_noise = param_noise
self.train_freq = train_freq
self.batch_size = batch_size
self.learning_starts = learning_starts
self.target_network_update_freq = target_network_update_freq
self.prioritized_replay = prioritized_replay
self.prioritized_replay_alpha = prioritized_replay_alpha
self.prioritized_replay_beta0 = prioritized_replay_beta0
self.prioritized_replay_beta_iters = prioritized_replay_beta_iters
self.prioritized_replay_eps = prioritized_replay_eps
out = layers.fully_connected(out, num_outputs=256, activation_fn=tf.nn.relu)
out = layers.fully_connected(out, num_outputs=256, activation_fn=tf.nn.relu)
out = layers.fully_connected(out, num_outputs=num_actions, activation_fn=None)
# out = layers.layer_norm(out, center=True, scale=True)
return out
if __name__ == '__main__':
with U.make_session(8):
# Create the environment
env = gym.make("CartPole-v0")
# Create all the functions necessary to train the model
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: U.BatchInput(env.observation_space.shape, name=name),
q_func=model,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
param_noise=False
)
# Create the replay buffer
replay_buffer = PrioritizedReplayBuffer(50000, alpha=0.6)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
exploration = LinearSchedule(schedule_timesteps=10000, initial_p=1.0, final_p=0.02)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
tvars = tf.trainable_variables()
tvars_vals = U.get_session().run(tvars)
def __init__(
self,
env,
# observation_space,
# action_space,
network=None,
scope='deepq',
seed=None,
lr=None, # Was 5e-4
lr_mc=5e-4,
total_episodes=None,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=None, # was 0.02
train_freq=1,
train_log_freq=100,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
# checkpoint_path=None,
learning_starts=1000,
gamma=None,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
save_path=None,
load_path=None,
save_reward_threshold=None,
**network_kwargs):
super().__init__(env, seed)
if train_log_freq % train_freq != 0:
raise ValueError(
'Train log frequency should be a multiple of train frequency')
elif checkpoint_freq % train_log_freq != 0:
raise ValueError(
'Checkpoint freq should be a multiple of train log frequency, or model saving will not be logged properly'
)
print('init dqnlearningagent')
self.train_log_freq = train_log_freq
self.scope = scope
self.learning_starts = learning_starts
self.save_reward_threshold = save_reward_threshold
self.batch_size = batch_size
self.train_freq = train_freq
self.total_episodes = total_episodes
self.total_timesteps = total_timesteps
# TODO: scope not doing anything.
if network is None and 'lunar' in env.unwrapped.spec.id.lower():
if lr is None:
lr = 1e-3
if exploration_final_eps is None:
exploration_final_eps = 0.02
#exploration_fraction = 0.1
#exploration_final_eps = 0.02
target_network_update_freq = 1500
#print_freq = 100
# num_cpu = 5
if gamma is None:
gamma = 0.99
network = 'mlp'
network_kwargs = {
'num_layers': 2,
'num_hidden': 64,
}
self.target_network_update_freq = target_network_update_freq
self.gamma = gamma
get_session()
# set_global_seeds(seed)
# TODO: Check whether below is ok to substitue for set_global_seeds.
try:
import tensorflow as tf
tf.set_random_seed(seed)
except ImportError:
pass
self.q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
act, self.train, self.train_mc, self.update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=self.q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
optimizer_mc=tf.train.AdamOptimizer(learning_rate=lr_mc),
gamma=gamma,
grad_norm_clipping=10,
param_noise=False,
scope=scope,
# reuse=reuse,
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': self.q_func,
'num_actions': env.action_space.n,
}
self._act = ActWrapper(act, act_params)
self.print_freq = print_freq
self.checkpoint_freq = checkpoint_freq
# Create the replay buffer
self.prioritized_replay = prioritized_replay
self.prioritized_replay_eps = prioritized_replay_eps
if self.prioritized_replay:
self.replay_buffer = PrioritizedReplayBuffer(
buffer_size,
alpha=prioritized_replay_alpha,
)
if prioritized_replay_beta_iters is None:
if total_episodes is not None:
raise NotImplementedError(
'Need to check how to set exploration based on episodes'
)
prioritized_replay_beta_iters = total_timesteps
self.beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0,
)
else:
self.replay_buffer = ReplayBuffer(buffer_size)
self.replay_buffer_mc = ReplayBuffer(buffer_size)
self.beta_schedule = None
# Create the schedule for exploration starting from 1.
self.exploration = LinearSchedule(
schedule_timesteps=int(
exploration_fraction *
total_timesteps if total_episodes is None else total_episodes),
initial_p=1.0,
final_p=exploration_final_eps,
)
# Initialize the parameters and copy them to the target network.
U.initialize()
self.update_target()
self.episode_lengths = [0]
self.episode_rewards = [0.0]
self.discounted_episode_rewards = [0.0]
self.start_values = [None]
self.lunar_crashes = [0]
self.lunar_goals = [0]
self.saved_mean_reward = None
self.td = None
if save_path is None:
self.td = tempfile.mkdtemp()
outdir = self.td
self.model_file = os.path.join(outdir, "model")
else:
outdir = os.path.dirname(save_path)
os.makedirs(outdir, exist_ok=True)
self.model_file = save_path
print('DQN agent saving to:', self.model_file)
self.model_saved = False
if tf.train.latest_checkpoint(outdir) is not None:
# TODO: Check scope addition
load_variables(self.model_file, scope=self.scope)
# load_variables(self.model_file)
logger.log('Loaded model from {}'.format(self.model_file))
self.model_saved = True
raise Exception('Check that we want to load previous model')
elif load_path is not None:
# TODO: Check scope addition
load_variables(load_path, scope=self.scope)
# load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
self.train_log_file = None
if save_path and load_path is None:
self.train_log_file = self.model_file + '.log.csv'
with open(self.train_log_file, 'w') as f:
cols = [
'episode',
't',
'td_max',
'td_mean',
'100ep_r_mean',
'100ep_r_mean_discounted',
'100ep_v_mean',
'100ep_n_crashes_mean',
'100ep_n_goals_mean',
'saved_model',
'smoothing',
]
f.write(','.join(cols) + '\n')
self.training_episode = 0
self.t = 0
self.episode_t = 0
"""
n = observation_space.n
m = action_space.n
self.Q = np.zeros((n, m))
self._lr_schedule = lr_schedule
self._eps_schedule = eps_schedule
self._boltzmann_schedule = boltzmann_schedule
"""
# Make placeholder for Q values
self.q_values = debug['q_values']
if mode == "1":
with open('expert_demonstrations_NN.csv', 'r', newline='') as csvfile:
data_reader = csv.reader(csvfile, delimiter=',')
exp_demo = ([r for r in data_reader])
elif mode == "2":
with open('expert_demonstrations_Human.csv', 'r', newline='') as csvfile:
data_reader = csv.reader(csvfile, delimiter=',')
exp_demo = ([r for r in data_reader])
else:
exp_demo = []
# Create all the functions necessary to train the model
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: ObservationInput(env.observation_space, name=name),
q_func=model,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
gamma=0.99,
)
# Create the replay buffer
replay_buffer = ReplayBuffer(50000)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
exploration = LinearSchedule(schedule_timesteps=10000, initial_p=1.0, final_p=0.02)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
obs = env.reset()
with open(os.path.join(savedir, 'args.json'), 'w') as f:
json.dump(vars(args), f)
with U.make_session(4) as sess:
# Create training graph and replay buffer
def model_wrapper(img_in, num_actions, scope, **kwargs):
actual_model = dueling_model if args.dueling else model
# return actual_model(img_in, num_actions, scope, layer_norm=args.layer_norm, **kwargs)
return actual_model(img_in, num_actions, scope, **kwargs)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: Uint8Input(env.observation_space.shape,
name=name),
q_func=model_wrapper,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr,
epsilon=1e-4),
gamma=0.99,
grad_norm_clipping=10,
double_q=args.double_q,
param_noise=args.param_noise)
approximate_num_iters = args.num_steps / 4
exploration = PiecewiseSchedule([(0, 1.0),
(approximate_num_iters / 50, 0.1),
(approximate_num_iters / 5, 0.01)],
outside_value=0.01)
if args.prioritized:
replay_buffer = PrioritizedReplayBuffer(args.replay_buffer_size,
args.prioritized_alpha)
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
thompson=True,
prior="no prior",
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
blr_params = BLRParams()
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
# q_func = build_q_func(network, **network_kwargs)
q_func = build_q_func_and_features(network,
hiddens=[blr_params.feat_dim],
**network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
#deep mind optimizer
# dm_opt = tf.train.RMSPropOptimizer(learning_rate=0.00025,decay=0.95,momentum=0.0,epsilon=0.00001,centered=True)
act, train, update_target, debug, blr_additions = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(
learning_rate=lr
), #tf.train.RMSPropOptimizer(learning_rate=lr,momentum=0.95),#
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise,
thompson=thompson,
double_q=thompson)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
# replay_buffer = ReplayBuffer(buffer_size)
replay_buffer = ReplayBufferPerActionNew(buffer_size,
env.action_space.n)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
num_actions = env.action_space.n
if thompson:
# Create parameters for Bayesian Regression
feat_dim = blr_additions['feat_dim']
num_models = 5
print("num models is: {}".format(num_models))
w_sample = np.random.normal(loc=0,
scale=blr_params.sigma,
size=(num_actions, num_models, feat_dim))
w_mu = np.zeros((num_actions, feat_dim))
w_cov = np.zeros((num_actions, feat_dim, feat_dim))
for i in range(num_actions):
w_cov[i] = blr_params.sigma * np.eye(feat_dim)
phiphiT = np.zeros((num_actions, feat_dim, feat_dim), dtype=np.float32)
phiphiT_inv = np.zeros((num_actions, feat_dim, feat_dim),
dtype=np.float32)
for i in range(num_actions):
phiphiT[i] = (1 / blr_params.sigma) * np.eye(feat_dim)
phiphiT_inv[i] = blr_params.sigma * np.eye(feat_dim)
old_phiphiT_inv = [phiphiT_inv for i in range(5)]
phiY = np.zeros((num_actions, feat_dim), dtype=np.float32)
YY = np.zeros(num_actions)
model_idx = np.random.randint(0, num_models, size=num_actions)
blr_ops = blr_additions['blr_ops']
blr_ops_old = blr_additions['blr_ops_old']
last_layer_weights = np.zeros((feat_dim, num_actions))
phiphiT0 = np.copy(phiphiT)
invgamma_a = [blr_params.a0 for _ in range(num_actions)]
invgamma_b = [blr_params.a0 for _ in range(num_actions)]
# Initialize the parameters and copy them to the target network.
U.initialize()
# update_target()
if thompson:
blr_additions['update_old']()
if isinstance(blr_additions['update_old_target'], list):
for update_net in reversed(blr_additions['update_old_target']):
update_net()
else:
blr_additions['update_old_target']()
if blr_additions['old_networks'] is not None:
for key in blr_additions['old_networks'].keys():
blr_additions['old_networks'][key]["update"]()
episode_rewards = [0.0]
# episode_Q_estimates = [0.0]
unclipped_episode_rewards = [0.0]
# eval_rewards = [0.0]
old_networks_num = 5
# episode_pseudo_count = [[0.0] for i in range(old_networks_num)]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
actions_hist = [0 for _ in range(num_actions)]
actions_hist_total = [0 for _ in range(num_actions)]
last_layer_weights_decaying_average = None
blr_counter = 0
action_buffers_size = 512
action_buffers = [
ReplayBuffer(action_buffers_size) for _ in range(num_actions)
]
eval_flag = False
eval_counter = 0
for t in tqdm(range(total_timesteps)):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
if thompson:
# for each action sample one of the num_models samples of w
model_idx = np.random.randint(0, num_models, size=num_actions)
cur_w = np.zeros((num_actions, feat_dim))
for i in range(num_actions):
cur_w[i] = w_sample[i, model_idx[i]]
action, estimate = act(np.array(obs)[None], cur_w[None])
actions_hist[int(action)] += 1
actions_hist_total[int(action)] += 1
else:
action, estimate = act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)
env_action = action
reset = False
new_obs, unclipped_rew, done_list, _ = env.step(env_action)
if isinstance(done_list, list):
done, real_done = done_list
else:
done, real_done = done_list, done_list
rew = np.sign(unclipped_rew)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
action_buffers[action].add(obs, action, rew, new_obs, float(done))
if action_buffers[action]._next_idx == 0:
obses_a, actions_a, rewards_a, obses_tp1_a, dones_a = replay_buffer.get_samples(
[i for i in range(action_buffers_size)])
phiphiT_a, phiY_a, YY_a = blr_ops_old(obses_a, actions_a,
rewards_a, obses_tp1_a,
dones_a)
phiphiT[action] += phiphiT_a
phiY[action] += phiY_a
YY[action] += YY_a
precision = phiphiT[action] + phiphiT0[action]
cov = np.linalg.pinv(precision)
mu = np.array(
np.dot(cov, (phiY[action] + np.dot(
phiphiT0[action], last_layer_weights[:, action]))))
invgamma_a[action] += 0.5 * action_buffers_size
b_upd = 0.5 * YY[action]
b_upd += 0.5 * np.dot(
last_layer_weights[:, action].T,
np.dot(phiphiT0[action], last_layer_weights[:, action]))
b_upd -= 0.5 * np.dot(mu.T, np.dot(precision, mu))
invgamma_b[action] += b_upd
# old_phiphiT_inv_a = [np.tile(oppTi[action], (action_buffers_size,1,1)) for oppTi in old_phiphiT_inv]
# old_pseudo_count = blr_additions['old_pseudo_counts'](obses_a, *old_phiphiT_inv_a)
# old_pseudo_count = np.sum(old_pseudo_count, axis=-1)
# for i in range(old_networks_num):
# idx = ((blr_counter-1)-i) % old_networks_num # arrange networks from newest to oldest
# episode_pseudo_count[i][-1] += old_pseudo_count[idx]
# if real_done:
# for a in range(num_actions):
# if action_buffers[a]._next_idx != 0:
# obses_a, actions_a, rewards_a, obses_tp1_a, dones_a = replay_buffer.get_samples([i for i in range(action_buffers[a]._next_idx)])
# nk = obses_a.shape[0]
#
# # old_phiphiT_inv_a = [np.tile(oppTi[action],(nk,1,1)) for oppTi in old_phiphiT_inv]
# # old_pseudo_count = blr_additions['old_pseudo_counts'](obses_a, *old_phiphiT_inv_a)
# # old_pseudo_count = np.sum(old_pseudo_count, axis=-1)
# # for i in range(old_networks_num):
# # idx = ((blr_counter-1)-i) % old_networks_num # arrange networks from newest to oldest
# # episode_pseudo_count[i][-1] += old_pseudo_count[idx]
#
# phiphiT_a, phiY_a, YY_a = blr_ops_old(obses_a, actions_a, rewards_a, obses_tp1_a, dones_a)
# phiphiT[a] += phiphiT_a
# phiY[a] += phiY_a
# YY[a] += YY_a
#
# action_buffers[a]._next_idx = 0
obs = new_obs
episode_rewards[-1] += rew
# episode_Q_estimates[-1] += estimate
unclipped_episode_rewards[-1] += unclipped_rew
if t % 250000 == 0 and t > 0:
eval_flag = True
if done:
obs = env.reset()
episode_rewards.append(0.0)
# episode_Q_estimates.append(0.0)
reset = True
if real_done:
unclipped_episode_rewards.append(0.0)
# for i in range(old_networks_num):
# episode_pseudo_count[i].append(0.0)
# every time full episode ends run eval episode
if eval_flag:
te = 0
print("running evaluation")
eval_rewards = [0.0]
while te < 125000:
# for te in range(125000):
real_done = False
print(te)
while not real_done:
action, _ = blr_additions['eval_act'](
np.array(obs)[None])
new_obs, unclipped_rew, done_list, _ = env.step(
action)
if isinstance(done_list, list):
done, real_done = done_list
else:
done, real_done = done_list, done_list
eval_rewards[-1] += unclipped_rew
obs = new_obs
te += 1
if done:
obs = env.reset()
if real_done:
eval_rewards.append(0.0)
obs = env.reset()
eval_rewards.pop()
mean_reward_eval = round(np.mean(eval_rewards), 2)
logger.record_tabular("mean eval episode reward",
mean_reward_eval)
logger.dump_tabular()
eval_flag = False
# eval_counter += 1
# if eval_counter % 10 == 0:
# if t > learning_starts:
# real_done = False
# while not real_done:
# action, _ = blr_additions['eval_act'](np.array(obs)[None])
# new_obs, unclipped_rew, done_list, _ = env.step(action)
# done, real_done = done_list
# eval_rewards[-1] += unclipped_rew
# obs = new_obs
# eval_rewards.append(0.0)
# obs = env.reset()
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if thompson:
if t > learning_starts and t % (
blr_params.update_w * target_network_update_freq) == 0:
phiphiT_inv = np.zeros_like(phiphiT)
for i in range(num_actions):
try:
phiphiT_inv[i] = np.linalg.inv(phiphiT[i])
except:
phiphiT_inv[i] = np.linalg.pinv(phiphiT[i])
old_phiphiT_inv[blr_counter % 5] = phiphiT_inv
llw = sess.run(blr_additions['last_layer_weights'])
phiphiT, phiY, phiphiT0, last_layer_weights, YY, invgamma_a, invgamma_b = BayesRegression(
phiphiT,
phiY,
replay_buffer,
blr_additions['feature_extractor'],
blr_additions['target_feature_extractor'],
num_actions,
blr_params,
w_mu,
w_cov,
llw,
prior=prior,
blr_ops=blr_additions['blr_ops'],
sdp_ops=blr_additions['sdp_ops'],
old_networks=blr_additions['old_networks'],
blr_counter=blr_counter,
old_feat=blr_additions['old_feature_extractor'],
a=invgamma_a)
blr_counter += 1
if seed is not None:
print('seed is {}'.format(seed))
blr_additions['update_old']()
if isinstance(blr_additions['update_old_target'], list):
for update_net in reversed(
blr_additions['update_old_target']):
update_net()
else:
blr_additions['update_old_target']()
if blr_additions['old_networks'] is not None:
blr_additions['old_networks'][blr_counter %
5]["update"]()
if thompson:
if t > 0 and t % blr_params.sample_w == 0:
# sampling num_models samples of w
if debug:
print(actions_hist)
else:
if t % 10000 == 0:
print(actions_hist)
actions_hist = [0 for _ in range(num_actions)]
# if t > 1000000:
adaptive_sigma = True
# else:
# adaptive_sigma = False
cov_norms = []
cov_norms_no_sigma = []
sampled_sigmas = []
for i in range(num_actions):
if prior == 'no prior' or last_layer_weights is None:
cov = np.linalg.inv(phiphiT[i])
mu = np.array(np.dot(cov, phiY[i]))
elif prior == 'last layer':
cov = np.linalg.inv(phiphiT[i])
mu = np.array(
np.dot(cov, (phiY[i] + (1 / blr_params.sigma) *
last_layer_weights[:, i])))
elif prior == 'single sdp':
try:
cov = np.linalg.inv(phiphiT[i] + phiphiT0)
except:
print("singular matrix using pseudo inverse")
cov = np.linalg.pinv(phiphiT[i] + phiphiT0)
mu = np.array(
np.dot(cov, (phiY[i] + np.dot(
phiphiT0, last_layer_weights[:, i]))))
elif prior == 'sdp' or prior == 'linear':
try:
cov = np.linalg.inv(phiphiT[i] + phiphiT0[i])
except:
# print("singular matrix")
cov = np.linalg.pinv(phiphiT[i] + phiphiT0[i])
mu = np.array(
np.dot(cov, (phiY[i] + np.dot(
phiphiT0[i], last_layer_weights[:, i]))))
else:
print("No valid prior")
exit(0)
for j in range(num_models):
if adaptive_sigma:
sigma = invgamma_b[i] * invgamma.rvs(
invgamma_a[i])
else:
sigma = blr_params.sigma
try:
w_sample[i, j] = np.random.multivariate_normal(
mu, sigma * cov)
except:
w_sample[i, j] = mu
cov_norms.append(np.linalg.norm(sigma * cov))
cov_norms_no_sigma.append(np.linalg.norm(cov))
sampled_sigmas.append(sigma)
if t % 7 == 0:
for i, cov_norm in enumerate(cov_norms):
print(
"cov*sigma norm for action {}: {}, visits: {}".
format(i, cov_norm,
len(replay_buffer.buffers[i])))
# if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
# print(update_target)
# update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
mean_10ep_reward = round(np.mean(episode_rewards[-11:-1]), 1)
mean_100ep_reward_unclipped = round(
np.mean(unclipped_episode_rewards[-101:-1]), 1)
mean_10ep_reward_unclipped = round(
np.mean(unclipped_episode_rewards[-11:-1]), 1)
# mean_100ep_reward_eval = round(np.mean(eval_rewards[-101:-1]), 1)
# mean_10ep_reward_eval = round(np.mean(eval_rewards[-11:-1]), 1)
# mean_100ep_est = round(np.mean(episode_Q_estimates[-101:-1]), 1)
# mean_10ep_est = round(np.mean(episode_Q_estimates[-11:-1]), 1)
num_episodes = len(episode_rewards)
# mean_10ep_pseudo_count = [0.0 for _ in range(old_networks_num)]
# mean_100ep_pseudo_count = [0.0 for _ in range(old_networks_num)]
# for i in range(old_networks_num):
# mean_10ep_pseudo_count[i] = round(np.log(np.mean(episode_pseudo_count[i][-11:-1])), 1)
# mean_100ep_pseudo_count[i] = round(np.log(np.mean(episode_pseudo_count[i][-101:-1])), 1)
# if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
if t % 10000 == 0 and t > 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("mean 10 episode reward",
mean_10ep_reward)
logger.record_tabular("mean 100 unclipped episode reward",
mean_100ep_reward_unclipped)
logger.record_tabular("mean 10 unclipped episode reward",
mean_10ep_reward_unclipped)
# logger.record_tabular("mean 100 eval episode reward", mean_100ep_reward_eval)
# logger.record_tabular("mean 10 eval episode reward", mean_10ep_reward_eval)
# for i in range(old_networks_num):
# logger.record_tabular("mean 10 episode pseudo count for -{} net".format(i+1), mean_10ep_pseudo_count[i])
# logger.record_tabular("mean 100 episode pseudo count for -{} net".format(i+1), mean_100ep_pseudo_count[i])
# logger.record_tabular("mean 100 episode Q estimates", mean_100ep_est)
# logger.record_tabular("mean 10 episode Q estimates", mean_10ep_est)
logger.dump_tabular()
if t % 7 == 0:
print("len(unclipped_episode_rewards)")
print(len(unclipped_episode_rewards))
print("len(episode_rewards)")
print(len(episode_rewards))
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act
def learn(env,
q_func,
num_actions=64*64,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
param_noise=False,
param_noise_threshold=0.05,
callback=None):
"""Train a deepq model.
Parameters
-------
env: pysc2.env.SC2Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
num_cpu: int
number of cpus to use for training
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
# Set up summary Ops
summary_ops, summary_vars = build_summaries()
writer = tf.summary.FileWriter(SUMMARY_DIR, sess.graph)
def make_obs_ph(name):
return U.BatchInput((64, 64), name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
episode_minerals = [0.0]
saved_mean_reward = None
path_memory = np.zeros((64,64))
obs = env.reset()
# Select all marines first
step_result = env.step(actions=[sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])])
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
obs = player_relative + path_memory
player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
if(player[0]>32):
obs = shift(LEFT, player[0]-32, obs)
elif(player[0]<32):
obs = shift(RIGHT, 32 - player[0], obs)
if(player[1]>32):
obs = shift(UP, player[1]-32, obs)
elif(player[1]<32):
obs = shift(DOWN, 32 - player[1], obs)
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
if param_noise_threshold >= 0.:
update_param_noise_threshold = param_noise_threshold
else:
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
reset = False
coord = [player[0], player[1]]
rew = 0
path_memory_ = np.array(path_memory, copy=True)
if(action == 0): #UP
if(player[1] >= 16):
coord = [player[0], player[1] - 16]
path_memory_[player[1] - 16 : player[1], player[0]] = -1
elif(player[1] > 0):
coord = [player[0], 0]
path_memory_[0 : player[1], player[0]] = -1
else:
rew -= 1
elif(action == 1): #DOWN
if(player[1] <= 47):
coord = [player[0], player[1] + 16]
path_memory_[player[1] : player[1] + 16, player[0]] = -1
elif(player[1] > 47):
coord = [player[0], 63]
path_memory_[player[1] : 63, player[0]] = -1
else:
rew -= 1
elif(action == 2): #LEFT
if(player[0] >= 16):
coord = [player[0] - 16, player[1]]
path_memory_[player[1], player[0] - 16 : player[0]] = -1
elif(player[0] < 16):
coord = [0, player[1]]
path_memory_[player[1], 0 : player[0]] = -1
else:
rew -= 1
elif(action == 3): #RIGHT
if(player[0] <= 47):
coord = [player[0] + 16, player[1]]
path_memory_[player[1], player[0] : player[0] + 16] = -1
elif(player[0] > 47):
coord = [63, player[1]]
path_memory_[player[1], player[0] : 63] = -1
else:
rew -= 1
else:
#Cannot move, give minus reward
rew -= 1
if(path_memory[coord[1],coord[0]] != 0):
rew -= 0.5
path_memory = np.array(path_memory_)
#print("action : %s Coord : %s" % (action, coord))
new_action = [sc2_actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, coord])]
step_result = env.step(actions=new_action)
player_relative = step_result[0].observation["screen"][_PLAYER_RELATIVE]
new_obs = player_relative + path_memory
player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
if(player[0]>32):
new_obs = shift(LEFT, player[0]-32, new_obs)
elif(player[0]<32):
new_obs = shift(RIGHT, 32 - player[0], new_obs)
if(player[1]>32):
new_obs = shift(UP, player[1]-32, new_obs)
elif(player[1]<32):
new_obs = shift(DOWN, 32 - player[1], new_obs)
rew += step_result[0].reward * 10
done = step_result[0].step_type == environment.StepType.LAST
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
episode_minerals[-1] += step_result[0].reward
if done:
obs = env.reset()
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
obs = player_relative + path_memory
player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
if(player[0]>32):
obs = shift(LEFT, player[0]-32, obs)
elif(player[0]<32):
obs = shift(RIGHT, 32 - player[0], obs)
if(player[1]>32):
obs = shift(UP, player[1]-32, obs)
elif(player[1]<32):
obs = shift(DOWN, 32 - player[1], obs)
# Select all marines first
env.step(actions=[sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])])
episode_rewards.append(0.0)
episode_minerals.append(0.0)
path_memory = np.zeros((64,64))
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
mean_100ep_mineral = round(np.mean(episode_minerals[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
summary_str = sess.run(summary_ops, feed_dict={
summary_vars[0]: mean_100ep_reward,
summary_vars[1]: mean_100ep_mineral
})
writer.add_summary(summary_str, num_episodes)
writer.flush()
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("mean 100 episode mineral", mean_100ep_mineral)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
U.load_state(model_file)
return ActWrapper(act)
def learn(env,
q_func,
num_actions=4,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
param_noise=False,
param_noise_threshold=0.05,
callback=None):
"""Train a deepq model.
Parameters
-------
env: pysc2.env.SC2Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
num_cpu: int
number of cpus to use for training
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
def make_obs_ph(name):
return U.BatchInput((32, 32), name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
scope="deepq")
#
# act_y, train_y, update_target_y, debug_y = deepq.build_train(
# make_obs_ph=make_obs_ph,
# q_func=q_func,
# num_actions=num_actions,
# optimizer=tf.train.AdamOptimizer(learning_rate=lr),
# gamma=gamma,
# grad_norm_clipping=10,
# scope="deepq_y"
# )
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
# replay_buffer_y = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(
prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
# beta_schedule_y = LinearSchedule(prioritized_replay_beta_iters,
# initial_p=prioritized_replay_beta0,
# final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
# replay_buffer_y = ReplayBuffer(buffer_size)
beta_schedule = None
# beta_schedule_y = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(
schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
# update_target_y()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
# Select all marines first
obs = env.step(
actions=[sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])])
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
screen = (player_relative == _PLAYER_NEUTRAL).astype(int) #+ path_memory
player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
if (player[0] > 16):
screen = shift(LEFT, player[0] - 16, screen)
elif (player[0] < 16):
screen = shift(RIGHT, 16 - player[0], screen)
if (player[1] > 16):
screen = shift(UP, player[1] - 16, screen)
elif (player[1] < 16):
screen = shift(DOWN, 16 - player[1], screen)
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join("model/", "mineral_shards")
print(model_file)
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
if param_noise_threshold >= 0.:
update_param_noise_threshold = param_noise_threshold
else:
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) +
exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(
np.array(screen)[None], update_eps=update_eps, **kwargs)[0]
# action_y = act_y(np.array(screen)[None], update_eps=update_eps, **kwargs)[0]
reset = False
coord = [player[0], player[1]]
rew = 0
if (action == 0): #UP
if (player[1] >= 8):
coord = [player[0], player[1] - 8]
#path_memory_[player[1] - 16 : player[1], player[0]] = -1
elif (player[1] > 0):
coord = [player[0], 0]
#path_memory_[0 : player[1], player[0]] = -1
#else:
# rew -= 1
elif (action == 1): #DOWN
if (player[1] <= 23):
coord = [player[0], player[1] + 8]
#path_memory_[player[1] : player[1] + 16, player[0]] = -1
elif (player[1] > 23):
coord = [player[0], 31]
#path_memory_[player[1] : 63, player[0]] = -1
#else:
# rew -= 1
elif (action == 2): #LEFT
if (player[0] >= 8):
coord = [player[0] - 8, player[1]]
#path_memory_[player[1], player[0] - 16 : player[0]] = -1
elif (player[0] < 8):
coord = [0, player[1]]
#path_memory_[player[1], 0 : player[0]] = -1
#else:
# rew -= 1
elif (action == 3): #RIGHT
if (player[0] <= 23):
coord = [player[0] + 8, player[1]]
#path_memory_[player[1], player[0] : player[0] + 16] = -1
elif (player[0] > 23):
coord = [31, player[1]]
#path_memory_[player[1], player[0] : 63] = -1
if _MOVE_SCREEN not in obs[0].observation["available_actions"]:
obs = env.step(actions=[
sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
])
new_action = [
sc2_actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, coord])
]
# else:
# new_action = [sc2_actions.FunctionCall(_NO_OP, [])]
obs = env.step(actions=new_action)
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
new_screen = (player_relative == _PLAYER_NEUTRAL).astype(
int) #+ path_memory
player_y, player_x = (
player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
if (player[0] > 16):
new_screen = shift(LEFT, player[0] - 16, new_screen)
elif (player[0] < 16):
new_screen = shift(RIGHT, 16 - player[0], new_screen)
if (player[1] > 16):
new_screen = shift(UP, player[1] - 16, new_screen)
elif (player[1] < 16):
new_screen = shift(DOWN, 16 - player[1], new_screen)
rew = obs[0].reward
done = obs[0].step_type == environment.StepType.LAST
# Store transition in the replay buffer.
replay_buffer.add(screen, action, rew, new_screen, float(done))
# replay_buffer_y.add(screen, action_y, rew, new_screen, float(done))
screen = new_screen
episode_rewards[-1] += rew
reward = episode_rewards[-1]
if done:
obs = env.reset()
player_relative = obs[0].observation["screen"][
_PLAYER_RELATIVE]
screen = (player_relative == _PLAYER_NEUTRAL).astype(
int) #+ path_memory
player_y, player_x = (
player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
# Select all marines first
env.step(actions=[
sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
])
episode_rewards.append(0.0)
#episode_minerals.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
# experience_y = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
# (obses_t_y, actions_y, rewards_y, obses_tp1_y, dones_y, weights_y, batch_idxes_y) = experience_y
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# obses_t_y, actions_y, rewards_y, obses_tp1_y, dones_y = replay_buffer_y.sample(batch_size)
# weights_y, batch_idxes_y = np.ones_like(rewards_y), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
# td_errors_y = train_x(obses_t_y, actions_y, rewards_y, obses_tp1_y, dones_y, weights_y)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
# new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
# replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
# update_target_y()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("reward", reward)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}".
format(saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
U.load_state(model_file)
return ActWrapper(act)
def train_policy(arglist):
with U.single_threaded_session():
# Create the environment
if arglist.use_dense_rewards:
print("Will use env MineRLNavigateDense-v0")
env = gym.make("MineRLNavigateDense-v0")
env_name = "MineRLNavigateDense-v0"
else:
print("Will use env MineRLNavigate-v0")
env = gym.make('MineRLNavigate-v0')
env_name = "MineRLNavigate-v0"
if arglist.force_forward:
env = MineCraftWrapperSimplified(env)
else:
env = MineCraftWrapper(env)
if not arglist.use_demonstrations:
# Use stack of last 4 frames as obs
env = FrameStack(env, 4)
# Create all the functions necessary to train the model
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: ObservationInput(env.observation_space,
name=name),
q_func=build_q_func('conv_only', dueling=True),
num_actions=env.action_space.n,
gamma=0.9,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
)
# Create the replay buffer(s) (TODO: Use prioritized replay buffer)
if arglist.use_demonstrations:
replay_buffer = ReplayBuffer(int(arglist.replay_buffer_len / 2))
demo_buffer = load_demo_buffer(env_name,
int(arglist.replay_buffer_len / 2))
else:
replay_buffer = ReplayBuffer(arglist.replay_buffer_len)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
exploration = LinearSchedule(
schedule_timesteps=arglist.num_exploration_steps *
arglist.num_episodes * arglist.max_episode_steps,
initial_p=1.0,
final_p=arglist.final_epsilon)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
n_episodes = 0
n_steps = 0
obs = env.reset()
log_path = "./learning_curves/minerl_" + str(date.today()) + "_" + str(
time.time()) + ".dat"
log_file = open(log_path, "a")
for episode in range(arglist.num_episodes):
print("Episode: ", str(episode))
done = False
episode_steps = 0
while not done:
# Take action and update exploration to the newest value
action = act(obs[None],
update_eps=exploration.value(n_steps))[0]
new_obs, rew, done, _ = env.step(action)
n_steps += 1
episode_steps += 1
# Break episode
if episode_steps > arglist.max_episode_steps:
done = True
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
# Store rewards
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0)
n_episodes += 1
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if (n_steps > arglist.learning_starts_at_steps) and (n_steps %
4 == 0):
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
32)
train(obses_t, actions, rewards, obses_tp1, dones,
np.ones_like(rewards))
if arglist.use_demonstrations:
if (n_steps < arglist.learning_starts_at_steps) and (
n_steps % 4 == 0):
obses_t, actions, rewards, obses_tp1, dones = demo_buffer.sample(
32)
train(obses_t, actions, rewards, obses_tp1, dones,
np.ones_like(rewards))
if (n_steps > arglist.learning_starts_at_steps) and (
n_steps % 4 == 0):
obses_t, actions, rewards, obses_tp1, dones = demo_buffer.sample(
32)
train(obses_t, actions, rewards, obses_tp1, dones,
np.ones_like(rewards))
# Update target network periodically.
if n_steps % arglist.target_net_update_freq == 0:
update_target()
# Log data for analysis
if done and len(episode_rewards) % 10 == 0:
logger.record_tabular("steps", n_steps)
logger.record_tabular("episodes", len(episode_rewards))
logger.record_tabular(
"mean episode reward",
round(np.mean(episode_rewards[-101:-1]), 1))
logger.record_tabular(
"% time spent exploring",
int(100 * exploration.value(n_steps)))
logger.dump_tabular()
#TODO: Save checkpoints
if n_steps % arglist.checkpoint_rate == 0:
checkpoint_path = "./checkpoints/minerl_" + str(
episode) + "_" + str(date.today()) + "_" + str(
time.time()) + ".pkl"
save_variables(checkpoint_path)
print("%s,%s,%s,%s" %
(n_steps, episode,
round(np.mean(episode_rewards[-101:-1]),
1), int(100 * exploration.value(n_steps))),
file=log_file)
log_file.close()
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
callback=None):
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
update_eps = exploration.value(t)
action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
# obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_state(model_file)
return act
def train_dqn(opts,
seed=None,
lr=1e-3,
total_timesteps=500000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
checkpoint_freq=500000,
learning_starts=1000,
gamma=1.000,
target_network_update_freq=3000,
load_path=None):
"""
Runs the main recorder by binding certain discrete actions to keys.
"""
if os.path.exists(opts.model_dir):
print('Path already exists. Remove? y for yes')
input_char = getch.getch()
if not input_char == 'y':
print('Exiting')
return
shutil.rmtree(opts.model_dir)
os.makedirs(opts.model_dir)
os.makedirs(os.path.join(opts.model_dir, 'logs'))
os.makedirs(os.path.join(opts.model_dir, 'weights'))
#env = gym.make('MountainCar-v0')
env = gym.make('LunarLander-v2')
env._max_episode_steps = 1200
sess = get_session()
set_global_seeds(seed)
train_writer = tf.summary.FileWriter(os.path.join(opts.model_dir, 'logs'),
sess.graph)
q_func = build_q_func('mlp')
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10)
replay_buffer = ReplayBuffer(buffer_size)
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
obs = env.reset()
for t in range(total_timesteps):
# Take action and update exploration to the newest value
env.render()
update_eps = exploration.value(t)
action = act(np.array(obs)[None], update_eps=update_eps)[0]
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
print("Exploration value: {}".format(exploration.value(t)))
print("Last 25 episode rewards: {}".format(episode_rewards[-25:]))
reward_summary = tf.Summary(value=[
tf.Summary.Value(tag='reward',
simple_value=episode_rewards[-1])
])
train_writer.add_summary(reward_summary, t)
obs = env.reset()
episode_rewards.append(0.0)
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors, summary = train(obses_t, actions, rewards, obses_tp1,
dones, weights)
train_writer.add_summary(summary, t)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
if t > learning_starts and t % checkpoint_freq == 0:
save_variables(
os.path.join(opts.model_dir, 'weights', '{}.model'.format(t)))
save_variables(os.path.join(opts.model_dir, 'weights', 'last.model'))
def learn(env,
q_func,
num_actions=4,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
param_noise=False,
param_noise_threshold=0.05,
callback=None):
# Create all the functions necessary to train the model
# Returns a session that will use <num_cpu> CPU's only
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
# Creates a placeholder for a batch of tensors of a given shape and dtyp
def make_obs_ph(name):
return U.BatchInput((64, 64), name=name)
# act, train, update_target are function, debug is dict
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10)
# Choose use prioritized replay buffer or normal replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# SC2的部分開始
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
path_memory = np.zeros((64, 64))
obs = env.reset()
# Select all marines
obs = env.step(
actions=[sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])])
# obs is tuple, obs[0] is 'pysc2.env.environment.TimeStep', obs[0].observation is dictionary.
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
# 利用path memory記憶曾經走過的軌跡
screen = player_relative + path_memory
# 取得兩個陸戰隊的中心位置
player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
if param_noise_threshold >= 0.:
update_param_noise_threshold = param_noise_threshold
else:
update_param_noise_threshold = -np.log(
1. - exploration.value(t) +
exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# np.array()[None] 是指多包一個維度在外面 e.g. [1] -> [[1]]
action = act(np.array(screen)[None],
update_eps=update_eps,
**kwargs)[0]
reset = False
coord = [player[0], player[1]]
rew = 0
# 只有四個action，分別是上下左右，走過之後在路徑上留下一整排-3，目的是與水晶碎片的id(=3)相抵銷，代表有順利採集到。
path_memory_ = np.array(path_memory, copy=True)
if (action == 0): # UP
if (player[1] >= 16):
coord = [player[0], player[1] - 16]
path_memory_[player[1] - 16:player[1], player[0]] = -3
elif (player[1] > 0):
coord = [player[0], 0]
path_memory_[0:player[1], player[0]] = -3
elif (action == 1): # DOWN
if (player[1] <= 47):
coord = [player[0], player[1] + 16]
path_memory_[player[1]:player[1] + 16, player[0]] = -3
elif (player[1] > 47):
coord = [player[0], 63]
path_memory_[player[1]:63, player[0]] = -3
elif (action == 2): # LEFT
if (player[0] >= 16):
coord = [player[0] - 16, player[1]]
path_memory_[player[1], player[0] - 16:player[0]] = -3
elif (player[0] < 16):
coord = [0, player[1]]
path_memory_[player[1], 0:player[0]] = -3
elif (action == 3): # RIGHT
if (player[0] <= 47):
coord = [player[0] + 16, player[1]]
path_memory_[player[1], player[0]:player[0] + 16] = -3
elif (player[0] > 47):
coord = [63, player[1]]
path_memory_[player[1], player[0]:63] = -3
# 更新path_memory
path_memory = np.array(path_memory_)
# 如果不能移動陸戰隊，想必是還沒圈選到陸戰隊，圈選他們
if _MOVE_SCREEN not in obs[0].observation["available_actions"]:
obs = env.step(actions=[
sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
])
# 移動陸戰隊
new_action = [
sc2_actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, coord])
]
# 取得環境給的observation
obs = env.step(actions=new_action)
# 這裡要重新取得player_relative，因為上一行的obs是個有複數資訊的tuple
# 但我們要存入replay_buffer的只有降維後的screen畫面
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
new_screen = player_relative + path_memory
# 取得reward
rew = obs[0].reward
# StepType.LAST 代表done的意思
done = obs[0].step_type == environment.StepType.LAST
# Store transition in the replay buffer
replay_buffer.add(screen, action, rew, new_screen, float(done))
# 確實存入之後就能以新screen取代舊screen
screen = new_screen
episode_rewards[-1] += rew
if done:
# 重新取得敵我中立關係位置圖
obs = env.reset()
# player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
# # 還是看不懂為何要加上path_memory
# screen = player_relative + path_memory
# player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
# player = [int(player_x.mean()), int(player_y.mean())]
# # 圈選全部的陸戰隊(為何要在done observation做這件事情?)
# env.step(actions=[sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])])
episode_rewards.append(0.0)
# 清空path_memory
path_memory = np.zeros((64, 64))
reset = True
# 定期從replay buffer中抽experience來訓練，以及train target network
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# 這裡的train來自deepq.build_train
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
# target network
if t > learning_starts and t % target_network_update_freq == 0:
# 同樣來自deepq.build_train
# Update target network periodically
update_target()
# 下LOG追蹤reward
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode", mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
# 當model進步時，就存檔下來
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
U.load_state(model_file)
return ActWrapper(act)
"""This model takes as input an observation and returns values of all actions."""
with tf.variable_scope(scope, reuse=reuse):
out = inpt
out = layers.fully_connected(out, num_outputs=64, activation_fn=tf.nn.tanh)
out = layers.fully_connected(out, num_outputs=num_actions, activation_fn=None)
return out
if __name__ == '__main__':
with U.make_session(8):
# Create the environment
env = gym.make("CartPole-v0")
# Create all the functions necessary to train the model
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: U.BatchInput(env.observation_space.shape, name=name),
q_func=model,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
)
# Create the replay buffer
replay_buffer = ReplayBuffer(50000)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
exploration = LinearSchedule(schedule_timesteps=10000, initial_p=1.0, final_p=0.02)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
obs = env.reset()
for t in itertools.count():
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=5,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batch sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the trained model from. (default: None)(used in test stage)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
med_libs = MedLibs()
'''Define Q network
inputs: observation place holder(make_obs_ph), num_actions, scope, reuse
outputs(tensor of shape batch_size*num_actions): values of each action, Q(s,a_{i})
'''
q_func = build_q_func(network, **network_kwargs)
''' To put observations into a placeholder '''
# TODO: Can only deal with Discrete and Box observation spaces for now
# observation_space = env.observation_space (default)
# Use sub_obs_space instead
observation_space = med_libs.subobs_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
''' Customize action '''
# TODO: subset of action space.
action_dim = med_libs.sub_act_dim
'''
Returns: deepq.build_train()
act: (tf.Variable, bool, float) -> tf.Variable
function to select and action given observation.
act is computed by [build_act] or [build_act_with_param_noise]
train: (object, np.array, np.array, object, np.array, np.array) -> np.array
optimize the error in Bellman's equation.
update_target: () -> ()
copy the parameters from optimized Q function to the target Q function.
debug: {str: function}
a bunch of functions to print debug data like q_values.
'''
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=action_dim,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
double_q=True,
grad_norm_clipping=10,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': action_dim,
}
'''Contruct an act object using ActWrapper'''
act = ActWrapper(act, act_params)
''' Create the replay buffer'''
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
'''Create the schedule for exploration starting from 1.'''
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
'''
Initialize all the uninitialized variables in the global scope and copy them to the target network.
'''
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
sub_obs = med_libs.custom_obs(obs) # TODO: customize observations
pre_obs = obs
reset = True
mydict = med_libs.action_dict
already_starts = False
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
# load_path: a trained model/policy
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
''' Training loop starts'''
t = 0
while t < total_timesteps:
if callback is not None:
if callback(locals(), globals()):
break
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
''' Choose action: take action and update exploration to the newest value
'''
# TODO: Mixed action strategy
# Normal status, action is easily determined by rules, use [obs]
action = med_libs.simple_case_action(obs)
# Distraction status, action is determined by Q, with [sub_obs]
if action == -10:
action = act(np.array(sub_obs)[None],
update_eps=update_eps,
**kwargs)[0]
action = med_libs.action_Q_env(
action
) # TODO:action_Q_env, from Q_action(0~2) to env_action(2~4)
reset = False
''' Step action '''
new_obs, rew, done, d_info = env.step(action)
d_att_last = int(pre_obs[0][0])
d_att_now = int(obs[0][0])
d_att_next = int(new_obs[0][0])
''' Store transition in the replay buffer.'''
pre_obs = obs
obs = new_obs
sub_new_obs = med_libs.custom_obs(new_obs)
if (d_att_last == 0 and d_att_now == 1) and not already_starts:
already_starts = True
if already_starts and d_att_now == 1:
replay_buffer.add(sub_obs, action, rew, sub_new_obs,
float(done))
episode_rewards[-1] += rew # Sum of rewards
t = t + 1
print(
'>> Iteration:{}, State[d_att,cd_activate,L4_available,ssl4_activate,f_dc]:{}'
.format(t, sub_obs))
print(
'Dis_Last:{}, Dis_Now:{}, Dis_Next:{},Reward+Cost:{}, Action:{}'
.format(
d_att_last, d_att_now, d_att_next, rew,
list(mydict.keys())[list(
mydict.values()).index(action)]))
# update sub_obs
sub_obs = sub_new_obs
# Done and Reset
if done:
print('Done infos: ', d_info)
print('======= end =======')
obs = env.reset()
sub_obs = med_libs.custom_obs(obs) # TODO: custom obs
pre_obs = obs # TODO: save obs at t-1
already_starts = False
episode_rewards.append(0.0)
reset = True
# Update the Q network parameters
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# Calculate td-errors
actions = med_libs.action_env_Q(
actions
) # TODO:action_env_Q, from env_action(2~4) to Q_action(0~2)
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically, copy weights of Q to target Q
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act
def update_ec(sequence):
Rt = 0.
for seq in reversed(sequence):
s, a, r = seq
z = np.dot(rp, s.flatten())
Rt = r + 0.99 * Rt
z = z.reshape((latent_dim))
q = ec_buffer[a].add(z, Rt)
# Create training graph and replay buffer
act, train, update_target, debug, get_q_t_selected = deepq.build_train(
make_obs_ph=lambda name: U.Uint8Input(env.observation_space.shape,
name=name),
q_func=dueling_model if args.dueling else model,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr,
epsilon=1e-4),
gamma=0.99,
grad_norm_clipping=10,
double_q=args.double_q,
emdqn=args.emdqn)
def test_agent(): #TODO
tenv, tenv_monitor = make_env(args.env)
tenv.unwrapped.seed(args.seed)
scores = []
for i in range(30):
tobs = tenv.reset()
while True:
action = act(np.array(tobs)[None], stochastic=0.05)[0]
tobs, rew, done, info = tenv.step(action)
def pok_learn(env,
q_func,
lr=5e-4,
max_timesteps=1000, #DP DEL 000
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1500,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space_shape = env.observation_space.shape #ok
def make_obs_ph(name):
return U.BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n, #ok
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n, #ok
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1. #DP - don't need this
# exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
# initial_p=1.0,
# final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
td_error_list = []
saved_mean_reward = None
saved_td_error = None
obs = env.reset() #ok
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()): #DP this somehow uses exploration
break
#DP - not needed
# Take action and update exploration to the newest value
# kwargs = {}
# if not param_noise:
# update_eps = exploration.value(t)
# update_param_noise_threshold = 0.
# else:
# update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
# update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
# kwargs['reset'] = reset
# kwargs['update_param_noise_threshold'] = update_param_noise_threshold
# kwargs['update_param_noise_scale'] = True
action = np.int64(env.action_space.sample()) #act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0] #DP this is what we replace - what does act do??
env_action = action #DP action
reset = False
new_obs, rew, done, _ = env.step(env_action) #ok
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
# #DP EDIT
# print("at step t " + str(t))
# print("printing obses_t, actions, rewards, obses_tp1, dones, weights")
# print(obses_t, actions, rewards, obses_tp1, dones, weights)
# print("%"*30)
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
td_error_list.append(np.mean(np.abs(td_errors)))
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
#DP - convert to TD errors?
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
if len(td_error_list) > 1000 / batch_size:
mean_1000step_tderror = round(np.mean(td_error_list[-int(round(100/batch_size)):-1]),5)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward (how to interpret)?", mean_100ep_reward)
if len(td_error_list) > 1000 / batch_size:
logger.record_tabular("mean abs 1000 td errs", mean_1000step_tderror) #DP
logger.record_tabular("0% time spent exploring since using handlogs", 0) #int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_td_error is None or mean_1000step_tderror < saved_td_error: #mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to new avg trailing td error: {} -> {}".format( #DP
saved_mean_reward, mean_1000step_tderror))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
saved_td_error = mean_1000step_tderror
import pdb; pdb.set_trace()
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward & error: {} and {}".format(saved_mean_reward, saved_td_error))
U.load_state(model_file)
return act
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps, **kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_state(model_file)
return act
else:
container = None
# Create and seed the env.
env, monitored_env = make_env(args.env)
if args.seed > 0:
set_global_seeds(args.seed)
env.unwrapped.seed(args.seed)
with U.make_session(120) as sess:
# Create training graph and replay buffer
if args.bootstrap:
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: U.Uint8Input(
env.observation_space.shape, name=name),
q_func=bootstrap_model,
bootstrap=args.bootstrap,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr),
gamma=0.99,
double_q=args.double_q,
noisy=args.noisy)
else:
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: U.Uint8Input(
env.observation_space.shape, name=name),
q_func=model,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr),
gamma=0.99,
double_q=args.double_q,
noisy=args.noisy)
env = gym.wrappers.Monitor(env, os.path.join(savedir, 'gym_monitor'), force=True)
if savedir:
with open(os.path.join(savedir, 'args.json'), 'w') as f:
json.dump(vars(args), f)
with U.make_session(4) as sess:
# Create training graph and replay buffer
def model_wrapper(img_in, num_actions, scope, **kwargs):
actual_model = dueling_model if args.dueling else model
return actual_model(img_in, num_actions, scope, layer_norm=args.layer_norm, **kwargs)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: U.Uint8Input(env.observation_space.shape, name=name),
q_func=model_wrapper,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=args.lr, epsilon=1e-4),
gamma=0.99,
grad_norm_clipping=10,
double_q=args.double_q,
param_noise=args.param_noise
)
approximate_num_iters = args.num_steps / 4
exploration = PiecewiseSchedule([
(0, 1.0),
(approximate_num_iters / 50, 0.1),
(approximate_num_iters / 5, 0.01)
], outside_value=0.01)
if args.prioritized:
replay_buffer = PrioritizedReplayBuffer(args.replay_buffer_size, args.prioritized_alpha)
beta_schedule = LinearSchedule(approximate_num_iters, initial_p=args.prioritized_beta0, final_p=1.0)
def main():
with U.make_session(8):
env = gym.make("Pendulum-v0")
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: U.BatchInput(env.observation_space.shape,
name=name),
q_func=model,
num_actions=env.action_space,
optimizer=tf.train.AdamOptimizer(learning_rate=5e-4),
)
# Create the replay buffer
replay_buffer = ReplayBuffer(50000)
# Create the schedule for exploration starting from 1 (every action is random) down to
# 0.02 (98% of actions are selected according to values predicted by the model).
exploration = LinearSchedule(schedule_timesteps=10000,
initial_p=1.0,
final_p=0.02)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
obs = env.reset()
for t in itertools.count():
env.render()
# Take action and update exploration to the newest value
action = act(obs[None], update_eps=exploration.value(t))[0]
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0)
is_solved = t > 100 and np.mean(episode_rewards[-101:-1]) >= 200
if is_solved:
# Show off the result
env.render()
else:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if t > 1000:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
32)
train(obses_t, actions, rewards, obses_tp1, dones,
np.ones_like(rewards))
# Update target network periodically.
if t % 1000 == 0:
update_target()
if done and len(episode_rewards) % 10 == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", len(episode_rewards))
logger.record_tabular(
"mean episode reward",
round(np.mean(episode_rewards[-101:-1]), 1))
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
bootstrap=False,
noisy=False,
greedy=False):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
def make_obs_ph(name):
return ObservationInput(env.observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
noisy=noisy,
bootstrap=bootstrap)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
logger.configure('models', ['json', 'stdout'])
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
head = np.random.randint(10) #Initial head initialisation
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_state(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
if bootstrap:
action = act(np.array(obs)[None],
head=head,
update_eps=update_eps)[0]
elif noisy:
action = act(np.array(obs)[None], stochastic=False)[0]
elif greedy:
action = act(np.array(obs)[None], stochastic=False)[0]
else:
action = act(np.array(obs)[None], update_eps=update_eps)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
ep_rew = episode_rewards[-1]
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
if bootstrap:
td_errors = train(obses_t, actions, rewards, obses_tp1,
dones, weights,
learning_rate.value(num_iters))
else:
td_errors = train(obses_t, actions, rewards, obses_tp1,
dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("reward", ep_rew)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
if bootstrap:
logger.record_tabular("head for episode", (head + 1))
logger.dump_tabular()
head = np.random.randint(10)
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_state(model_file)
return act
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None):
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
if(env.is_single):
observation_space_shape = env.observation_space.shape
num_actions = env.action_space.n
else:
observation_space_shape = env.observation_space[0].shape
num_actions = env.action_space[0].n
num_agents=env.agentSize
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size*num_agents, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size*num_agents)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) + exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action=[]
qval=[]
for i in range(num_agents):
prediction=act(np.array(obs[i])[None], update_eps=update_eps, **kwargs)
#print(prediction[0],prediction[1][0])
action.append(prediction[0][0])
qval.append(prediction[1][0])
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action,qval)
# Store transition in the replay buffer.
for i in range(num_agents):
replay_buffer.add(obs[i], action[i], rew, new_obs[i], float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t*num_agents % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
#print(obses_t.shape,actions.shape,rewards.shape,obses_tp1.shape,dones.shape)
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
load_state(model_file)
return act,episode_rewards
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.01,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=50,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
callback=None,
num_optimisation_steps=40):
"""Train a deepq model.
Parameters
-------
env : gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
num_cpu: int
number of cpus to use for training
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
def make_obs_ph(name):
return U.BatchInput((env.observation_space.shape[0] * 2, ), name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_max_rewards = [env.reward_max]
episode_rewards = [0.0]
saved_mean_reward_diff = None # difference in saved reward
obs = env.reset(seed=np.random.randint(0, 1000))
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
episode_buffer = [None] * env.n
episode_timestep = 0
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
action = act(np.concatenate([obs, env.goal])[None],
update_eps=exploration.value(t))[0]
new_obs, rew, done, _ = env.step(action)
# Store transition in the replay buffer.
episode_buffer[episode_timestep] = (obs, action, rew, new_obs,
float(done))
episode_timestep += 1
replay_buffer.add(np.concatenate([obs, env.goal]), action, rew,
np.concatenate([new_obs, env.goal]), float(done))
obs = new_obs
episode_rewards[-1] += rew
num_episodes = len(episode_rewards)
#######end of episode
if done:
for episode in range(episode_timestep):
obs1, action1, _, new_obs1, done1 = episode_buffer[episode]
goal_prime = new_obs1
rew1 = env.calculate_reward(new_obs1, goal_prime)
replay_buffer.add(np.concatenate([obs1, goal_prime]),
action1, rew1,
np.concatenate([new_obs1, goal_prime]),
float(done1))
episode_timestep = 0
obs = env.reset(seed=np.random.randint(0, 1000))
episode_rewards.append(0.0)
episode_max_rewards.append(env.reward_max)
#############Training Q
if t > learning_starts and num_episodes % train_freq == 0:
for i in range(num_optimisation_steps):
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones,
weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1,
dones, weights)
if prioritized_replay:
new_priorities = np.abs(
td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(
batch_idxes, new_priorities)
#############Training Q target
if t > learning_starts and num_episodes % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = np.mean(episode_rewards[-101:-1])
mean_100ep_max_reward = np.mean(episode_max_rewards[-101:-1])
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("mean 100 episode max reward",
mean_100ep_max_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100
and num_episodes % checkpoint_freq == 0):
if saved_mean_reward_diff is None or mean_100ep_max_reward - mean_100ep_reward < saved_mean_reward_diff:
if print_freq is not None:
logger.log(
"Saving model due to mean reward difference decrease: {} -> {}"
.format(saved_mean_reward_diff,
mean_100ep_max_reward - mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward_diff = mean_100ep_max_reward - mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward_diff))
U.load_state(model_file)
return ActWrapper(act, act_params)
def main():
with tf_util.make_session(4) as session:
act_fn, train_fn, target_update_fn, debug_fn = deepq.build_train(
make_obs_ph=lambda name: Uint8Input([input_height, input_width], name=name),
q_func=q_function_nn,
num_actions=action_space_size,
optimizer=tf.train.AdamOptimizer(learning_rate=0.001),
gamma=0.99,
grad_norm_clipping=10,
double_q=False)
epsilon = PiecewiseSchedule([(0, 1.0),
(10000, 1.0), # since we start training at 10000 steps
(20000, 0.4),
(50000, 0.2),
(100000, 0.1),
(500000, 0.05)], outside_value=0.01)
replay_memory = PrioritizedReplayBuffer(replay_memory_size, replay_alpha)
beta = LinearSchedule(int(NUM_STEPS/4), initial_p=replay_beta, final_p=1.0)
tf_util.initialize()
target_update_fn()
state = env.reset()
state = preprocess_frame(state)
watch_train = False
dq = [] # a queue to store episode rewards
start_step = 1
episode = 1
if is_load_model:
dict_state = load_model()
replay_memory = dict_state["replay_memory"]
dq = dict_state["dq"]
start_step = dict_state["step"] + 1
for step in itertools.count(start=start_step):
action = act_fn(state[np.newaxis], update_eps=epsilon.value(step))[0]
state_tplus1, reward, is_finished, _ = env.step(action)
dq.append(reward)
if watch_flag:
env.render()
time.sleep(1.0/fps)
state_tplus1 = preprocess_frame(state_tplus1)
replay_memory.add(state, action, reward, state_tplus1, float(is_finished))
state = state_tplus1
if is_finished:
ep_reward = sum(dq)
log.logkv("Steps", step)
log.logkv("Episode reward", ep_reward)
log.logkv("Episode number", episode)
log.dumpkvs()
print("Step", step, ". Finished episode", episode, "with reward ", ep_reward)
dq = []
state = preprocess_frame(env.reset())
episode += 1
for _ in range(30):
# NOOP for ~90 frames to skip the start screen. Range 30 used because each
# step executed for 3 frames on average. Action 0 stands for doing nothing
env.step(0)
if watch_flag:
env.render()
if step > 10000 and step % learn_freq == 0:
# only start training after 10000 steps are completed
batch = replay_memory.sample(batch_size, beta=beta.value(step))
states = batch[0]
actions = batch[1]
rewards = batch[2]
states_tplus1 = batch[3]
finished_vars = batch[4]
weights = batch[5]
state_indeces = batch[6]
errors = train_fn(states, actions, rewards, states_tplus1, finished_vars, weights)
priority_order_new = np.abs(errors) + replay_epsilon
replay_memory.update_priorities(state_indeces, priority_order_new)
if step % save_freq == 0:
print("State save", step)
dict_state = {
"step": step,
"replay_memory": replay_memory,
"dq": dq
}
save_model(dict_state)
if step > NUM_STEPS:
print("Finished training. Saving model to ./saved_model/model.ckpt")
dict_state = {
"step": step,
"replay_memory": replay_memory,
"dq": dq
}
save_model(dict_state)
break
def learn(env,
q_func,
num_actions=3,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
param_noise=False,
param_noise_threshold=0.05,
callback=None,
demo_replay=[]
):
"""Train a deepq model.
Parameters
-------
env: pysc2.env.SC2Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
num_cpu: int
number of cpus to use for training
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
def make_obs_ph(name):
return U.BatchInput((64, 64), name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
# Select all marines first
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
screen = player_relative
obs = common.init(env, obs)
group_id = 0
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
if param_noise_threshold >= 0.:
update_param_noise_threshold = param_noise_threshold
else:
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(t) + exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# custom process for DefeatZerglingsAndBanelings
obs, screen, player = common.select_marine(env, obs)
action = act(np.array(screen)[None], update_eps=update_eps, **kwargs)[0]
reset = False
rew = 0
new_action = None
obs, new_action = common.marine_action(env, obs, player, action)
army_count = env._obs.observation.player_common.army_count
try:
if army_count > 0 and _ATTACK_SCREEN in obs[0].observation["available_actions"]:
obs = env.step(actions=new_action)
else:
new_action = [sc2_actions.FunctionCall(_NO_OP, [])]
obs = env.step(actions=new_action)
except Exception as e:
#print(e)
1 # Do nothing
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
new_screen = player_relative
rew += obs[0].reward
done = obs[0].step_type == environment.StepType.LAST
selected = obs[0].observation["screen"][_SELECTED]
player_y, player_x = (selected == _PLAYER_FRIENDLY).nonzero()
if(len(player_y)>0):
player = [int(player_x.mean()), int(player_y.mean())]
if(len(player) == 2):
if(player[0]>32):
new_screen = common.shift(LEFT, player[0]-32, new_screen)
elif(player[0]<32):
new_screen = common.shift(RIGHT, 32 - player[0], new_screen)
if(player[1]>32):
new_screen = common.shift(UP, player[1]-32, new_screen)
elif(player[1]<32):
new_screen = common.shift(DOWN, 32 - player[1], new_screen)
# Store transition in the replay buffer.
replay_buffer.add(screen, action, rew, new_screen, float(done))
screen = new_screen
episode_rewards[-1] += rew
reward = episode_rewards[-1]
if done:
print("Episode Reward : %s" % episode_rewards[-1])
obs = env.reset()
player_relative = obs[0].observation["screen"][_PLAYER_RELATIVE]
screen = player_relative
group_list = common.init(env, obs)
# Select all marines first
#env.step(actions=[sc2_actions.FunctionCall(_SELECT_UNIT, [_SELECT_ALL])])
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones, weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes, new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("reward", reward)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
U.load_state(model_file)
return ActWrapper(act)
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=3000,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=3000,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batch sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=lambda name: ObservationInput(env.observation_space, name=name),
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=0.99,
double_q=False
#grad_norm_clipping=10,
# param_noise=param_noise
)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(10000),
initial_p=1.0,
final_p=0.02)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
old_state = None
formula_LTLf_1 = "!d U(g)"
monitoring_RightToLeft = MonitoringSpecification(
ltlf_formula=formula_LTLf_1,
r=0,
c=-0.01,
s=10,
f=-10
)
formula_LTLf_2 = "F(G(bb)) " # break brick
monitoring_BreakBrick = MonitoringSpecification(
ltlf_formula=formula_LTLf_2,
r=10,
c=-0.01,
s=10,
f=0
)
monitoring_specifications = [monitoring_BreakBrick, monitoring_RightToLeft]
def RightToLeftConversion(observation) -> TraceStep:
done=False
global old_state
if arrays_equal(observation[-9:], np.zeros((len(observation[-9:])))): ### Checking if all Bricks are broken
# print('goal reached')
goal = True # all bricks are broken
done = True
else:
goal = False
dead = False
if done and not goal:
dead = True
order = check_ordered(observation[-9:])
if not order:
# print('wrong order', state[5:])
dead=True
done = True
if old_state is not None: # if not the first state
if not arrays_equal(old_state[-9:], observation[-9:]):
brick_broken = True
# check_ordered(state[-9:])
# print(' a brick is broken')
else:
brick_broken = False
else:
brick_broken = False
dictionary={'g': goal, 'd': dead, 'o': order, 'bb':brick_broken}
#print(dictionary)
return dictionary
multi_monitor = MultiRewardMonitor(
monitoring_specifications=monitoring_specifications,
obs_to_trace_step=RightToLeftConversion
)
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
# initialize
done = False
#monitor.get_reward(None, False) # add first state in trace
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
episodeCounter=0
num_episodes=0
for t in itertools.count():
# Take action and update exploration to the newest value
action = act(obs[None], update_eps=exploration.value(t))[0]
#print(action)
#print(action)
new_obs, rew, done, _ = env.step(action)
done=False
#done=False ## FOR FIRE ONLY
#print(new_obs)
#new_obs.append()
start_time = time.time()
rew, is_perm = multi_monitor(new_obs)
#print("--- %s seconds ---" % (time.time() - start_time))
old_state=new_obs
#print(rew)
done=done or is_perm
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
is_solved = t > 100 and np.mean(episode_rewards[-101:-1]) >= 200
if episodeCounter % 100 == 0 or episodeCounter<1:
# Show off the result
#print("coming here Again and Again")
env.render()
if done:
episodeCounter+=1
num_episodes+=1
obs = env.reset()
old_state=None
episode_rewards.append(0)
multi_monitor.reset()
#monitor.get_reward(None, False)
else:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if t > 1000:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(64)
train(obses_t, actions, rewards, obses_tp1, dones, np.ones_like(rewards))
# Update target network periodically.
if t % 1000 == 0:
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
if done and len(episode_rewards) % 10 == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", len(episode_rewards))
logger.record_tabular("currentEpisodeReward", episode_rewards[-1])
logger.record_tabular("mean 100 episode reward", round(np.mean(episode_rewards[-101:-1]), 1))
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts and
num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log("Saving model due to mean reward increase: {} -> {}".format(
saved_mean_reward, mean_100ep_reward))
act.save_act()
#save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
# if model_saved:
# if print_freq is not None:
# logger.log("Restored model with mean reward: {}".format(saved_mean_reward))
# load_variables(model_file)
return act
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
for t in range(total_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
return act, debug['q_func'], debug['obs']
def learn(env,
q_func,
num_actions=4,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=1,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
num_cpu=16,
param_noise=False,
param_noise_threshold=0.05,
callback=None):
"""Train a deepq model.
Parameters
-------
env: pysc2.env.SC2Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
num_cpu: int
number of cpus to use for training
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = U.make_session(num_cpu=num_cpu)
sess.__enter__()
#def make_obs_ph(name):
#return U.BatchInput((16, 16), name=name)
obs_spec = env.observation_spec()[0]
screen_dim = obs_spec['feature_screen'][1:3]
def make_obs_ph(name):
#return ObservationInput(ob_space, name=name)
return ObservationInput(Box(low=0.0,
high=screen_dim[0],
shape=(screen_dim[0], screen_dim[1], 1)),
name=name)
act_x, train_x, update_target_x, debug_x = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
scope="deepq_x")
act_y, train_y, update_target_y, debug_y = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=num_actions,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
scope="deepq_y")
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': num_actions,
}
# Create the replay buffer
if prioritized_replay:
replay_buffer_x = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
replay_buffer_y = PrioritizedReplayBuffer(
buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule_x = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
beta_schedule_y = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer_x = ReplayBuffer(buffer_size)
replay_buffer_y = ReplayBuffer(buffer_size)
beta_schedule_x = None
beta_schedule_y = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target_x()
update_target_y()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
# Select all marines first
obs = env.step(
actions=[sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])])
print(obs[0].observation.keys())
player_relative = obs[0].observation["feature_screen"][_PLAYER_RELATIVE]
screen = (player_relative == _PLAYER_NEUTRAL).astype(int) #+ path_memory
player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join("model/", "mineral_shards")
print(model_file)
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
if param_noise_threshold >= 0.:
update_param_noise_threshold = param_noise_threshold
else:
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) +
exploration.value(t) / float(num_actions))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action_x = act_x(np.array(screen)[None],
update_eps=update_eps,
**kwargs)[0]
action_y = act_y(np.array(screen)[None],
update_eps=update_eps,
**kwargs)[0]
reset = False
coord = [player[0], player[1]]
rew = 0
coord = [action_x, action_y]
if _MOVE_SCREEN not in obs[0].observation["available_actions"]:
obs = env.step(actions=[
sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
])
new_action = [
sc2_actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, coord])
]
# else:
# new_action = [sc2_actions.FunctionCall(_NO_OP, [])]
obs = env.step(actions=new_action)
player_relative = obs[0].observation["feature_screen"][
_PLAYER_RELATIVE]
new_screen = (player_relative == _PLAYER_NEUTRAL).astype(int)
player_y, player_x = (
player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
rew = obs[0].reward
done = obs[0].step_type == environment.StepType.LAST
# Store transition in the replay buffer.
replay_buffer_x.add(screen, action_x, rew, new_screen, float(done))
replay_buffer_y.add(screen, action_y, rew, new_screen, float(done))
screen = new_screen
episode_rewards[-1] += rew
reward = episode_rewards[-1]
if done:
obs = env.reset()
player_relative = obs[0].observation["feature_screen"][
_PLAYER_RELATIVE]
screent = (player_relative == _PLAYER_NEUTRAL).astype(int)
player_y, player_x = (
player_relative == _PLAYER_FRIENDLY).nonzero()
player = [int(player_x.mean()), int(player_y.mean())]
# Select all marines first
env.step(actions=[
sc2_actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL])
])
episode_rewards.append(0.0)
#episode_minerals.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience_x = replay_buffer_x.sample(
batch_size, beta=beta_schedule_x.value(t))
(obses_t_x, actions_x, rewards_x, obses_tp1_x, dones_x,
weights_x, batch_idxes_x) = experience_x
experience_y = replay_buffer_y.sample(
batch_size, beta=beta_schedule_y.value(t))
(obses_t_y, actions_y, rewards_y, obses_tp1_y, dones_y,
weights_y, batch_idxes_y) = experience_y
else:
obses_t_x, actions_x, rewards_x, obses_tp1_x, dones_x = replay_buffer_x.sample(
batch_size)
weights_x, batch_idxes_x = np.ones_like(rewards_x), None
obses_t_y, actions_y, rewards_y, obses_tp1_y, dones_y = replay_buffer_y.sample(
batch_size)
weights_y, batch_idxes_y = np.ones_like(rewards_y), None
td_errors_x = train_x(obses_t_x, actions_x, rewards_x,
obses_tp1_x, dones_x, weights_x)
td_errors_y = train_x(obses_t_y, actions_y, rewards_y,
obses_tp1_y, dones_y, weights_y)
if prioritized_replay:
new_priorities_x = np.abs(
td_errors_x) + prioritized_replay_eps
new_priorities_y = np.abs(
td_errors_y) + prioritized_replay_eps
replay_buffer_x.update_priorities(batch_idxes_x,
new_priorities_x)
replay_buffer_y.update_priorities(batch_idxes_y,
new_priorities_y)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target_x()
update_target_y()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("reward", reward)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
U.save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
U.load_state(model_file)
return ActWrapper(act_x), ActWrapper(act_y)
