class Agent:
def __init__(self,
n_actions,
input_dims,
gamma=0.99,
alpha=0.0003,
gae_lambda=0.95,
policy_clip=0.2,
batch_size=64,
n_epochs=10,
chkpt_dir='models/'):
self.gamma = gamma
self.policy_clip = policy_clip
self.n_epochs = n_epochs
self.gae_lambda = gae_lambda
self.chkpt_dir = chkpt_dir
self.actor = ActorNetwork(n_actions)
self.actor.compile(optimizer=Adam(learning_rate=alpha))
self.critic = CriticNetwork()
self.critic.compile(optimizer=Adam(learning_rate=alpha))
self.memory = PPOMemory(batch_size)
def store_transition(self, state, action, probs, vals, reward, done):
self.memory.store_memory(state, action, probs, vals, reward, done)
def save_models(self):
print('... saving models ...')
self.actor.save(self.chkpt_dir + 'actor')
self.critic.save(self.chkpt_dir + 'critic')
def load_models(self):
print('... loading models ...')
self.actor = keras.models.load_model(self.chkpt_dir + 'actor')
self.critic = keras.models.load_model(self.chkpt_dir + 'critic')
def choose_action(self, observation):
state = tf.convert_to_tensor([observation])
probs = self.actor(state)
dist = tfp.distributions.Categorical(probs)
action = dist.sample()
log_prob = dist.log_prob(action)
value = self.critic(state)
action = action.numpy()[0]
value = value.numpy()[0]
log_prob = log_prob.numpy()[0]
return action, log_prob, value
def learn(self):
for _ in range(self.n_epochs):
state_arr, action_arr, old_prob_arr, vals_arr,\
reward_arr, dones_arr, batches = \
self.memory.generate_batches()
values = vals_arr
advantage = np.zeros(len(reward_arr), dtype=np.float32)
for t in range(len(reward_arr) - 1):
discount = 1
a_t = 0
for k in range(t, len(reward_arr) - 1):
a_t += discount * (reward_arr[k] +
self.gamma * values[k + 1] *
(1 - int(dones_arr[k])) - values[k])
discount *= self.gamma * self.gae_lambda
advantage[t] = a_t
for batch in batches:
with tf.GradientTape(persistent=True) as tape:
states = tf.convert_to_tensor(state_arr[batch])
old_probs = tf.convert_to_tensor(old_prob_arr[batch])
actions = tf.convert_to_tensor(action_arr[batch])
probs = self.actor(states)
dist = tfp.distributions.Categorical(probs)
new_probs = dist.log_prob(actions)
critic_value = self.critic(states)
critic_value = tf.squeeze(critic_value, 1)
prob_ratio = tf.math.exp(new_probs - old_probs)
weighted_probs = advantage[batch] * prob_ratio
clipped_probs = tf.clip_by_value(prob_ratio,
1 - self.policy_clip,
1 + self.policy_clip)
weighted_clipped_probs = clipped_probs * advantage[batch]
actor_loss = -tf.math.minimum(weighted_probs,
weighted_clipped_probs)
actor_loss = tf.math.reduce_mean(actor_loss)
returns = advantage[batch] + values[batch]
# critic_loss = tf.math.reduce_mean(tf.math.pow(
# returns-critic_value, 2))
critic_loss = keras.losses.MSE(critic_value, returns)
actor_params = self.actor.trainable_variables
actor_grads = tape.gradient(actor_loss, actor_params)
critic_params = self.critic.trainable_variables
critic_grads = tape.gradient(critic_loss, critic_params)
self.actor.optimizer.apply_gradients(
zip(actor_grads, actor_params))
self.critic.optimizer.apply_gradients(
zip(critic_grads, critic_params))
self.memory.clear_memory()
class Agent:
def __init__(self,
alpha=0.0003,
beta=0.0003,
input_dims=[8],
env=None,
gamma=0.99,
n_actions=2,
max_size=1000000,
tau=0.005,
layer1_size=256,
layer2_size=256,
batch_size=256,
reward_scale=2):
self.gamma = gamma
self.tau = tau
self.memory = ReplayBuffer(max_size, input_dims, n_actions)
self.batch_size = batch_size
self.n_actions = n_actions
self.actor = ActorNetwork(n_actions=n_actions,
name='actor',
max_action=env.action_space.high)
self.critic_1 = CriticNetwork(n_actions=n_actions, name='critic_1')
self.critic_2 = CriticNetwork(n_actions=n_actions, name='critic_2')
self.value = ValueNetwork(name='value')
self.target_value = ValueNetwork(name='target_value')
self.actor.compile(optimizer=Adam(learning_rate=alpha))
self.critic_1.compile(optimizer=Adam(learning_rate=beta))
self.critic_2.compile(optimizer=Adam(learning_rate=beta))
self.value.compile(optimizer=Adam(learning_rate=beta))
self.target_value.compile(optimizer=Adam(learning_rate=beta))
self.scale = reward_scale
self.update_network_parameters(tau=1)
def choose_action(self, observation):
state = tf.convert_to_tensor([observation])
actions, _ = self.actor.sample_normal(state, reparameterize=False)
return actions[0]
def remember(self, state, action, reward, new_state, done):
self.memory.store_transition(state, action, reward, new_state, done)
def update_network_parameters(self, tau=None):
if tau is None:
tau = self.tau
weights = []
targets = self.target_value.weights
for i, weight in enumerate(self.value.weights):
weights.append(weight * tau + targets[i] * (1 - tau))
self.target_value.set_weights(weights)
def save_models(self):
print('... saving models ...')
self.actor.save_weights(self.actor.checkpoint_file)
self.critic_1.save_weights(self.critic_1.checkpoint_file)
self.critic_2.save_weights(self.critic_2.checkpoint_file)
self.value.save_weights(self.value.checkpoint_file)
self.target_value.save_weights(self.target_value.checkpoint_file)
def load_models(self):
print('... loading models ...')
self.actor.load_weights(self.actor.checkpoint_file)
self.critic_1.load_weights(self.critic_1.checkpoint_file)
self.critic_2.load_weights(self.critic_2.checkpoint_file)
self.value.load_weights(self.value.checkpoint_file)
self.target_value.load_weights(self.target_value.checkpoint_file)
def learn(self):
if self.memory.mem_cntr < self.batch_size:
return
state, action, reward, new_state, done = \
self.memory.sample_buffer(self.batch_size)
states = tf.convert_to_tensor(state, dtype=tf.float32)
states_ = tf.convert_to_tensor(new_state, dtype=tf.float32)
rewards = tf.convert_to_tensor(reward, dtype=tf.float32)
actions = tf.convert_to_tensor(action, dtype=tf.float32)
with tf.GradientTape() as tape:
value = tf.squeeze(self.value(states), 1)
value_ = tf.squeeze(self.target_value(states_), 1)
current_policy_actions, log_probs = self.actor.sample_normal(
states, reparameterize=False)
log_probs = tf.squeeze(log_probs, 1)
q1_new_policy = self.critic_1(states, current_policy_actions)
q2_new_policy = self.critic_2(states, current_policy_actions)
critic_value = tf.squeeze(
tf.math.minimum(q1_new_policy, q2_new_policy), 1)
value_target = critic_value - log_probs
value_loss = 0.5 * keras.losses.MSE(value, value_target)
value_network_gradient = tape.gradient(value_loss,
self.value.trainable_variables)
self.value.optimizer.apply_gradients(
zip(value_network_gradient, self.value.trainable_variables))
with tf.GradientTape() as tape:
# in the original paper, they reparameterize here. We don't implement
# this so it's just the usual action.
new_policy_actions, log_probs = self.actor.sample_normal(
states, reparameterize=True)
log_probs = tf.squeeze(log_probs, 1)
q1_new_policy = self.critic_1(states, new_policy_actions)
q2_new_policy = self.critic_2(states, new_policy_actions)
critic_value = tf.squeeze(
tf.math.minimum(q1_new_policy, q2_new_policy), 1)
actor_loss = log_probs - critic_value
actor_loss = tf.math.reduce_mean(actor_loss)
actor_network_gradient = tape.gradient(actor_loss,
self.actor.trainable_variables)
self.actor.optimizer.apply_gradients(
zip(actor_network_gradient, self.actor.trainable_variables))
with tf.GradientTape(persistent=True) as tape:
# I didn't know that these context managers shared values?
q_hat = self.scale * reward + self.gamma * value_ * (1 - done)
q1_old_policy = tf.squeeze(self.critic_1(state, action), 1)
q2_old_policy = tf.squeeze(self.critic_2(state, action), 1)
critic_1_loss = 0.5 * keras.losses.MSE(q1_old_policy, q_hat)
critic_2_loss = 0.5 * keras.losses.MSE(q2_old_policy, q_hat)
critic_1_network_gradient = tape.gradient(
critic_1_loss, self.critic_1.trainable_variables)
critic_2_network_gradient = tape.gradient(
critic_2_loss, self.critic_2.trainable_variables)
self.critic_1.optimizer.apply_gradients(
zip(critic_1_network_gradient, self.critic_1.trainable_variables))
self.critic_2.optimizer.apply_gradients(
zip(critic_2_network_gradient, self.critic_2.trainable_variables))
self.update_network_parameters()
class Agent:
def __init__(self,
input_dims,
alpha=0.001,
beta=0.002,
env=None,
gamma=0.99,
n_actions=2,
max_size=1000000,
tau=0.005,
fc1=400,
fc2=300,
batch_size=64,
noise=0.1):
self.gamma = gamma
self.tau = tau
self.memory = ReplayBuffer(max_size, input_dims, n_actions)
self.batch_size = batch_size
self.n_actions = n_actions
self.noise = noise
self.max_action = env.action_space.high[0]
self.min_action = env.action_space.low[0]
self.actor = ActorNetwork(n_actions=n_actions, name='actor')
self.critic = CriticNetwork(name='critic')
self.target_actor = ActorNetwork(n_actions=n_actions,
name='target_actor')
self.target_critic = CriticNetwork(name='target_critic')
self.actor.compile(optimizer=Adam(learning_rate=alpha))
self.critic.compile(optimizer=Adam(learning_rate=beta))
self.target_actor.compile(optimizer=Adam(learning_rate=alpha))
self.target_critic.compile(optimizer=Adam(learning_rate=beta))
self.update_network_parameters(tau=1)
def update_network_parameters(self, tau=None):
if tau is None:
tau = self.tau
weights = []
targets = self.target_actor.weights
for i, weight in enumerate(self.actor.weights):
weights.append(weight * tau + targets[i] * (1 - tau))
self.target_actor.set_weights(weights)
weights = []
targets = self.target_critic.weights
for i, weight in enumerate(self.critic.weights):
weights.append(weight * tau + targets[i] * (1 - tau))
self.target_critic.set_weights(weights)
def remember(self, state, action, reward, new_state, done):
self.memory.store_transition(state, action, reward, new_state, done)
def save_models(self):
print('... saving models ...')
self.actor.save_weights(self.actor.checkpoint_file)
self.target_actor.save_weights(self.target_actor.checkpoint_file)
self.critic.save_weights(self.critic.checkpoint_file)
self.target_critic.save_weights(self.target_critic.checkpoint_file)
def load_models(self):
print('... loading models ...')
self.actor.load_weights(self.actor.checkpoint_file)
self.target_actor.load_weights(self.target_actor.checkpoint_file)
self.critic.load_weights(self.critic.checkpoint_file)
self.target_critic.load_weights(self.target_critic.checkpoint_file)
def choose_action(self, observation, evaluate=False):
state = tf.convert_to_tensor([observation], dtype=tf.float32)
actions = self.actor(state)
if not evaluate:
actions += tf.random.normal(shape=[self.n_actions],
mean=0.0,
stddev=self.noise)
# note that if the env has an action > 1, we have to multiply by
# max action at some point
actions = tf.clip_by_value(actions, self.min_action, self.max_action)
return actions[0]
def learn(self):
if self.memory.mem_cntr < self.batch_size:
return
state, action, reward, new_state, done = \
self.memory.sample_buffer(self.batch_size)
states = tf.convert_to_tensor(state, dtype=tf.float32)
states_ = tf.convert_to_tensor(new_state, dtype=tf.float32)
rewards = tf.convert_to_tensor(reward, dtype=tf.float32)
actions = tf.convert_to_tensor(action, dtype=tf.float32)
with tf.GradientTape() as tape:
target_actions = self.target_actor(states_)
critic_value_ = tf.squeeze(
self.target_critic(states_, target_actions), 1)
critic_value = tf.squeeze(self.critic(states, actions), 1)
target = rewards + self.gamma * critic_value_ * (1 - done)
critic_loss = keras.losses.MSE(target, critic_value)
critic_network_gradient = tape.gradient(
critic_loss, self.critic.trainable_variables)
self.critic.optimizer.apply_gradients(
zip(critic_network_gradient, self.critic.trainable_variables))
with tf.GradientTape() as tape:
new_policy_actions = self.actor(states)
actor_loss = -self.critic(states, new_policy_actions)
actor_loss = tf.math.reduce_mean(actor_loss)
actor_network_gradient = tape.gradient(actor_loss,
self.actor.trainable_variables)
self.actor.optimizer.apply_gradients(
zip(actor_network_gradient, self.actor.trainable_variables))
self.update_network_parameters()
class Agent:
""" 2019 State-of-the-Art Implementation of SAC with optimized temperature
"""
def __init__(self,
env,
lr_Q=3e-4,
lr_actor=3e-4,
lr_a=3e-4,
gamma=0.99,
tau=0.005,
layer1_size=256,
layer2_size=256,
batch_size=256,
max_size=1000000,
warmup=1000,
policy_delay=1,
minimum_entropy=None):
self.env = env
self.action_range = [env.action_space.low, env.action_space.high]
self.n_states = env.observation_space.shape[0]
self.n_actions = env.action_space.shape[0]
self.min_action = env.action_space.low
self.max_action = env.action_space.high
self.gamma = gamma
self.tau = tau
self.batch_size = batch_size
self.warmup = warmup
self.time_step = 0
self.update_step = 0
self.policy_delay = policy_delay
self.policy_net = ActorNetwork(n_states=self.n_states,
n_actions=self.n_actions,
fc1_dims=layer1_size,
fc2_dims=layer2_size,
network_name='Actor')
self.q_net1 = CriticNetwork(n_states=self.n_states,
n_actions=self.n_actions,
hidden_neurons_1=layer1_size,
hidden_neurons_2=layer2_size,
network_name='Critic_1')
self.q_net2 = CriticNetwork(n_states=self.n_states,
n_actions=self.n_actions,
hidden_neurons_1=layer1_size,
hidden_neurons_2=layer2_size,
network_name='Critic_2')
self.target_q_net1 = CriticNetwork(n_states=self.n_states,
n_actions=self.n_actions,
hidden_neurons_1=layer1_size,
hidden_neurons_2=layer2_size,
network_name='Target_Critic_1')
self.target_q_net2 = CriticNetwork(n_states=self.n_states,
n_actions=self.n_actions,
hidden_neurons_1=layer1_size,
hidden_neurons_2=layer2_size,
network_name='Target_Critic_2')
self.replay_buffer = ReplayBuffer(n_actions=self.n_actions,
n_states=self.n_states,
memory_size=max_size)
self.policy_net.compile(optimizer=tf.keras.optimizers.Adam(
lr=lr_actor))
self.q_net1.compile(optimizer=tf.keras.optimizers.Adam(lr=lr_Q))
self.q_net2.compile(optimizer=tf.keras.optimizers.Adam(lr=lr_Q))
self.update_target_networks(
tau=1) # copy parameters to target networks
# entropy temperature parameter alpha
# self.log_alpha = tf.Variable(0.0, dtype=tf.float32)
print(-tf.constant(env.action_space.shape[0], dtype=tf.float32))
self.log_alpha = tf.Variable(tf.zeros(1), trainable=True)
self.minimum_entropy = -tf.reduce_prod(
tf.convert_to_tensor(env.action_space.shape, dtype=tf.float32))
self.minimum_entropy = -tf.reduce_prod(
tf.convert_to_tensor(env.action_space.shape, dtype=tf.float32)
) if minimum_entropy is None else minimum_entropy
print('Minimum Entropy set to: ', self.minimum_entropy)
self.alpha_optimizer = tf.keras.optimizers.Adam(learning_rate=lr_a)
self.alpha = tf.exp(self.log_alpha).numpy()
print('alpha: ', self.alpha)
def choose_action(self, state):
if self.time_step < self.warmup:
actions = np.random.uniform(
low=-1.0, high=1.0, size=self.n_actions
) # "random uniform distribution over all valid actions"
actions = tf.convert_to_tensor(actions, dtype=tf.float32)
else:
state = tf.convert_to_tensor(state, dtype=tf.float32)
state = tf.expand_dims(state, axis=0)
actions, _ = self.policy_net(state)
self.time_step += 1
if self.time_step == self.warmup:
print('No warmup anymore!')
a = self.rescale_action(actions[0].numpy())
return a
def scale_action(self, action):
""" Scale all actions to [-1., +1.]
:param action: unscaled actions
:return: scaled actions all in range -1. .. +1.
"""
# old = 2 * (action - self.min_action) / (self.max_action - self.min_action) - 1.0
scale = (2 * action - (self.action_range[1] + self.action_range[0])) / \
(self.action_range[1] - self.action_range[0])
return scale
def rescale_action(self, action):
""" Rescale all scaled actions to environment actionspace values
:param action: scaled actions
:return: rescaled actions all in range min_action .. max_action
"""
# old = (action + 1.0) * (self.max_action - self.min_action) / 2.0 + self.min_action
rescale = action * (self.action_range[1] - self.action_range[0]) / 2.0 + \
(self.action_range[1] + self.action_range[0]) / 2.0
return rescale
def remember(self, state, action, reward, new_state, done):
action = self.scale_action(action) # ÄNDERUNG! Funktioniert das mit?
self.replay_buffer.store_environment_transition(
state, action, reward, new_state, done)
def update_target_networks(self, tau=None):
if tau is None:
tau = self.tau
weights = []
for theta_target, theta in zip(self.target_q_net1.get_weights(),
self.q_net1.get_weights()):
theta_target = tau * theta + (1 - tau) * theta_target
weights.append(theta_target)
self.target_q_net1.set_weights(weights)
weights = []
for theta_target, theta in zip(self.target_q_net2.get_weights(),
self.q_net2.get_weights()):
theta_target = tau * theta + (1 - tau) * theta_target
weights.append(theta_target)
self.target_q_net2.set_weights(weights)
# weights = []
# theta_target = self.target_q_net1.weights
# for i, theta in enumerate(self.q_net1.weights):
# weights.append(tau*theta + (1-tau)*theta_target[i])
# self.target_q_net1.set_weights(weights)
#
# weights = []
# theta_target = self.target_q_net2.weights
# for i, theta in enumerate(self.q_net2.weights):
# weights.append(tau*theta + (1-tau)*theta_target[i])
# self.target_q_net2.set_weights(weights)
def save_models(self):
print('models saved') # To Do!
def load_models(self):
print('models loaded') # To Do!
def learn(self):
if self.replay_buffer.count < self.batch_size:
return
elif self.replay_buffer.count == self.batch_size:
print('Buffer Size equals batch Size! - Learning begins!')
return
# sample batch from replay buffer
states, actions, rewards, next_states, dones = self.replay_buffer.sample_from_buffer(
batch_size=self.batch_size)
# convert batchs from 2D numpy arrays to tensorflow tensors
states = tf.convert_to_tensor(states, dtype=tf.float32)
actions = tf.convert_to_tensor(actions, dtype=tf.float32)
next_states = tf.convert_to_tensor(next_states, dtype=tf.float32)
# expand rewards and dones from 1D numpy arrays to 2D tensors and reshape them
rewards = tf.convert_to_tensor(rewards, dtype=tf.float32)
rewards = tf.expand_dims(rewards, axis=0)
rewards = tf.reshape(rewards, [self.batch_size, 1])
dones = tf.convert_to_tensor(dones, dtype=tf.float32)
dones = tf.expand_dims(dones, axis=0)
dones = tf.reshape(dones, [self.batch_size, 1])
## Update critic networks Q1 & Q2
with tf.GradientTape(persistent=True) as tape_Q:
next_actions, next_log_pi = self.policy_net(next_states)
Q1_next = self.target_q_net1(next_states, next_actions)
Q2_next = self.target_q_net2(next_states, next_actions)
next_q_target = tf.minimum(Q1_next,
Q2_next) - self.alpha * next_log_pi
expected_q = tf.stop_gradient(rewards + (1 - dones) * self.gamma *
next_q_target)
curr_q1 = self.q_net1(states, actions)
curr_q2 = self.q_net2(states, actions)
q1_loss = tf.reduce_mean((curr_q1 - expected_q)**2)
q2_loss = tf.reduce_mean((curr_q2 - expected_q)**2) # tf.square()
q_loss = q1_loss + q2_loss
grad_Q1 = tape_Q.gradient(q_loss, self.q_net1.trainable_variables)
grad_Q2 = tape_Q.gradient(q_loss, self.q_net2.trainable_variables)
self.q_net1.optimizer.apply_gradients(
zip(grad_Q1, self.q_net1.trainable_variables))
self.q_net2.optimizer.apply_gradients(
zip(grad_Q2, self.q_net2.trainable_variables))
## Update policy network and polyak update target Q networks less frequently (like in TD3 --> "Delayed SAC")
if self.update_step % self.policy_delay == 0:
with tf.GradientTape() as tape_policy:
new_actions, log_pi = self.policy_net(states)
Q1 = self.q_net1(states, new_actions)
Q2 = self.q_net2(states, new_actions)
Q_min = tf.minimum(Q1, Q2)
loss_policy = tf.reduce_mean(self.alpha * log_pi - Q_min)
grad_policy = tape_policy.gradient(
loss_policy, self.policy_net.trainable_variables)
self.policy_net.optimizer.apply_gradients(
zip(grad_policy, self.policy_net.trainable_variables))
self.update_target_networks(
) # update target networks with polyak averaging
## Update temperature parameter alpha
with tf.GradientTape() as tape:
_, log_pi_a = self.policy_net(states)
alpha_loss = tf.reduce_mean(self.log_alpha *
(-log_pi_a - self.minimum_entropy))
grads = tape.gradient(alpha_loss, [self.log_alpha])
self.alpha_optimizer.apply_gradients(zip(grads, [self.log_alpha]))
self.alpha = tf.exp(self.log_alpha).numpy()
self.update_step += 1 # Keep track of the number of network updates
