class TD3Agent(Agent): def __init__(self, action_space, observation_space, gamma=0.99, nb_steps_warmup=2000, sigma=0.3, polyak=0.995, pi_lr=0.001, q_lr=0.001, batch_size=100, action_noise=0.1, target_noise=0.2, noise_clip=0.5, policy_delay=2, memory_size=10000, training=True): super().__init__() self.gamma = gamma self.sigma = sigma self.polyak = polyak self.pi_lr = pi_lr self.q_lr = q_lr self.batch_size = batch_size self.action_noise = action_noise self.target_noise = target_noise self.noise_clip = noise_clip self.policy_delay = policy_delay self.action_space = action_space self.nb_actions = action_space.shape[0] self.observation_shape = observation_space.shape self.nb_steps_warmup = nb_steps_warmup self.training = training self.memory = Memory(capacity=memory_size, observation_shape=self.observation_shape, action_shape=self.action_space.shape) self.actor_model, self.critic_model1, self.critic_model2 = self._build_network( ) self.target_actor_model, self.target_critic_model1, self.target_critic_model2 = self._build_network( ) self.target_actor_model.set_weights(self.actor_model.get_weights()) self.target_critic_model1.set_weights(self.critic_model1.get_weights()) self.target_critic_model2.set_weights(self.critic_model2.get_weights()) self.step_count = 0 def _build_network(self): action_tensor = tf.keras.layers.Input(shape=(self.nb_actions, ), dtype=tf.float64) observation_tensor = tf.keras.layers.Input( shape=self.observation_shape, dtype=tf.float64) # 创建Actor模型 y = tf.keras.layers.Flatten()(observation_tensor) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(self.nb_actions, activation='tanh')(y) actor_model = tf.keras.Model(inputs=observation_tensor, outputs=y) actor_model.compile(optimizer=tf.keras.optimizers.Adam(lr=self.pi_lr), loss='mse') # 创建Critic1模型 critic_model1 = self._build_critic_network(observation_tensor, action_tensor) # 创建Critic2模型 critic_model2 = self._build_critic_network(observation_tensor, action_tensor) return actor_model, critic_model1, critic_model2 def _build_critic_network(self, observation_tensor, action_tensor): y = tf.keras.layers.Concatenate()([observation_tensor, action_tensor]) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(1, activation='linear')(y) critic_model = tf.keras.Model( inputs=[observation_tensor, action_tensor], outputs=y) critic_model.compile(optimizer=tf.keras.optimizers.Adam(lr=self.q_lr), loss='mse') return critic_model def forward(self, observation): self.step_count += 1 if self.step_count < self.nb_steps_warmup: return self.action_space.sample() else: observation = np.expand_dims(observation, axis=0) action = self.actor_model.predict(observation) action = action.reshape(self.nb_actions) if self.training: action = action + np.clip( np.random.normal(0.0, self.action_noise, self.nb_actions), -self.noise_clip, self.noise_clip) return action def backward(self, observation, action, reward, terminal, next_observation): self.memory.store_transition(observation, action, reward, terminal, next_observation) if self.step_count < self.nb_steps_warmup: return else: self._update() def _update(self): observations, actions, rewards, terminals, next_observations = self.memory.sample_batch( ) self._update_critic(observations, actions, rewards, terminals, next_observations) self._update_actor(observations) if self.step_count % self.policy_delay == 0: # 更新critic的target网络 new_target_critic_weights_list = polyak_averaging( self.critic_model1.get_weights(), self.target_critic_model1.get_weights(), self.polyak) self.target_critic_model1.set_weights( new_target_critic_weights_list) new_target_critic_weights_list = polyak_averaging( self.critic_model2.get_weights(), self.target_critic_model2.get_weights(), self.polyak) self.target_critic_model2.set_weights( new_target_critic_weights_list) # 更新actor的target网络 new_target_actor_weights_list = polyak_averaging( self.actor_model.get_weights(), self.target_actor_model.get_weights(), self.polyak) self.target_actor_model.set_weights(new_target_actor_weights_list) def _update_critic(self, observations, actions, rewards, terminals, next_observations): batch_size = observations.shape[0] q_values_next1 = self.target_critic_model1( [next_observations, self.actor_model(next_observations)]) target1_noise = tf.clip_by_value( tf.random.normal(mean=0.0, stddev=self.target_noise, shape=(batch_size, 1), dtype=tf.float64), -self.noise_clip, self.noise_clip) target_q_values1 = rewards + self.gamma * q_values_next1 + target1_noise q_values_next2 = self.target_critic_model2( [next_observations, self.actor_model(next_observations)]) target2_noise = tf.clip_by_value( tf.random.normal(mean=0.0, stddev=self.target_noise, shape=(batch_size, 1), dtype=tf.float64), -self.noise_clip, self.noise_clip) target_q_values2 = rewards + self.gamma * q_values_next2 + target2_noise target_q_values = tf.minimum(target_q_values1, target_q_values2) self.critic_model1.fit([observations, actions], target_q_values, verbose=0) self.critic_model2.fit([observations, actions], target_q_values, verbose=0) @tf.function def _update_actor(self, observations): with tf.GradientTape() as tape: tape.watch(self.actor_model.trainable_weights) q_values = self.target_critic_model1( [observations, self.actor_model(observations)]) loss = -tf.reduce_mean(q_values) actor_grads = tape.gradient(loss, self.actor_model.trainable_weights) self.actor_model.optimizer.apply_gradients( zip(actor_grads, self.actor_model.trainable_weights))
class DqnCacheEnv(CacheEnv): def __init__(self, capacity, request_path, top_k, time_slot_length, gamma=0.99, memory_size=1000, target_model_update=10): super().__init__(capacity, request_path, top_k, time_slot_length) self.nb_actions = capacity + top_k self.observation_shape = (self.nb_actions, ) # DQN参数 self.gamma = gamma self.memory_size = memory_size self.target_model_update = target_model_update # ReplayBuffer self.memory = Memory(capacity=memory_size, observation_shape=self.observation_shape, action_shape=(self.nb_actions, )) # 创建DQN网络 self.q_net = self._build_q_net() self.target_q_net = self._build_q_net() self.target_q_net.set_weights(self.q_net.get_weights()) self.step_count = 0 self.last_observation = None self.last_action = None self.eps = 1 self.eps_decay = 0.95 self.min_eps = 0.05 def _build_q_net(self): model = tf.keras.Sequential([ tf.keras.layers.Flatten(input_shape=self.observation_shape), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dense(32, activation='relu'), tf.keras.layers.Dense(self.nb_actions, activation='linear') ]) model.compile(optimizer='adam', loss=tf.keras.losses.mean_squared_error) return model def _get_new_cache_content(self, cache_content, top_k_missed_videos, hit_rate): candidates = np.concatenate([cache_content, top_k_missed_videos]) observation = self.loader.get_frequencies(candidates) reward = hit_rate if self.last_observation is not None and self.last_action is not None: self.backward(self.last_observation, self.last_action, reward, False, observation) action = self.forward(observation) new_cache_content = candidates[action] self.last_action = action self.last_observation = observation return new_cache_content def forward(self, observation): self.step_count += 1 action_mask = np.zeros_like(observation, dtype=np.bool) if np.random.random() < max(self.min_eps, self.eps): action = np.random.choice(np.arange(self.capacity + self.top_k), self.capacity, replace=False) else: self.eps *= self.eps_decay if observation.ndim == 1: observation = np.expand_dims(observation, axis=0) q_values = self.q_net.predict(observation).squeeze(0) action = np.argpartition(q_values, -self.capacity)[-self.capacity:] action_mask[action] = True return action_mask def backward(self, observation, action, reward, terminal, next_observation): self.memory.store_transition(observation, action, reward, terminal, next_observation) observations, actions, rewards, terminals, next_observations = self.memory.sample_batch( ) q_values = self.q_net.predict(observations) actions = actions.astype(np.bool) target_q_values = self.target_q_net.predict(next_observations) new_q_values = rewards + self.gamma * target_q_values * (~terminals) q_values[actions] = new_q_values[actions] self.q_net.fit(observations, q_values, verbose=0) if self.step_count % self.target_model_update == 0: self.target_q_net.set_weights(self.q_net.get_weights())
class DQNAgent(Agent): def __init__(self, action_space, observation_space, gamma=0.99, target_model_update=100, memory_size=10000): super().__init__() self.gamma = gamma self.action_space = action_space self.observation_space = observation_space self.nb_actions = action_space.n self.observation_shape = observation_space.shape self.target_model_update = target_model_update self.memory = Memory(capacity=memory_size, action_shape=(1, ), observation_shape=self.observation_shape) self.model = self._build_network() self.target_model = self._build_network() self.target_model.set_weights(self.model.get_weights()) self.update_count = 0 def _build_network(self): model = tf.keras.Sequential([ tf.keras.layers.Flatten(input_shape=self.observation_shape), tf.keras.layers.Dense(128, activation='relu'), tf.keras.layers.Dense(32, activation='relu'), tf.keras.layers.Dense(self.nb_actions, activation='linear') ]) model.compile(optimizer='adam', metrics=['mse'], loss=tf.keras.losses.mean_squared_error) return model def forward(self, observation): observation = np.expand_dims(observation, axis=0) q_values = self.model.predict(observation) return q_values def backward(self, observation, action, reward, terminal, next_observation): self.memory.store_transition(observation, action, reward, terminal, next_observation) observations, actions, rewards, terminals, next_observations = self.memory.sample_batch( ) actions = tf.keras.utils.to_categorical( actions, num_classes=self.nb_actions).astype(np.bool) q_values = self.model.predict(observations) target_q_values = np.max(self.target_model.predict(next_observations), axis=1, keepdims=True) q_values[ actions, np.newaxis] = rewards + self.gamma * target_q_values * (~terminals) self.model.fit(observations, q_values, verbose=0) self.update_count += 1 if self.update_count % self.target_model_update == 0: self.target_model.set_weights(self.model.get_weights())
class DDPGAgent(Agent): def __init__(self, action_space, observation_space, gamma=0.99, nb_steps_warmup=2000, training=True, polyak=0.99, memory_size=10000): super().__init__() self.gamma = gamma self.polyak = polyak self.action_space = action_space self.nb_actions = action_space.shape[0] self.observation_shape = observation_space.shape self.nb_steps_warmup = nb_steps_warmup self.training = training self.memory = Memory(capacity=memory_size, observation_shape=self.observation_shape, action_shape=self.action_space.shape) self.actor_model, self.critic_model = self._build_network() self.target_actor_model, self.target_critic_model = self._build_network( ) self.target_actor_model.set_weights(self.actor_model.get_weights()) self.target_critic_model.set_weights(self.critic_model.get_weights()) self.step_count = 0 self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions, theta=0.15, mu=0., sigma=0.3) def _build_network(self): action_tensor = tf.keras.layers.Input(shape=(self.nb_actions, ), dtype=tf.float64) observation_tensor = tf.keras.layers.Input( shape=self.observation_shape, dtype=tf.float64) # 创建Actor模型 y = tf.keras.layers.Flatten()(observation_tensor) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(self.nb_actions, activation='tanh')(y) actor_model = tf.keras.Model(inputs=observation_tensor, outputs=y) actor_model.compile(optimizer=tf.keras.optimizers.Adam(lr=3e-4), loss='mse') # 创建Critic模型 y = tf.keras.layers.Concatenate()([observation_tensor, action_tensor]) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(32, activation='relu')(y) y = tf.keras.layers.Dense(1, activation='linear')(y) critic_model = tf.keras.Model( inputs=[observation_tensor, action_tensor], outputs=y) critic_model.compile(optimizer=tf.keras.optimizers.Adam(lr=3e-4), loss='mse') return actor_model, critic_model def forward(self, observation): self.step_count += 1 if self.step_count < self.nb_steps_warmup: return self.action_space.sample() else: observation = np.expand_dims(observation, axis=0) action = self.actor_model.predict(observation) action = action.reshape(self.nb_actions) if self.training: action = action + self.random_process.sample() return action def backward(self, observation, action, reward, terminal, next_observation): self.memory.store_transition(observation, action, reward, terminal, next_observation) if self.step_count < self.nb_steps_warmup: return else: self._update() def _update(self): observations, actions, rewards, terminals, next_observations = self.memory.sample_batch( ) self._update_critic(observations, actions, rewards, terminals, next_observations) self._update_actor(observations) # 更新critic的target网络 new_target_critic_weights_list = polyak_averaging( self.critic_model.get_weights(), self.target_critic_model.get_weights(), self.polyak) self.target_critic_model.set_weights(new_target_critic_weights_list) # 更新actor的target网络 new_target_actor_weights_list = polyak_averaging( self.actor_model.get_weights(), self.target_actor_model.get_weights(), self.polyak) self.target_actor_model.set_weights(new_target_actor_weights_list) def polyak_averaging(self, weights_list, target_weights_list): new_target_weights_list = [] for weights, target_weights in zip(weights_list, target_weights_list): new_target_weights = self.polyak * target_weights + ( 1 - self.polyak) * weights new_target_weights_list.append(new_target_weights) return new_target_weights_list def _update_critic(self, observations, actions, rewards, terminals, next_observations): q_values_next = self.target_critic_model( [next_observations, self.actor_model(next_observations)]) target_q_values = rewards + self.gamma * q_values_next self.critic_model.fit([observations, actions], target_q_values, verbose=0) @tf.function def _update_actor(self, observations): with tf.GradientTape() as tape: tape.watch(self.actor_model.trainable_weights) q_values = self.target_critic_model( [observations, self.actor_model(observations)]) loss = -tf.reduce_mean(q_values) actor_grads = tape.gradient(loss, self.actor_model.trainable_weights) self.actor_model.optimizer.apply_gradients( zip(actor_grads, self.actor_model.trainable_weights))
class SacCacheEnv(CacheEnv): def __init__(self, capacity, request_path, top_k, time_slot_length, gamma=0.99, nb_steps_warmup=100, alpha=0.2, polyak=0.995, lr=3e-4, log_std_min=-20, log_std_max=2, memory_size=10000 ): super().__init__(capacity, request_path, top_k, time_slot_length) self.nb_actions = capacity + top_k self.action_dim = 3 self.observation_shape = (self.nb_actions, self.action_dim) self.gamma = gamma self.alpha = alpha self.polyak = polyak self.nb_steps_warmup = nb_steps_warmup self.lr = lr self.step_count = 0 self.log_std_min = log_std_min self.log_std_max = log_std_max self.memory_size = memory_size self.memory = Memory(capacity=memory_size, observation_shape=self.observation_shape, action_shape=(self.nb_actions,)) self.policy_net = self._build_policy_network() self.soft_q_net1 = self._build_q_net() self.soft_q_net2 = self._build_q_net() self.value_net = self._build_value_net() self.target_value_net = self._build_value_net() self.target_value_net.set_weights(self.value_net.get_weights()) self.step_count = 0 self.last_observation = None self.last_action = None def _get_new_cache_content(self, cache_content, top_k_missed_videos, hit_rate): candidates = np.concatenate([cache_content, top_k_missed_videos]) observation = self.loader.get_frequencies2(candidates, [200, 1000, 2000]) reward = hit_rate if self.last_observation is not None and self.last_action is not None: self.backward(self.last_observation, self.last_action, reward, False, observation) action = self.forward(observation) new_cache_content = candidates[action] self.last_action = action self.last_observation = observation return new_cache_content def forward(self, observation): self.step_count += 1 action_mask = np.zeros((self.observation_shape[0],), dtype=np.bool) if observation.ndim == 2: observation = np.expand_dims(observation, axis=0) mean, log_std = self.policy_net.predict(observation) mean = mean.squeeze(0) log_std = log_std.squeeze(0) log_prob = gaussian_likelihood(observation.squeeze(0), mean, log_std) action = np.random.choice(np.arange(self.nb_actions), size=self.capacity, p=softmax(log_prob), replace=False) action_mask[action] = True return action_mask def backward(self, observation, action, reward, terminal, next_observation): self.memory.store_transition(observation, action, reward, terminal, next_observation) if self.step_count >= self.nb_steps_warmup: self._update() new_target_weights = polyak_averaging( self.value_net.get_weights(), self.target_value_net.get_weights(), self.polyak ) self.target_value_net.set_weights(new_target_weights) def _build_policy_network(self): layers = tf.keras.layers observation_tensor = layers.Input(shape=self.observation_shape) y = layers.Flatten()(observation_tensor) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) mean = layers.Dense(self.action_dim, activation='tanh')(y) log_std = layers.Dense(self.action_dim, activation='tanh')(y) log_std = self.log_std_min + 0.5 * (self.log_std_max - self.log_std_min) * (log_std + 1) model = tf.keras.models.Model(inputs=observation_tensor, outputs=[mean, log_std]) model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(self.lr)) return model def _build_q_net(self): layers = tf.keras.layers observation_tensor = layers.Input(shape=self.observation_shape) action_tensor = layers.Input(shape=(self.action_dim,)) observation_tensor_flattened = layers.Flatten()(observation_tensor) y = layers.Concatenate()([observation_tensor_flattened, action_tensor]) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(1)(y) model = tf.keras.models.Model(inputs=[observation_tensor, action_tensor], outputs=y) model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=self.lr)) return model def _build_value_net(self): layers = tf.keras.layers model = tf.keras.models.Sequential([ layers.Flatten(input_shape=self.observation_shape), layers.Dense(32, activation='relu'), layers.Dense(32, activation='relu'), layers.Dense(32, activation='relu'), layers.Dense(1) ]) model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=self.lr)) return model @tf.function def _evaluate(self, observations): mean, log_std = self.policy_net(observations) std = tf.math.exp(log_std) z = mean + tf.random.normal(tf.shape(mean)) * std action = tf.math.tanh(z) log_prob = gaussian_likelihood(z, mean, log_std) log_prob -= tf.math.reduce_sum(tf.math.log(1 - action ** 2 + 1e-6), axis=1) action = tf.cast(action, dtype=tf.float64) return action, log_prob def _update(self): observations, actions, rewards, terminals, next_observations = self.memory.sample_batch() target_q_value = rewards + self.gamma * self.target_value_net.predict(next_observations) soft_actions, log_probs = self._evaluate(observations) soft_q_value1 = self.soft_q_net1.predict([observations, soft_actions]) soft_q_value2 = self.soft_q_net2.predict([observations, soft_actions]) target_value = tf.minimum(soft_q_value1, soft_q_value2) - self.alpha * log_probs # Update soft Q network batch_size = observations.shape[0] actions = actions.astype(np.bool) actions = observations[actions].reshape((batch_size, self.capacity, self.action_dim)).mean(axis=1) self.soft_q_net1.fit([observations, actions], target_q_value, verbose=0) self.soft_q_net2.fit([observations, actions], target_q_value, verbose=0) # Update value network self.value_net.fit(observations, target_value, verbose=0) # Update policy network with tf.GradientTape() as tape: tape.watch(self.policy_net.trainable_weights) soft_actions, log_probs = self._evaluate(observations) soft_q_value = self.soft_q_net1([observations, soft_actions]) loss = -tf.reduce_mean(soft_q_value - self.alpha * log_probs) actor_grads = tape.gradient(loss, self.policy_net.trainable_weights) self.policy_net.optimizer.apply_gradients(zip(actor_grads, self.policy_net.trainable_weights))
class SACAgent(Agent): def __init__(self, action_space, observation_space, gamma=0.99, nb_steps_warmup=2000, alpha=0.2, polyak=0.995, lr=3e-4, log_std_min=-20, log_std_max=2, memory_size=10000 ): super().__init__() self.action_space = action_space self.observation_space = observation_space self.gamma = gamma self.alpha = alpha self.polyak = polyak self.nb_steps_warmup = nb_steps_warmup self.lr = lr self.step_count = 0 self.log_std_min = log_std_min self.log_std_max = log_std_max self.min_entropy = -self.action_space.shape[0] self.memory = Memory(capacity=memory_size, observation_shape=observation_space.shape, action_shape=action_space.shape) self.value_net = self._build_value_network() self.target_value_net = self._build_value_network() self.target_value_net.set_weights(self.value_net.get_weights()) self.soft_q_net1 = self._build_soft_q_network() self.soft_q_net2 = self._build_soft_q_network() self.policy_net = self._build_policy_network() def forward(self, observation): self.step_count += 1 if self.step_count < self.nb_steps_warmup: return self.action_space.sample() else: if observation.ndim == 1: observation = np.expand_dims(observation, axis=0) mean, log_std = self.policy_net.predict(observation) std = tf.math.exp(log_std) action = mean + tf.random.normal(tf.shape(mean)) * std return action def backward(self, observation, action, reward, terminal, next_observation): self.memory.store_transition(observation, action, reward, terminal, next_observation) if self.step_count >= self.nb_steps_warmup: self._update() new_target_weights = polyak_averaging( self.value_net.get_weights(), self.target_value_net.get_weights(), self.polyak ) self.target_value_net.set_weights(new_target_weights) def _update(self): observations, actions, rewards, terminals, next_observations = self.memory.sample_batch() target_q_value = rewards + self.gamma * self.target_value_net.predict(next_observations) soft_actions, log_probs = self.evaluate(observations) soft_q_value1 = self.soft_q_net1.predict([observations, soft_actions]) soft_q_value2 = self.soft_q_net2.predict([observations, soft_actions]) target_value = tf.minimum(soft_q_value1, soft_q_value2) - self.alpha * log_probs # Update soft Q network self.soft_q_net1.fit([observations, actions], target_q_value, verbose=0) self.soft_q_net2.fit([observations, actions], target_q_value, verbose=0) # Update value network self.value_net.fit(observations, target_value, verbose=0) # Update policy network with tf.GradientTape() as tape: tape.watch(self.policy_net.trainable_weights) soft_actions, log_probs = self.evaluate(observations) soft_q_value = self.soft_q_net1([observations, soft_actions]) loss = -tf.reduce_mean(soft_q_value - self.alpha * log_probs) actor_grads = tape.gradient(loss, self.policy_net.trainable_weights) self.policy_net.optimizer.apply_gradients(zip(actor_grads, self.policy_net.trainable_weights)) @tf.function def evaluate(self, observations): mean, log_std = self.policy_net(observations) std = tf.math.exp(log_std) z = mean + tf.random.normal(tf.shape(mean)) * std action = tf.math.tanh(z) log_prob = gaussian_likelihood(z, mean, log_std) log_prob -= tf.math.reduce_sum(tf.math.log(1 - action ** 2 + 1e-6), axis=1) action = tf.cast(action, dtype=tf.float64) return action, log_prob def _build_value_network(self): observation_shape = self.observation_space.shape layers = tf.keras.layers model = tf.keras.models.Sequential([ layers.Flatten(input_shape=observation_shape), layers.Dense(32, activation='relu'), layers.Dense(32, activation='relu'), layers.Dense(32, activation='relu'), layers.Dense(1) ]) model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=self.lr)) return model def _build_soft_q_network(self): observation_shape = self.observation_space.shape nb_actions = self.action_space.shape[0] layers = tf.keras.layers observation_tensor = layers.Input(shape=observation_shape) action_tensor = layers.Input(shape=(nb_actions,)) observation_tensor_flattened = layers.Flatten()(observation_tensor) y = layers.Concatenate()([observation_tensor_flattened, action_tensor]) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(1)(y) model = tf.keras.models.Model(inputs=[observation_tensor, action_tensor], outputs=y) model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(lr=self.lr)) return model def _build_policy_network(self): observation_shape = self.observation_space.shape nb_actions = self.action_space.shape[0] layers = tf.keras.layers observation_tensor = layers.Input(shape=observation_shape) y = layers.Flatten()(observation_tensor) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) y = layers.Dense(32, activation='relu')(y) mean = layers.Dense(nb_actions, activation='tanh')(y) log_std = layers.Dense(nb_actions, activation='tanh')(y) log_std = self.log_std_min + 0.5 * (self.log_std_max - self.log_std_min) * (log_std + 1) model = tf.keras.models.Model(inputs=observation_tensor, outputs=[mean, log_std]) model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(self.lr)) return model