class DQN: def __init__(self, state_size, action_size, memory_size): self.state_size = state_size self.action_size = action_size self.memory = ReplayBuffer(memory_size) self.gamma = 0.95 # discount rate self.epsilon = 1.0 # exploration rate self.epsilon_min = 0.01 self.epsilon_max = 1 self.decay_step = 0 self.epsilon_decay = 0.0001 self.learning_rate = 0.001 self.model = self._build_model() self.target_model = self._build_model() self.update_target_model() def _huber_loss(self, y_true, y_pred, clip_delta=1.0): error = y_true - y_pred cond = K.abs(error) <= clip_delta squared_loss = 0.5 * K.square(error) quadratic_loss = 0.5 * K.square(clip_delta) + clip_delta * (K.abs(error) - clip_delta) return K.mean(tf.where(cond, squared_loss, quadratic_loss)) def _build_model(self): # Neural Net for Deep-Q learning Model model = models.Sequential() model.add(layers.Conv2D(32, (8, 8), strides=(4,4), activation='elu', input_shape=self.state_size)) model.add(layers.Conv2D(64, (4,4), strides=(2,2), activation='elu')) model.add(layers.Flatten()) model.add(layers.Dense(64, activation='relu')) model.add(layers.Dense(self.action_size, activation='linear')) model.compile(loss=self._huber_loss, optimizer=Adam(lr=self.learning_rate)) return model def update_target_model(self): # Copy weights from model to target_model print('Updating target model...') self.target_model.set_weights(self.model.get_weights()) def remember(self, experience): self.memory.store_experience(experience) def act(self, state): if np.random.rand() <= self.epsilon: return random.randrange(self.action_size) state = np.reshape(state, (1, 42, 42, self.state_size[2])) act_values = self.model.predict(state) return np.argmax(act_values[0]) # returns action def replay(self, batch_size): minibatch = self.memory.get_experiences(batch_size) states, targets_f = [], [] for state, action, reward, next_state, done in minibatch: state = np.reshape(state, (1, 42, 42, self.state_size[2])) next_state = np.reshape(next_state, (1, 42, 42, self.state_size[2])) target = self.model.predict(state) if done: target[0][action] = reward else: t = self.target_model.predict(next_state)[0] target[0][action] = reward + self.gamma * np.amax(t) states.append(state[0]) targets_f.append(target[0]) self.model.fit(np.array(states), np.array(targets_f), batch_size=batch_size, epochs=1, verbose=0) if self.epsilon > self.epsilon_min: self.epsilon = self.epsilon_min + (self.epsilon_max - self.epsilon_min) * np.exp(-self.epsilon_decay * self.decay_step) self.decay_step += 1 def load(self, name): self.model.load_weights(name) def save(self, name): self.model.save_weights(name)