def test_fit_observations():
memory = SequentialMemory(100,
window_length=2,
ignore_episode_boundaries=False)
agent = TestAgent(memory)
env = TestEnv()
agent.compile()
agent.fit(env, 20, verbose=0)
# Inspect memory to see if observations are correct.
experiencies = memory.sample(batch_size=8, batch_idxs=range(8))
assert experiencies[0].reward == .2
assert experiencies[0].action == 1
assert_allclose(experiencies[0].state0, np.array([0, 1]))
assert_allclose(experiencies[0].state1, np.array([1, 2]))
assert experiencies[0].terminal1 is False
assert experiencies[1].reward == .3
assert experiencies[1].action == 2
assert_allclose(experiencies[1].state0, np.array([1, 2]))
assert_allclose(experiencies[1].state1, np.array([2, 3]))
assert experiencies[1].terminal1 is False
assert experiencies[2].reward == .4
assert experiencies[2].action == 3
assert_allclose(experiencies[2].state0, np.array([2, 3]))
assert_allclose(experiencies[2].state1, np.array([3, 4]))
assert experiencies[2].terminal1 is False
assert experiencies[3].reward == .5
assert experiencies[3].action == 4
assert_allclose(experiencies[3].state0, np.array([3, 4]))
assert_allclose(experiencies[3].state1, np.array([4, 5]))
assert experiencies[3].terminal1 is False
assert experiencies[4].reward == .6
assert experiencies[4].action == 5
assert_allclose(experiencies[4].state0, np.array([4, 5]))
assert_allclose(experiencies[4].state1, np.array([5, 6]))
assert experiencies[4].terminal1 is True
# Experience 5 has been re-sampled since since state0 would be terminal in which case we
# cannot really have a meaningful transition because the environment gets reset. We thus
# just ensure that state0 is not terminal.
assert not np.all(experiencies[5].state0 == np.array([5, 6]))
assert experiencies[6].reward == .2
assert experiencies[6].action == 1
assert_allclose(experiencies[6].state0, np.array([0, 1]))
assert_allclose(experiencies[6].state1, np.array([1, 2]))
assert experiencies[6].terminal1 is False
assert experiencies[7].reward == .3
assert experiencies[7].action == 2
assert_allclose(experiencies[7].state0, np.array([1, 2]))
assert_allclose(experiencies[7].state1, np.array([2, 3]))
assert experiencies[7].terminal1 is False
def test_fit_observations():
memory = SequentialMemory(100, window_length=2, ignore_episode_boundaries=False)
agent = TestAgent(memory)
env = TestEnv()
agent.compile()
agent.fit(env, 20, verbose=0)
# Inspect memory to see if observations are correct.
experiencies = memory.sample(batch_size=6, batch_idxs=range(2, 8))
assert experiencies[0].reward == .4
assert experiencies[0].action == 3
assert_allclose(experiencies[0].state0, np.array([2, 3]))
assert_allclose(experiencies[0].state1, np.array([3, 4]))
assert experiencies[0].terminal1 is False
assert experiencies[1].reward == .5
assert experiencies[1].action == 4
assert_allclose(experiencies[1].state0, np.array([3, 4]))
assert_allclose(experiencies[1].state1, np.array([4, 5]))
assert experiencies[1].terminal1 is False
assert experiencies[2].reward == .6
assert experiencies[2].action == 5
assert_allclose(experiencies[2].state0, np.array([4, 5]))
assert_allclose(experiencies[2].state1, np.array([5, 6]))
assert experiencies[2].terminal1 is True
# Experience 3 has been re-sampled since since state0 would be terminal in which case we
# cannot really have a meaningful transition because the environment gets reset. We thus
# just ensure that state0 is not terminal.
assert not np.all(experiencies[3].state0 == np.array([5, 6]))
assert experiencies[4].reward == .2
assert experiencies[4].action == 1
assert_allclose(experiencies[4].state0, np.array([0, 1]))
assert_allclose(experiencies[4].state1, np.array([1, 2]))
assert experiencies[4].terminal1 is False
assert experiencies[5].reward == .3
assert experiencies[5].action == 2
assert_allclose(experiencies[5].state0, np.array([1, 2]))
assert_allclose(experiencies[5].state1, np.array([2, 3]))
assert experiencies[5].terminal1 is False
def test_sampling():
memory = SequentialMemory(100, window_length=2, ignore_episode_boundaries=False)
obs_size = (3, 4)
actions = range(5)
obs0 = np.random.random(obs_size)
terminal0 = False
action0 = np.random.choice(actions)
reward0 = np.random.random()
obs1 = np.random.random(obs_size)
terminal1 = False
action1 = np.random.choice(actions)
reward1 = np.random.random()
obs2 = np.random.random(obs_size)
terminal2 = False
action2 = np.random.choice(actions)
reward2 = np.random.random()
obs3 = np.random.random(obs_size)
terminal3 = True
action3 = np.random.choice(actions)
reward3 = np.random.random()
obs4 = np.random.random(obs_size)
terminal4 = False
action4 = np.random.choice(actions)
reward4 = np.random.random()
obs5 = np.random.random(obs_size)
terminal5 = False
action5 = np.random.choice(actions)
reward5 = np.random.random()
obs6 = np.random.random(obs_size)
terminal6 = False
action6 = np.random.choice(actions)
reward6 = np.random.random()
# memory.append takes the current observation, the reward after taking an action and if
# the *new* observation is terminal, thus `obs0` and `terminal1` is correct.
memory.append(obs0, action0, reward0, terminal1)
memory.append(obs1, action1, reward1, terminal2)
memory.append(obs2, action2, reward2, terminal3)
memory.append(obs3, action3, reward3, terminal4)
memory.append(obs4, action4, reward4, terminal5)
memory.append(obs5, action5, reward5, terminal6)
assert memory.nb_entries == 6
experiences = memory.sample(batch_size=5, batch_idxs=[0, 1, 2, 3, 4])
assert len(experiences) == 5
assert_allclose(experiences[0].state0, np.array([np.zeros(obs_size), obs0]))
assert_allclose(experiences[0].state1, np.array([obs0, obs1]))
assert experiences[0].action == action0
assert experiences[0].reward == reward0
assert experiences[0].terminal1 is False
assert_allclose(experiences[1].state0, np.array([obs0, obs1]))
assert_allclose(experiences[1].state1, np.array([obs1, obs2]))
assert experiences[1].action == action1
assert experiences[1].reward == reward1
assert experiences[1].terminal1 is False
assert_allclose(experiences[2].state0, np.array([obs1, obs2]))
assert_allclose(experiences[2].state1, np.array([obs2, obs3]))
assert experiences[2].action == action2
assert experiences[2].reward == reward2
assert experiences[2].terminal1 is True
# Next experience has been re-sampled since since state0 would be terminal in which case we
# cannot really have a meaningful transition because the environment gets reset. We thus
# just ensure that state0 is not terminal.
assert not np.all(experiences[3].state0 == np.array([obs2, obs3]))
assert_allclose(experiences[4].state0, np.array([np.zeros(obs_size), obs4]))
assert_allclose(experiences[4].state1, np.array([obs4, obs5]))
assert experiences[4].action == action4
assert experiences[4].reward == reward4
assert experiences[4].terminal1 is False
class MACE(Agent):
def __init__(self, env: gym.Env, **kwargs):
super(MACE, self).__init__(**kwargs)
self.nb_actions = env.action_space.shape[0]
obs_input_actor = Input(shape=(1, ) + env.observation_space.shape,
name='observation_input')
x_ac = Flatten()(obs_input_actor)
x_ac = Dense(units=256, activation='relu')(x_ac)
obs_input_critic = Input(shape=(1, ) + env.observation_space.shape,
name='observation_input')
x_cr = Flatten()(obs_input_critic)
x_cr = Dense(units=256, activation='relu')(x_cr)
x_critic = Dense(units=128, activation='relu')(x_cr)
value = Dense(units=1)(x_critic)
x_actor = Dense(units=128, activation='relu')(x_ac)
action = Dense(units=self.nb_actions, activation='tanh')(x_actor)
actor = Model(inputs=obs_input_actor, outputs=action)
critic = Model(inputs=obs_input_critic, outputs=value)
metrics = []
metrics += [mean_q]
critic_metrics = metrics
critic_optimizer = Adam(lr=1e-3)
actor_optimizer = Adam(lr=1e-3)
# critic_optimizer = SGD(lr=1e-4, momentum=0.9)
# actor_optimizer = SGD(lr=1e-3, momentum=0.9)
self.actor = actor
self.critic = critic
self.target_actor = clone_model(self.actor)
self.target_actor.compile(optimizer='sgd', loss='mse')
self.target_critic = clone_model(self.critic)
self.target_critic.compile(optimizer='sgd', loss='mse')
self.target_model_update = 1e-3
#self.target_model_update=500
if self.target_model_update < 1.:
# We use the `AdditionalUpdatesOptimizer` to efficiently soft-update the target model.
critic_updates = get_soft_target_model_updates(
self.target_critic, self.critic, self.target_model_update)
critic_optimizer = AdditionalUpdatesOptimizer(
critic_optimizer, critic_updates)
actor_updates = get_soft_target_model_updates(
self.target_actor, self.actor, self.target_model_update)
actor_optimizer = AdditionalUpdatesOptimizer(
actor_optimizer, actor_updates)
self.delta_clip = np.inf
def clipped_error(y_true, y_pred):
return K.mean(huber_loss(y_true, y_pred, self.delta_clip), axis=-1)
actor.compile(actor_optimizer, loss='mse')
critic.compile(critic_optimizer, loss='mse', metrics=critic_metrics)
self.compiled = True
self.memory = SequentialMemory(limit=100000, window_length=1)
self.memory_interval = 1
self.memory_actor = SequentialMemory(limit=100000, window_length=1)
self.memory_critic = SequentialMemory(limit=100000, window_length=1)
self.nb_steps_warmup = 50000
self.train_interval = 4
self.batch_size = 64
self.gamma = 0.99
self.processor = None
self.random_process = OrnsteinUhlenbeckProcess(theta=.15,
mu=0.,
sigma=.3,
size=self.nb_actions)
self.eps = 0.9
def process_state_batch(self, batch):
batch = np.array(batch)
if self.processor is None:
return batch
return self.processor.process_state_batch(batch)
def select_action(self, state):
batch = [state]
action = self.actor.predict_on_batch(np.asarray(batch)).flatten()
# Apply noise, if a random process is set.
if self.training and self.random_process is not None:
#Actor exploration Bernoulli
# rd = np.random.rand()
# if rd<self.eps:
noise = self.random_process.sample()
assert noise.shape == action.shape
action += noise
# self.action_exploration=True
# else:
# self.action_exploration=False
return action
def forward(self, observation):
# Select an action.
state = self.memory.get_recent_state(observation)
action = self.select_action(state) # TODO: move this into policy
# Book-keeping.
self.recent_observation = observation
self.recent_action = action
return action
def backward(self, reward, terminal=False):
# Store most recent experience in memory.
if self.step % self.memory_interval == 0:
self.memory.append(self.recent_observation,
self.recent_action,
reward,
terminal,
training=self.training)
metrics = [np.nan for _ in self.metrics_names]
if not self.training:
# We're done here. No need to update the experience memory since we only use the working
# memory to obtain the state over the most recent observations.
return metrics
# Train the network on a single stochastic batch.
can_train_either = self.step > self.nb_steps_warmup
if can_train_either and self.step % self.train_interval == 0:
experiences = self.memory.sample(self.batch_size)
assert len(experiences) == self.batch_size
# Start by extracting the necessary parameters (we use a vectorized implementation).
state0_batch = []
reward_batch = []
action_batch = []
terminal1_batch = []
state1_batch = []
for e in experiences:
state0_batch.append(e.state0)
state1_batch.append(e.state1)
reward_batch.append(e.reward)
action_batch.append(e.action)
terminal1_batch.append(0. if e.terminal1 else 1.)
# Prepare and validate parameters.
state0_batch = self.process_state_batch(state0_batch)
state1_batch = self.process_state_batch(state1_batch)
terminal1_batch = np.array(terminal1_batch)
reward_batch = np.array(reward_batch)
action_batch = np.array(action_batch)
assert reward_batch.shape == (self.batch_size, )
assert terminal1_batch.shape == reward_batch.shape
assert action_batch.shape == (self.batch_size, self.nb_actions)
# Update actor and critic, if warm up is over.
if self.step > self.nb_steps_warmup:
if len(self.critic.inputs) >= 3:
state1_batch_with_action = state1_batch[:]
else:
state1_batch_with_action = [state1_batch]
target_q_values = self.target_critic.predict_on_batch(
state1_batch_with_action).flatten()
assert target_q_values.shape == (self.batch_size, )
# Compute r_t + gamma * max_a Q(s_t+1, a) and update the target ys accordingly,
# but only for the affected output units (as given by action_batch).
discounted_reward_batch = self.gamma * target_q_values
discounted_reward_batch *= terminal1_batch
assert discounted_reward_batch.shape == reward_batch.shape
targets = (reward_batch + discounted_reward_batch).reshape(
self.batch_size, 1)
# Perform a single batch update on the critic network.
if len(self.critic.inputs) >= 3:
state0_batch_with_action = state0_batch[:]
else:
state0_batch_with_action = [state0_batch]
#state0_batch_with_action.insert(self.critic_action_input_idx, action_batch)
metrics = self.critic.train_on_batch(state0_batch_with_action,
targets)
if self.processor is not None:
metrics += self.processor.metrics
#Actor
experiences = self.memory.sample(self.batch_size)
assert len(experiences) == self.batch_size
# Start by extracting the necessary parameters (we use a vectorized implementation).
state0_batch = []
reward_batch = []
action_batch = []
terminal1_batch = []
state1_batch = []
for e in experiences:
state0_batch.append(e.state0)
state1_batch.append(e.state1)
reward_batch.append(e.reward)
action_batch.append(e.action)
terminal1_batch.append(0. if e.terminal1 else 1.)
# Prepare and validate parameters.
state0_batch = self.process_state_batch(state0_batch)
state1_batch = self.process_state_batch(state1_batch)
terminal1_batch = np.array(terminal1_batch)
reward_batch = np.array(reward_batch)
action_batch = np.array(action_batch)
assert reward_batch.shape == (self.batch_size, )
assert terminal1_batch.shape == reward_batch.shape
assert action_batch.shape == (self.batch_size, self.nb_actions)
if self.step > self.nb_steps_warmup:
#Actor
target_q_values1 = self.target_critic.predict_on_batch(
state1_batch_with_action).flatten()
discounted_reward_batch = self.gamma * target_q_values1
discounted_reward_batch *= terminal1_batch
targets = (reward_batch + discounted_reward_batch)
target_q_values0 = self.target_critic.predict_on_batch(
state0_batch_with_action).flatten()
delta = targets - target_q_values0
if len(self.actor.inputs) >= 2:
inputs = state0_batch[:]
else:
#inputs = [state0_batch]
inputs = state0_batch
pos_dif = delta > 0
# if self.step%1000==0:
# print(np.sum(pos_dif))
inputs = np.asarray(inputs)[pos_dif]
actions_target = action_batch[pos_dif]
#state0_batch_with_action.insert(self.critic_action_input_idx, action_batch)
self.actor.train_on_batch(inputs, actions_target)
if self.target_model_update >= 1 and self.step % self.target_model_update == 0:
self.update_target_models_hard()
return metrics
def reset_states(self):
if self.random_process is not None:
self.random_process.reset_states()
self.recent_action = None
self.recent_observation = None
if self.compiled:
self.actor.reset_states()
self.critic.reset_states()
self.target_actor.reset_states()
self.target_critic.reset_states()
def update_target_models_hard(self):
self.target_critic.set_weights(self.critic.get_weights())
self.target_actor.set_weights(self.actor.get_weights())
@property
def metrics_names(self):
names = self.critic.metrics_names[:]
if self.processor is not None:
names += self.processor.metrics_names[:]
return names
class deepAMDP():
def __init__(self,
inputDim=16,
alpha=1e-4,
gamma=0.99,
epsilon=0.1,
numberOfActions=0,
tau=1e-1):
self.predictionModel = Sequential()
self.predictionModel.add(
Dense(16, input_dim=inputDim, activation='relu'))
self.predictionModel.add(Dense(16, activation='relu'))
self.predictionModel.add(Dense(numberOfActions, activation='linear'))
self.predictionModel.compile(loss="mse", optimizer=Adam(lr=alpha))
self.targetModel = Sequential()
self.targetModel.add(Dense(16, input_dim=inputDim))
self.targetModel.add(Dense(16, activation='relu'))
self.targetModel.add(Dense(numberOfActions, activation="linear"))
self.targetModel.compile(loss="mse", optimizer=Adam(lr=alpha))
self.memory = SequentialMemory(limit=100000, window_length=1)
self.otherMemory = deque(maxlen=2000)
self.numberOfActions = numberOfActions
self.alpha = alpha
self.gamma = gamma
self.tau = tau
self.epsilon = epsilon
self.initialEpsilon = 0.1
self.finalEpsilon = 0.01
self.currentEpsilon = self.initialEpsilon
self.episodesToDecay = 500
def addExperience(self, latentState, action, reward, done):
self.memory.append(latentState, action, reward, done)
#print(latentState, action, reward)
def memorize(self, state, action, reward, next_state, done):
self.otherMemory.append((state, action, reward, next_state, done))
def action(self, state):
if np.random.random() < self.currentEpsilon:
return np.random.randint(self.numberOfActions)
state = state.reshape(1, -1)
qValues = self.predictionModel.predict(state)
return np.argmax(self.predictionModel.predict(state)[0])
def replay(self, batchSize=8):
#print("replay")
#if len(self.memory) < batchSize:
# return
experiences = self.memory.sample(batchSize)
# Start by extracting the necessary parameters (we use a vectorized implementation).
state0Batch = []
rewardBatch = []
actionBatch = []
terminal1Batch = []
state1Batch = []
for e in experiences:
# print(e.state0, e.state1, e.reward, e.action)
state0Batch.append(e.state0[0])
state1Batch.append(e.state1[0])
rewardBatch.append(e.reward)
actionBatch.append(e.action)
terminal1Batch.append(0. if e.terminal1 else 1.)
state0Batch = np.array(state0Batch)
rewardBatch = np.array(rewardBatch)
actionBatch = np.array(actionBatch)
terminal1Batch = np.array(terminal1Batch)
state1Batch = np.array(state1Batch)
#state0Batch = normalize(state0Batch, axis=-1)
#state1Batch = normalize(state1Batch, axis=-1)
targetQValues = self.targetModel.predict_on_batch(state1Batch)
#print("Target Q Values")
#print(targetQValues)
#print("Target Q 1")
#print(state1Batch[0])
qBatch = np.max(targetQValues, axis=1).flatten()
#targets = np.zeros((batchSize, self.numberOfActions))
#targets = np.random.rand(batchSize, self.numberOfActions)
targets = targetQValues
discountedRewardBatch = self.gamma * qBatch
discountedRewardBatch *= terminal1Batch
Rs = rewardBatch + discountedRewardBatch
#print("Our Targets")
#print(targets)
for (target, r, action) in zip(targets, Rs, actionBatch):
target[action] = r
#print("Updated Targets")
#print(targets)
self.predictionModel.fit(state0Batch, targets, verbose=0)
#self.updateTargetModel()
def otherReplay(self, batch_size=8):
minibatch = random.sample(self.otherMemory, batch_size)
for state, action, reward, next_state, done in minibatch:
target = reward
state = state.reshape(1, -1)
next_state = next_state.reshape(1, -1)
if not done:
target = (reward + self.gamma *
np.amax(self.predictionModel.predict(next_state)[0]))
target_f = self.predictionModel.predict(state)
target_f[0][action] = target
self.predictionModel.fit(state, target_f, epochs=1, verbose=0)
#self.updateTargetModel()
def updateTargetModel(self):
predictionWeights = self.predictionModel.get_weights()
targetWeights = self.targetModel.get_weights()
for i in range(0, len(targetWeights)):
targetWeights[i] = predictionWeights[i] * self.tau + targetWeights[
i] * (1 - self.tau)
self.targetModel.set_weights(targetWeights)
