def test_simple_importance_sampling(self):
"""
Test importance sampling when the buffer is never
changed after the initial build-up.
"""
np.random.seed(1337)
buf = PrioritizedReplayBuffer(capacity=10,
alpha=1.5,
beta=1.3,
epsilon=0.5)
for i in range(10):
sample = {
'obs': 0,
'action': 0,
'reward': 0,
'new_obs': 0,
'steps': 1,
'idx': i
}
buf.add_sample(sample, init_weight=i)
weights = np.power(np.arange(10).astype('float64') + 0.5, 1.5)
weights /= np.sum(weights)
weights = np.power(weights * len(weights), -1.3)
weights /= np.max(weights)
for i in range(1000):
samples = buf.sample(3)
for sample in samples:
self.assertTrue(
np.allclose(weights[sample['idx']], sample['weight']))
def test_uniform_sampling(self):
"""
Test the buffer when it's configured to sample
uniformly.
"""
np.random.seed(1337)
buf = PrioritizedReplayBuffer(capacity=10, alpha=0, beta=1)
for i in range(10):
sample = {
'obs': 0,
'action': 0,
'reward': 0,
'new_obs': 0,
'steps': 1,
'idx': i
}
buf.add_sample(sample)
sampled_idxs = []
for _ in range(10000):
samples = buf.sample(3)
sampled_idxs.extend([s['idx'] for s in samples])
buf.update_weights(samples, [s['idx'] for s in samples])
counts = Counter(sampled_idxs)
for i in range(10):
frac = counts[i] / len(sampled_idxs)
self.assertGreater(frac, 0.09)
self.assertLess(frac, 0.11)
def test_prioritized_sampling(self):
"""
Test the buffer in a simple prioritized setting.
"""
np.random.seed(1337)
buf = PrioritizedReplayBuffer(capacity=10,
alpha=1.5,
beta=1,
epsilon=0.5)
for i in range(10):
sample = {
'obs': 0,
'action': 0,
'reward': 0,
'new_obs': 0,
'steps': 1,
'idx': i
}
buf.add_sample(sample, init_weight=i)
sampled_idxs = []
for i in range(50000):
for sample in buf.sample(3):
sampled_idxs.append(sample['idx'])
counts = Counter(sampled_idxs)
probs = np.power(np.arange(10).astype('float64') + 0.5, 1.5)
probs /= np.sum(probs)
for i, prob in enumerate(probs):
frac = counts[i] / len(sampled_idxs)
self.assertGreater(frac, prob - 0.01)
self.assertLess(frac, prob + 0.01)
def main():
env_name = 'MineRLNavigateDense-v0'
"""Run DQN until the environment throws an exception."""
base_env = [SimpleNavigateEnvWrapper(get_env(env_name)) for _ in range(1)]
env = BatchedFrameStack(BatchedGymEnv([base_env]),
num_images=4,
concat=True)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # pylint: disable=E1101
with tf.Session(config=config) as sess:
online, target = mine_rainbow_online_target(mine_cnn,
sess,
env.action_space.n,
gym_space_vectorizer(
env.observation_space),
min_val=-200,
max_val=200)
dqn = DQN(online, target)
player = NStepPlayer(BatchedPlayer(env, dqn.online_net), 3)
optimize = dqn.optimize(learning_rate=1e-4)
sess.run(tf.global_variables_initializer())
buffer_capacity = 5000
replay_buffer = PrioritizedReplayBuffer(buffer_capacity,
0.5,
0.4,
epsilon=0.1)
iter = non_bugged_data_arr(env_name, num_trajs=100)
expert_player = NStepPlayer(ImitationPlayer(iter, 200), 3)
for traj in expert_player.play():
replay_buffer.add_sample(traj, init_weight=1)
print('starting training')
dqn.train(num_steps=200,
player=player,
replay_buffer=replay_buffer,
optimize_op=optimize,
train_interval=1,
target_interval=8192,
batch_size=32,
min_buffer_size=20000)
print('starting eval')
player._cur_states = None
score = evaluate(player)
print(score)
