def __init__(self, size, alpha=0.6, beta_start=0.4, beta_frames=100000):
"""Create Prioritized Replay buffer.
Parameters
----------
size: int
Max number of transitions to store in the buffer. When the buffer
overflows the old memories are dropped.
alpha: float
how much prioritization is used
(0 - no prioritization, 1 - full prioritization)
See Also
--------
ReplayBuffer.__init__
"""
super(PrioritizedReplayMemory, self).__init__()
self._storage = []
self._maxsize = size
self._next_idx = 0
assert alpha >= 0
self._alpha = alpha
self.beta_start = beta_start
self.beta_frames = beta_frames
self.frame = 1
it_capacity = 1
while it_capacity < size:
it_capacity *= 2
self._it_sum = SumSegmentTree(it_capacity)
self._it_min = MinSegmentTree(it_capacity)
self._max_priority = 1.0
def __init__(self,
capacity,
alpha=0.6,
beta_start=0.4,
beta_frames=100000):
"""
初始化继承经验回抽机制
:param capacity: 经验回收容量
:param alpha: 优先级随机采样重要性
:param beta_start: 使用重要性的相关性 0不相关 1完全相关
:param beta_frames: 计算beta折扣所用，超过此数后beta=1，表示更新权重与优先级取样的损失完全相关
"""
super(PrioritizedReplayMemory, self).__init__(capacity)
# # 回放内存容量
# self.capacity = capacity
# # 存储数组
# self.memory = []
self._next_idx = 0
assert alpha > 0
self._alpha = alpha
self.beta_start = beta_start
self.beta_frames = beta_frames
self.frame = 1
it_capacity = 1
while it_capacity < capacity:
it_capacity *= 2
self._it_sum = SumSegmentTree(it_capacity)
self._it_min = MinSegmentTree(it_capacity)
self._max_priority = 1.0
def __init__(self, size, alpha=0.6, beta_start=0.4, beta_frames=100000):
super(PrioritizedReplayMemory, self).__init__()
self._storage = []
self._maxsize = size
self._next_idx = 0
assert alpha >= 0
self._alpha = alpha
self.beta_start = beta_start
self.beta_frames = beta_frames
self.frame=1
it_capacity = 1
while it_capacity < size:
it_capacity *= 2
self._it_sum = SumSegmentTree(it_capacity)
self._it_min = MinSegmentTree(it_capacity)
self._max_priority = 1.0
class PrioritizedReplayMemory(object):
def __init__(self, size, alpha=0.6, beta_start=0.4, beta_frames=100000):
"""Create Prioritized Replay buffer.
Parameters
----------
size: int
Max number of transitions to store in the buffer. When the buffer
overflows the old memories are dropped.
alpha: float
how much prioritization is used
(0 - no prioritization, 1 - full prioritization)
See Also
--------
ReplayBuffer.__init__
"""
super(PrioritizedReplayMemory, self).__init__()
self._storage = []
self._maxsize = size
self._next_idx = 0
assert alpha >= 0
self._alpha = alpha
self.beta_start = beta_start
self.beta_frames = beta_frames
self.frame = 1
it_capacity = 1
while it_capacity < size:
it_capacity *= 2
self._it_sum = SumSegmentTree(it_capacity)
self._it_min = MinSegmentTree(it_capacity)
self._max_priority = 1.0
def beta_by_frame(self, frame_idx):
return min(
1.0, self.beta_start + frame_idx *
(1.0 - self.beta_start) / self.beta_frames)
def push(self, data):
"""See ReplayBuffer.store_effect"""
idx = self._next_idx
if self._next_idx >= len(self._storage):
self._storage.append(data)
else:
self._storage[self._next_idx] = data
self._next_idx = (self._next_idx + 1) % self._maxsize
self._it_sum[idx] = self._max_priority**self._alpha
self._it_min[idx] = self._max_priority**self._alpha
def _encode_sample(self, idxes):
return [self._storage[i] for i in idxes]
def _sample_proportional(self, batch_size):
res = []
for _ in range(batch_size):
# TODO(szymon): should we ensure no repeats?
mass = random.random() * self._it_sum.sum(0,
len(self._storage) - 1)
idx = self._it_sum.find_prefixsum_idx(mass)
res.append(idx)
return res
def sample(self, batch_size):
"""Sample a batch of experiences.
compared to ReplayBuffer.sample
it also returns importance weights and idxes
of sampled experiences.
Parameters
----------
batch_size: int
How many transitions to sample.
beta: float
To what degree to use importance weights
(0 - no corrections, 1 - full correction)
Returns
-------
obs_batch: np.array
batch of observations
act_batch: np.array
batch of actions executed given obs_batch
rew_batch: np.array
rewards received as results of executing act_batch
next_obs_batch: np.array
next set of observations seen after executing act_batch
done_mask: np.array
done_mask[i] = 1 if executing act_batch[i] resulted in
the end of an episode and 0 otherwise.
weights: np.array
Array of shape (batch_size,) and dtype np.float32
denoting importance weight of each sampled transition
idxes: np.array
Array of shape (batch_size,) and dtype np.int32
idexes in buffer of sampled experiences
"""
idxes = self._sample_proportional(batch_size)
weights = []
#find smallest sampling prob: p_min = smallest priority^alpha / sum of priorities^alpha
p_min = self._it_min.min() / self._it_sum.sum()
beta = self.beta_by_frame(self.frame)
self.frame += 1
#max_weight given to smallest prob
max_weight = (p_min * len(self._storage))**(-beta)
for idx in idxes:
p_sample = self._it_sum[idx] / self._it_sum.sum()
weight = (p_sample * len(self._storage))**(-beta)
weights.append(weight / max_weight)
weights = torch.tensor(weights, device=device, dtype=torch.float)
encoded_sample = self._encode_sample(idxes)
return encoded_sample, idxes, weights
def update_priorities(self, idxes, priorities):
"""Update priorities of sampled transitions.
sets priority of transition at index idxes[i] in buffer
to priorities[i].
Parameters
----------
idxes: [int]
List of idxes of sampled transitions
priorities: [float]
List of updated priorities corresponding to
transitions at the sampled idxes denoted by
variable `idxes`.
"""
assert len(idxes) == len(priorities)
for idx, priority in zip(idxes, priorities):
assert 0 <= idx < len(self._storage)
self._it_sum[idx] = (priority + 1e-5)**self._alpha
self._it_min[idx] = (priority + 1e-5)**self._alpha
self._max_priority = max(self._max_priority, (priority + 1e-5))
class PrioritizedReplayMemory(object):
def __init__(self, size, alpha=0.6, beta_start=0.4, beta_frames=100000):
super(PrioritizedReplayMemory, self).__init__()
self._storage = []
self._maxsize = size
self._next_idx = 0
assert alpha >= 0
self._alpha = alpha
self.beta_start = beta_start
self.beta_frames = beta_frames
self.frame=1
it_capacity = 1
while it_capacity < size:
it_capacity *= 2
self._it_sum = SumSegmentTree(it_capacity)
self._it_min = MinSegmentTree(it_capacity)
self._max_priority = 1.0
def beta_by_frame(self, frame_idx):
return min(1.0, self.beta_start + frame_idx * (1.0 - self.beta_start) / self.beta_frames)
def push(self, data):
idx = self._next_idx
if self._next_idx >= len(self._storage):
self._storage.append(data)
else:
self._storage[self._next_idx] = data
self._next_idx = (self._next_idx + 1) % self._maxsize
self._it_sum[idx] = self._max_priority ** self._alpha
self._it_min[idx] = self._max_priority ** self._alpha
def _encode_sample(self, idxes):
return [self._storage[i] for i in idxes]
def _sample_proportional(self, batch_size):
res = []
for _ in range(batch_size):
mass = random.random() * self._it_sum.sum(0, len(self._storage) - 1)
idx = self._it_sum.find_prefixsum_idx(mass)
res.append(idx)
return res
def sample(self, batch_size):
idxes = self._sample_proportional(batch_size)
weights = []
#find smallest sampling prob: p_min = smallest priority^alpha / sum of priorities^alpha
p_min = self._it_min.min() / self._it_sum.sum()
beta = self.beta_by_frame(self.frame)
self.frame+=1
#max_weight given to smallest prob
max_weight = (p_min * len(self._storage)) ** (-beta)
for idx in idxes:
p_sample = self._it_sum[idx] / self._it_sum.sum()
weight = (p_sample * len(self._storage)) ** (-beta)
weights.append(weight / max_weight)
weights = torch.tensor(weights, device=device, dtype=torch.float)
encoded_sample = self._encode_sample(idxes)
return encoded_sample, idxes, weights
def update_priorities(self, idxes, priorities):
assert len(idxes) == len(priorities)
for idx, priority in zip(idxes, priorities):
assert 0 <= idx < len(self._storage)
self._it_sum[idx] = (priority+1e-5) ** self._alpha
self._it_min[idx] = (priority+1e-5) ** self._alpha
self._max_priority = max(self._max_priority, (priority+1e-5))
def __len__(self):
return len(self._storage)
class PrioritizedReplayMemory(ExperienceReplayMemory):
def __init__(self,
capacity,
alpha=0.6,
beta_start=0.4,
beta_frames=100000):
"""
初始化继承经验回抽机制
:param capacity: 经验回收容量
:param alpha: 优先级随机采样重要性
:param beta_start: 使用重要性的相关性 0不相关 1完全相关
:param beta_frames: 计算beta折扣所用，超过此数后beta=1，表示更新权重与优先级取样的损失完全相关
"""
super(PrioritizedReplayMemory, self).__init__(capacity)
# # 回放内存容量
# self.capacity = capacity
# # 存储数组
# self.memory = []
self._next_idx = 0
assert alpha > 0
self._alpha = alpha
self.beta_start = beta_start
self.beta_frames = beta_frames
self.frame = 1
it_capacity = 1
while it_capacity < capacity:
it_capacity *= 2
self._it_sum = SumSegmentTree(it_capacity)
self._it_min = MinSegmentTree(it_capacity)
self._max_priority = 1.0
def beta_by_frame(self, frame_idx):
return min(
1.0, self.beta_start + frame_idx *
(1.0 - self.beta_start) / self.beta_frames)
def push(self, transition):
"""
以循环队列的方式存储经验
:param transition:
:return:
"""
# 下一个id
idx = self._next_idx
# 若循环队列未填满则添加，否则替换
if self._next_idx >= len(self.memory):
self.memory.append(transition)
else:
self.memory[self._next_idx] = transition
self._next_idx = (self._next_idx + 1) % self.capacity
# 将新放入的权重设为最大
self._it_sum[idx] = self._max_priority**self._alpha
self._it_min[idx] = self._max_priority**self._alpha
# 返回记忆队列的指定位置的场景
def _encode_sample(self, indices):
return [self.memory[i] for i in indices]
# 以比例方式抽样
def _sample_proportional(self, batch_size):
res = []
# print(len(self.memory), self._it_sum.sum(0, len(self.memory) - 1))
# print(self._it_sum.sum(0,))
for _ in range(batch_size):
mass = random.random() * self._it_sum.sum(0, len(self.memory) - 1)
idx = self._it_sum.find_prefixsum_idx(mass)
res.append(idx)
return res
# 根据batch大小优先级取样
def sample(self, batch_size):
# 回放机制中抽样的位置
indices = self._sample_proportional(batch_size)
# 抽样结果的权重
weights = list()
# 获取最小权重的概率
p_min = self._it_min.min() / self._it_sum.sum()
# 获取beta值，表示权重对取样的重要性
beta = self.beta_by_frame(self.frame)
self.frame += 1
# 获取最大权重
max_weight = (p_min * len(self.memory))**(-beta)
for idx in indices:
p_sample = self._it_sum[idx] / self._it_sum.sum()
weight = (p_sample * len(self.memory))**(-beta)
weights.append(weight / max_weight)
weights = t.tensor(weights, device=cfg.device, dtype=t.float)
encode_sample = self._encode_sample(indices)
return encode_sample, indices, weights
# 更新优先级
def update_priorities(self, indices, priorities):
assert len(indices) == len(priorities)
# 对所有抽样出来的经验更新其优先级
for idx, priority in zip(indices, priorities):
assert 0 <= idx < len(self.memory)
# 优先级为priority**alpha,其中alpha表示优先级重要性
self._it_sum[idx] = (priority + 1e-5)**self._alpha
self._it_min[idx] = (priority + 1e-5)**self._alpha
self._max_priority = max(self._max_priority, (priority + 1e-5))
# 保存经验回放内存到文件
def save_replay(self):
pickle.dump(
self.memory,
open(cfg.model_dir + 'saved_agents/exp_replay_agent.dump', 'wb'))
# 从文件中加载经验回放内存
def load_replay(self):
replay_name = cfg.model_dir + 'saved_agents/exp_replay_agent.dump' 'saved_agents/exp_replay_agent.dump'
if os.path.isfile(replay_name):
self.memory = pickle.load(open(replay_name, 'rb'))
# 重写len方法保持能够返回长度
def __len__(self):
return len(self.memory)