class ReplayBuffer(AbstractReplayBuffer): def __init__(self, capacity=None): self.memory = RandomAccessQueue(maxlen=capacity) def append(self, state, action, reward, next_state=None, next_action=None, is_state_terminal=False): experience = dict(state=state, action=action, reward=reward, next_state=next_state, next_action=next_action, is_state_terminal=is_state_terminal) self.memory.append(experience) def sample(self, n): assert len(self.memory) >= n return self.memory.sample(n) def __len__(self): return len(self.memory) def save(self, filename): with open(filename, 'wb') as f: pickle.dump(self.memory, f) def load(self, filename): with open(filename, 'rb') as f: self.memory = pickle.load(f) if isinstance(self.memory, collections.deque): # Load v0.2 self.memory = RandomAccessQueue( self.memory, maxlen=self.memory.maxlen) def stop_current_episode(self): pass
class ReplayBuffer(object): def __init__(self, capacity=None): self.memory = RandomAccessQueue(maxlen=capacity) def append(self, state, action, reward, next_state=None, next_action=None, is_state_terminal=False): """Append a transition to this replay buffer Args: state: s_t action: a_t reward: r_t next_state: s_{t+1} (can be None if terminal) next_action: a_{t+1} (can be None for off-policy algorithms) is_state_terminal (bool) """ experience = dict(state=state, action=action, reward=reward, next_state=next_state, next_action=next_action, is_state_terminal=is_state_terminal) self.memory.append(experience) def sample(self, n): """Sample n unique samples from this replay buffer""" assert len(self.memory) >= n return self.memory.sample(n) def __len__(self): return len(self.memory) def save(self, filename): with open(filename, 'wb') as f: pickle.dump(self.memory, f) def load(self, filename): with open(filename, 'rb') as f: self.memory = pickle.load(f) def stop_current_episode(self): pass
class EpisodicReplayBuffer(AbstractEpisodicReplayBuffer): def __init__(self, capacity=None): self.current_episode = [] self.episodic_memory = RandomAccessQueue() self.memory = RandomAccessQueue() self.capacity = capacity def append(self, state, action, reward, next_state=None, next_action=None, is_state_terminal=False, **kwargs): experience = dict(state=state, action=action, reward=reward, next_state=next_state, next_action=next_action, is_state_terminal=is_state_terminal, **kwargs) self.current_episode.append(experience) if is_state_terminal: self.stop_current_episode() def sample(self, n): assert len(self.memory) >= n return self.memory.sample(n) def sample_episodes(self, n_episodes, max_len=None): assert len(self.episodic_memory) >= n_episodes episodes = self.episodic_memory.sample(n_episodes) if max_len is not None: return [random_subseq(ep, max_len) for ep in episodes] else: return episodes def __len__(self): return len(self.memory) @property def n_episodes(self): return len(self.episodic_memory) def save(self, filename): with open(filename, 'wb') as f: pickle.dump((self.memory, self.episodic_memory), f) def load(self, filename): with open(filename, 'rb') as f: memory = pickle.load(f) if isinstance(memory, tuple): self.memory, self.episodic_memory = memory else: # Load v0.2 # FIXME: The code works with EpisodicReplayBuffer # but not with PrioritizedEpisodicReplayBuffer self.memory = RandomAccessQueue(memory) self.episodic_memory = RandomAccessQueue() # Recover episodic_memory with best effort. episode = [] for item in self.memory: episode.append(item) if item['is_state_terminal']: self.episodic_memory.append(episode) episode = [] def stop_current_episode(self): if self.current_episode: self.episodic_memory.append(self.current_episode) self.memory.extend(self.current_episode) self.current_episode = [] while self.capacity is not None and \ len(self.memory) > self.capacity: discarded_episode = self.episodic_memory.popleft() for _ in range(len(discarded_episode)): self.memory.popleft() assert not self.current_episode
class TestRandomAccessQueue(unittest.TestCase): def setUp(self): if self.init_seq: self.y_queue = RandomAccessQueue(self.init_seq, maxlen=self.maxlen) self.t_queue = collections.deque(self.init_seq, maxlen=self.maxlen) else: self.y_queue = RandomAccessQueue(maxlen=self.maxlen) self.t_queue = collections.deque(maxlen=self.maxlen) def test1(self): self.check_all() self.check_popleft() self.do_append(10) self.check_all() self.check_popleft() self.check_popleft() self.do_append(11) self.check_all() # test negative indices n = len(self.t_queue) for i in range(-n, 0): self.check_getitem(i) for k in range(4): self.do_extend(range(k)) self.check_all() for k in range(4): self.check_popleft() self.do_extend(range(k)) self.check_all() for k in range(10): self.do_append(20 + k) self.check_popleft() self.check_popleft() self.check_all() for _ in range(100): self.check_popleft() def check_all(self): self.check_len() n = len(self.t_queue) for i in range(n): self.check_getitem(i) def check_len(self): self.assertEqual(len(self.y_queue), len(self.t_queue)) def check_getitem(self, i): self.assertEqual(self.y_queue[i], self.t_queue[i]) def do_setitem(self, i, x): self.y_queue[i] = x self.t_queue[i] = x def do_append(self, x): self.y_queue.append(x) self.t_queue.append(x) def do_extend(self, xs): self.y_queue.extend(xs) self.t_queue.extend(xs) def check_popleft(self): try: t = self.t_queue.popleft() except IndexError: with self.assertRaises(IndexError): self.y_queue.popleft() else: self.assertEqual(self.y_queue.popleft(), t)
class ValueBuffer(with_metaclass(ABCMeta, object)): """non-parametricQ値を出力するためのbuffer""" def __init__(self, capacity = 2000, lookup_k = 5, n_action = None, key_size = 256, xp = np): self.capacity = capacity self.memory = RandomAccessQueue(maxlen=capacity) self.lookup_k = lookup_k self.xp = xp self.num_action = n_action self.key_size = key_size assert self.num_action self.tmp_emb_arr = self.xp.empty((0, self.key_size), dtype='float32') self.knn = knn.ArgsortKnn(capacity = self.capacity, dimension=key_size, xp = self.xp) def __len__(self): return len(self.memory) def store(self, embedding, q_np): # value bufferに保存する self._store(dict(embedding = embedding, action_value = q_np)) #knnにembeddingを送る self.knn.add(embedding) assert len(self.knn) == len(self.memory) assert self.memory[0]['embedding'][0,0] == self.knn.head_emb() if len(self.memory) == self.capacity: assert self.memory[-1]['embedding'][-1,0] == self.knn.end_emb() # 戻り値はなし (必要ならつける) return def _store(self, dictionaries): # 蓄える(容量いっぱいのときなどの処理は場合分け) self.memory.append(dictionaries) while self.capacity is not None and \ len(self.memory) > self.capacity: self.memory.popleft() def compute_q(self, embedding): """ if len(self.memory) < self.lookup_k: k = len(self.memory) else: k = self.lookup_k """ index_list = self.knn.search(embedding, self.lookup_k) tmp_vbuf = self.xp.asarray([self.memory[i]['action_value'] for i in index_list], dtype=self.xp.float32) q_np = self.xp.average(tmp_vbuf, axis=0) return q_np
class ReplayBuffer(replay_buffer.AbstractReplayBuffer): """Experience Replay Buffer As described in https://storage.googleapis.com/deepmind-media/dqn/DQNNaturePaper.pdf. Args: capacity (int): capacity in terms of number of transitions num_steps (int): Number of timesteps per stored transition (for N-step updates) """ def __init__(self, capacity=None, num_steps=1): self.capacity = capacity assert num_steps > 0 self.num_steps = num_steps self.memory = RandomAccessQueue(maxlen=capacity) self.last_n_transitions = collections.defaultdict( lambda: collections.deque([], maxlen=num_steps)) def append(self, state, action, reward, next_state=None, next_action=None, is_state_terminal=False, env_id=0, **kwargs): last_n_transitions = self.last_n_transitions[env_id] experience = dict(state=state, action=action, reward=reward, next_state=next_state, next_action=next_action, is_state_terminal=is_state_terminal, **kwargs) last_n_transitions.append(experience) if is_state_terminal: while last_n_transitions: self.memory.append(list(last_n_transitions)) del last_n_transitions[0] assert len(last_n_transitions) == 0 else: if len(last_n_transitions) == self.num_steps: self.memory.append(list(last_n_transitions)) def stop_current_episode(self, env_id=0): last_n_transitions = self.last_n_transitions[env_id] # if n-step transition hist is not full, add transition; # if n-step hist is indeed full, transition has already been added; if 0 < len(last_n_transitions) < self.num_steps: self.memory.append(list(last_n_transitions)) # avoid duplicate entry if 0 < len(last_n_transitions) <= self.num_steps: del last_n_transitions[0] while last_n_transitions: self.memory.append(list(last_n_transitions)) del last_n_transitions[0] assert len(last_n_transitions) == 0 def sample(self, num_experiences): assert len(self.memory) >= num_experiences return self.memory.sample(num_experiences) def __len__(self): return len(self.memory) def save(self, filename): with open(filename, 'wb') as f: pickle.dump(self.memory, f) def load(self, filename): with open(filename, 'rb') as f: self.memory = pickle.load(f) if isinstance(self.memory, collections.deque): # Load v0.2 self.memory = RandomAccessQueue(self.memory, maxlen=self.memory.maxlen)
class ReplayBuffer(AbstractReplayBuffer): def __init__(self, capacity=None, num_steps=1): self.capacity = capacity assert num_steps > 0 self.num_steps = num_steps self.memory = RandomAccessQueue(maxlen=capacity) self.last_n_transitions = collections.deque([], maxlen=num_steps) def append(self, state, action, reward, next_state=None, next_action=None, is_state_terminal=False): experience = dict(state=state, action=action, reward=reward, next_state=next_state, next_action=next_action, is_state_terminal=is_state_terminal) self.last_n_transitions.append(experience) if is_state_terminal: while self.last_n_transitions: self.memory.append(list(self.last_n_transitions)) del self.last_n_transitions[0] assert len(self.last_n_transitions) == 0 else: if len(self.last_n_transitions) == self.num_steps: self.memory.append(list(self.last_n_transitions)) def stop_current_episode(self): # if n-step transition hist is not full, add transition; # if n-step hist is indeed full, transition has already been added; if 0 < len(self.last_n_transitions) < self.num_steps: self.memory.append(list(self.last_n_transitions)) # avoid duplicate entry if 0 < len(self.last_n_transitions) <= self.num_steps: del self.last_n_transitions[0] while self.last_n_transitions: self.memory.append(list(self.last_n_transitions)) del self.last_n_transitions[0] assert len(self.last_n_transitions) == 0 def sample(self, num_experiences): assert len(self.memory) >= num_experiences return self.memory.sample(num_experiences) def __len__(self): return len(self.memory) def save(self, filename): with open(filename, 'wb') as f: pickle.dump(self.memory, f) def load(self, filename): with open(filename, 'rb') as f: self.memory = pickle.load(f) if isinstance(self.memory, collections.deque): # Load v0.2 self.memory = RandomAccessQueue(self.memory, maxlen=self.memory.maxlen)
class EpisodicReplayBuffer(object): def __init__(self, capacity=None): self.current_episode = [] self.episodic_memory = RandomAccessQueue() self.memory = RandomAccessQueue() self.capacity = capacity def append(self, state, action, reward, next_state=None, next_action=None, is_state_terminal=False, **kwargs): """Append a transition to this replay buffer Args: state: s_t action: a_t reward: r_t next_state: s_{t+1} (can be None if terminal) next_action: a_{t+1} (can be None for off-policy algorithms) is_state_terminal (bool) """ experience = dict(state=state, action=action, reward=reward, next_state=next_state, next_action=next_action, is_state_terminal=is_state_terminal, **kwargs) self.current_episode.append(experience) if is_state_terminal: self.stop_current_episode() def sample(self, n): """Sample n unique samples from this replay buffer""" assert len(self.memory) >= n return self.memory.sample(n) def sample_episodes(self, n_episodes, max_len=None): """Sample n unique samples from this replay buffer""" assert len(self.episodic_memory) >= n_episodes episodes = self.episodic_memory.sample(n_episodes) if max_len is not None: return [random_subseq(ep, max_len) for ep in episodes] else: return episodes def __len__(self): return len(self.episodic_memory) def save(self, filename): with open(filename, 'wb') as f: pickle.dump((self.memory, self.episodic_memory), f) def load(self, filename): with open(filename, 'rb') as f: self.memory, self.episodic_memory = pickle.load(f) def stop_current_episode(self): if self.current_episode: self.episodic_memory.append(self.current_episode) self.memory.extend(self.current_episode) self.current_episode = [] while self.capacity is not None and \ len(self.memory) > self.capacity: discarded_episode = self.episodic_memory.popleft() for _ in range(len(discarded_episode)): self.memory.popleft() assert not self.current_episode