def __init__(self, state_size, action_size, seed, QNetwork):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
QNetwork: a class inheriting from torch.nn.Module that define the structure of the neural network
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
# Q-Network
self.qnetwork_local = QNetwork(state_size, action_size, seed).to(device)
self.qnetwork_target = QNetwork(state_size, action_size, seed).to(device)
#when using a dropout the qnetwork_target should be put in eval mode
self.qnetwork_target.eval()
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = PrioritizedReplayBuffer(seed, device, action_size, BUFFER_SIZE, BATCH_SIZE, DEFAULT_PRIORITY, PRIORITY_FACTOR)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.u_step = 0
self.t_step = 0
self.up_step = 0
# To control the importance sampling weight. As the network is converging, b should move toward 1
self.b = torch.tensor(1., device=device, requires_grad=False)
self.b_decay = torch.tensor(0.00015, device=device, requires_grad=False)
def __init__(self, create_network, state_size, action_size, seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
# Q-Network
self.qnetwork_local = create_network(state_size, action_size,
seed).to(device)
self.qnetwork_target = create_network(state_size, action_size,
seed).to(device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = PrioritizedReplayBuffer(BUFFER_SIZE, seed)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
# For debugging
self.learn_step = 0
def clear_rb(self):
if self.subpolicies > 1:
if self.replay_buffer_type == "rb":
self.replay_buffers = [[ReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(self.env.n)]
else:
self.replay_buffers = [
[PrioritizedReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(self.env.n)]
else:
if self.replay_buffer_type == "rb":
self.replay_buffers = ReplayBuffer(self.replay_buffer_size)
else:
self.replay_buffers = [
[PrioritizedReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in
range(self.env.n)]
def __init__(
self,
algorithm,
device,
obs_dim,
action_dim,
env: gym.Env,
memory_size: int,
batch_size: int,
target_update: int,
n_step: int = 3,
gamma: float = 0.99,
# PER parameters
alpha: float = 0.2,
beta: float = 0.6,
):
self.env = env
self.algorithm = algorithm
self.batch_size = batch_size
self.target_update = target_update
self.gamma = gamma
# NoisyNet: All attributes related to epsilon are removed
# PER
# memory for 1-step Learning
self.beta = beta
self.memory = PrioritizedReplayBuffer(obs_dim,
memory_size,
batch_size,
alpha=alpha)
# memory for N-step Learning
self.use_n_step = True if n_step > 1 else False
if self.use_n_step:
self.n_step = n_step
self.memory_n = ReplayBuffer(obs_dim,
memory_size,
batch_size,
n_step=n_step,
gamma=gamma)
# transition to store in memory
self.transition = list()
# mode: train / test
self.is_test = False
def __init__(self, env, env_obs, gamma=0.99, tau=0.001, lr_actor=1e-3, lr_critic=1e-3, weight_decay=0.1, batch_size=64, subpolicies=1, action_shape=2, replay_buffer_size=5000, replay_buffer_type="rb", noise=0.1, noise_decay=0.999, max_action=1, min_action=-1, teacher=False, alpha=0.1, bc=None):
self.env = env
self.subpolicies = subpolicies
self.total_obs = np.sum(env_obs)
self.weight_decay = weight_decay
self.env_obs = env_obs
self.max_action = max_action
self.min_action = min_action
self.action_shape = action_shape
self.gamma = gamma
self.tau = tau
self.batch_size = batch_size
self.replay_buffer_type = replay_buffer_type
self.replay_buffer_size = replay_buffer_size
self.init_noise = noise
self.noise = noise
self.noise_decay = noise_decay
self.teacher = teacher
self.bc = bc
self.alpha = alpha
self.mul = 1 if self.teacher is False else 2
self.actors = [[Actor(self.mul * env_obs[agent], action_shape) for i in range(self.subpolicies)] for agent in range(env.n)]
self.actors_targets = [[Actor(self.mul * env_obs[agent], action_shape) for i in range(self.subpolicies)] for agent in range(env.n)]
self.critics = [Critic(self.mul * self.total_obs + action_shape * len(env.agents)) for _ in env.agents]
self.critics_targets = [Critic(self.mul * self.total_obs + action_shape * len(env.agents)) for _ in env.agents]
self.actors_optimizers = [[torch.optim.RMSprop(self.actors[agent][i].parameters(), lr=lr_actor, weight_decay=weight_decay) for i in range(self.subpolicies)] for agent in range(len(env.agents))]
self.critics_optimisers = [torch.optim.RMSprop(self.critics[agent].parameters(), lr=lr_critic ,weight_decay=weight_decay) for agent in range(len(env.agents))]
if self.subpolicies > 1:
if self.replay_buffer_type == "rb":
self.replay_buffers = [[ReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(env.n)]
else:
self.replay_buffers = [[PrioritizedReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(env.n)]
else:
if self.replay_buffer_type == "rb":
self.replay_buffers = ReplayBuffer(self.replay_buffer_size)
else:
self.replay_buffers = [[PrioritizedReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(env.n)]
class TestPrioritizedReplayBuffer(unittest.TestCase):
def setUp(self):
self.buffer = PrioritizedReplayBuffer(prioritized_memory=self.memory,
buffer_size=4,
seed=0,
device="cpu")
self.memory = self.buffer.memory
def test_add(self):
self.buffer.add(state=[1, 2, 3],
action=0,
reward=0,
next_state=[4, 5, 6],
done=False)
priority_one = self.buffer._calculate_priority(1)
np.testing.assert_array_equal(
[priority_one, priority_one, 0, priority_one, 0, 0, 0],
self.memory.tree)
def test_sample(self):
self.buffer.add(state=[1, 2, 3],
action=0,
reward=0,
next_state=[4, 5, 6],
done=False)
self.buffer.add(state=[4, 5, 6],
action=0,
reward=0,
next_state=[7, 8, 9],
done=False)
self.buffer.add(state=[7, 8, 9],
action=0,
reward=0,
next_state=[10, 11, 12],
done=False)
sample = self.buffer.sample(2)
print(sample)
def __init__(self, state_size, action_size, seed, lr_decay=0.9999):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
lr_decay (float): multiplicative factor of learning rate decay
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
print("Running on: "+str(device))
# Q-Network
hidden_layers = [128, 32]
if USE_DUELING_NETWORK:
hidden_state_value = [64, 32]
self.qnetwork_local = DuelingQNetwork(state_size, action_size, seed, hidden_layers, hidden_state_value).to(device)
self.qnetwork_target = DuelingQNetwork(state_size, action_size, seed, hidden_layers, hidden_state_value).to(device)
self.qnetwork_target.eval()
else:
self.qnetwork_local = QNetwork(state_size, action_size, seed, hidden_layers).to(device)
self.qnetwork_target = QNetwork(state_size, action_size, seed, hidden_layers).to(device)
self.qnetwork_target.eval()
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
self.lr_scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, lr_decay)
# Replay memory
if USE_PRIORITIZED_REPLAY:
self.memory = PrioritizedReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed, device,
alpha=0.6, beta=0.4, beta_scheduler=1.0)
else:
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed, device)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def __init__(self, state_size, action_size, seed, double_dqn=False, dueling_network=False, prioritized_replay=False):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
double_dqn (bool): use Double DQN method
dueling_network (bool): use Dueling Network
prioritized_replay (bool): use Prioritized Replay Buffer
"""
self.state_size = state_size
self.action_size = action_size
self.dueling_network = dueling_network
self.double_dqn = double_dqn
self.prioritized_replay = prioritized_replay
random.seed(seed)
# Q-Network
hidden_layers = [128, 32]
if self.dueling_network:
hidden_state_value = [64, 32]
self.qnetwork_local = DuelingQNetwork(state_size, action_size, seed, hidden_layers, hidden_state_value).to(device)
self.qnetwork_target = DuelingQNetwork(state_size, action_size, seed, hidden_layers, hidden_state_value).to(device)
self.qnetwork_target.eval()
else:
self.qnetwork_local = QNetwork(state_size, action_size, seed, hidden_layers).to(device)
self.qnetwork_target = QNetwork(state_size, action_size, seed, hidden_layers).to(device)
self.qnetwork_target.eval()
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
self.lr_scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer, LR_DECAY)
# Replay memory
if prioritized_replay:
self.memory = PrioritizedReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed, device, alpha=0.6, beta=0.4, beta_scheduler=1.0)
else:
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed, device)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, seed, QNetwork):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
QNetwork: a class inheriting from torch.nn.Module that define the structure of the neural network
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
# Q-Network
self.qnetwork_local = QNetwork(state_size, action_size, seed).to(device)
self.qnetwork_target = QNetwork(state_size, action_size, seed).to(device)
#when using a dropout the qnetwork_target should be put in eval mode
self.qnetwork_target.eval()
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = PrioritizedReplayBuffer(seed, device, action_size, BUFFER_SIZE, BATCH_SIZE, DEFAULT_PRIORITY, PRIORITY_FACTOR)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.u_step = 0
self.t_step = 0
self.up_step = 0
# To control the importance sampling weight. As the network is converging, b should move toward 1
self.b = torch.tensor(1., device=device, requires_grad=False)
self.b_decay = torch.tensor(0.00015, device=device, requires_grad=False)
def act(self, state, eps=0.):
"""Returns actions for given state as per current policy.
Params
======
state (array_like): current state
eps (float): epsilon, for epsilon-greedy action selection
"""
state = torch.from_numpy(state).float().unsqueeze(0).to(device)
self.qnetwork_local.eval()
with torch.no_grad():
action_values = self.qnetwork_local(state)
# Epsilon-greedy action selection
if random.random() > eps:
return np.argmax(action_values.cpu().data.numpy()).astype(np.int_)
else:
return random.choice(np.arange(self.action_size)).astype(np.int_)
def step(self, state, action, reward, next_state, done):
# Save experience in replay memory
self.memory.add(state, action, reward, next_state, done)
self.t_step = (self.t_step + 1) % TRANSFER_EVERY
self.u_step = (self.u_step + 1) % UPDATE_EVERY
self.up_step = (self.up_step + 1) % UPDATE_PRIORITY_EVERY
# Learn from experiences
if len(self.memory) > BATCH_SIZE and self.u_step == 0:
# sample the experiences from the memory based on their priority
experiences = self.memory.sample()
self.learn(experiences)
# Transfer the knowledge from the local network to the fixed on
if len(self.memory) > BATCH_SIZE and self.t_step == 0:
self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)
# Update the priorities in the memory to alter the sampling
# Ideally, this should be done before the sampling is taking place
# But, for sake of performance, it might be better to recalculate them less often
if len(self.memory) > 1 and self.up_step == 0:
for experiences in self.memory.get_all_experiences(512):
with torch.no_grad():
self.qnetwork_local.eval()
current_estimate, from_env = self.get_target_estimate(experiences)
# update the priorities based on newly learned errors
self.memory.update(experiences[-1], (from_env - current_estimate).squeeze())
def get_target_estimate(self, experiences):
states, actions, rewards, next_states, dones, probabilities, selected = experiences
with torch.no_grad():
best_actions = self.qnetwork_local(next_states).detach().max(1)[1].unsqueeze(1)
evaluations = self.qnetwork_target(next_states).gather(1,best_actions)
from_env = rewards + GAMMA*evaluations*(1 - dones)
return self.qnetwork_local(states).gather(1, actions), from_env
def learn(self, experiences):
self.qnetwork_local.train()
current_estimate,from_env = self.get_target_estimate(experiences)
probabilities = experiences[-2]
errors = (from_env - current_estimate)
# Since the experiences were retrieved based on a given probabilities, such experience will biase the network
# Therefore, we introduce here an importance sampling weight
loss = (errors * errors / (len(self.memory) * probabilities) * self.b).mean()
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
self.b = min(self.b + self.b_decay,1)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
θ_target = θ_target + τ*(θ_local - θ_target)
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
def reinitialize_memory(self):
self.memory = PrioritizedReplayBuffer(
self.num_actions, BUFFER_SIZE, BATCH_SIZE, EXPERIENCES_PER_SAMPLING, self.seed, self.compute_weights)
def setUp(self):
self.buffer = PrioritizedReplayBuffer(prioritized_memory=self.memory,
buffer_size=4,
seed=0,
device="cpu")
self.memory = self.buffer.memory
def __init__(self, movie_dict=None, act_set=None, slot_set=None, params=None, seed=1, compute_weights=False):
self.movie_dict = movie_dict
self.act_set = act_set
self.slot_set = slot_set
self.act_cardinality = len(act_set.keys())
self.slot_cardinality = len(slot_set.keys())
self.seed = seed
self.compute_weights = compute_weights
self.feasible_actions = dialog_config.feasible_actions
self.num_actions = len(self.feasible_actions)
self.movie_dict = movie_dict
self.act_set = act_set
self.slot_set = slot_set
self.act_cardinality = len(act_set.keys())
self.slot_cardinality = len(slot_set.keys())
self.feasible_actions = dialog_config.feasible_actions
self.num_actions = len(self.feasible_actions)
self.epsilon = params['epsilon']
self.agent_run_mode = params['agent_run_mode']
self.agent_act_level = params['agent_act_level']
self.experience_replay_pool = [] # experience replay pool <s_t, a_t, r_t, s_t+1>
# Replay memory
self.memory = PrioritizedReplayBuffer(
self.num_actions, BUFFER_SIZE, BATCH_SIZE, EXPERIENCES_PER_SAMPLING, seed, compute_weights)
# Initialize time step (for updating every UPDATE_NN_EVERY steps)
self.t_step_nn = 0
# Initialize time step (for updating every UPDATE_MEM_PAR_EVERY steps)
self.t_step_mem_par = 0
# Initialize time step (for updating every UPDATE_MEM_EVERY steps)
self.t_step_mem = 0
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.experience_replay_pool_size = params.get(
'experience_replay_pool_size', 1000)
self.hidden_size = params.get('dqn_hidden_size', 60)
self.gamma = params.get('gamma', 0.9)
self.predict_mode = params.get('predict_mode', False)
self.warm_start = params.get('warm_start', 0)
self.max_turn = params['max_turn'] + 4
self.state_dimension = 2 * self.act_cardinality + \
7 * self.slot_cardinality + 3 + self.max_turn
self.qnetwork_local = QNetwork(
self.state_dimension, self.num_actions, seed).to(self.device)
self.qnetwork_target = QNetwork(
self.state_dimension, self.num_actions, seed).to(self.device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=5e-4)
self.cur_bellman_err = 0
# Prediction Mode: load trained DQN model
if params['trained_model_path'] != None:
self.dqn.model = copy.deepcopy(
self.load_trained_DQN(params['trained_model_path']))
self.clone_dqn = copy.deepcopy(self.dqn)
self.predict_mode = True
self.warm_start = 2
class PrioritizedDQNAgent(Agent):
"""Interacts with and learns from the environment."""
def __init__(self, movie_dict=None, act_set=None, slot_set=None, params=None, seed=1, compute_weights=False):
self.movie_dict = movie_dict
self.act_set = act_set
self.slot_set = slot_set
self.act_cardinality = len(act_set.keys())
self.slot_cardinality = len(slot_set.keys())
self.seed = seed
self.compute_weights = compute_weights
self.feasible_actions = dialog_config.feasible_actions
self.num_actions = len(self.feasible_actions)
self.movie_dict = movie_dict
self.act_set = act_set
self.slot_set = slot_set
self.act_cardinality = len(act_set.keys())
self.slot_cardinality = len(slot_set.keys())
self.feasible_actions = dialog_config.feasible_actions
self.num_actions = len(self.feasible_actions)
self.epsilon = params['epsilon']
self.agent_run_mode = params['agent_run_mode']
self.agent_act_level = params['agent_act_level']
self.experience_replay_pool = [] # experience replay pool <s_t, a_t, r_t, s_t+1>
# Replay memory
self.memory = PrioritizedReplayBuffer(
self.num_actions, BUFFER_SIZE, BATCH_SIZE, EXPERIENCES_PER_SAMPLING, seed, compute_weights)
# Initialize time step (for updating every UPDATE_NN_EVERY steps)
self.t_step_nn = 0
# Initialize time step (for updating every UPDATE_MEM_PAR_EVERY steps)
self.t_step_mem_par = 0
# Initialize time step (for updating every UPDATE_MEM_EVERY steps)
self.t_step_mem = 0
self.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
self.experience_replay_pool_size = params.get(
'experience_replay_pool_size', 1000)
self.hidden_size = params.get('dqn_hidden_size', 60)
self.gamma = params.get('gamma', 0.9)
self.predict_mode = params.get('predict_mode', False)
self.warm_start = params.get('warm_start', 0)
self.max_turn = params['max_turn'] + 4
self.state_dimension = 2 * self.act_cardinality + \
7 * self.slot_cardinality + 3 + self.max_turn
self.qnetwork_local = QNetwork(
self.state_dimension, self.num_actions, seed).to(self.device)
self.qnetwork_target = QNetwork(
self.state_dimension, self.num_actions, seed).to(self.device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=5e-4)
self.cur_bellman_err = 0
# Prediction Mode: load trained DQN model
if params['trained_model_path'] != None:
self.dqn.model = copy.deepcopy(
self.load_trained_DQN(params['trained_model_path']))
self.clone_dqn = copy.deepcopy(self.dqn)
self.predict_mode = True
self.warm_start = 2
def initialize_episode(self):
""" Initialize a new episode. This function is called every time a new episode is run. """
self.current_slot_id = 0
self.phase = 0
self.request_set = ['moviename', 'starttime',
'city', 'date', 'theater', 'numberofpeople']
def state_to_action(self, state):
""" DQN: Input state, output action """
self.representation = self.prepare_state_representation(state)
self.action = self.run_policy(self.representation)
act_slot_response = copy.deepcopy(self.feasible_actions[self.action])
return {'act_slot_response': act_slot_response, 'act_slot_value_response': None}
def prepare_state_representation(self, state):
""" Create the representation for each state """
user_action = state['user_action']
current_slots = state['current_slots']
kb_results_dict = state['kb_results_dict']
agent_last = state['agent_action']
########################################################################
# Create one-hot of acts to represent the current user action
########################################################################
user_act_rep = np.zeros((1, self.act_cardinality))
user_act_rep[0, self.act_set[user_action['diaact']]] = 1.0
########################################################################
# Create bag of inform slots representation to represent the current user action
########################################################################
user_inform_slots_rep = np.zeros((1, self.slot_cardinality))
for slot in user_action['inform_slots'].keys():
user_inform_slots_rep[0, self.slot_set[slot]] = 1.0
########################################################################
# Create bag of request slots representation to represent the current user action
########################################################################
user_request_slots_rep = np.zeros((1, self.slot_cardinality))
for slot in user_action['request_slots'].keys():
user_request_slots_rep[0, self.slot_set[slot]] = 1.0
########################################################################
# Creat bag of filled_in slots based on the current_slots
########################################################################
current_slots_rep = np.zeros((1, self.slot_cardinality))
for slot in current_slots['inform_slots']:
current_slots_rep[0, self.slot_set[slot]] = 1.0
########################################################################
# Encode last agent act
########################################################################
agent_act_rep = np.zeros((1, self.act_cardinality))
if agent_last:
agent_act_rep[0, self.act_set[agent_last['diaact']]] = 1.0
########################################################################
# Encode last agent inform slots
########################################################################
agent_inform_slots_rep = np.zeros((1, self.slot_cardinality))
if agent_last:
for slot in agent_last['inform_slots'].keys():
agent_inform_slots_rep[0, self.slot_set[slot]] = 1.0
########################################################################
# Encode last agent request slots
########################################################################
agent_request_slots_rep = np.zeros((1, self.slot_cardinality))
if agent_last:
for slot in agent_last['request_slots'].keys():
agent_request_slots_rep[0, self.slot_set[slot]] = 1.0
turn_rep = np.zeros((1, 1)) + state['turn'] / 10.
########################################################################
# One-hot representation of the turn count?
########################################################################
turn_onehot_rep = np.zeros((1, self.max_turn))
turn_onehot_rep[0, state['turn']] = 1.0
########################################################################
# Representation of KB results (scaled counts)
########################################################################
kb_count_rep = np.zeros((1, self.slot_cardinality + 1)) + \
kb_results_dict['matching_all_constraints'] / 100.
for slot in kb_results_dict:
if slot in self.slot_set:
kb_count_rep[0, self.slot_set[slot]
] = kb_results_dict[slot] / 100.
########################################################################
# Representation of KB results (binary)
########################################################################
kb_binary_rep = np.zeros((1, self.slot_cardinality + 1)) + \
np.sum(kb_results_dict['matching_all_constraints'] > 0.)
for slot in kb_results_dict:
if slot in self.slot_set:
kb_binary_rep[0, self.slot_set[slot]] = np.sum(
kb_results_dict[slot] > 0.)
self.final_representation = np.hstack([user_act_rep, user_inform_slots_rep, user_request_slots_rep, agent_act_rep,
agent_inform_slots_rep, agent_request_slots_rep, current_slots_rep, turn_rep, turn_onehot_rep, kb_binary_rep, kb_count_rep])
return self.final_representation
def run_policy(self, state):
""" epsilon-greedy policy """
if random.random() < self.epsilon:
return random.randint(0, self.num_actions - 1)
else:
if self.warm_start == 1:
if len(self.experience_replay_pool) > self.experience_replay_pool_size:
self.warm_start = 2
return self.rule_policy()
else:
state = torch.from_numpy(
state).float().unsqueeze(0).to(self.device)
self.qnetwork_local.eval()
with torch.no_grad():
action_values = self.qnetwork_local(state)
self.qnetwork_local.train()
return np.argmax(action_values.cpu().data.numpy())
def rule_policy(self):
""" Rule Policy """
if self.current_slot_id < len(self.request_set):
slot = self.request_set[self.current_slot_id]
self.current_slot_id += 1
act_slot_response = {}
act_slot_response['diaact'] = "request"
act_slot_response['inform_slots'] = {}
act_slot_response['request_slots'] = {slot: "UNK"}
elif self.phase == 0:
act_slot_response = {'diaact': "inform", 'inform_slots': {
'taskcomplete': "PLACEHOLDER"}, 'request_slots': {}}
self.phase += 1
elif self.phase == 1:
act_slot_response = {'diaact': "thanks",
'inform_slots': {}, 'request_slots': {}}
return self.action_index(act_slot_response)
def action_index(self, act_slot_response):
""" Return the index of action """
for (i, action) in enumerate(self.feasible_actions):
if act_slot_response == action:
return i
print act_slot_response
raise Exception("action index not found")
return None
def register_experience_replay_tuple(self, s_t, a_t, reward, s_tplus1, episode_over):
""" Register feedback from the environment, to be stored as future training data """
state = self.prepare_state_representation(s_t)
action = self.action
reward = reward
next_state = self.prepare_state_representation(s_tplus1)
done = episode_over
if self.predict_mode == False: # Training Mode
if self.warm_start == 1:
self.memory.add(state, action, reward, next_state, done)
else: # Prediction Mode
self.memory.add(state, action, reward, next_state, done)
def train(self, batch_size=16, num_batches=100, gamma = 0.99):
""" Train DQN with experience replay """
for iter_batch in range(num_batches):
self.cur_bellman_err = 0
self.memory.update_memory_sampling()
self.memory.update_parameters()
for iter in range(int(EXPERIENCES_PER_SAMPLING/batch_size)):
experiences = self.memory.sample()
self.learn(experiences, gamma)
def learn(self, sampling, gamma):
"""Update value parameters using given batch of experience tuples.
Params
======
sampling (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones, weights, indices = sampling
## TODO: compute and minimize the loss
q_target = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)
expected_values = rewards + gamma * q_target * (1 - dones)
output = self.qnetwork_local(states).gather(1, actions)
loss = F.mse_loss(output, expected_values)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# ------------------- update target network ------------------- #
self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)
# ------------------- update priorities ------------------- #
delta = abs(expected_values - output.detach()).numpy()
self.memory.update_priorities(delta, indices)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
def reinitialize_memory(self):
self.memory = PrioritizedReplayBuffer(
self.num_actions, BUFFER_SIZE, BATCH_SIZE, EXPERIENCES_PER_SAMPLING, self.seed, self.compute_weights)
################################################################################
# Debug Functions
################################################################################
def save_experience_replay_to_file(self, path):
""" Save the experience replay pool to a file """
try:
pickle.dump(self.experience_replay_pool, open(path, "wb"))
print 'saved model in %s' % (path, )
except Exception, e:
print 'Error: Writing model fails: %s' % (path, )
print e
class Agent():
"""An agent that interacts with and learns from its environment. As its baseline it
uses Deep Q-Learning (DQN). The following can optionally be enabled in all possible
combinations:
* Double DQN (DDQN)
* Prioritized Experience Replay
* Dueling Network Architecture
"""
def __init__(self,
state_size,
action_size,
buffer_size=int(1e5),
batch_size=64,
gamma=0.99,
tau=0.025,
learning_rate=5e-4,
update_every=4,
enable_double_dqn=False,
enable_prioritized_experience_replay=False,
enable_dueling_network=False,
alpha=0.6):
"""Initialize an Agent object.
Params
======
state_size (int): the dimension of each state
action_size (int): the dimension of each action
buffer_size (int): the replay buffer size
batch_size (int): the minibatch size
gamma (float): the reward discount factor
tau (float): for soft updates of the target parameters
learning_rate (float): the learning rate
update_every (int): controls how regularly the network learns
enable_double_dqn (bool): enables Double DQN (DDQN)
enable_prioritized__experience_replay (bool): enables Prioritized Experience Replay
enable_dueling_network (bool): enables a Dueling Network architecture
alpha (float): the priority dampening effect in Prioritized Experience Replay
"""
self.state_size = state_size
self.action_size = action_size
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.learning_rate = learning_rate
self.update_every = update_every
self.enable_double_dqn = enable_double_dqn
self.enable_prioritized_experience_replay = enable_prioritized_experience_replay
self.enable_dueling_network = enable_dueling_network
self.alpha = alpha
# Instantiate the local and target networks.
Network = QStandardNetwork if not self.enable_dueling_network else QDuelingNetwork
self.qnetwork_local = Network(state_size, action_size).to(device)
self.qnetwork_target = Network(state_size, action_size).to(device)
# Starting off with the same random weights in self.qnetwork_local and self.qnetwork_target.
self._perform_soft_update(self.qnetwork_local,
self.qnetwork_target,
tau=1)
# Instantiate the experience memory.
if not self.enable_prioritized_experience_replay:
self.memory = ReplayBuffer(self.buffer_size)
else:
self.memory = PrioritizedReplayBuffer(self.buffer_size,
alpha=self.alpha)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(),
lr=self.learning_rate)
# Start counting the steps.
self.t_step = 0
# Clear the performance timers.
self.reset_timers()
def act(self, state, epsilon):
"""Returns an action for the given state and epison-greedy value as per the current policy.
Params
======
state (array_like): the current state
epsilon (float): the epsilon-greedy value
"""
state = torch.from_numpy(state).float().unsqueeze(0).to(device)
self.qnetwork_local.eval()
with torch.no_grad():
action_values = self.qnetwork_local(state)
self.qnetwork_local.train()
# Epsilon-greedy action selection
if random.random() > epsilon:
return np.argmax(action_values.cpu().data.numpy())
else:
return random.choice(np.arange(self.action_size))
def step(self, state, action, reward, next_state, done, beta=None):
"""Updates the policy based on the state, action, reward and next_state.
Also takes into account that the episode might be done.
For Prioritized Experience Replay, also receives the beta value that should
be used for the de-biasing factor.
Params
======
state (array_like): the current state
action (int): the action taken
reward (float): the reward received for taking the action in the state
next_state (array_like): the resulting state
done (bool): indicates whether the episode is done or not
beta (float): For Prioritized Experience Replay, the beta value that
should be used next for the de-biasing factor
"""
# Save experience in replay memory
self.memory.add(state, action, reward, next_state, done)
# Learn every UPDATE_EVERY time steps.
self.t_step = (self.t_step + 1) % self.update_every
if self.t_step == 0:
# If enough samples are available in memory, get random subset and learn
if len(self.memory) > self.batch_size:
experiences = self.memory.sample(self.batch_size)
self._learn(experiences, beta)
def save(self, path):
torch.save(self.qnetwork_local.state_dict(), path)
def reset_timers(self):
"""Resets performance timers.
"""
self.time_1 = 0
self.time_2 = 0
self.time_3 = 0
self.time_4 = 0
def _learn(self, experiences, beta):
"""Updates Q by learning from a batch of experiences.
Params
======
experiences (tuple): A batch of sampled experiences.
"""
predictions, targets, nodes = self._calculate_predictions_and_targets(
experiences)
if not self.enable_prioritized_experience_replay:
self._learn_from_experience_replay(predictions, targets)
else:
self._learn_from_prioritized_experience_replay(
predictions, targets, nodes, beta)
def _calculate_predictions_and_targets(self, experiences):
"""From a batch of sampled experiences, calculates the predictions and targets.
Also returns the nodes of the samples.
Params
======
experiences (tuple): a batch of sampled experiences
Returns
=======
A tuple of predictions, targets and nodes.
"""
in_states, in_actions, in_rewards, in_next_states, in_dones, nodes = experiences
states = torch.from_numpy(in_states).float().to(device)
actions = torch.from_numpy(in_actions).long().to(device)
rewards = torch.from_numpy(in_rewards).float().to(device)
next_states = torch.from_numpy(in_next_states).float().to(device)
dones = torch.from_numpy(in_dones).float().to(device)
predictions = self.qnetwork_local(states)[
torch.range(0, states.shape[0] - 1, dtype=torch.long),
torch.squeeze(actions)].to(device)
with torch.no_grad():
if not self.enable_double_dqn:
inputs_for_targets = self.qnetwork_target(next_states).to(
device)
targets = (torch.squeeze(rewards) +
(1.0 - torch.squeeze(dones)) * self.gamma *
inputs_for_targets.max(1)[0]).to(device)
else:
temp_1 = self.qnetwork_local(next_states).to(device)
temp_2 = temp_1.max(1)[1].to(device)
temp_3 = self.qnetwork_target(next_states)[
torch.range(0, next_states.shape[0] - 1, dtype=torch.long),
temp_2].to(device)
targets = (torch.squeeze(rewards) +
(1.0 - torch.squeeze(dones)) * self.gamma *
temp_3).to(device)
return (predictions, targets, nodes)
def _learn_from_experience_replay(self, predictions, targets):
"""Updates Q by learning from (non-prioritized) Experience Replay.
Params
======
predictions (array_like): batch-size predictions
targets (array_like): batch-size targets
"""
assert not self.enable_prioritized_experience_replay
td_errors = targets - predictions
torch.Tensor.clamp_(td_errors, min=-1, max=1)
loss = torch.mean(torch.pow(td_errors, 2))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update the target network:
self._perform_soft_update(self.qnetwork_local,
self.qnetwork_target,
tau=self.tau)
def _learn_from_prioritized_experience_replay(self, predictions, targets,
nodes, beta):
"""Updates Q by learning from Prioritized Experience Replay.
Params
======
predictions (array_like): batch-size predictions
targets (array_like): batch-size targets
nodes (array_like): the nodes associated with the predictions and targets
beta (float): The beta value that should be used next for the de-biasing factor
"""
assert self.enable_prioritized_experience_replay
# Calculate the gradient weights:
time_1_start = time.process_time()
root = nodes[0].get_root()
total_weight = root.get_max_of_weights_overall(
) # 'alpha' has already been applied.
sampled_weights = np.array([n.own_weight for n in nodes
]) # 'alpha' has already been applied.
scaled_weights = sampled_weights / total_weight # P
gradient_weights = np.power(self.buffer_size * scaled_weights, -beta)
gradient_weights = gradient_weights / np.max(gradient_weights)
gradient_weights = torch.from_numpy(gradient_weights).float().to(
device)
self.time_1 += time.process_time(
) - time_1_start # Measure the performance.
# Calculate the TD errors and loss; update the local network weights:
time_2_start = time.process_time()
td_errors = targets - predictions
torch.Tensor.clamp_(td_errors, min=-1,
max=1) # Clip the TD errors for greater stability.
loss = torch.mean(torch.pow(td_errors, 2) * gradient_weights)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# An alternative but less elegant and slower approach involves applying the
# gradient weights to the gradient components instead of the loss components
# as was done above:
# weighted_gradients = {}
# for k in range(self.batch_size):
# loss_single = self.loss_function(predictions[k], targets[k]) # Where for equivalent self.loss_function is MSE.
# self.qnetwork_local.zero_grad()
# loss_single.backward(retain_graph=True)
# with torch.no_grad():
# for name, param in self.qnetwork_local.named_parameters():
# if name not in weighted_gradients:
# weighted_gradients[name] = param.grad * gradient_weights[k]
# else:
# weighted_gradients[name] += param.grad * gradient_weights[k]
# with torch.no_grad():
# for name, param in self.qnetwork_local.named_parameters():
# param.data -= self.learning_rate * weighted_gradients[name]
self.time_2 += time.process_time(
) - time_2_start # Measure the performance.
# Update the target network:
time_3_start = time.process_time()
self._perform_soft_update(self.qnetwork_local,
self.qnetwork_target,
tau=self.tau)
self.time_3 += time.process_time(
) - time_3_start # Measure the performance.
# Update the node weights:
time_4_start = time.process_time()
with torch.no_grad():
for node, td_error in zip(nodes, td_errors.cpu().numpy()):
weight = self.memory.calculate_weight_from_raw_weight(td_error)
node.update_weight(weight)
self.time_4 += time.process_time(
) - time_4_start # Measure the performance.
def _perform_soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
def __init__(self,
state_size,
action_size,
buffer_size=int(1e5),
batch_size=64,
gamma=0.99,
tau=0.025,
learning_rate=5e-4,
update_every=4,
enable_double_dqn=False,
enable_prioritized_experience_replay=False,
enable_dueling_network=False,
alpha=0.6):
"""Initialize an Agent object.
Params
======
state_size (int): the dimension of each state
action_size (int): the dimension of each action
buffer_size (int): the replay buffer size
batch_size (int): the minibatch size
gamma (float): the reward discount factor
tau (float): for soft updates of the target parameters
learning_rate (float): the learning rate
update_every (int): controls how regularly the network learns
enable_double_dqn (bool): enables Double DQN (DDQN)
enable_prioritized__experience_replay (bool): enables Prioritized Experience Replay
enable_dueling_network (bool): enables a Dueling Network architecture
alpha (float): the priority dampening effect in Prioritized Experience Replay
"""
self.state_size = state_size
self.action_size = action_size
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.learning_rate = learning_rate
self.update_every = update_every
self.enable_double_dqn = enable_double_dqn
self.enable_prioritized_experience_replay = enable_prioritized_experience_replay
self.enable_dueling_network = enable_dueling_network
self.alpha = alpha
# Instantiate the local and target networks.
Network = QStandardNetwork if not self.enable_dueling_network else QDuelingNetwork
self.qnetwork_local = Network(state_size, action_size).to(device)
self.qnetwork_target = Network(state_size, action_size).to(device)
# Starting off with the same random weights in self.qnetwork_local and self.qnetwork_target.
self._perform_soft_update(self.qnetwork_local,
self.qnetwork_target,
tau=1)
# Instantiate the experience memory.
if not self.enable_prioritized_experience_replay:
self.memory = ReplayBuffer(self.buffer_size)
else:
self.memory = PrioritizedReplayBuffer(self.buffer_size,
alpha=self.alpha)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(),
lr=self.learning_rate)
# Start counting the steps.
self.t_step = 0
# Clear the performance timers.
self.reset_timers()
'__iter__') and not isinstance(obj,
(str, bytes, bytearray)):
size += sum([get_size(i, seen) for i in obj])
return size
##############################
##### Initializing items #####
##############################
# Initializing saver class (for loading & saving data)
saver = saver.Saver(sess, filename)
# Initializing replay buffer, with the option of a prioritized replay buffer
if Settings.PRIORITY_REPLAY_BUFFER:
# Loading is not implemented on prioritized replay buffer
replay_buffer = PrioritizedReplayBuffer()
else:
replay_buffer = ReplayBuffer(filename)
# Initializing thread & process list
threads = []
environment_processes = []
# Event()s are used to communicate with threads while they run.
# In this case, it is used to signal to the threads when it is time to stop gracefully.
stop_run_flag = threading.Event() # Flag to stop all threads
replay_buffer_dump_flag = threading.Event(
) # Flag to pause data writing to the replay buffer
replay_buffer_dump_flag.set(
) # Set the flag to initially be True so that the agents will write data
class D3QNetAgent(AgentWithConverter):
def __init__(self,
observation_space,
action_space,
num_frames=4,
batch_size=32,
learning_rate=1e-5,
learning_rate_decay_steps=10000,
learning_rate_decay_rate=0.95,
discount_factor=0.95,
tau=1e-2,
lam=1,
per_size=50000,
per_alpha=0.6,
per_beta=0.4,
per_anneal_rate=1.5e6,
epsilon=0.99,
decay_epsilon=1024 * 32,
final_epsilon=0.0001):
# initializes AgentWithConverter class to handle action conversiions
AgentWithConverter.__init__(self,
action_space,
action_space_converter=IdToAct)
self.obs_space = observation_space
self.act_space = action_space
self.num_frames = num_frames
self.batch_size = batch_size
self.lr = learning_rate
self.lr_decay_steps = learning_rate_decay_steps
self.lr_decay_rate = learning_rate_decay_rate
self.gamma = discount_factor
self.lam = lam
self.tau = tau
# epsilon is the degree of exploraation
self.initial_epsilon = epsilon
# Adaptive epsilon decay constants
self.decay_epsilon = decay_epsilon
self.final_epsilon = final_epsilon
#PER data
self.buff_size = per_size
self.alpha = per_alpha
self.beta = per_beta
self.anneal = per_anneal_rate
self.observation_size = self.obs_space.size_obs()
self.action_size = self.act_space.size()
self.d3qn = D3QNet(self.action_size, self.observation_size,
self.num_frames, self.lr, self.lr_decay_steps,
self.lr_decay_rate, self.batch_size, self.gamma,
self.tau)
#State variables
self.obs = None
self.done = None
self.epsilon = self.initial_epsilon
self.state = []
self.frames = []
self.next_frames = []
self.replay_buffer = PrioritizedReplayBuffer(self.buff_size,
self.alpha, self.beta,
self.anneal)
return
## Helper Functions
# Adds to current frame buffer, enforfes length to num_frames
def update_curr_frame(self, obs):
self.frames.append(obs.copy())
if (len(self.frames) > self.num_frames):
self.frames.pop(0)
return
# Adds next frame to next frame buffer, enforces length to num_frames
def update_next_frame(self, next_obs):
self.next_frames.append(next_obs.copy())
if (len(self.next_frames) > self.num_frames):
self.next_frames.pop(0)
return
# Adaptive epsilon decay determines next epsilon based off number of steps
# completed and curren epsilon
def set_next_epsilon(self, current_step):
ada_div = self.decay_epsilon / 10.0
step_off = current_step + ada_div
ada_eps = self.initial_epsilon * -math.log10(
(step_off + 1) / (self.decay_epsilon + ada_div))
ada_eps_up_clip = min(self.initial_epsilon, ada_eps)
ada_eps_low_clip = max(self.final_epsilon, ada_eps_up_clip)
self.epsilon = ada_eps_low_clip
return
## Agent Interface
#Adapted from l2rpn-baselines from RTE-France
# Vectorizes observations from grid2op environment for neural network uses
def convert_obs(self, obs):
li_vect = []
for el in obs.attr_list_vect:
v = obs._get_array_from_attr_name(el).astype(np.float32)
v_fix = np.nan_to_num(v)
v_norm = np.linalg.norm(v_fix)
if v_norm > 1e6:
v_res = (v_fix / v_norm) * 10.0
else:
v_res = v_fix
li_vect.append(v_res)
return np.concatenate(li_vect)
# converts encoded action number to action used to interact with grid2op
# environment
def convert_act(self, encoded_act):
return super().convert_act(encoded_act)
# Required for agent evaluation
# Returns random action or best action as estimated by Q network based on
# exploration parameter (espilon)
def my_act(self, state, reward, done):
if (len(self.frames) < self.num_frames): return 0 #do nothing
random_act = random.randint(0, self.action_size)
self.update_curr_frame(state)
qnet_act, _ = self.dqn.model_action(np.array(self.frames))
if (np.random.rand(1) < self.epsilon):
return random_act
else:
return qnet_act
## Training Loop
def learn(self,
env,
num_epochs,
num_steps,
soft_update_freq=250,
hard_update_freq=1000):
#pre-training to fill buffer
print("Starting Pretraining...\n")
self.done = True
# Plays random moves and saves the resulting (s, a, r, s', d) pair to
# replay buffer. Resets environment when done and continues
while (len(self.replay_buffer) < self.buff_size):
if (self.done):
# reset environment and state parameters
new_env = env.reset()
self.frames = []
self.next_frames = []
self.done = False
self.obs = new_env
self.state = self.convert_obs(self.obs)
self.update_curr_frame(self.state)
# action is random
encoded_act = np.random.randint(0, self.action_size)
act = self.convert_act(encoded_act)
new_obs, reward, self.done, info = env.step(act)
gplay_reward = info['rewards']['gameplay']
adj_reward = self.lam * reward + (1 - self.lam) * gplay_reward
new_state = self.convert_obs(new_obs)
self.update_next_frame(new_state)
# only add to buffer if num_frames states are seen
if (len(self.frames) == self.num_frames
and len(self.next_frames) == self.num_frames):
agg_state = np.array(self.frames)
agg_next_state = np.array(self.next_frames)
#(s,a,r,s',d) pair
self.replay_buffer.add(agg_state, encoded_act, adj_reward,
agg_next_state, self.done)
self.obs = new_obs
self.state = new_state
epoch = 0 # number of complete runs through environment
total_steps = 0 # total number of training steps across all epochs
losses = [] # losses[i] is loss from dqn at total_step i
avg_losses = [
] # avg_losses[i] is avg loss during training during epcoh i
net_reward = [] #net_reward[i] is total reward during epoch i
alive = [] # alive[i] is number of steps survived for at epoch i
print("Starting training...\n")
# Trains a minimum of num_steps or num_epochs
while (total_steps < num_steps or epoch < num_epochs):
total_reward = 0
curr_steps = 0
total_loss = []
# Reset state parameters
self.frames = []
self.next_frames = []
self.done = False
self.obs = env.reset()
self.state = self.convert_obs(self.obs)
# continues until failure
while (not self.done):
self.update_curr_frame(self.state)
# Determine action
if (len(self.frames) < self.num_frames):
enc_act = 0 # do nothing
elif (np.random.rand(1) < self.epsilon):
enc_act = np.random.randint(0, self.action_size)
else:
input = np.array(self.frames)
enc_act, _ = self.d3qn.model_action(input)
# converts action and steps in environment
act = self.convert_act(enc_act)
new_obs, reward, self.done, info = env.step(act)
gplay_reward = info['rewards']['gameplay']
adj_reward = self.lam * reward + (1 - self.lam) * gplay_reward
new_state = self.convert_obs(new_obs)
# updates next_state frame
self.update_next_frame(new_state)
if (len(self.frames) == self.num_frames
and len(self.next_frames) == self.num_frames):
agg_state = np.array(self.frames)
agg_next_state = np.array(self.next_frames)
# Adds (s,a,r,s',d) tuple to replay buffer
self.replay_buffer.add(agg_state, encoded_act, adj_reward,
agg_next_state, self.done)
# finds the next epsilon
self.set_next_epsilon(total_steps)
# samples a batch_size number of experience samples from replay
# buffer
(s_batch, a_batch, r_batch, s_next_batch, d_batch, w_batch,
ind_batch) = (self.replay_buffer.sample(
self.batch_size, total_steps))
# updates network estimates based on replay
loss = self.d3qn.train_on_minibatch(s_batch, a_batch, r_batch,
s_next_batch, d_batch,
w_batch)
priorities = self.d3qn.prio
self.replay_buffer.update_priorities(ind_batch, priorities)
# periodically hard updates the network
if (total_steps % hard_update_freq):
self.d3qn.hard_update_target_network()
# periodically soft update the network
elif (total_steps % soft_update_freq):
self.d3qn.soft_update_target_network()
# update state variables
self.obs = new_obs
self.state = new_state
# increase steps, updates metrics
curr_steps += 1
total_steps += 1
total_reward += reward
losses.append(loss)
total_loss.append(loss)
# updates metrics throughout epoch
alive.append(curr_steps)
net_reward.append(total_reward)
avg_losses.append(np.average(np.array(total_loss)))
epoch += 1
# sanity check to ensure it's working
if (epoch % 100 == 0):
print("Completed epoch {}".format(epoch))
print("Total steps: {}".format(total_steps))
return (epoch, total_steps, losses, avg_losses, net_reward, alive)
def __init__(self,
observation_space,
action_space,
num_frames=4,
batch_size=32,
learning_rate=1e-5,
learning_rate_decay_steps=10000,
learning_rate_decay_rate=0.95,
discount_factor=0.95,
tau=1e-2,
lam=1,
per_size=50000,
per_alpha=0.6,
per_beta=0.4,
per_anneal_rate=1.5e6,
epsilon=0.99,
decay_epsilon=1024 * 32,
final_epsilon=0.0001):
# initializes AgentWithConverter class to handle action conversiions
AgentWithConverter.__init__(self,
action_space,
action_space_converter=IdToAct)
self.obs_space = observation_space
self.act_space = action_space
self.num_frames = num_frames
self.batch_size = batch_size
self.lr = learning_rate
self.lr_decay_steps = learning_rate_decay_steps
self.lr_decay_rate = learning_rate_decay_rate
self.gamma = discount_factor
self.lam = lam
self.tau = tau
# epsilon is the degree of exploraation
self.initial_epsilon = epsilon
# Adaptive epsilon decay constants
self.decay_epsilon = decay_epsilon
self.final_epsilon = final_epsilon
#PER data
self.buff_size = per_size
self.alpha = per_alpha
self.beta = per_beta
self.anneal = per_anneal_rate
self.observation_size = self.obs_space.size_obs()
self.action_size = self.act_space.size()
self.d3qn = D3QNet(self.action_size, self.observation_size,
self.num_frames, self.lr, self.lr_decay_steps,
self.lr_decay_rate, self.batch_size, self.gamma,
self.tau)
#State variables
self.obs = None
self.done = None
self.epsilon = self.initial_epsilon
self.state = []
self.frames = []
self.next_frames = []
self.replay_buffer = PrioritizedReplayBuffer(self.buff_size,
self.alpha, self.beta,
self.anneal)
return
class MADDPG():
actors = None
actors_targets = None
actors_optimisers = None
critics = None
critics_targets = None
critics_optimisers = None
env = None
gamma = None
def __init__(self,
env,
env_obs,
gamma=0.99,
tau=0.001,
lr_actor=1e-3,
lr_critic=1e-3,
weight_decay=0.1,
batch_size=64,
subpolicies=1,
action_shape=2,
replay_buffer_size=5000,
replay_buffer_type="rb",
noise=0.1,
noise_decay=0.999,
max_action=1,
min_action=-1,
teacher=False,
bc=None):
self.env = env
self.subpolicies = subpolicies
self.total_obs = np.sum(env_obs)
self.weight_decay = weight_decay
self.max_action = max_action
self.min_action = min_action
self.action_shape = action_shape
self.gamma = gamma
self.tau = tau
self.batch_size = batch_size
self.replay_buffer_type = replay_buffer_type
self.replay_buffer_size = replay_buffer_size
self.init_noise = noise
self.noise = noise
self.noise_decay = noise_decay
self.teacher = teacher
self.mul = 1 if self.teacher is False else 2
self.actors = [[
Actor(self.mul * env_obs[agent], action_shape)
for i in range(self.subpolicies)
] for agent in range(env.n)]
self.actors_targets = [[
Actor(self.mul * env_obs[agent], action_shape)
for i in range(self.subpolicies)
] for agent in range(env.n)]
self.critics = [
Critic(self.mul * self.total_obs + action_shape * len(env.agents))
for _ in env.agents
]
self.critics_targets = [
Critic(self.mul * self.total_obs + action_shape * len(env.agents))
for _ in env.agents
]
self.actors_optimizers = [[
torch.optim.RMSprop(self.actors[agent][i].parameters(),
lr=lr_actor,
weight_decay=weight_decay)
for i in range(self.subpolicies)
] for agent in range(len(env.agents))]
self.critics_optimisers = [
torch.optim.RMSprop(self.critics[agent].parameters(),
lr=lr_critic,
weight_decay=weight_decay)
for agent in range(len(env.agents))
]
if self.subpolicies > 1:
if self.replay_buffer_type == "rb":
self.replay_buffers = [[
ReplayBuffer(self.replay_buffer_size)
for _ in range(self.subpolicies)
] for _ in range(env.n)]
else:
self.replay_buffers = [[
PrioritizedReplayBuffer(self.replay_buffer_size)
for _ in range(self.subpolicies)
] for _ in range(env.n)]
else:
if self.replay_buffer_type == "rb":
self.replay_buffers = ReplayBuffer(self.replay_buffer_size)
else:
self.replay_buffers = PrioritizedReplayBuffer(
self.replay_buffer_size)
def save(self, path):
for agent in range(self.env.n):
for sub in range(self.subpolicies):
torch.save(
self.actors[agent][sub].state_dict(),
path + '/actor_{}_subpolicy_{}.pt'.format(agent, sub))
def load(self, path):
for agent in range(self.env.n):
for sub in range(self.subpolicies):
self.actors[agent][sub].load_state_dict(
torch.load(path +
"/actor_{}_subpolicy_{}.pt".format(agent, sub)))
def push_sample(self, s, a, r, d, s_t, subs):
if self.replay_buffer_type == "rb":
if self.subpolicies > 1:
for agent in range(self.env.n):
self.replay_buffers[agent][subs[agent]].push(
s, a, r, d, s_t, subs)
else:
self.replay_buffers.push(s, a, r, d, s_t, subs)
else:
errors = self.get_errors(s, a, r, d, s_t, subs)
if self.subpolicies > 1:
for agent in range(self.env.n):
self.replay_buffers[agent][subs[agent]].push(
s, a, r, d, s_t, subs, errors[agent])
else:
self.replay_buffers.push(s, a, r, d, s_t, subs,
np.mean(errors))
def get_errors(self, s, a, r, d, s_t, subs):
errors = []
per_agent_obs = [s[agent] for agent in range(self.env.n)]
t_agent_obs = [
torch.Tensor(list(per_agent_obs[agent][:])).view(1, -1)
for agent in range(self.env.n)
]
obs = torch.cat(t_agent_obs, 1)
per_agent_new_obs = [s_t[agent] for agent in range(self.env.n)]
t_agent_new_obs = [
torch.Tensor(list(per_agent_new_obs[agent][:])).view(1, -1)
for agent in range(self.env.n)
]
new_obs = torch.cat(t_agent_new_obs, 1)
action = torch.Tensor(a)
reward = torch.Tensor(r)
new_actions = []
for agent in range(self.env.n):
current_new_action = self.actors_targets[agent][subs[agent]](
t_agent_new_obs[agent]).clamp(self.min_action, self.max_action)
new_actions.append(current_new_action)
new_actions = torch.stack(new_actions, 1)
for i in range(len(self.env.agents)):
with torch.no_grad():
target_Q_input = torch.cat((new_obs, new_actions.view(1, -1)),
1)
target_Q = reward[i] + (self.gamma * self.critics_targets[i]
(target_Q_input) * (1 - d))
current_Q_input = torch.cat((obs, action.view(1, -1)), 1)
current_Q = self.critics[i](current_Q_input)
critic_loss = F.mse_loss(current_Q, target_Q)
errors.append(critic_loss.item())
return errors
def apply_noise_decay(self):
self.noise = self.noise * self.noise_decay
def reset_noise(self, noise_value=None):
if noise_value is None:
self.noise = self.init_noise
else:
self.noise = noise_value
def random_act(self):
return np.array([
random.uniform(self.min_action, self.max_action)
for _ in range(self.env.n * self.action_shape)
]).reshape(self.env.n, self.action_shape)
def act(self, s, subs, noise=True):
actions = []
input = torch.Tensor(s)
for agent in range(self.env.n):
action = self.actors[agent][subs[agent]](input[agent])
if noise is True:
action = action + torch.FloatTensor(action.shape).uniform_(
-self.noise, self.noise)
action = action.clamp(self.min_action, self.max_action)
actions.append(action)
return np.array([action.detach().numpy() for action in actions
]).reshape(self.env.n, self.action_shape)
def train(self, subs):
for i in range(len(self.env.agents)):
self.critics_optimisers[i].zero_grad()
if self.subpolicies > 1:
minibatch = self.replay_buffers[i][subs[i]].sample(
self.batch_size)
else:
minibatch = self.replay_buffers.sample(self.batch_size)
if self.replay_buffer_type == "rb":
obs, actions, rewards, dones, new_obs, _ = minibatch
else:
(batch, index, is_weight) = minibatch
is_weight = torch.Tensor(is_weight)
obs, actions, rewards, dones, new_obs, _ = batch
""" Handle heterogeneous observation sizes by splitting agent observations """
per_agent_obs = [obs[:, agent] for agent in range(self.env.n)]
t_agent_obs = [
torch.Tensor(list(per_agent_obs[agent][:]))
for agent in range(self.env.n)
]
obs = torch.cat(t_agent_obs, 1)
per_agent_new_obs = [
new_obs[:, agent] for agent in range(self.env.n)
]
t_agent_new_obs = [
torch.Tensor(list(per_agent_new_obs[agent][:]))
for agent in range(self.env.n)
]
new_obs = torch.cat(t_agent_new_obs, 1)
actions = torch.Tensor(actions).clone()
rewards = torch.Tensor(rewards)
dones = torch.Tensor(dones).view(self.batch_size, -1)
""" Compute a(st + 1) for each agent """
new_actions = []
for agent in range(self.env.n):
current_new_action = self.actors_targets[agent][subs[agent]](
t_agent_new_obs[agent]).clamp(self.min_action,
self.max_action)
new_actions.append(current_new_action)
new_actions = torch.stack(new_actions, 1)
target_Q_input = torch.cat(
(new_obs, new_actions.view(self.batch_size, -1)), 1)
current_Q_input = torch.cat(
(obs, actions.view(self.batch_size, -1)), 1)
with torch.no_grad():
target_Q = rewards[:, i].view(self.batch_size, -1) + (
self.gamma * self.critics_targets[i](target_Q_input) *
(1 - dones))
target_Q.detach()
current_Q = self.critics[i](current_Q_input)
if self.replay_buffer_type == "per":
error = ((current_Q - target_Q)**2).reshape(
is_weight.shape) * is_weight
error = error.detach().numpy()
for sample_index in range(self.batch_size):
if self.subpolicies > 1:
self.replay_buffers[i][subs[i]].update(
index[sample_index], error[sample_index])
else:
self.replay_buffers.update(index[sample_index],
error[sample_index])
critic_loss = F.mse_loss(current_Q, target_Q)
critic_loss.backward()
self.critics_optimisers[i].step()
self.actors_optimizers[i][subs[i]].zero_grad()
t_agent_obs[i].requires_grad = True
actions[:, i] = self.actors[i][subs[i]](t_agent_obs[i])
_input = torch.cat((obs.view(
self.batch_size, -1), actions.view(self.batch_size, -1)), 1)
actor_loss = -self.critics[i](_input).mean()
actor_loss.backward()
self.actors_optimizers[i][subs[i]].step()
# Update the frozen target models
for param, target_param in zip(
self.critics[i].parameters(),
self.critics_targets[i].parameters()):
target_param.data.copy_(self.tau * param.data +
(1 - self.tau) * target_param.data)
for param, target_param in zip(
self.actors[i][subs[i]].parameters(),
self.actors_targets[i][subs[i]].parameters()):
target_param.data.copy_(self.tau * param.data +
(1 - self.tau) * target_param.data)
class DQNAgent(parl.Agent):
"""DQN Agent interacting with environment.
Attribute:
env (gym.Env): openAI Gym environment
memory (PrioritizedReplayBuffer): replay memory to store transitions
batch_size (int): batch size for sampling
target_update (int): period for target model's hard update
gamma (float): discount factor
dqn (Network): model to train and select actions
dqn_target (Network): target model to update
optimizer (torch.optim): optimizer for training dqn
transition (list): transition information including
state, action, reward, next_state, done
v_min (float): min value of support
v_max (float): max value of support
atom_size (int): the unit number of support
support (torch.Tensor): support for categorical dqn
use_n_step (bool): whether to use n_step memory
n_step (int): step number to calculate n-step td error
memory_n (ReplayBuffer): n-step replay buffer
"""
def __init__(
self,
algorithm,
device,
obs_dim,
action_dim,
env: gym.Env,
memory_size: int,
batch_size: int,
target_update: int,
n_step: int = 3,
gamma: float = 0.99,
# PER parameters
alpha: float = 0.2,
beta: float = 0.6,
):
self.env = env
self.algorithm = algorithm
self.batch_size = batch_size
self.target_update = target_update
self.gamma = gamma
# NoisyNet: All attributes related to epsilon are removed
# PER
# memory for 1-step Learning
self.beta = beta
self.memory = PrioritizedReplayBuffer(obs_dim,
memory_size,
batch_size,
alpha=alpha)
# memory for N-step Learning
self.use_n_step = True if n_step > 1 else False
if self.use_n_step:
self.n_step = n_step
self.memory_n = ReplayBuffer(obs_dim,
memory_size,
batch_size,
n_step=n_step,
gamma=gamma)
# transition to store in memory
self.transition = list()
# mode: train / test
self.is_test = False
def sample(self, state: np.ndarray):
state, selected_action = self.algorithm.sample(state)
self.transition = [state, selected_action]
return selected_action
def predict(self, state: np.ndarray):
return self.algorithm.predict(state)
def step(self, action: np.ndarray) -> Tuple[np.ndarray, np.float64, bool]:
"""Take an action and return the response of the env."""
next_state, reward, done, _ = self.env.step(action)
if not self.is_test:
self.transition += [reward, next_state, done]
# N-step transition
if self.use_n_step:
one_step_transition = self.memory_n.store(*self.transition)
# 1-step transition
else:
one_step_transition = self.transition
# add a single step transition
if one_step_transition:
self.memory.store(*one_step_transition)
return next_state, reward, done
def learn(self) -> torch.Tensor:
if self.use_n_step:
return self.algorithm.learn(self.memory, self.memory_n)
else:
return self.algorithm.learn(self.memory, memory_n=None)
def train(self,
num_frames: int,
plotting_interval: int = 200,
plot: bool = False):
"""Train the agent."""
self.is_test = False
state = self.env.reset()
update_cnt = 0
losses = []
scores = []
score = 0
for frame_idx in range(1, num_frames + 1):
action = self.sample(state)
next_state, reward, done = self.step(action)
state = next_state
score += reward
# NoisyNet: removed decrease of epsilon
# PER: increase beta
fraction = min(frame_idx / num_frames, 1.0)
self.beta = self.beta + fraction * (1.0 - self.beta)
# if episode ends
if done:
state = self.env.reset()
scores.append(score)
tensorboard.add_scalar('score', score)
score = 0
# if training is ready
if len(self.memory) >= self.batch_size:
loss = self.learn()
losses.append(loss)
tensorboard.add_scalar('loss', loss)
update_cnt += 1
# if hard update is needed
if update_cnt % self.target_update == 0:
self.algorithm._target_hard_update()
# plotting
if frame_idx % plotting_interval == 0:
self._plot(frame_idx, scores, losses, plot=plot)
self.env.close()
def test(self, render: bool) -> None:
"""Test the agent."""
self.is_test = True
state = self.env.reset()
done = False
score = 0
while not done:
# self.env.render()
action = self.sample(state)
next_state, reward, done = self.step(action)
state = next_state
score += reward
logger.info("score: {}".format(score))
self.env.close()
def _plot(self, frame_idx: int, scores: List[float], losses: List[float],
plot: bool):
"""Plot the training progresses."""
logger.info("Frame: {}, Score: {:.1f}, loss: {:.2f}".format(
frame_idx, np.mean(scores[-10:]), np.mean(losses)))
if plot:
clear_output(True)
plt.figure(figsize=(20, 5))
plt.subplot(131)
plt.title('frame %s. score: %s' %
(frame_idx, np.mean(scores[-10:])))
plt.plot(scores)
plt.subplot(132)
plt.title('loss')
plt.plot(losses)
plt.show()
plt.close()
if Settings.KINEMATIC_NOISE:
print(
"Noise is being applied to the kinematics during training to simulate a poor controller\n"
)
##############################
##### Initializing items #####
##############################
# Initializing saver class (for loading & saving data)
saver = saver.Saver(sess, filename)
# Initializing replay buffer, with the option of a prioritized replay buffer
if Settings.PRIORITY_REPLAY_BUFFER:
replay_buffer = PrioritizedReplayBuffer()
else:
replay_buffer = ReplayBuffer()
# Initializing thread & process list
threads = []
environment_processes = []
# Event()s are used to communicate with threads while they run.
# In this case, it is used to signal to the threads when it is time to stop gracefully.
stop_run_flag = threading.Event() # Flag to stop all threads
replay_buffer_dump_flag = threading.Event(
) # Flag to pause data writing to the replay buffer
replay_buffer_dump_flag.set(
) # Set the flag to initially be True so that the agents will write data
class Agent:
"""Interacts with and learns from the environment."""
def __init__(self, create_network, state_size, action_size, seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
# Q-Network
self.qnetwork_local = create_network(state_size, action_size,
seed).to(device)
self.qnetwork_target = create_network(state_size, action_size,
seed).to(device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = PrioritizedReplayBuffer(BUFFER_SIZE, seed)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
# For debugging
self.learn_step = 0
def step(self, state, action, reward, next_state, done):
# Save experience in replay memory
self.memory.add(state, action, reward, next_state, done)
# Learn every UPDATE_EVERY time steps.
self.t_step = (self.t_step + 1) % UPDATE_EVERY
if self.t_step == 0:
# If enough samples are available in memory, get random subset and learn
if len(self.memory) > BATCH_SIZE:
batch = self.memory.sample(BATCH_SIZE)
self.learn(batch, GAMMA)
if done:
self.memory.increase_beta()
def act(self, state, eps=0.):
"""Returns actions for given state as per current policy.
Params
======
state (array_like): current state
eps (float): epsilon, for epsilon-greedy action selection
"""
state = torch.from_numpy(state).float().unsqueeze(0).to(device)
self.qnetwork_local.eval()
with torch.no_grad():
action_values = self.qnetwork_local(state)
self.qnetwork_local.train()
# Epsilon-greedy action selection
if random.random() > eps:
return np.argmax(action_values.cpu().data.numpy())
else:
return random.choice(np.arange(self.action_size))
def learn(self, batch, gamma):
"""Update value parameters using given batch of experience tuples.
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
experiences, indices, w_is = batch
states, actions, rewards, next_states, dones = self._vstack_experiences(
experiences)
w_is = torch.from_numpy(w_is).float().to(device)
## TODO: compute and minimize the loss
"*** YOUR CODE HERE ***"
max_actions = self.qnetwork_local(next_states).detach().argmax(
dim=1, keepdim=True)
next_Q_values = self.qnetwork_target(next_states).detach().gather(
dim=1, index=max_actions)
targets = rewards + gamma * next_Q_values * (1.0 - dones)
Q_values = self.qnetwork_local.forward(states).gather(dim=1,
index=actions)
errors = torch.abs(Q_values - targets).squeeze(1)
self.memory.update_errors(indices, errors.data.numpy())
loss = ((errors**2) * w_is).mean()
# print(f"w_IS: {w_is}")
# print(f"errors: {errors}")
# print(f"loss: {loss}")
# self.learn_step += 1
# if self.learn_step > 10:
# raise Exception("stop here")
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# ------------------- update target network ------------------- #
self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)
def _vstack_experiences(self, experiences):
states = torch.from_numpy(
np.vstack([e.state for e in experiences
if e is not None])).float().to(device)
actions = torch.from_numpy(
np.vstack([e.action for e in experiences
if e is not None])).long().to(device)
rewards = torch.from_numpy(
np.vstack([e.reward for e in experiences
if e is not None])).float().to(device)
next_states = torch.from_numpy(np.vstack([e.next_state for e in experiences if e is not None])) \
.float().to(device)
dones = torch.from_numpy(np.vstack([e.done for e in experiences if e is not None]).astype(np.uint8)) \
.float().to(device)
return states, actions, rewards, next_states, dones
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)