def __init__(self, config):
super().__init__(config)
self.training_mode = True
self.num_skills = config.hyperparameters["num_skills"] # check
self.unsupervised_episodes = config.hyperparameters["num_unsupservised_episodes"] # check
self.supervised_episodes = config.num_episodes_to_run - self.unsupervised_episodes
assert self.hyperparameters["DISCRIMINATOR"]["final_layer_activation"] == None, "Final layer activation for disciminator should be None" # check
self.discriminator = self.create_NN(self.state_size, self.num_skills, key_to_use="DISCRIMINATOR")
self.discriminator_optimizer = optim.Adam(self.discriminator.parameters(),
lr=self.hyperparameters["DISCRIMINATOR"]["learning_rate"]) # check
self.agent_config = copy.deepcopy(config)
self.agent_config.environment = DIAYN_Skill_Wrapper(copy.deepcopy(self.environment), self.num_skills, self)
self.agent_config.hyperparameters = self.agent_config.hyperparameters["AGENT"]
self.agent_config.hyperparameters["do_evaluation_iterations"] = False # check
if 'Discrete' in config.environment[0].action_space:
self.agent = SAC_Discrete(self.agent_config)
else:
self.agent = SAC(self.agent_config) #We have to use SAC because it involves maximising the policy's entropy over actions which is also a part of DIAYN
self.timesteps_to_give_up_control_for = self.hyperparameters["MANAGER"]["timesteps_to_give_up_control_for"] # check
self.manager_agent_config = copy.deepcopy(config)
self.manager_agent_config.environment = DIAYN_Manager_Agent_Wrapper(copy.deepcopy(self.environment), self.agent,
self.timesteps_to_give_up_control_for, self.num_skills)
self.manager_agent_config.hyperparameters = self.manager_agent_config.hyperparameters["MANAGER"]
self.manager_agent = DDQN(self.manager_agent_config)
def __init__(self, config):
Base_Agent.__init__(self, config)
self.controller_config = copy.deepcopy(config)
self.controller_config.hyperparameters = self.controller_config.hyperparameters[
"CONTROLLER"]
self.controller = DDQN(self.controller_config)
self.controller.q_network_local = self.create_NN(
input_dim=self.state_size * 2,
output_dim=self.action_size,
key_to_use="CONTROLLER")
self.controller.q_network_target = self.create_NN(
input_dim=self.state_size * 2,
output_dim=self.action_size,
key_to_use="CONTROLLER")
self.meta_controller_config = copy.deepcopy(config)
self.meta_controller_config.hyperparameters = self.meta_controller_config.hyperparameters[
"META_CONTROLLER"]
# self.meta_controller = DDQN(self.meta_controller_config)
# self.meta_controller.q_network_local = self.create_NN(input_dim=self.state_size, output_dim=config.environment.observation_space.n,
# key_to_use="META_CONTROLLER")
# self.meta_controller.q_network_target = self.create_NN(input_dim=self.state_size, output_dim=config.environment.observation_space.n,
# key_to_use="META_CONTROLLER")
self.list_meta_controller = [
DDQN(self.meta_controller_config) for _ in range(5)
]
self.lq_network_local = []
self.lq_network_target = []
for m in self.list_meta_controller:
m.q_network_local = self.create_NN(
input_dim=self.state_size,
output_dim=config.environment.observation_space.n,
key_to_use="META_CONTROLLER")
self.lq_network_local.append(m.q_network_local)
m.q_network_target = self.create_NN(
input_dim=self.state_size,
output_dim=config.environment.observation_space.n,
key_to_use="META_CONTROLLER")
self.lq_network_target.append(m.q_network_target)
self.rolling_intrinsic_rewards = []
self.goals_seen = []
self.controller_learnt_enough = False
self.controller_actions = []
def create_skill_training_agent(self):
"""Creates and instantiates a pre-training environment for the agent to learn skills in and then instantiates
and agent to learn in this environment"""
self.skill_agent_config.environment = Skill_Wrapper(copy.deepcopy(self.environment), self.environment.observation_space.n,
self.num_skills,
self.skill_agent_config.hyperparameters[
"regularisation_weight"], self.skill_agent_config.hyperparameters["visitations_decay"])
return DDQN(self.skill_agent_config)
def __init__(self, config):
Base_Agent.__init__(self, config)
self.controller_config = copy.deepcopy(config)
self.controller_config.hyperparameters = self.controller_config.hyperparameters[
"CONTROLLER"]
self.controller = DDQN(self.controller_config)
self.controller.q_network_local = self.create_NN(
input_dim=self.state_size * 2,
output_dim=self.action_size,
key_to_use="CONTROLLER")
self.meta_controller_config = copy.deepcopy(config)
self.meta_controller_config.hyperparameters = self.meta_controller_config.hyperparameters[
"META_CONTROLLER"]
self.meta_controller = DDQN(self.meta_controller_config)
self.meta_controller.q_network_local = self.create_NN(
input_dim=self.state_size,
output_dim=config.environment.observation_space.n,
key_to_use="META_CONTROLLER")
self.rolling_intrinsic_rewards = []
self.goals_seen = []
self.controller_learnt_enough = False
self.controller_actions = []
def create_manager_agent(self, skill_agent):
"""Instantiates a manager agent"""
self.manager_config.environment = Manager_Frozen_Worker_Wrapper(
copy.deepcopy(self.environment), self.num_skills,
self.timesteps_before_changing_skill, skill_agent)
return DDQN(self.manager_config)