def test_appo_compilation(self): """Test whether an APPOTrainer can be built with both frameworks.""" config = ppo.appo.DEFAULT_CONFIG.copy() config["num_workers"] = 1 num_iterations = 2 for _ in framework_iterator(config): print("w/o v-trace") _config = config.copy() _config["vtrace"] = False trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0") for i in range(num_iterations): results = trainer.train() check_train_results(results) print(results) check_compute_single_action(trainer) trainer.stop() print("w/ v-trace") _config = config.copy() _config["vtrace"] = True trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0") for i in range(num_iterations): results = trainer.train() check_train_results(results) print(results) check_compute_single_action(trainer) trainer.stop()
def test_appo_compilation(self): """Test whether an APPOTrainer can be built with both frameworks.""" config = ppo.appo.DEFAULT_CONFIG.copy() config["num_workers"] = 1 num_iterations = 2 for _ in framework_iterator(config, frameworks=("torch", "tf")): _config = config.copy() trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0") for i in range(num_iterations): print(trainer.train()) _config = config.copy() _config["vtrace"] = True trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0") for i in range(num_iterations): print(trainer.train())
def __new__(cls, config={}): name = config.pop('agent', None) if name == "DQN": return dqn.DQNTrainer(config=config) elif name == "PPO": return ppo.APPOTrainer(config=config) else: raise Exception("{} agent is not supported".format(name))
def test(self, algo, path, lr, fc_hid, fc_act): """Test trained agent for a single episode. Return the episode reward""" # instantiate env class unused_shared = [] unused_own = [] unsatisfied_shared = [] unsatisfied_own = [] episode_reward = 0 #self.config["num_workers"] = 0 self.config["lr"] = lr self.config['model']["fcnet_hiddens"] = fc_hid self.config['model']["fcnet_activation"] = fc_act if algo == "ppo": self.agent = ppo.PPOTrainer(config=self.config) if algo == "ddpg": self.agent = ddpg.DDPGTrainer(config=self.config) if algo == "a3c": self.agent = a3c.A3CTrainer(config=self.config) if algo == "impala": self.agent = impala.ImpalaTrainer(config=self.config) if algo == "appo": self.agent = ppo.APPOTrainer(config=self.config) if algo == "td3": self.agent = ddpg.TD3Trainer(config=self.config) self.agent.restore(path) env = caching_vM(config=self.config) obs = env.reset() done = False action = {} for agent_id, agent_obs in obs.items(): policy_id = self.config['multiagent']['policy_mapping_fn']( agent_id) action[agent_id] = self.agent.compute_action(agent_obs, policy_id=policy_id) obs, reward, done, info = env.step(action) done = done['__all__'] for x in range(len(info)): res = ast.literal_eval(info[x]) unused_shared.append(res[0]) unused_own.append(res[1]) unsatisfied_shared.append(res[2]) unsatisfied_own.append(res[3]) print("reward == ", reward) # sum up reward for all agents episode_reward += sum(reward.values()) return episode_reward, unused_shared, unused_own, unsatisfied_shared, unsatisfied_own
def test(self,algo, path, lr, fc_hid, fc_act): """Test trained agent for a single episode. Return the episode reward""" # instantiate env class unused_shared = [] unused_own = [] unsatisfied_shared = [] unsatisfied_own = [] episode_reward = 0 self.config_test["num_workers"] = 0 self.config_test["lr"] = lr self.config_test['model']["fcnet_hiddens"] = fc_hid self.config_test['model']["fcnet_activation"] = fc_act if algo == "ppo": self.agent = ppo.PPOTrainer(config=self.config_test) if algo == "ddpg": self.agent = ddpg.DDPGTrainer(config=self.config_test) if algo == "a3c": self.agent = a3c.A3CTrainer(config=self.config_test) if algo == "impala": self.agent = impala.ImpalaTrainer(config=self.config_test) if algo == "appo": self.agent = ppo.APPOTrainer(config=self.config_test) if algo == "td3": self.agent = ddpg.TD3Trainer(config=self.config_test) self.agent.restore(path) #env = self.agent.workers.local_worker().env #env = self.env_class(self.env_config) #env = ContentCaching(*self.config_train) #env = self.config_train["env"]#env_config) #env = self.env_class(3) #env = ContentCaching #env = self.env #self.env = ContentCaching #env = self.config_train["env"] obs = ContentCaching.reset() done = False while not done: action = self.agent.compute_action(obs) obs, reward, done, info = self.env.step(action) episode_reward += reward unused_shared.append(info["unused_shared"]) unused_own.append(info["unused_own"]) unsatisfied_shared.append(info["unsatisfied_shared"]) unsatisfied_own.append(info["unsatisfied_own"]) return episode_reward, unused_shared, unused_own, unsatisfied_shared, unsatisfied_own
def test_appo_compilation_use_kl_loss(self): """Test whether an APPOTrainer can be built with kl_loss enabled.""" config = ppo.appo.DEFAULT_CONFIG.copy() config["num_workers"] = 1 config["use_kl_loss"] = True num_iterations = 2 for _ in framework_iterator(config, with_eager_tracing=True): trainer = ppo.APPOTrainer(config=config, env="CartPole-v0") for i in range(num_iterations): results = trainer.train() check_train_results(results) print(results) check_compute_single_action(trainer) trainer.stop()
def get_rl_agent(agent_name, config, env_to_agent): if agent_name == A2C: import ray.rllib.agents.a3c as a2c agent = a2c.A2CTrainer(config=config, env=env_to_agent) elif agent_name == A3C: import ray.rllib.agents.a3c as a3c agent = a3c.A3CTrainer(config=config, env=env_to_agent) elif agent_name == BC: import ray.rllib.agents.marwil as bc agent = bc.BCTrainer(config=config, env=env_to_agent) elif agent_name == DQN: import ray.rllib.agents.dqn as dqn agent = dqn.DQNTrainer(config=config, env=env_to_agent) elif agent_name == APEX_DQN: import ray.rllib.agents.dqn as dqn agent = dqn.ApexTrainer(config=config, env=env_to_agent) elif agent_name == IMPALA: import ray.rllib.agents.impala as impala agent = impala.ImpalaTrainer(config=config, env=env_to_agent) elif agent_name == MARWIL: import ray.rllib.agents.marwil as marwil agent = marwil.MARWILTrainer(config=config, env=env_to_agent) elif agent_name == PG: import ray.rllib.agents.pg as pg agent = pg.PGTrainer(config=config, env=env_to_agent) elif agent_name == PPO: import ray.rllib.agents.ppo as ppo agent = ppo.PPOTrainer(config=config, env=env_to_agent) elif agent_name == APPO: import ray.rllib.agents.ppo as ppo agent = ppo.APPOTrainer(config=config, env=env_to_agent) elif agent_name == SAC: import ray.rllib.agents.sac as sac agent = sac.SACTrainer(config=config, env=env_to_agent) elif agent_name == LIN_UCB: import ray.rllib.contrib.bandits.agents.lin_ucb as lin_ucb agent = lin_ucb.LinUCBTrainer(config=config, env=env_to_agent) elif agent_name == LIN_TS: import ray.rllib.contrib.bandits.agents.lin_ts as lin_ts agent = lin_ts.LinTSTrainer(config=config, env=env_to_agent) else: raise Exception("Not valid agent name") return agent
def get_rllib_agent(agent_name, env_name, env, env_to_agent): config = get_config(env_name, env, 1) if is_rllib_agent(agent_name) else {} if agent_name == RLLIB_A2C: import ray.rllib.agents.a3c as a2c agent = a2c.A2CTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_A3C: import ray.rllib.agents.a3c as a3c agent = a3c.A3CTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_BC: import ray.rllib.agents.marwil as bc agent = bc.BCTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_DQN: import ray.rllib.agents.dqn as dqn agent = dqn.DQNTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_APEX_DQN: import ray.rllib.agents.dqn as dqn agent = dqn.ApexTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_IMPALA: import ray.rllib.agents.impala as impala agent = impala.ImpalaTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_MARWIL: import ray.rllib.agents.marwil as marwil agent = marwil.MARWILTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_PG: import ray.rllib.agents.pg as pg agent = pg.PGTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_PPO: import ray.rllib.agents.ppo as ppo agent = ppo.PPOTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_APPO: import ray.rllib.agents.ppo as ppo agent = ppo.APPOTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_SAC: import ray.rllib.agents.sac as sac agent = sac.SACTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_LIN_UCB: import ray.rllib.contrib.bandits.agents.lin_ucb as lin_ucb agent = lin_ucb.LinUCBTrainer(config=config, env=env_to_agent) elif agent_name == RLLIB_LIN_TS: import ray.rllib.contrib.bandits.agents.lin_ts as lin_ts agent = lin_ts.LinTSTrainer(config=config, env=env_to_agent) return agent
def test_appo_entropy_coeff_schedule(self): config = ppo.appo.DEFAULT_CONFIG.copy() config["num_workers"] = 1 config["num_gpus"] = 0 config["train_batch_size"] = 20 config["batch_mode"] = "truncate_episodes" config["rollout_fragment_length"] = 10 config["timesteps_per_iteration"] = 20 # 0 metrics reporting delay, this makes sure timestep, # which entropy coeff depends on, is updated after each worker rollout. config["min_iter_time_s"] = 0 # Initial lr, doesn't really matter because of the schedule below. config["entropy_coeff"] = 0.01 schedule = [ [0, 0.01], [120, 0.0001], ] config["entropy_coeff_schedule"] = schedule def _step_n_times(trainer, n: int): """Step trainer n times. Returns: learning rate at the end of the execution. """ for _ in range(n): results = trainer.train() return results["info"][LEARNER_INFO][DEFAULT_POLICY_ID][ LEARNER_STATS_KEY]["entropy_coeff"] for _ in framework_iterator(config): trainer = ppo.APPOTrainer(config=config, env="CartPole-v0") coeff = _step_n_times(trainer, 1) # 20 timesteps # Should be close to the starting coeff of 0.01. self.assertGreaterEqual(coeff, 0.005) coeff = _step_n_times(trainer, 10) # 200 timesteps # Should have annealed to the final coeff of 0.0001. self.assertLessEqual(coeff, 0.00011) trainer.stop()
def test_appo_two_tf_optimizers(self): config = ppo.appo.DEFAULT_CONFIG.copy() config["num_workers"] = 1 # Not explicitly setting this should cause a warning, but not fail. # config["_tf_policy_handles_more_than_one_loss"] = True config["_separate_vf_optimizer"] = True config["_lr_vf"] = 0.0002 # Make sure we have two completely separate models for policy and # value function. config["model"]["vf_share_layers"] = False num_iterations = 2 # Only supported for tf so far. for _ in framework_iterator(config, frameworks="tf"): trainer = ppo.APPOTrainer(config=config, env="CartPole-v0") for i in range(num_iterations): print(trainer.train()) check_compute_single_action(trainer) trainer.stop()