def test_appo_compilation(self):
"""Test whether an APPOTrainer can be built with both frameworks."""
config = ppo.appo.DEFAULT_CONFIG.copy()
config["num_workers"] = 1
num_iterations = 2
for _ in framework_iterator(config):
print("w/o v-trace")
_config = config.copy()
_config["vtrace"] = False
trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0")
for i in range(num_iterations):
results = trainer.train()
check_train_results(results)
print(results)
check_compute_single_action(trainer)
trainer.stop()
print("w/ v-trace")
_config = config.copy()
_config["vtrace"] = True
trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0")
for i in range(num_iterations):
results = trainer.train()
check_train_results(results)
print(results)
check_compute_single_action(trainer)
trainer.stop()
def test_appo_compilation(self):
"""Test whether an APPOTrainer can be built with both frameworks."""
config = ppo.appo.DEFAULT_CONFIG.copy()
config["num_workers"] = 1
num_iterations = 2
for _ in framework_iterator(config, frameworks=("torch", "tf")):
_config = config.copy()
trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0")
for i in range(num_iterations):
print(trainer.train())
_config = config.copy()
_config["vtrace"] = True
trainer = ppo.APPOTrainer(config=_config, env="CartPole-v0")
for i in range(num_iterations):
print(trainer.train())
def __new__(cls, config={}):
name = config.pop('agent', None)
if name == "DQN":
return dqn.DQNTrainer(config=config)
elif name == "PPO":
return ppo.APPOTrainer(config=config)
else:
raise Exception("{} agent is not supported".format(name))
def test(self, algo, path, lr, fc_hid, fc_act):
"""Test trained agent for a single episode. Return the episode reward"""
# instantiate env class
unused_shared = []
unused_own = []
unsatisfied_shared = []
unsatisfied_own = []
episode_reward = 0
#self.config["num_workers"] = 0
self.config["lr"] = lr
self.config['model']["fcnet_hiddens"] = fc_hid
self.config['model']["fcnet_activation"] = fc_act
if algo == "ppo":
self.agent = ppo.PPOTrainer(config=self.config)
if algo == "ddpg":
self.agent = ddpg.DDPGTrainer(config=self.config)
if algo == "a3c":
self.agent = a3c.A3CTrainer(config=self.config)
if algo == "impala":
self.agent = impala.ImpalaTrainer(config=self.config)
if algo == "appo":
self.agent = ppo.APPOTrainer(config=self.config)
if algo == "td3":
self.agent = ddpg.TD3Trainer(config=self.config)
self.agent.restore(path)
env = caching_vM(config=self.config)
obs = env.reset()
done = False
action = {}
for agent_id, agent_obs in obs.items():
policy_id = self.config['multiagent']['policy_mapping_fn'](
agent_id)
action[agent_id] = self.agent.compute_action(agent_obs,
policy_id=policy_id)
obs, reward, done, info = env.step(action)
done = done['__all__']
for x in range(len(info)):
res = ast.literal_eval(info[x])
unused_shared.append(res[0])
unused_own.append(res[1])
unsatisfied_shared.append(res[2])
unsatisfied_own.append(res[3])
print("reward == ", reward)
# sum up reward for all agents
episode_reward += sum(reward.values())
return episode_reward, unused_shared, unused_own, unsatisfied_shared, unsatisfied_own
def test(self,algo, path, lr, fc_hid, fc_act):
"""Test trained agent for a single episode. Return the episode reward"""
# instantiate env class
unused_shared = []
unused_own = []
unsatisfied_shared = []
unsatisfied_own = []
episode_reward = 0
self.config_test["num_workers"] = 0
self.config_test["lr"] = lr
self.config_test['model']["fcnet_hiddens"] = fc_hid
self.config_test['model']["fcnet_activation"] = fc_act
if algo == "ppo":
self.agent = ppo.PPOTrainer(config=self.config_test)
if algo == "ddpg":
self.agent = ddpg.DDPGTrainer(config=self.config_test)
if algo == "a3c":
self.agent = a3c.A3CTrainer(config=self.config_test)
if algo == "impala":
self.agent = impala.ImpalaTrainer(config=self.config_test)
if algo == "appo":
self.agent = ppo.APPOTrainer(config=self.config_test)
if algo == "td3":
self.agent = ddpg.TD3Trainer(config=self.config_test)
self.agent.restore(path)
#env = self.agent.workers.local_worker().env
#env = self.env_class(self.env_config)
#env = ContentCaching(*self.config_train)
#env = self.config_train["env"]#env_config)
#env = self.env_class(3)
#env = ContentCaching
#env = self.env
#self.env = ContentCaching
#env = self.config_train["env"]
obs = ContentCaching.reset()
done = False
while not done:
action = self.agent.compute_action(obs)
obs, reward, done, info = self.env.step(action)
episode_reward += reward
unused_shared.append(info["unused_shared"])
unused_own.append(info["unused_own"])
unsatisfied_shared.append(info["unsatisfied_shared"])
unsatisfied_own.append(info["unsatisfied_own"])
return episode_reward, unused_shared, unused_own, unsatisfied_shared, unsatisfied_own
def test_appo_compilation_use_kl_loss(self):
"""Test whether an APPOTrainer can be built with kl_loss enabled."""
config = ppo.appo.DEFAULT_CONFIG.copy()
config["num_workers"] = 1
config["use_kl_loss"] = True
num_iterations = 2
for _ in framework_iterator(config, with_eager_tracing=True):
trainer = ppo.APPOTrainer(config=config, env="CartPole-v0")
for i in range(num_iterations):
results = trainer.train()
check_train_results(results)
print(results)
check_compute_single_action(trainer)
trainer.stop()
def get_rl_agent(agent_name, config, env_to_agent):
if agent_name == A2C:
import ray.rllib.agents.a3c as a2c
agent = a2c.A2CTrainer(config=config, env=env_to_agent)
elif agent_name == A3C:
import ray.rllib.agents.a3c as a3c
agent = a3c.A3CTrainer(config=config, env=env_to_agent)
elif agent_name == BC:
import ray.rllib.agents.marwil as bc
agent = bc.BCTrainer(config=config, env=env_to_agent)
elif agent_name == DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.DQNTrainer(config=config, env=env_to_agent)
elif agent_name == APEX_DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.ApexTrainer(config=config, env=env_to_agent)
elif agent_name == IMPALA:
import ray.rllib.agents.impala as impala
agent = impala.ImpalaTrainer(config=config, env=env_to_agent)
elif agent_name == MARWIL:
import ray.rllib.agents.marwil as marwil
agent = marwil.MARWILTrainer(config=config, env=env_to_agent)
elif agent_name == PG:
import ray.rllib.agents.pg as pg
agent = pg.PGTrainer(config=config, env=env_to_agent)
elif agent_name == PPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.PPOTrainer(config=config, env=env_to_agent)
elif agent_name == APPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.APPOTrainer(config=config, env=env_to_agent)
elif agent_name == SAC:
import ray.rllib.agents.sac as sac
agent = sac.SACTrainer(config=config, env=env_to_agent)
elif agent_name == LIN_UCB:
import ray.rllib.contrib.bandits.agents.lin_ucb as lin_ucb
agent = lin_ucb.LinUCBTrainer(config=config, env=env_to_agent)
elif agent_name == LIN_TS:
import ray.rllib.contrib.bandits.agents.lin_ts as lin_ts
agent = lin_ts.LinTSTrainer(config=config, env=env_to_agent)
else:
raise Exception("Not valid agent name")
return agent
def get_rllib_agent(agent_name, env_name, env, env_to_agent):
config = get_config(env_name, env, 1) if is_rllib_agent(agent_name) else {}
if agent_name == RLLIB_A2C:
import ray.rllib.agents.a3c as a2c
agent = a2c.A2CTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_A3C:
import ray.rllib.agents.a3c as a3c
agent = a3c.A3CTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_BC:
import ray.rllib.agents.marwil as bc
agent = bc.BCTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.DQNTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_APEX_DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.ApexTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_IMPALA:
import ray.rllib.agents.impala as impala
agent = impala.ImpalaTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_MARWIL:
import ray.rllib.agents.marwil as marwil
agent = marwil.MARWILTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_PG:
import ray.rllib.agents.pg as pg
agent = pg.PGTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_PPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.PPOTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_APPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.APPOTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_SAC:
import ray.rllib.agents.sac as sac
agent = sac.SACTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_LIN_UCB:
import ray.rllib.contrib.bandits.agents.lin_ucb as lin_ucb
agent = lin_ucb.LinUCBTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_LIN_TS:
import ray.rllib.contrib.bandits.agents.lin_ts as lin_ts
agent = lin_ts.LinTSTrainer(config=config, env=env_to_agent)
return agent
def test_appo_entropy_coeff_schedule(self):
config = ppo.appo.DEFAULT_CONFIG.copy()
config["num_workers"] = 1
config["num_gpus"] = 0
config["train_batch_size"] = 20
config["batch_mode"] = "truncate_episodes"
config["rollout_fragment_length"] = 10
config["timesteps_per_iteration"] = 20
# 0 metrics reporting delay, this makes sure timestep,
# which entropy coeff depends on, is updated after each worker rollout.
config["min_iter_time_s"] = 0
# Initial lr, doesn't really matter because of the schedule below.
config["entropy_coeff"] = 0.01
schedule = [
[0, 0.01],
[120, 0.0001],
]
config["entropy_coeff_schedule"] = schedule
def _step_n_times(trainer, n: int):
"""Step trainer n times.
Returns:
learning rate at the end of the execution.
"""
for _ in range(n):
results = trainer.train()
return results["info"][LEARNER_INFO][DEFAULT_POLICY_ID][
LEARNER_STATS_KEY]["entropy_coeff"]
for _ in framework_iterator(config):
trainer = ppo.APPOTrainer(config=config, env="CartPole-v0")
coeff = _step_n_times(trainer, 1) # 20 timesteps
# Should be close to the starting coeff of 0.01.
self.assertGreaterEqual(coeff, 0.005)
coeff = _step_n_times(trainer, 10) # 200 timesteps
# Should have annealed to the final coeff of 0.0001.
self.assertLessEqual(coeff, 0.00011)
trainer.stop()
def test_appo_two_tf_optimizers(self):
config = ppo.appo.DEFAULT_CONFIG.copy()
config["num_workers"] = 1
# Not explicitly setting this should cause a warning, but not fail.
# config["_tf_policy_handles_more_than_one_loss"] = True
config["_separate_vf_optimizer"] = True
config["_lr_vf"] = 0.0002
# Make sure we have two completely separate models for policy and
# value function.
config["model"]["vf_share_layers"] = False
num_iterations = 2
# Only supported for tf so far.
for _ in framework_iterator(config, frameworks="tf"):
trainer = ppo.APPOTrainer(config=config, env="CartPole-v0")
for i in range(num_iterations):
print(trainer.train())
check_compute_single_action(trainer)
trainer.stop()
