def test_evaluation_wo_evaluation_worker_set(self):
config = a3c.DEFAULT_CONFIG.copy()
config.update({
"env": "CartPole-v0",
# Switch off evaluation (this should already be the default).
"evaluation_interval": None,
})
for _ in framework_iterator(frameworks=("tf", "torch")):
# Setup trainer w/o evaluation worker set and still call
# evaluate() -> Expect error.
trainer_wo_env_on_driver = a3c.A3CTrainer(config=config)
self.assertRaisesRegexp(
ValueError, "Cannot evaluate w/o an evaluation worker set",
trainer_wo_env_on_driver.evaluate)
trainer_wo_env_on_driver.stop()
# Try again using `create_env_on_driver=True`.
# This force-adds the env on the local-worker, so this Trainer
# can `evaluate` even though, it doesn't have an evaluation-worker
# set.
config["create_env_on_driver"] = True
trainer_w_env_on_driver = a3c.A3CTrainer(config=config)
results = trainer_w_env_on_driver.evaluate()
assert "evaluation" in results
assert "episode_reward_mean" in results["evaluation"]
trainer_w_env_on_driver.stop()
config["create_env_on_driver"] = False
def check_learned(self):
"""
check the learned agent
"""
ray.init(local_mode=True)
if self.algorithm == 'PPO':
agent = ppo.PPOTrainer(config=self.ray_config,
env=self.env.__class__)
elif self.algorithm == 'A3C':
agent = a3c.A3CTrainer(config=self.ray_config,
env=self.env.__class__)
elif self.algorithm == 'PG':
agent = pg.PGTrainer(config=self.ray_config,
env=self.env.__class__)
agent.restore(self.checkpoint_path)
# run until episode ends
episode_reward = 0
done = False
obs = self.env.reset()
while True:
self.env.render()
action = agent.compute_action(obs)
obs, reward, done, info = self.env.step(action)
# print(f"obs:\n{obs}")
print(f"reward:\n{reward}")
print(f"info:\n{info}")
episode_reward += reward
def test(self, algo, path, lr, fc_hid, fc_act):
"""Test trained agent for a single episode. Return the episode reward"""
# instantiate env class
unused_shared = []
unused_own = []
unsatisfied_shared = []
unsatisfied_own = []
episode_reward = 0
#self.config["num_workers"] = 0
self.config["lr"] = lr
self.config['model']["fcnet_hiddens"] = fc_hid
self.config['model']["fcnet_activation"] = fc_act
if algo == "ppo":
self.agent = ppo.PPOTrainer(config=self.config)
if algo == "ddpg":
self.agent = ddpg.DDPGTrainer(config=self.config)
if algo == "a3c":
self.agent = a3c.A3CTrainer(config=self.config)
if algo == "impala":
self.agent = impala.ImpalaTrainer(config=self.config)
if algo == "appo":
self.agent = ppo.APPOTrainer(config=self.config)
if algo == "td3":
self.agent = ddpg.TD3Trainer(config=self.config)
self.agent.restore(path)
env = caching_vM(config=self.config)
obs = env.reset()
done = False
action = {}
for agent_id, agent_obs in obs.items():
policy_id = self.config['multiagent']['policy_mapping_fn'](
agent_id)
action[agent_id] = self.agent.compute_action(agent_obs,
policy_id=policy_id)
obs, reward, done, info = env.step(action)
done = done['__all__']
for x in range(len(info)):
res = ast.literal_eval(info[x])
unused_shared.append(res[0])
unused_own.append(res[1])
unsatisfied_shared.append(res[2])
unsatisfied_own.append(res[3])
print("reward == ", reward)
# sum up reward for all agents
episode_reward += sum(reward.values())
return episode_reward, unused_shared, unused_own, unsatisfied_shared, unsatisfied_own
def test(self,algo, path, lr, fc_hid, fc_act):
"""Test trained agent for a single episode. Return the episode reward"""
# instantiate env class
unused_shared = []
unused_own = []
unsatisfied_shared = []
unsatisfied_own = []
episode_reward = 0
self.config_test["num_workers"] = 0
self.config_test["lr"] = lr
self.config_test['model']["fcnet_hiddens"] = fc_hid
self.config_test['model']["fcnet_activation"] = fc_act
if algo == "ppo":
self.agent = ppo.PPOTrainer(config=self.config_test)
if algo == "ddpg":
self.agent = ddpg.DDPGTrainer(config=self.config_test)
if algo == "a3c":
self.agent = a3c.A3CTrainer(config=self.config_test)
if algo == "impala":
self.agent = impala.ImpalaTrainer(config=self.config_test)
if algo == "appo":
self.agent = ppo.APPOTrainer(config=self.config_test)
if algo == "td3":
self.agent = ddpg.TD3Trainer(config=self.config_test)
self.agent.restore(path)
#env = self.agent.workers.local_worker().env
#env = self.env_class(self.env_config)
#env = ContentCaching(*self.config_train)
#env = self.config_train["env"]#env_config)
#env = self.env_class(3)
#env = ContentCaching
#env = self.env
#self.env = ContentCaching
#env = self.config_train["env"]
obs = ContentCaching.reset()
done = False
while not done:
action = self.agent.compute_action(obs)
obs, reward, done, info = self.env.step(action)
episode_reward += reward
unused_shared.append(info["unused_shared"])
unused_own.append(info["unused_own"])
unsatisfied_shared.append(info["unsatisfied_shared"])
unsatisfied_own.append(info["unsatisfied_own"])
return episode_reward, unused_shared, unused_own, unsatisfied_shared, unsatisfied_own
def test_a3c_compilation(self):
"""Test whether an A3CTrainer can be built with both frameworks."""
config = a3c.DEFAULT_CONFIG.copy()
config["num_workers"] = 2
config["num_envs_per_worker"] = 2
num_iterations = 1
# Test against all frameworks.
for fw in framework_iterator(config, ("tf", "torch")):
config["sample_async"] = fw == "tf"
for env in ["CartPole-v0", "Pendulum-v0", "PongDeterministic-v0"]:
trainer = a3c.A3CTrainer(config=config, env=env)
for i in range(num_iterations):
results = trainer.train()
print(results)
check_compute_action(trainer)
def test_a3c_compilation(self):
"""Test whether an A3CTrainer can be built with both frameworks."""
config = a3c.DEFAULT_CONFIG.copy()
config["num_workers"] = 2
config["num_envs_per_worker"] = 2
num_iterations = 1
# Test against all frameworks.
for _ in framework_iterator(config):
for env in ["CartPole-v0", "Pendulum-v0", "PongDeterministic-v0"]:
print("env={}".format(env))
trainer = a3c.A3CTrainer(config=config, env=env)
for i in range(num_iterations):
results = trainer.train()
print(results)
check_compute_single_action(trainer)
trainer.stop()
def test_a3c_entropy_coeff_schedule(self):
"""Test A3CTrainer entropy coeff schedule support."""
config = a3c.DEFAULT_CONFIG.copy()
config["num_workers"] = 1
config["num_envs_per_worker"] = 1
config["train_batch_size"] = 20
config["batch_mode"] = "truncate_episodes"
config["rollout_fragment_length"] = 10
config["timesteps_per_iteration"] = 20
# 0 metrics reporting delay, this makes sure timestep,
# which entropy coeff depends on, is updated after each worker rollout.
config["min_time_s_per_reporting"] = 0
# Initial lr, doesn't really matter because of the schedule below.
config["entropy_coeff"] = 0.01
schedule = [
[0, 0.01],
[120, 0.0001],
]
config["entropy_coeff_schedule"] = schedule
def _step_n_times(trainer, n: int):
"""Step trainer n times.
Returns:
learning rate at the end of the execution.
"""
for _ in range(n):
results = trainer.train()
return results["info"][LEARNER_INFO][DEFAULT_POLICY_ID][
LEARNER_STATS_KEY]["entropy_coeff"]
# Test against all frameworks.
for _ in framework_iterator(config):
trainer = a3c.A3CTrainer(config=config, env="CartPole-v1")
coeff = _step_n_times(trainer, 1) # 20 timesteps
# Should be close to the starting coeff of 0.01
self.assertGreaterEqual(coeff, 0.005)
coeff = _step_n_times(trainer, 10) # 200 timesteps
# Should have annealed to the final coeff of 0.0001.
self.assertLessEqual(coeff, 0.00011)
trainer.stop()
def get_rl_agent(agent_name, config, env_to_agent):
if agent_name == A2C:
import ray.rllib.agents.a3c as a2c
agent = a2c.A2CTrainer(config=config, env=env_to_agent)
elif agent_name == A3C:
import ray.rllib.agents.a3c as a3c
agent = a3c.A3CTrainer(config=config, env=env_to_agent)
elif agent_name == BC:
import ray.rllib.agents.marwil as bc
agent = bc.BCTrainer(config=config, env=env_to_agent)
elif agent_name == DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.DQNTrainer(config=config, env=env_to_agent)
elif agent_name == APEX_DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.ApexTrainer(config=config, env=env_to_agent)
elif agent_name == IMPALA:
import ray.rllib.agents.impala as impala
agent = impala.ImpalaTrainer(config=config, env=env_to_agent)
elif agent_name == MARWIL:
import ray.rllib.agents.marwil as marwil
agent = marwil.MARWILTrainer(config=config, env=env_to_agent)
elif agent_name == PG:
import ray.rllib.agents.pg as pg
agent = pg.PGTrainer(config=config, env=env_to_agent)
elif agent_name == PPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.PPOTrainer(config=config, env=env_to_agent)
elif agent_name == APPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.APPOTrainer(config=config, env=env_to_agent)
elif agent_name == SAC:
import ray.rllib.agents.sac as sac
agent = sac.SACTrainer(config=config, env=env_to_agent)
elif agent_name == LIN_UCB:
import ray.rllib.contrib.bandits.agents.lin_ucb as lin_ucb
agent = lin_ucb.LinUCBTrainer(config=config, env=env_to_agent)
elif agent_name == LIN_TS:
import ray.rllib.contrib.bandits.agents.lin_ts as lin_ts
agent = lin_ts.LinTSTrainer(config=config, env=env_to_agent)
else:
raise Exception("Not valid agent name")
return agent
def get_rllib_agent(agent_name, env_name, env, env_to_agent):
config = get_config(env_name, env, 1) if is_rllib_agent(agent_name) else {}
if agent_name == RLLIB_A2C:
import ray.rllib.agents.a3c as a2c
agent = a2c.A2CTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_A3C:
import ray.rllib.agents.a3c as a3c
agent = a3c.A3CTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_BC:
import ray.rllib.agents.marwil as bc
agent = bc.BCTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.DQNTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_APEX_DQN:
import ray.rllib.agents.dqn as dqn
agent = dqn.ApexTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_IMPALA:
import ray.rllib.agents.impala as impala
agent = impala.ImpalaTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_MARWIL:
import ray.rllib.agents.marwil as marwil
agent = marwil.MARWILTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_PG:
import ray.rllib.agents.pg as pg
agent = pg.PGTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_PPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.PPOTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_APPO:
import ray.rllib.agents.ppo as ppo
agent = ppo.APPOTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_SAC:
import ray.rllib.agents.sac as sac
agent = sac.SACTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_LIN_UCB:
import ray.rllib.contrib.bandits.agents.lin_ucb as lin_ucb
agent = lin_ucb.LinUCBTrainer(config=config, env=env_to_agent)
elif agent_name == RLLIB_LIN_TS:
import ray.rllib.contrib.bandits.agents.lin_ts as lin_ts
agent = lin_ts.LinTSTrainer(config=config, env=env_to_agent)
return agent
def test_a3c_compilation(self):
"""Test whether an A3CTrainer can be built with both frameworks."""
config = a3c.DEFAULT_CONFIG.copy()
config["num_workers"] = 2
config["num_envs_per_worker"] = 2
num_iterations = 2
# Test against all frameworks.
for _ in framework_iterator(config, with_eager_tracing=True):
for env in ["CartPole-v1", "Pendulum-v1", "PongDeterministic-v0"]:
print("env={}".format(env))
config["model"]["use_lstm"] = env == "CartPole-v1"
trainer = a3c.A3CTrainer(config=config, env=env)
for i in range(num_iterations):
results = trainer.train()
check_train_results(results)
print(results)
check_compute_single_action(
trainer, include_state=config["model"]["use_lstm"])
trainer.stop()
def main():
ray.init()
'''config = ppo.DEFAULT_CONFIG.copy()
config["num_gpus"] = 0
config["num_workers"] = 1
trainer = ppo.PPOTrainer(config=config, env=SingleplayerGym)'''
config = a3c.DEFAULT_CONFIG.copy()
config["num_gpus"] = 2
config["num_workers"] = 6
trainer = a3c.A3CTrainer(config=config, env=SingleplayerGym)
# Can optionally call trainer.restore(path) to load a checkpoint.
for i in range(1000):
# Perform one iteration of training the policy with PPO
result = trainer.train()
print(pretty_print(result))
if i % 100 == 0:
checkpoint = trainer.save()
print("checkpoint saved at", checkpoint)
'reward_scale': 0.01,
'max_steps': 1000,
'is_continuous': True,
'catch_distance': 0.1
}
ray.init(include_dashboard=False)
ModelCatalog.register_custom_model("CartpoleModel", PredatorVictimModel)
PVEnv = gym.make("PredatorVictim-v0", params=params)
register_env("PredatorVictimEnv", lambda _: PVEnv)
trainer = a3c.A3CTrainer(env="PredatorVictimEnv",
config={
"multiagent": {
"policies": {
"policy_predator": gen_policy(PVEnv, 0),
"policy_victim": gen_policy(PVEnv, 1)
},
"policy_mapping_fn": policy_mapping_fn,
},
})
if os.path.isfile(model_file):
weights = pickle.load(open(model_file, "rb"))
trainer.restore_from_object(weights)
keyboard.on_press_key("q", press_key_exit)
while True:
if ready_to_exit:
break
rest = trainer.train()
print(rest['policy_reward_mean'])
agent_delta_config = delta_config['agent']
for key, value in agent_delta_config.items():
print('Agent config: ', key, ' --> ', value)
agent_config[key] = value
# Load parameters that control the training regime
training_config = delta_config['training']
evaluation_steps = training_config['evaluation_steps']
checkpoint_path = training_config['checkpoint_path']
# Register the custom items
ModelCatalog.register_custom_model(model_name, Agent)
print('Agent config:\n', agent_config)
# agent_config['gamma'] = 0.0
agent = a3c.A3CTrainer(agent_config,
env=meta_env_type) # Note use of custom Env creator fn
agent.restore(checkpoint_path)
# Use this line uncommented to see the whole config and all options
# print('\n\n\nPOLICY CONFIG',agent.get_policy().config,"\n\n\n")
# Evaluate the model
def find_json_value(key_path, json, delimiter='.'):
paths = key_path.split(delimiter)
data = json
for i in range(0, len(paths)):
data = data[paths[i]]
return data
"env": NewsWorld
}
sac_config = {
"env": NewsWorld
}
parser = argparse.ArgumentParser()
parser.add_argument("--iterations", type=int, default=10)
if __name__ == "__main__":
args = parser.parse_args()
ray.init()
register_env(
"NewsLearn",
#lambda _: HeartsEnv()
lambda _: ExternalWorld(env=NewsWorld(dict()))
)
trainer = a3c.A3CTrainer(env="NewsLearn", config=dict())
for i in range(args.iterations):
result = trainer.train()
print("Iteration {}, Episodes {}, Mean Reward {}, Mean Length {}".format(
i, result['episodes_this_iter'], result['episode_reward_mean'], result['episode_len_mean']
))
i += 1
ray.shutdown()
import os
# os.environ["TUNE_RESULT_DIR"] = "/media/drake/BlackPassport/ray_results/"
import time
import ray
import ray.rllib.agents.a3c as a3c
from ray.tune.logger import pretty_print
ray.init()
config = a3c.DEFAULT_CONFIG.copy()
config["num_gpus"] = 0
config["num_workers"] = 5
config["num_envs_per_worker"] = 5
trainer = a3c.A3CTrainer(config=config, env="Blackjack-v0")
# Can optionally call trainer.restore(path) to load a checkpoint.
start = int(round(time.time()))
while True:
# Perform one iteration of training the policy with PPO
elapsed = int(round(time.time())) - start
if elapsed > 30:
break
result = trainer.train()
print(pretty_print(result))
def forward(self, input_dict, state, seq_lens):
model_out, self._value_out = self.base_model(input_dict["obs"])
return model_out, state
def value_function(self):
return tf.reshape(self._value_out, [-1])
ray.init(include_dashboard=False)
ModelCatalog.register_custom_model("CartpoleModel", CartpoleModel)
CartpoleEnv = gym.make('CartPole-v0')
CartpoleEnv = ScaleReward(CartpoleEnv)
register_env("CP", lambda _:CartpoleEnv)
trainer = a3c.A3CTrainer(env="CP", config={"model": {"custom_model": "CartpoleModel"}})
if os.path.isfile('CartPole.pickle'):
weights = pickle.load(open("CartPole.pickle", "rb"))
trainer.restore_from_object(weights)
keyboard.on_press_key("q", press_key_exit)
while True:
if ready_to_exit:
break
rest = trainer.train()
print(rest["episode_reward_mean"])
weights = trainer.save_to_object()
pickle.dump(weights, open('CartPole.pickle', 'wb'))
print('Model saved')
self.has_virus = self.get_virus()
self.destinations = self.get_position()
states = [self.encode_state(i) for i in range(self.agent_num)]
self.s = states
return np.array(agent_matrix).astype(int)
if __name__ == "__main__":
population = [
np.load("./data/seocho.npy"),
np.load("./data/daechi.npy"),
np.load("./data/dogok.npy"),
np.load("./data/yangjae.npy"),
np.load("./data/sunreung.npy"),
np.load("./data/nambu.npy")
]
ray.init()
trainer = a3c.A3CTrainer(
env=EpidemicMultiEnv,
config={
"env_config": {
'agent_num': 200,
'population': population
}, # config to pass to env class
})
for _ in range(90000):
print(trainer.train())
import ray
from model import EpiNN
import ray.rllib.agents.a3c as a3c
from ray.tune.logger import pretty_print
from ray.rllib.models import ModelCatalog
ray.init(num_gpus=2)
config = a3c.DEFAULT_CONFIG.copy()
VirtLocalCDC = EpiNN()
ModelCatalog.register_custom_model("CDC_model", VirtLocalCDC)
config["num_gpus"] = 2
config["num_workers"] = 10
config["eager"] = True
config["model"] = "CDC_model"
trainer = a3c.A3CTrainer(config=config,env=)
for i in range(1000):
result = trainer.train()
print(pretty_print(result))
if i % 100 == 0:
checkpoint = trainer.save()
print("checkpoint saved at", checkpoint)
def render(checkpoint, home_path):
"""
Renders pybullet and mujoco environments.
"""
alg = re.match('.+?(?=_)', os.path.basename(os.path.normpath(home_path))).group(0)
current_env = re.search("(?<=_).*?(?=_)", os.path.basename(os.path.normpath(home_path))).group(0)
checkpoint_path = home_path + "checkpoint_" + str(checkpoint) + "/checkpoint-" + str(checkpoint)
config = json.load(open(home_path + "params.json"))
config_bin = pickle.load(open(home_path + "params.pkl", "rb"))
ray.shutdown()
import pybullet_envs
ray.init()
ModelCatalog.register_custom_model("RBF", RBFModel)
ModelCatalog.register_custom_model("MLP_2_64", MLP)
ModelCatalog.register_custom_model("linear", Linear)
if alg == "PPO":
trainer = ppo.PPOTrainer(config_bin)
if alg == "SAC":
trainer = sac.SACTrainer(config)
if alg == "DDPG":
trainer = ddpg.DDPGTrainer(config)
if alg == "PG":
trainer = pg.PGTrainer(config)
if alg == "A3C":
trainer = a3c.A3CTrainer(config)
if alg == "TD3":
trainer = td3.TD3Trainer(config)
if alg == "ES":
trainer = es.ESTrainer(config)
if alg == "ARS":
trainer = ars.ARSTrainer(config)
# "normalize_actions": true,
trainer.restore(checkpoint_path)
if "Bullet" in current_env:
env = gym.make(current_env, render=True)
else:
env = gym.make(current_env)
#env.unwrapped.reset_model = det_reset_model
env._max_episode_steps = 10000
obs = env.reset()
action_hist = []
m_act_hist = []
state_hist = []
obs_hist = []
reward_hist = []
done = False
step = 0
for t in range(10000):
# for some algorithms you can get the sample mean out, need to change the value on the index to match your env for now
# mean_actions = out_dict['behaviour_logits'][:17]
# actions = trainer.compute_action(obs.flatten())
# sampled_actions, _ , out_dict = trainer.compute_action(obs.flatten(),full_fetch=True)
sampled_actions = trainer.compute_action(obs.flatten())
# sampled_actions, _ , out_dict = trainer.compute_action(obs.flatten(),full_fetch=True)
actions = sampled_actions
obs, reward, done, _ = env.step(np.asarray(actions))
# env.camera_adjust()
env.render(mode='human')
time.sleep(0.01)
# env.render()
# env.render(mode='rgb_array', close = True)
# p.computeViewMatrix(cameraEyePosition=[0,10,5], cameraTargetPosition=[0,0,0], cameraUpVector=[0,0,0])
# if step % 1000 == 0:
# env.reset()
# step += 1
action_hist.append(np.copy(actions))
obs_hist.append(np.copy(obs))
reward_hist.append(np.copy(reward))
if done:
obs = env.reset()
# print(sum(reward_hist))
# print((obs_hist))
#plt.plot(action_hist)
#plt.figure()
#plt.figure()
#plt.plot(obs_hist)
#plt.figure()
# Reminder that the bahavior logits that come out are the mean and logstd (not log mean, despite the name logit)
# trainer.compute_action(obs, full_fetch=True)
trainer.compute_action(obs)
self.model.add(layers.Dense(2, name='l5', activation='relu'))
return self.model.get_layer("l5").output, self.model.get_layer("l4").output
ray.init()
ModelCatalog.register_custom_model("CartpoleModel", CartpoleModel)
CartpoleEnv = gym.make('CartPole-v0')
CartpoleEnv=ScaleReward(CartpoleEnv)
register_env("CP", lambda _:CartpoleEnv)
trainer = a3c.A3CTrainer(env="CP", config={
#"model": {"custom_model": "CartpoleModel"},
#"observation_filter": "MeanStdFilter",
#"vf_share_layers": True,
}, logger_creator=lambda _:ray.tune.logger.NoopLogger({},None))
if os.path.isfile('weights.pickle'):
weights = pickle.load(open("weights.pickle", "rb"))
trainer.restore_from_object(weights)
a_list = []
_thread.start_new_thread(input_thread, (a_list,))
while not a_list:
rest=trainer.train()
print(rest["episode_reward_mean"])
weights=trainer.save_to_object()
