def main():
parser = argparse.ArgumentParser(description="Watch a continuous agent.")
parser.add_argument("env", help="ID of the Environment")
parser.add_argument("filename", help="File where the model was saved.")
parser.add_argument(
"--device",
default="cuda",
help=
"The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
parser.add_argument(
"--fps",
default=120,
help="Playback speed",
)
args = parser.parse_args()
if args.env in ENVS:
env = GymEnvironment(args.env, device=args.device)
elif 'BulletEnv' in args.env or args.env in PybulletEnvironment.short_names:
env = PybulletEnvironment(args.env, device=args.device)
else:
env = GymEnvironment(args.env, device=args.device)
load_and_watch(args.filename, env, fps=args.fps)
def main():
# run on gpu
device = 'cuda'
def get_agents(preset):
agents = [
getattr(preset, agent_name)
for agent_name in classic_control.__all__
]
return [agent(device=device) for agent in agents]
SlurmExperiment(get_agents(atari),
AtariEnvironment('Breakout', device=device),
2e7,
sbatch_args={'partition': '1080ti-long'})
SlurmExperiment(get_agents(classic_control),
GymEnvironment('CartPole-v0', device=device),
100000,
sbatch_args={'partition': '1080ti-short'})
SlurmExperiment(get_agents(continuous),
GymEnvironment('LunarLanderContinuous-v2', device=device),
500000,
sbatch_args={'partition': '1080ti-short'})
def setUp(self):
self.env = GymEnvironment('LunarLanderContinuous-v2')
self.env.reset()
self.parallel_env = DuplicateEnvironment([
GymEnvironment('LunarLanderContinuous-v2'),
GymEnvironment('LunarLanderContinuous-v2'),
])
self.parallel_env.reset()
def main():
parser = argparse.ArgumentParser(description="Run a continuous actions benchmark.")
parser.add_argument("env", help="Name of the env (e.g. 'lander', 'cheetah')")
parser.add_argument(
"agent", help="Name of the agent (e.g. ddpg). See presets for available agents."
)
parser.add_argument(
"--device",
default="cuda",
help="The name of the device to run the agent on (e.g. cpu, cuda, cuda:0).",
)
parser.add_argument(
"--frames", type=int, default=2e6, help="The number of training frames."
)
parser.add_argument(
"--render", action="store_true", default=False, help="Render the environment."
)
parser.add_argument(
"--logdir", default='runs', help="The base logging directory."
)
parser.add_argument("--writer", default='tensorboard', help="The backend used for tracking experiment metrics.")
parser.add_argument(
'--hyperparameters',
default=[],
nargs='*',
help="Custom hyperparameters, in the format hyperparameter1=value1 hyperparameter2=value2 etc."
)
args = parser.parse_args()
if args.env in ENVS:
env = GymEnvironment(args.env, device=args.device)
elif 'BulletEnv' in args.env or args.env in PybulletEnvironment.short_names:
env = PybulletEnvironment(args.env, device=args.device)
else:
env = GymEnvironment(args.env, device=args.device)
agent_name = args.agent
agent = getattr(continuous, agent_name)
agent = agent.device(args.device)
# parse hyperparameters
hyperparameters = {}
for hp in args.hyperparameters:
key, value = hp.split('=')
hyperparameters[key] = type(agent.default_hyperparameters[key])(value)
agent = agent.hyperparameters(**hyperparameters)
run_experiment(
agent,
env,
frames=args.frames,
render=args.render,
logdir=args.logdir,
writer=args.writer,
)
def main():
device = 'cpu'
timesteps = 40000
run_experiment(
[dqn(), a2c()],
[
GymEnvironment('CartPole-v0', device),
GymEnvironment('Acrobot-v1', device)
],
timesteps,
)
plot_returns_100('runs', timesteps=timesteps)
def setUp(self):
np.random.seed(0)
torch.manual_seed(0)
self.env = GymEnvironment('CartPole-v0')
self.experiment = MockExperiment(a2c(), self.env, quiet=True)
for i, env in enumerate(self.experiment._envs):
env.seed(i)
def main():
parser = argparse.ArgumentParser(description="Watch a continuous agent.")
parser.add_argument("env", help="ID of the Environment")
parser.add_argument("dir",
help="Directory where the agent's model was saved.")
parser.add_argument(
"--device",
default="cpu",
help=
"The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
parser.add_argument(
"--fps",
default=120,
help="Playback speed",
)
args = parser.parse_args()
if args.env in ENVS:
env_id = ENVS[args.env]
else:
env_id = args.env
env = GymEnvironment(env_id, device=args.device)
agent = TimeFeature(GreedyAgent.load(args.dir, env))
watch(agent, env, fps=args.fps)
def main():
parser = argparse.ArgumentParser(
description="Run a classic control benchmark.")
parser.add_argument("env", help="Name of the env (e.g. CartPole-v1).")
parser.add_argument(
"agent",
help="Name of the agent (e.g. dqn). See presets for available agents.")
parser.add_argument(
"--device",
default="cuda",
help=
"The name of the device to run the agent on (e.g. cpu, cuda, cuda:0).",
)
parser.add_argument("--frames",
type=int,
default=20000,
help="The number of training frames.")
parser.add_argument("--render",
type=bool,
default=False,
help="Render the environment.")
args = parser.parse_args()
env = GymEnvironment(args.env, device=args.device)
agent_name = args.agent
agent = getattr(classic_control, agent_name)
run_experiment(agent(device=args.device),
env,
args.frames,
render=args.render)
def run_atari():
parser = argparse.ArgumentParser(description="Run a continuous actions benchmark.")
parser.add_argument("env", help="Name of the env (see envs)")
parser.add_argument(
"agent",
help="Name of the agent (e.g. actor_critic). See presets for available agents.",
)
parser.add_argument(
"--frames", type=int, default=2e6, help="The number of training frames"
)
parser.add_argument(
"--device",
default="cuda",
help="The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
parser.add_argument(
"--render", default=False, help="Whether to render the environment."
)
args = parser.parse_args()
if args.env in envs:
env_id = envs[args.env]
else:
env_id = args.env
env = GymEnvironment(env_id, device=args.device)
agent_name = args.agent
agent = getattr(continuous, agent_name)
experiment = Experiment(
agent(device=args.device), env, frames=args.frames, render=args.render
)
def setUp(self):
self.agent = MockAgent()
self.env = GymEnvironment('PongNoFrameskip-v4')
self.body = DeepmindAtariBody(ToLegacyBody(self.agent),
self.env,
noop_max=0,
frame_stack=1)
def test_reset(self):
env = GymEnvironment('CartPole-v0')
state = env.reset()
self.assertEqual(state.observation.shape, (4, ))
self.assertEqual(state.reward, 0)
self.assertFalse(state.done)
self.assertEqual(state.mask, 1)
def test_reset_preconstructed_env(self):
env = GymEnvironment(gym.make('CartPole-v0'))
state = env.reset()
self.assertEqual(state.observation.shape, (4, ))
self.assertEqual(state.reward, 0)
self.assertFalse(state.done)
self.assertEqual(state.mask, 1)
def setUp(self):
np.random.seed(0)
torch.manual_seed(0)
self.env = GymEnvironment('CartPole-v0')
self.env.seed(0)
self.experiment = MockExperiment(self.make_agent(), self.env, quiet=True)
self.experiment._env.seed(0)
def __init__(self):
super().__init__()
if Settings.CUDA:
self.device = "cuda"
else:
self.device = "cpu"
self.env = GymEnvironment(Settings.GYM_ENVIRONMENT, device=self.device)
self.agent = None
def test_step_until_done(self):
env = GymEnvironment('CartPole-v0')
env.reset()
for _ in range(100):
state = env.step(1)
if state.done:
break
self.assertEqual(state.observation.shape, (4, ))
self.assertEqual(state.reward, 1.)
self.assertTrue(state.done)
self.assertEqual(state.mask, 0)
def main():
DEVICE = 'cpu'
# DEVICE = 'cuda' # uncomment for gpu support
timesteps = 40000
run_experiment(
[
# DQN with default hyperparameters
dqn.device(DEVICE),
# DQN with a custom hyperparameters and a custom name.
dqn.device(DEVICE).hyperparameters(replay_buffer_size=100
).name('dqn-small-buffer'),
# A2C with a custom name
a2c.device(DEVICE).name('not-dqn')
],
[
GymEnvironment('CartPole-v0', DEVICE),
GymEnvironment('Acrobot-v1', DEVICE)
],
timesteps,
)
plot_returns_100('runs', timesteps=timesteps)
def run():
parser = argparse.ArgumentParser(description="Run a continuous actions benchmark.")
parser.add_argument("env", help="Name of the env (see envs)")
parser.add_argument("agent", help="Name of the agent (e.g. cacla). See presets for available agents")
parser.add_argument(
"--episodes", type=int, default=2000, help="The number of training episodes"
)
parser.add_argument(
"--frames", type=int, default=6e10, help="The number of training frames"
)
parser.add_argument(
"--device",
default="cuda",
help="The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
parser.add_argument(
"--render", default=False, help="Whether to render the environment."
)
args = parser.parse_args()
# create the environment
env = GymEnvironment(args.env, device=args.device)
agent_name = args.agent
agent = getattr(presets, agent_name)
# configure desired baseline (run sequentially)
run_baseline = False
baseline_agent_name = "cacla"
baseline_agent = getattr(presets, baseline_agent_name)
result_dir = create_result_dir(agent_name, args.env)
num_repeats = 20
for i in range(num_repeats):
# run the experiment
OptimisationExperiment(
agent(device=args.device),
env,
episodes=args.episodes,
frames=args.frames,
render=args.render,
writer=_make_writer(agent_name, env.name, True, result_dir),
write_episode_return=True
)
if run_baseline:
# run the baseline agent for comparison
OptimisationExperiment(
baseline_agent(device=args.device), env, episodes=args.episodes, frames=args.frames, render=args.render
)
def watch():
parser = argparse.ArgumentParser(description="Run an Atari benchmark.")
parser.add_argument(
"env", help="Name of the environment (e.g. RoboschoolHalfCheetah-v1")
parser.add_argument("dir",
help="Directory where the agent's model was saved.")
parser.add_argument(
"--device",
default="cpu",
help=
"The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
args = parser.parse_args()
env = GymEnvironment(args.env, device=args.device)
load_and_watch(args.dir, env)
def __init__(self, env, frames=None, episodes=None):
if frames is None:
frames = np.inf
if episodes is None:
episodes = np.inf
if isinstance(env, str):
self.env = GymEnvironment(env)
else:
self.env = env
self._max_frames = frames
self._max_episodes = episodes
self._agent = None
self._episode = None
self._frames = None
self._writer = None
self._render = None
self._console = None
def make_env(using_lander_reward_shaping=False):
env = gym.make('LunarLanderContinuous-v2')
env.action_space = spaces.Discrete(n_act_dim)
#override the step function. Before run the originally step function, run disc_to_cont to convert
#the discrete action to continous action.
env.unwrapped._step_orig = env.unwrapped.step
def _step(self, action):
obs, r, done, info = self._step_orig(disc_to_cont(action))
return obs, r, done, info
env.unwrapped.step = types.MethodType(_step, env.unwrapped)
env.unwrapped.using_lander_reward_shaping = using_lander_reward_shaping
env = GymEnvironment(env, device="cuda")
return env
def evaluate_dqn_all(num_test_episodes):
from all.experiments.watch import GreedyAgent
from all.environments import GymEnvironment
if Settings.CUDA:
device = "cuda"
else:
device = "cpu"
env = GymEnvironment(Settings.GYM_ENVIRONMENT, device=device)
agent = GreedyAgent.load('models', env)
num_crashed = 0
num_arrived = 0
action = None
iteration = 0
rlstats = StatsAggregator()
episode_reward = 0
def add_reward(state):
return {"reward": episode_reward}
rlstats.add_custom_stat_callback(add_reward)
rewards = []
while iteration < num_test_episodes:
if env.done:
actualEnv = env.env
stats = actualEnv.get_stats()
if len(stats["position_history"]) != 0:
rlstats.add_episode_stats(stats)
num_crashed += stats["crashed"]
num_arrived += stats["merged"]
iteration += 1
print(iteration)
rewards.append(episode_reward)
episode_reward = 0
env.reset()
else:
env.step(action)
action = agent.eval(env.state, env.reward)
episode_reward += env.reward
logging.info("Rewards: {}".format(rewards))
rlstats.print_stats()
def main():
parser = argparse.ArgumentParser(description="Run an Atari benchmark.")
parser.add_argument(
"env", help="Name of the environment (e.g. RoboschoolHalfCheetah-v1")
parser.add_argument("filename", help="File where the model was saved.")
parser.add_argument(
"--device",
default="cuda",
help=
"The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
parser.add_argument(
"--fps",
default=60,
help="Playback speed",
)
args = parser.parse_args()
env = GymEnvironment(args.env, device=args.device)
load_and_watch(args.filename, env, fps=args.fps)
def train_dqn_all():
from all.environments import GymEnvironment
from all.presets.classic_control import ddqn
from all.experiments import SingleEnvExperiment
if Settings.CUDA:
device = "cuda"
else:
device = "cpu"
env = GymEnvironment(Settings.GYM_ENVIRONMENT, device=device)
preset = ddqn(device=device,
lr=Settings.LEARNING_RATE,
initial_exploration=Settings.EPS_START,
final_exploration=Settings.EPS_END)
experiment = SingleEnvExperiment(preset, env)
experiment.train(1E6)
default_log_dir = experiment._writer.log_dir
copy_tree(default_log_dir, Settings.FULL_LOG_DIR)
rmtree(default_log_dir)
def main():
parser = argparse.ArgumentParser(
description="Run a continuous actions benchmark.")
parser.add_argument("env", help="Name of the env (see envs)")
parser.add_argument(
"agent",
help="Name of the agent (e.g. ddpg). See presets for available agents."
)
parser.add_argument(
"--device",
default="cuda",
help=
"The name of the device to run the agent on (e.g. cpu, cuda, cuda:0).",
)
parser.add_argument("--frames",
type=int,
default=2e6,
help="The number of training frames.")
parser.add_argument("--render",
type=bool,
default=False,
help="Render the environment.")
parser.add_argument("--logdir",
default='runs',
help="The base logging directory.")
args = parser.parse_args()
if args.env in ENVS:
env_id = ENVS[args.env]
else:
env_id = args.env
env = GymEnvironment(env_id, device=args.device)
agent_name = args.agent
agent = getattr(continuous, agent_name)
run_experiment(agent(device=args.device),
env,
frames=args.frames,
render=args.render,
logdir=args.logdir)
def fitness(self, individual):
print("Running individual: {0}".format(individual))
returns = []
# TODO - average past fitness values
# TODO - loop env
run_idxs = []
for i in range(2):
# create the environment and agent
env = GymEnvironment(self.args.env, device=self.args.device)
experiment = OptimisationExperiment(
self.agent(device=args.device,
lr_v=individual[0],
lr_pi=individual[1],
trace_decay=individual[2],
log=args.log), env,
episodes=args.episodes,
frames=args.frames,
render=args.render,
log=args.log,
quiet=True,
write_loss=False,
write_episode_return=args.log,
writer=self._make_writer(self.agent_name, env.name, self._write_loss, self.result_dir),
)
episodes_returns = np.array(experiment.runner.rewards) # returns against episodes
solved_return_value = np.array([100.0 for x in range(len(episodes_returns))])
fitness = sum(abs(solved_return_value - episodes_returns))
returns.append(fitness)
run_idxs.append(copy(self.run_count))
self.run_count += 1
print("runs fitnesses: {0}".format(returns))
avg_fitness = sum(returns) / len(returns)
for idx in run_idxs:
self.f_fitness_run_map.write("{0}\t{1}\t{2}\n".format(self.individual_id, idx, avg_fitness))
self.f_fitness_run_map.flush()
self.individual_id += 1
return avg_fitness
def run_classic():
parser = argparse.ArgumentParser(description="Run a classic control benchmark.")
parser.add_argument("env", help="Name of the env (e.g. CartPole-v1)")
parser.add_argument(
"agent",
help="Name of the agent (e.g. sarsa). See presets for available agents.",
)
parser.add_argument(
"--episodes", type=int, default=2000, help="The number of training frames"
)
parser.add_argument(
"--device",
default="cuda",
help="The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)",
)
args = parser.parse_args()
env = GymEnvironment(args.env, device=args.device)
agent_name = args.agent
agent = getattr(classic_control, agent_name)
Experiment(agent(device=args.device), env, episodes=args.episodes)
def test_same_as_duplicate(self):
n_envs = 3
torch.manual_seed(42)
env1 = DuplicateEnvironment([GymEnvironment('CartPole-v0') for i in range(n_envs)])
env2 = GymVectorEnvironment(make_vec_env(n_envs), "CartPole-v0")
env1.seed(42)
env2.seed(42)
state1 = env1.reset()
state2 = env2.reset()
self.assertEqual(env1.name, env2.name)
self.assertEqual(env1.action_space.n, env2.action_space.n)
self.assertEqual(env1.observation_space.shape, env2.observation_space.shape)
self.assertEqual(env1.num_envs, 3)
self.assertEqual(env2.num_envs, 3)
act_space = env1.action_space
for i in range(2):
self.assertTrue(torch.all(torch.eq(state1.observation, state2.observation)))
self.assertTrue(torch.all(torch.eq(state1.reward, state2.reward)))
self.assertTrue(torch.all(torch.eq(state1.done, state2.done)))
self.assertTrue(torch.all(torch.eq(state1.mask, state2.mask)))
actions = torch.tensor([act_space.sample() for i in range(n_envs)])
state1 = env1.step(actions)
state2 = env2.step(actions)
def main():
device = 'cuda'
frames = int(1e7)
agents = [
ddpg(last_frame=frames),
ppo(last_frame=frames),
sac(last_frame=frames)
]
envs = [
GymEnvironment(env, device) for env in [
'AntBulletEnv-v0', "HalfCheetahBulletEnv-v0",
'HumanoidBulletEnv-v0', 'HopperBulletEnv-v0',
'Walker2DBulletEnv-v0'
]
]
SlurmExperiment(agents,
envs,
frames,
sbatch_args={'partition': '1080ti-long'})
def run_atari():
parser = argparse.ArgumentParser(
description='Run a classic control benchmark.')
parser.add_argument('env', help='Name of the env (e.g. CartPole-v1)')
parser.add_argument(
'agent', help="Name of the agent (e.g. sarsa). See presets for available agents.")
parser.add_argument('--episodes', type=int, default=1000,
help='The number of training frames')
parser.add_argument(
'--device', default='cuda',
help='The name of the device to run the agent on (e.g. cpu, cuda, cuda:0)'
)
args = parser.parse_args()
env = GymEnvironment(args.env, device=args.device)
agent_name = args.agent
agent = getattr(classic_control, agent_name)
experiment = Experiment(
env,
episodes=args.episodes
)
experiment.run(agent(device=args.device), label=agent_name)
def resume_dqn_all():
from all.presets.classic_control import ddqn
from all.environments import GymEnvironment
from all.experiments import SingleEnvExperiment
if Settings.CUDA:
device = "cuda"
else:
device = "cpu"
env = GymEnvironment('sumo-jerk-v0', device=device)
lr = 1e-5
agent = ddqn(device=device, lr=lr)
q_module = torch.load(os.path.join('models', "q.pt"),
map_location='cpu').to(device)
experiment = SingleEnvExperiment(agent, env)
agent = experiment._agent
old_q = agent.q
old_q.model.load_state_dict(q_module.state_dict())
experiment.train(frames=1e6)
default_log_dir = experiment._writer.log_dir
copy_tree(default_log_dir, Settings.FULL_LOG_DIR)
rmtree(default_log_dir)