def main(gpu, envid, seed, lam_fraction, final_lam, path):
with tf.device('/device:GPU:%s' % gpu):
logger.configure(dir=dirs + '%s/%s/' % (path, seed))
env = gym.make(envid)
env = bench.Monitor(env, logger.get_dir())
env = DummyVecEnv([lambda: env])
kwargs = dict(network='mlp',
env=env,
total_timesteps=1000000,
seed=seed,
nsteps=2048,
nminibatches=32,
lam=0.1,
lam_fraction=lam_fraction,
final_lam=final_lam,
gamma=0.99,
noptepochs=10,
log_interval=1,
ent_coef=0.0,
lr=lambda f: 3e-4 * f,
cliprange=0.2,
value_network='copy')
f = open(dirs + '%s/%s/params.txt' % (path, seed), 'w')
f.write(str(kwargs))
f.close()
model = ppo2.learn(**kwargs)
model.save(dirs + '%s/%s/model.pkl' % (path, seed))
env.close()
def make_vec_envs(env_name,
seed,
num_processes,
num_frame_stack=1,
downsample=True,
color=False,
gamma=0.99,
log_dir='./tmp/',
device=torch.device('cpu')):
Path(log_dir).mkdir(parents=True, exist_ok=True)
envs = [
make_env(env_name, seed, i, log_dir, downsample, color)
for i in range(num_processes)
]
if len(envs) > 1:
envs = SubprocVecEnv(envs, context='fork')
else:
envs = DummyVecEnv(envs)
if len(envs.observation_space.shape) == 1:
if gamma is None:
envs = VecNormalize(envs, ret=False)
else:
envs = VecNormalize(envs, gamma=gamma)
envs = VecPyTorch(envs, device)
if num_frame_stack > 1:
envs = VecPyTorchFrameStack(envs, num_frame_stack, device)
return envs
def ppo2(task, config, eval=False):
assert config["act_discrete"] == False
dummy_env = DummyVecEnv([lambda: task])
learn_args = {
"env": dummy_env,
"network": config["model_network"],
"total_timesteps": config["train_steps"],
"nsteps": config["train_steps_update"],
"nminibatches": config["train_minibatches_update"],
"noptepochs": config["train_epoch_update"],
"vf_coef": config["train_value_fn_coeff"],
"ent_coef": config["train_entropy_coeff"],
"gamma": config["train_gamma"],
"lam": config["train_lambda"],
"lr": float(config["train_learning_rate"]),
"save_interval": config["ckp_model_save_interval"],
# Network parameters.
"nactions": task.action_space.shape[0]
}
if eval:
learn_args["total_timesteps"] = config["eval_steps"]
learn_args["nsteps"] = config["eval_steps"]
learn_args["nminibatches"] = config["eval_steps"]
learn_args["lr"] = 0.0
models_path = config["working_dir"] + "/checkpoints/"
models_path = models_path + str(
max([f for f in os.listdir(models_path)]))
learn_args["load_path"] = models_path
return ppo2_learn(**learn_args)
def make_vec_env(env_id,
env_type,
num_env,
seed,
wrapper_kwargs=None,
start_index=0,
reward_scale=1.0,
flatten_dict_observations=True,
gamestate=None):
"""
Create a wrapped, monitored SubprocVecEnv for Atari and MuJoCo.
"""
wrapper_kwargs = wrapper_kwargs or {}
mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0
seed = seed + 10000 * mpi_rank if seed is not None else None
logger_dir = logger.get_dir()
def make_thunk(rank):
return lambda: make_env(env_id=env_id,
env_type=env_type,
mpi_rank=mpi_rank,
subrank=rank,
seed=seed,
reward_scale=reward_scale,
gamestate=gamestate,
flatten_dict_observations=
flatten_dict_observations,
wrapper_kwargs=wrapper_kwargs,
logger_dir=logger_dir)
set_global_seeds(seed)
return DummyVecEnv(
[make_thunk(i + start_index) for i in range(num_env)])
def deepq(task, config, eval=False):
assert config["act_discrete"] == True
dummy_env = DummyVecEnv([lambda: task])
learn_args = {
"env": dummy_env,
"network": config["model_network"],
"total_timesteps": config["train_steps"],
"gamma": config["train_gamma"],
"buffer_size": config["train_buffer_size"],
"exploration_fraction": config["train_exploration_fraction"],
"exploration_final_eps": config["train_exploration_final_eps"],
"prioritized_replay": config["train_prioritized_replay"],
"lr": float(config["train_learning_rate"]),
"print_freq": config["ckp_model_save_interval"],
"checkpoint_freq": config["ckp_model_save_interval"],
"checkpoint_path": config["working_dir"],
"learning_starts": -1,
# Network parameters.
"nactions": 1
}
if eval:
learn_args["total_timesteps"] = config["eval_steps"]
learn_args["learning_starts"] = config["eval_steps"]
learn_args["exploration_fraction"] = 1.0 / config["eval_steps"]
learn_args["exploration_final_eps"] = 0.0
learn_args["lr"] = 0.0
models_path = config["working_dir"] + "/checkpoints/model"
learn_args["load_path"] = models_path
return deepq_learn(**learn_args)
def make_vec_envs(env_name,
seed,
num_processes,
num_frame_stack=1,
downsample=True,
color=False,
gamma=0.99,
log_dir='./tmp/',
device=torch.device('cpu'),
use_extended_wrapper=False,
train_mode="train_encoder"):
try:
Path(log_dir).mkdir(parents=True, exist_ok=True)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
pass
envs = [
make_env(env_name,
seed,
i,
log_dir,
downsample,
color,
frame_stack=num_frame_stack,
use_extended_wrapper=use_extended_wrapper,
train_mode=train_mode) for i in range(num_processes)
]
if len(envs) > 1:
envs = SubprocVecEnv(envs, context='fork')
else:
envs = DummyVecEnv(envs)
if len(envs.observation_space.shape) == 1:
if gamma is None:
envs = VecNormalize(envs, ret=False)
else:
envs = VecNormalize(envs, gamma=gamma)
envs = VecPyTorch(envs, device)
# if num_frame_stack > 1:
# envs = VecPyTorchFrameStack(envs, num_frame_stack, device)
return envs
def make_envs(env_id, device, seed=0, num_envs=1, frame_stack=1, **kwargs):
envs = [
env_generator(env_id, seed=seed + 1000 * i, **kwargs)
for i in range(num_envs)
]
if len(envs) > 1:
envs = SubprocVecEnv(envs)
else:
envs = DummyVecEnv(envs)
envs = VecPyTorch(envs, device)
if frame_stack > 1:
envs = VecPyTorchFrameStack(envs, frame_stack, device)
return envs
def main():
steering_angles = np.array([-0.7, -0.5, -0.25, 0.0, 0.25, 0.5, 0.7])
env = AirSimGym(continuous=False, off_road_dist=2.9, max_speed=4.5, scale_reward=True, steering_angles=steering_angles)
env = DummyVecEnv([lambda: env])
# the noise objects for DDPG
n_actions = env.action_space.shape[-1]
param_noise = None
action_noise = OrnsteinUhlenbeckActionNoise(mean=np.zeros(n_actions), sigma=float(0.5) * np.ones(n_actions))
model = DDPG(MlpPolicy, env, verbose=1, param_noise=param_noise, action_noise=action_noise)
model.learn(total_timesteps=40000)
del model # remove to demonstrate saving and loading
obs = env.reset()
while True:
action, _states = model.predict(obs)
obs, rewards, dones, info = env.step(action)
def run_baselines(env, seed, log_dir):
"""Create baselines model and training.
Replace the ppo and its training with the algorithm you want to run.
Args:
env (gym.Env): Environment of the task.
seed (int): Random seed for the trial.
log_dir (str): Log dir path.
Returns:
str: The log file path.
"""
seed = seed + 1000000
set_global_seeds(seed)
env.seed(seed)
# Set up logger for baselines
configure(dir=log_dir, format_strs=['stdout', 'log', 'csv', 'tensorboard'])
baselines_logger.info('seed={}, logdir={}'.format(
seed, baselines_logger.get_dir()))
env = DummyVecEnv([
lambda: bench.Monitor(
env, baselines_logger.get_dir(), allow_early_resets=True)
])
ddpg.learn(network='mlp',
env=env,
nb_epochs=params['n_epochs'],
nb_epoch_cycles=params['steps_per_epoch'],
normalize_observations=False,
critic_l2_reg=0,
actor_lr=params['policy_lr'],
critic_lr=params['qf_lr'],
gamma=params['discount'],
nb_train_steps=params['n_train_steps'],
nb_rollout_steps=params['n_rollout_steps'],
nb_eval_steps=100)
return osp.join(log_dir, 'progress.csv')
def main():
vecEnv = DummyVecEnv([makeEnv])
env = vecEnv.envs[0]
model = ppo2.learn(
network=models.mlp(num_hidden=20, num_layers=1),
env=vecEnv,
total_timesteps=1000000
)
# play model using shorter change rate
env.change_rate = 100
play(env, model, 1000)
# play sine curve
env.change_rate = 1
env.scaling_env_options['input'] = INPUTS['SINE_CURVE']
play(env, model, 1000)
env.close()
def make_rl_envs(env_id, seed, n_envs, device, frame_stack=4, add_video=False, add_frames=False, vid_path=None, **kwargs):
envs = [env_generator(env_id, seed=seed+1000*i) for i in range(n_envs)]
if len(envs) > 1:
envs = SubprocVecEnv(envs)
else:
envs = DummyVecEnv(envs)
if add_video:
assert vid_path is not None
envs = VecConcatVideo(envs, vid_path, ordered=True)
elif add_frames:
assert vid_path is not None
envs = VecConcatVideo(envs, vid_path, ordered=False)
envs = VecPyTorch(envs, device)
if frame_stack > 1:
envs = VecPyTorchFrameStack(envs, frame_stack, device)
return envs
def get_envs(factory, dummy=False):
# assign an environment for each core
num_envs = len(os.sched_getaffinity(0))
# initialize (1) singular environment for metadata fetching and (2) vector of environments
env = factory.make_env()
env.seed(1)
def make_env():
def _thunk():
env = factory.make_env()
env.seed(1)
return env
return _thunk
envs = [make_env() for i in range(num_envs)]
if dummy:
envs = DummyVecEnv(envs)
else:
envs = ShmemVecEnv(envs)
return env, envs
def make_env(device, camera, policies, pose_estimator):
env_fn = [make_env_fn()]
vec_env = DummyVecEnv(env_fn)
base_env = vec_env.envs[0]
low = base_env.normalize_low
high = base_env.normalize_high
state_to_est = base_env.state_to_estimate
vec_env = wrap_initial_policies(vec_env, device, policies)
vec_env = RealImageObsVecEnvWrapper(vec_env, (128, 128), camera)
vec_env = VecPyTorch(vec_env, device)
vec_env = PoseEstimatorVecEnvWrapper(vec_env,
device,
pose_estimator,
state_to_est,
low,
high,
abs_to_rel=True)
return vec_env
def test_from_baselines_env(env_id):
env_fn = lambda: gym.make(env_id)
e = gym3.FromBaselinesVecEnv(DummyVecEnv([env_fn]))
gym3_rollout(e)
def train_save_baseline():
env = get_env()
model = learn('cnn_small',
DummyVecEnv([lambda: env]),
total_timesteps=int(80e6))
model.save(save_path=cache_path)
def load_baseline():
env = get_env()
model = learn('cnn_small', DummyVecEnv([lambda: env]), total_timesteps=0)
model.load(load_path=cache_path)
env.close()
return model
return model
# Avoid division error when calculate the mean (in our case if epinfo is empty returns np.nan, not return an error)
def safemean(xs):
return np.nan if len(xs) == 0 else np.mean(xs)
#### RANDOM SEED
print('RANDOM SEED: 666')
env = gym.make("metacar-level3-continuous-v0")
env = TerminateWrapper(env)
env = ClipRewardsWrapper(env)
env = WorldModelWrapper(env)
env = DummyVecEnv([lambda: env])
env = VecFrameStack(env, nstack=8)
start = time.time()
model_666 = learn(network='world_model_mlp',
env=env,
total_timesteps=1000,
seed=666,
nsteps=16,
save_model_path='')
print('Time elapsed {}'.format(time.time() - start))
model_666.train_model.value_network.save_weights(
'models/world_model_value_net_666.h5')
model_666.train_model.policy_network.save_weights(
'models/world_model_policy_net_666.h5')
DummyVecEnv,
)
import Maze
LOG_DIR = './maze_train_folder'
#Goes much faster without gpu for some reason...
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
def env_fn():
return gym.make('maze-v0', start_level=0, num_levels=50)
envs = [env_fn for x in range(64)]
venv = DummyVecEnv(envs)
config = tf.ConfigProto()
sess = tf.Session(config=config)
sess.__enter__()
conv_fn = lambda x: build_impala_cnn(x, depths=[16, 32, 32], emb_size=256)
final_model = ppo2.learn(
env=venv,
network=conv_fn,
total_timesteps=0,
mpi_rank_weight=0,
update_fn=None,
init_fn=None,
)
ppo_epochs = 3
clip_range = .2
use_vf_clipping = True
format_strs = ['csv', 'stdout']
logger.configure(dir=LOG_DIR, format_strs=format_strs)
logger.info("creating environment")
def env_fn():
return gym.make('maze-v0', start_level=0, num_levels=50)
envs = [env_fn for x in range(64)]
venv = DummyVecEnv(envs)
def eval_env_fn():
return gym.make('maze-v0', start_level=0, num_levels=0)
eval_envs = [eval_env_fn for x in range(64)]
eval_venv = DummyVecEnv(eval_envs)
venv = VecMonitor(
venv=venv,
filename=None,
keep_buf=100,
)
ppo_epochs = 3
clip_range = .2
use_vf_clipping = True
format_strs = ['csv', 'stdout']
logger.configure(dir=LOG_DIR, format_strs=format_strs)
logger.info("creating environment")
def env_fn():
return gym.make('snake-v0')
envs = [env_fn for x in range(64)]
venv = DummyVecEnv(envs)
venv = VecMonitor(
venv=venv,
filename=None,
keep_buf=100,
)
logger.info("creating tf session")
config = tf.ConfigProto()
sess = tf.Session(config=config)
sess.__enter__()
conv_fn = lambda x: build_impala_cnn(x, depths=[16, 32, 32], emb_size=256)
logger.info("training")
def __init__(self, env_fns):
DummyVecEnv.__init__(self, env_fns)
self.init_storage()
def train(*, env_id, num_env, hps, num_timesteps, seed, use_reward, ep_path,
dmlab):
venv = VecFrameStack(
DummyVecEnv([
lambda: CollectGymDataset(Minecraft('MineRLTreechop-v0', 'treechop'
),
ep_path,
atari=False)
]), hps.pop('frame_stack'))
venv.score_multiple = 1
venv.record_obs = True if env_id == 'SolarisNoFrameskip-v4' else False
ob_space = venv.observation_space
ac_space = venv.action_space
gamma = hps.pop('gamma')
policy = {'rnn': CnnGruPolicy, 'cnn': CnnPolicy}[hps.pop('policy')]
print('Running train ==========================================')
agent = PpoAgent(
scope='ppo',
ob_space=ob_space,
ac_space=ac_space,
stochpol_fn=functools.partial(
policy,
scope='pol',
ob_space=ob_space,
ac_space=ac_space,
update_ob_stats_independently_per_gpu=hps.pop(
'update_ob_stats_independently_per_gpu'),
proportion_of_exp_used_for_predictor_update=hps.pop(
'proportion_of_exp_used_for_predictor_update'),
dynamics_bonus=hps.pop("dynamics_bonus")),
gamma=gamma,
gamma_ext=hps.pop('gamma_ext'),
lam=hps.pop('lam'),
nepochs=hps.pop('nepochs'),
nminibatches=hps.pop('nminibatches'),
lr=hps.pop('lr'),
cliprange=0.1,
nsteps=128,
ent_coef=0.001,
max_grad_norm=hps.pop('max_grad_norm'),
use_news=hps.pop("use_news"),
comm=MPI.COMM_WORLD if MPI.COMM_WORLD.Get_size() > 1 else None,
update_ob_stats_every_step=hps.pop('update_ob_stats_every_step'),
int_coeff=hps.pop('int_coeff'),
ext_coeff=hps.pop('ext_coeff'))
agent.start_interaction([venv])
if hps.pop('update_ob_stats_from_random_agent'):
agent.collect_random_statistics(num_timesteps=128 * 50)
assert len(hps) == 0, "Unused hyperparameters: %s" % list(hps.keys())
counter = 0
while True:
info = agent.step()
if info['update']:
logger.logkvs(info['update'])
logger.dumpkvs()
counter += 1
if agent.I.stats['tcount'] > num_timesteps:
break
agent.stop_interaction()
def get_env(
env_name: str,
num_processes: int = 1,
seed: int = 1,
time_limit: int = None,
normalize_transition: bool = True,
normalize_first_n: int = None,
allow_early_resets: bool = False,
same_np_seed: bool = False,
**kwargs: Dict[str, Any],
) -> Env:
"""
Return environment object from environment name, with wrappers for added
functionality, such as multiprocessing and observation/reward normalization. Extra
arguments are passed to environment constructor.
Parameters
----------
env_name : str
Name of environment to create.
num_processes: int
Number of asynchronous copies of the environment to run simultaneously.
seed : int
Random seed for environment.
time_limit : int
Limit on number of steps for environment.
normalize_transition : bool
Whether or not to add environment wrapper to normalize observations and rewards.
normalize_first_n: int
If not equal to None, only normalize the first ``normalize_first_n`` elements of
the observation. If ``normalize_transition`` is False then this value is
ignored.
allow_early_resets: bool
Whether or not to allow environments before done=True is returned.
same_np_seed : bool
Whether or not to use the same numpy random seed across each process. This
should really only be used when training on MetaWorld, as it allows for multiple
processes to generate/act over the same set of goals.
Returns
-------
env : Env
Environment object.
"""
# Create vectorized environment.
seeds = [seed + i for i in range(num_processes)]
if same_np_seed:
np_seeds = [seed] * num_processes
else:
np_seeds = list(seeds)
env_creators = [
get_single_env_creator(env_name, seeds[i], np_seeds[i], time_limit,
allow_early_resets, **kwargs)
for i in range(num_processes)
]
if num_processes > 1:
env = ShmemVecEnv(env_creators, context="fork")
elif num_processes == 1:
# Use DummyVecEnv if num_processes is 1 to avoid multiprocessing overhead.
env = DummyVecEnv(env_creators)
else:
raise ValueError("Invalid num_processes value: %s" % num_processes)
# Add environment wrappers to normalize observations/rewards and convert between
# numpy arrays and torch.Tensors.
if normalize_transition:
env = VecNormalizeEnv(env, first_n=normalize_first_n)
env = VecPyTorchEnv(env)
return env