def main(args):
"""
Run a trained model for the cartpole problem
:param args: (ArgumentParser) the input arguments
"""
logger.configure()
env = make_mujoco_env(args.env, 0)
model = TRPO.load(args.dir, env)
while True:
obs, done = env.reset(), False
episode_rew = 0
while not done:
if not args.no_render:
img = env.render(mode="rgb_array")
img = np.array(img) / 255.
img = np.transpose(img, (2, 0, 1))
vis.image(img, win='frame')
action, _, _, _ = model.policy_pi.step(obs.reshape(-1, *obs.shape),
deterministic=True)
obs, rew, done, _ = env.step(action)
episode_rew += rew
print("Episode reward", episode_rew)
# No render is only used for automatic testing
if args.no_render:
break
def main():
"""
Runs the test
"""
logger.configure()
parser = mujoco_arg_parser()
parser.add_argument('--model-path', default=os.path.join(logger.get_dir(), 'humanoid_policy'))
parser.set_defaults(num_timesteps=int(2e7))
args = parser.parse_args()
if not args.play:
# train the model
train(num_timesteps=args.num_timesteps, seed=args.seed, model_path=args.model_path)
else:
# construct the model object, load pre-trained model and render
model = train(num_timesteps=1, seed=args.seed)
tf_util.load_state(args.model_path)
env = make_mujoco_env('Humanoid-v2', seed=0)
obs = env.reset()
while True:
action = model.policy.act(stochastic=False, obs=obs)[0]
obs, _, done, _ = env.step(action)
env.render()
if done:
obs = env.reset()
def train(env_id, num_timesteps, seed):
"""
train an ACKTR model on atari
:param env_id: (str) Environment ID
:param num_timesteps: (int) The total number of samples
:param seed: (int) The initial seed for training
"""
env = make_mujoco_env(env_id, seed)
with tf.Session(config=tf.ConfigProto()):
ob_dim = env.observation_space.shape[0]
ac_dim = env.action_space.shape[0]
with tf.variable_scope("vf"):
value_fn = NeuralNetValueFunction(ob_dim, ac_dim)
with tf.variable_scope("pi"):
policy = GaussianMlpPolicy(ob_dim, ac_dim)
learn(env,
policy=policy,
value_fn=value_fn,
gamma=0.99,
lam=0.97,
timesteps_per_batch=2500,
desired_kl=0.002,
num_timesteps=num_timesteps,
animate=False)
env.close()
def train(env_id, num_timesteps, seed):
"""
Train TRPO model for the mujoco environment, for testing purposes
:param env_id: (str) Environment ID
:param num_timesteps: (int) The total number of samples
:param seed: (int) The initial seed for training
"""
with tf_util.single_threaded_session():
rank = MPI.COMM_WORLD.Get_rank()
if rank == 0:
logger.configure()
else:
logger.configure(format_strs=[])
logger.set_level(logger.DISABLED)
workerseed = seed + 10000 * MPI.COMM_WORLD.Get_rank()
tblog = "/cvgl2/u/surajn/workspace/tb_logs/reacher/"
env = make_mujoco_env(env_id, workerseed)
model = TRPO(MlpPolicy,
env,
timesteps_per_batch=1024,
max_kl=0.01,
cg_iters=10,
cg_damping=0.1,
entcoeff=0.0,
gamma=0.99,
lam=0.98,
vf_iters=5,
vf_stepsize=1e-3,
tensorboard_log)
model.learn(total_timesteps=num_timesteps)
env.close()
def train(env_id, num_timesteps, run, kappa, vf_phi_update_interval, log):
"""
Train TRPO model for the mujoco environment, for testing purposes
:param env_id: (str) Environment ID
:param num_timesteps: (int) The total number of samples
:param seed: (int) The initial seed for training
"""
with tf_util.single_threaded_session():
rank = MPI.COMM_WORLD.Get_rank()
log_path = './experiments/'+str(env_id)+'./updated_nkappa_x7_ent_0.01_new/'+str(kappa)+'_'+str(vf_phi_update_interval)+'_'+str(run)
if not log:
if rank == 0:
logger.configure(log_path)
else:
logger.configure(log_path, format_strs=[])
logger.set_level(logger.DISABLED)
else:
if rank == 0:
logger.configure()
else:
logger.configure(format_strs=[])
logger.set_level(logger.DISABLED)
seed = run
workerseed = seed + 10000 * MPI.COMM_WORLD.Get_rank()
#set_global_seeds(run)
env = make_mujoco_env(env_id, workerseed)
test_env = None#make_mujoco_env(env_id, workerseed)
model = TRPO(MlpPolicy, env, test_env=test_env, timesteps_per_batch=1024, max_kl=0.01, cg_iters=10, cg_damping=0.1, entcoeff=0.01,
gamma=0.99, kappa=kappa, vf_iters=5, vf_stepsize=1e-3, verbose=1, vf_phi_update_interval=vf_phi_update_interval, seed=run)
model.learn(total_timesteps=int(2e6), seed=run)
#model.save("./"+str(env_id)+"./models/"+str(kappa)+"_"+str(run)+'_xnew_longer_slower'+str(vf_phi_update_interval)+'.pkl')
env.close()
def train_SAC(env, out_dir, seed=None, **kwargs):
# Logs will be saved in log_dir/monitor.csv
global output_dir
output_dir = out_dir
log_dir = os.path.join(out_dir, 'log')
os.makedirs(log_dir, exist_ok=True)
env = make_mujoco_env(env, 0)
env = Monitor(env, log_dir + "/")
continue_train = False
if continue_train:
# Continue training
print("Loading pretrained agent")
model = SAC.load(os.path.join(out_dir, 'final_model.pkl'),
env=env,
tensorboard_log=os.path.join(log_dir, 'tb'),
verbose=1,
**kwargs)
else:
model = SAC(
policy,
env, #action_noise=action_noise,
verbose=1,
tensorboard_log=os.path.join(log_dir, 'tb'),
full_tensorboard_log=False,
**kwargs)
model.learn(total_timesteps=n_timesteps,
seed=seed,
callback=callback,
log_interval=10)
return model
def main():
"""
Runs the test
"""
args = mujoco_arg_parser().parse_args()
logger.configure()
train(args.env, num_timesteps=args.num_timesteps, seed=args.seed)
env = make_mujoco_env(args.env, args.seed)
model = PPO1(MlpPolicy,
env,
timesteps_per_actorbatch=2048,
clip_param=0.2,
entcoeff=0.0,
optim_epochs=10,
optim_stepsize=3e-4,
optim_batchsize=64,
gamma=0.99,
lam=0.95,
schedule='linear')
model.learn(total_timesteps=args.num_timesteps)
model.save("ppo1")
# env.close()
del model # remove to demonstrate saving and loading
# env = make_mujoco_env(args.env, args.seed)
model = PPO1.load("ppo1")
logger.log("~!!!!!!!!")
episode_rew = 0
obs = env.reset()
while True:
action, _states = model.predict(obs)
ob, reward, done, info = env.step(action)
episode_rew += reward
env.render()
if done:
print(f'episode_rew={episode_rew}')
episode_rew = 0
obs = env.reset()
def train(env_id, num_timesteps, seed, algorithm, model_save_file=None, log_dir=None):
with tf_util.single_threaded_session():
logger.configure(folder=log_dir, format_strs=['stdout', 'log', 'csv'])
workerseed = seed + MPI.COMM_WORLD.Get_rank()
env = make_mujoco_env(env_id, workerseed)
if algorithm == "TRPO":
model = TRPO(MlpPolicy, env, seed=workerseed, verbose=1)
else:
# Algorithm is PPO
model = PPO1(MlpPolicy, env, seed=workerseed, verbose=1)
model.learn(total_timesteps=num_timesteps)
if model_save_file is not None:
model.save(model_save_file)
env.close()
def train(env_id, num_timesteps, seed):
"""
Train PPO1 model for the Mujoco environment, for testing purposes
:param env_id: (str) Environment ID
:param num_timesteps: (int) The total number of samples
:param seed: (int) The initial seed for training
"""
env = make_mujoco_env(env_id, seed)
model = PPO1(MlpPolicy,
env,
timesteps_per_actorbatch=2048,
clip_param=0.2,
entcoeff=0.0,
optim_epochs=10,
optim_stepsize=3e-4,
optim_batchsize=64,
gamma=0.99,
lam=0.95,
schedule='linear')
model.learn(total_timesteps=num_timesteps)
env.close()
def train(num_timesteps, seed, model_path=None):
"""
Train PPO1 model for the Humanoid environment, for testing purposes
:param num_timesteps: (int) The total number of samples
:param seed: (int) The initial seed for training
:param model_path: (str) path to the model
"""
env_id = 'Humanoid-v2'
env = make_mujoco_env(env_id, seed)
# parameters below were the best found in a simple random search
# these are good enough to make humanoid walk, but whether those are
# an absolute best or not is not certain
env = RewScale(env, 0.1)
model = PPO1(MlpPolicy, env, timesteps_per_actorbatch=2048, clip_param=0.2, entcoeff=0.0, optim_epochs=10,
optim_stepsize=3e-4, optim_batchsize=64, gamma=0.99, lam=0.95, schedule='linear')
model.learn(total_timesteps=num_timesteps)
env.close()
if model_path:
tf_util.save_state(model_path)
return model
import gym
from stable_baselines.common.policies import MlpPolicy
from stable_baselines.common.cmd_util import make_mujoco_env
#from stable_baselines.common.vec_env.vec_normalize import VecNormalize
from stable_baselines.common.vec_env import DummyVecEnv
from stable_baselines import PPO2
import pybullet_envs
#env = gym.make('CartPole-v1')
env = make_mujoco_env('AntBulletEnv-v0', seed=0)
#env = VecNormalize(env0, norm_obs = True, norm_reward = False)
#env = DummyVecEnv([lambda: AHUenv() for i in range(1)])
# Optional: PPO2 requires a vectorized environment to run
# the env is now wrapped automatically when passing it to the constructor
#env = DummyVecEnv([lambda: env])
model = PPO2(MlpPolicy, env, verbose=1)
model.learn(total_timesteps=500)
print("end training =========================================")
#obs = env.reset()
#for i in range(1000):
# action, _states = model.predict(obs)
# obs, rewards, dones, info = env.step(action)
# env.render()
model.save("ppo2_cartpole")
print("saved model =========================================")
#env.close()
actor_options = {
'learning_rate': lr,
'gamma': 1.,
'verbose': 0,
'n_steps': 100,
'ent_coef': 0.,
'max_grad_norm': 1e2,
}
description = ','.join(
['{}={}'.format(k, v) for k, v in actor_options.items()])
description += ',num_env={},norm_obs={},norm_reward={}'.format(
num_env, norm_obs, norm_reward)
learning_options = {'total_timesteps': int(1e6)}
# Wrap in a try statement to close the environment properly in case of keyboard interrupt.
try:
envs = [make_mujoco_env(env_name, 2) for _ in range(num_env)]
# env = DummyVecEnv([lambda: env for env in envs])
env = SubprocVecEnv([lambda: env for env in envs])
env = VecNormalize(env, norm_obs=norm_obs, norm_reward=norm_reward)
# Create the actor and learn
actor_options['tensorboard_log'] = os.path.join(
tensorboard_logdir, env_name)
model = PPO2(MlpPolicy, env, **actor_options)
# model = PPO2(MlpLstmPolicy, env, **actor_options)
model.learn(**learning_options, tb_log_name=description)
finally:
env.close()