def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env', help='environment ID', type=str, default='CartPole-v1')
parser.add_argument('-f', '--folder', help='Log folder', type=str, default='trained_agents')
parser.add_argument('--algo', help='RL Algorithm', default='ppo2',
type=str, required=False, choices=list(ALGOS.keys()))
parser.add_argument('-n', '--n-timesteps', help='number of timesteps', default=1000,
type=int)
parser.add_argument('--n-envs', help='number of environments', default=1,
type=int)
parser.add_argument('--exp-id', help='Experiment ID (default: -1, no exp folder, 0: latest)', default=-1,
type=int)
parser.add_argument('--verbose', help='Verbose mode (0: no output, 1: INFO)', default=1,
type=int)
parser.add_argument('--no-render', action='store_true', default=False,
help='Do not render the environment (useful for tests)')
parser.add_argument('--deterministic', action='store_true', default=False,
help='Use deterministic actions')
parser.add_argument('--stochastic', action='store_true', default=False,
help='Use stochastic actions (for DDPG/DQN/SAC)')
parser.add_argument('--load-best', action='store_true', default=False,
help='Load best model instead of last model if available')
parser.add_argument('--norm-reward', action='store_true', default=False,
help='Normalize reward if applicable (trained with VecNormalize)')
parser.add_argument('--seed', help='Random generator seed', type=int, default=0)
parser.add_argument('--reward-log', help='Where to log reward', default='', type=str)
parser.add_argument('--gym-packages', type=str, nargs='+', default=[], help='Additional external Gym environemnt package modules to import (e.g. gym_minigrid)')
parser.add_argument('--render-pybullet', help='Slow down Pybullet simulation to render', default=False) # added by Pierre
parser.add_argument('--random-pol', help='Random policy', default=False) # added by Pierre
args = parser.parse_args()
plot_bool = True
plot_dim = 2
log_bool = False
if plot_bool:
if plot_dim == 2:
fig, (ax1, ax2) = plt.subplots(2, 1, sharey=True, figsize=(5, 10))
elif plot_dim == 3:
fig = plt.figure()
ax = fig.gca(projection='3d')
if log_bool:
output_df = pd.DataFrame()
# Going through custom gym packages to let them register in the global registory
for env_module in args.gym_packages:
importlib.import_module(env_module)
env_id = args.env
algo = args.algo
folder = args.folder
if args.exp_id == 0:
args.exp_id = get_latest_run_id(os.path.join(folder, algo), env_id)
print('Loading latest experiment, id={}'.format(args.exp_id))
# Sanity checks
if args.exp_id > 0:
log_path = os.path.join(folder, algo, '{}_{}'.format(env_id, args.exp_id))
else:
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(log_path)
if not args.random_pol: # added by Pierre
model_path = find_saved_model(algo, log_path, env_id, load_best=args.load_best)
if algo in ['dqn', 'ddpg', 'sac', 'td3']:
args.n_envs = 1
set_global_seeds(args.seed)
is_atari = 'NoFrameskip' in env_id
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(stats_path, norm_reward=args.norm_reward, test_mode=True)
log_dir = args.reward_log if args.reward_log != '' else None
env = create_test_env(env_id, n_envs=args.n_envs, is_atari=is_atari,
stats_path=stats_path, seed=args.seed, log_dir=log_dir,
should_render=not args.no_render,
hyperparams=hyperparams)
# ACER raises errors because the environment passed must have
# the same number of environments as the model was trained on.
load_env = None if algo == 'acer' else env
if not args.random_pol: # added by Pierre
model = ALGOS[algo].load(model_path, env=load_env)
# if not args.no_render:
# env.render(mode="human") # added by Pierre (to work with ReachingJaco-v1)
obs = env.reset()
# Force deterministic for DQN, DDPG, SAC and HER (that is a wrapper around)
deterministic = args.deterministic or algo in ['dqn', 'ddpg', 'sac', 'her', 'td3'] and not args.stochastic
episode_reward = 0.0
episode_rewards, episode_lengths = [], []
ep_len = 0
episode = 0
# success_threshold_001 = 0.01
# success_list_001, reachtime_list_001, episode_success_list_001 = [], [], []
# success_threshold_0002 = 0.002
# success_list_0002, reachtime_list_0002, episode_success_list_0002 = [], [], []
# success_threshold_0001 = 0.001
# success_list_0001, reachtime_list_0001, episode_success_list_0001 = [], [], []
# success_threshold_00005 = 0.0005
# success_list_00005, reachtime_list_00005, episode_success_list_00005 = [], [], []
# changed for the paper
success_threshold_50 = 0.05
success_list_50, reachtime_list_50, episode_success_list_50 = [], [], []
success_threshold_20 = 0.02
success_list_20, reachtime_list_20, episode_success_list_20 = [], [], []
success_threshold_10 = 0.01
success_list_10, reachtime_list_10, episode_success_list_10 = [], [], []
success_threshold_5 = 0.005
success_list_5, reachtime_list_5, episode_success_list_5 = [], [], []
# For HER, monitor success rate
successes = []
state = None
for _ in range(args.n_timesteps):
if args.random_pol:
# Random Agent
action = [env.action_space.sample()]
else:
action, state = model.predict(obs, state=state, deterministic=deterministic)
# Clip Action to avoid out of bound errors
if isinstance(env.action_space, gym.spaces.Box):
action = np.clip(action, env.action_space.low, env.action_space.high)
obs, reward, done, infos = env.step(action)
if args.render_pybullet:
time.sleep(1./30.) # added by Pierre (slow down Pybullet for rendering)
if infos[0]['total_distance'] <= success_threshold_50:
episode_success_list_50.append(1)
else:
episode_success_list_50.append(0)
if infos[0]['total_distance'] <= success_threshold_20:
episode_success_list_20.append(1)
else:
episode_success_list_20.append(0)
if infos[0]['total_distance'] <= success_threshold_10:
episode_success_list_10.append(1)
else:
episode_success_list_10.append(0)
if infos[0]['total_distance'] <= success_threshold_5:
episode_success_list_5.append(1)
else:
episode_success_list_5.append(0)
if plot_bool:
goal = infos[0]['goal position']
tip = infos[0]['tip position']
if plot_dim == 2:
ax1.cla()
ax1.plot(goal[0], goal[2], marker='x', color='b', linestyle='', markersize=10, label="goal", mew=3)
ax1.plot(tip[0], tip[2], marker='o', color='r', linestyle='', markersize=10, label="end effector")
circ_1_50 = plt.Circle((goal[0], goal[2]), radius=success_threshold_50, edgecolor='g', facecolor='w', linestyle='--', label="50 mm")
circ_1_20 = plt.Circle((goal[0], goal[2]), radius=success_threshold_20, edgecolor='b', facecolor='w', linestyle='--', label="20 mm")
circ_1_10 = plt.Circle((goal[0], goal[2]), radius=success_threshold_10, edgecolor='m', facecolor='w', linestyle='--', label="10 mm")
circ_1_5 = plt.Circle((goal[0], goal[2]), radius=success_threshold_5, edgecolor='r', facecolor='w', linestyle='--', label="5 mm")
ax1.add_patch(circ_1_50)
ax1.add_patch(circ_1_20)
ax1.add_patch(circ_1_10)
ax1.add_patch(circ_1_5)
ax1.set_xlim([-0.25, 0.25])
ax1.set_ylim([0, 0.5])
ax1.set_xlabel("x (m)", fontsize=15)
ax1.set_ylabel("z (m)", fontsize=15)
ax2.cla()
ax2.plot(goal[1], goal[2], marker='x', color='b', linestyle='', markersize=10, mew=3)
ax2.plot(tip[1], tip[2], marker='o', color='r', linestyle='', markersize=10)
circ_2_50 = plt.Circle((goal[1], goal[2]), radius=success_threshold_50, edgecolor='g', facecolor='w', linestyle='--')
circ_2_20 = plt.Circle((goal[1], goal[2]), radius=success_threshold_20, edgecolor='b', facecolor='w', linestyle='--')
circ_2_10 = plt.Circle((goal[1], goal[2]), radius=success_threshold_10, edgecolor='m', facecolor='w', linestyle='--')
circ_2_5 = plt.Circle((goal[1], goal[2]), radius=success_threshold_5, edgecolor='r', facecolor='w', linestyle='--')
ax2.add_patch(circ_2_50)
ax2.add_patch(circ_2_20)
ax2.add_patch(circ_2_10)
ax2.add_patch(circ_2_5)
ax2.set_xlim([-0.25, 0.25])
ax2.set_ylim([0, 0.5])
ax2.set_xlabel("y (m)", fontsize=15)
ax2.set_ylabel("z (m)", fontsize=15)
ax1.legend(loc='upper left', bbox_to_anchor=(0, 1.2), ncol=3, fancybox=True, shadow=True)
elif plot_dim == 3:
ax.cla()
ax.plot([tip[0]], [tip[1]], zs=[tip[2]], marker='x', color='b')
ax.plot([goal[0]], [goal[1]], zs=[goal[2]], marker='o', color='r', linestyle="None")
ax.set_xlim([-0.2, 0.2])
ax.set_ylim([-0.2, 0.2])
ax.set_zlim([0, 0.5])
ax.set_xlabel("x (m)", fontsize=15)
ax.set_ylabel("y (m)", fontsize=15)
ax.set_zlabel("z (m)", fontsize=15)
fig.suptitle("timestep "+str(ep_len)+" | distance to target: "+str(round(infos[0]['total_distance']*1000, 1))+" mm")
plt.pause(0.01)
# plt.show()
if log_bool:
dict_log = infos[0]
dict_log['action'] = action[0]
dict_log['obs'] = obs[0]
dict_log['reward'] = reward[0]
dict_log['done'] = done[0]
dict_log['timestep'] = ep_len
dict_log['episode'] = episode
output_df = output_df.append(dict_log, ignore_index=True)
# if not args.no_render:
# env.render('human')
episode_reward += reward[0]
ep_len += 1
if args.n_envs == 1:
# For atari the return reward is not the atari score
# so we have to get it from the infos dict
if is_atari and infos is not None and args.verbose >= 1:
episode_infos = infos[0].get('episode')
if episode_infos is not None:
print("Atari Episode Score: {:.2f}".format(episode_infos['r']))
print("Atari Episode Length", episode_infos['l'])
if done and not is_atari and args.verbose > 0:
# NOTE: for env using VecNormalize, the mean reward
# is a normalized reward when `--norm_reward` flag is passed
print("Episode nb: {} | Episode Reward: {:.2f} | Episode Length: {}".format(episode, episode_reward, ep_len))
# print("Episode Length", ep_len) # commented by Pierre
state = None
episode_rewards.append(episode_reward)
episode_lengths.append(ep_len)
# append the last element of the episode success list when episode is done
success_list_50.append(episode_success_list_50[-1])
success_list_20.append(episode_success_list_20[-1])
success_list_10.append(episode_success_list_10[-1])
success_list_5.append(episode_success_list_5[-1])
# if the episode is successful and it starts from an unsucessful step, calculate reach time
if episode_success_list_50[-1] == True and episode_success_list_50[0] == False:
idx = 0
while episode_success_list_50[idx] == False:
idx += 1
reachtime_list_50.append(idx)
if episode_success_list_20[-1] == True and episode_success_list_20[0] == False:
idx = 0
while episode_success_list_20[idx] == False:
idx += 1
reachtime_list_20.append(idx)
if episode_success_list_10[-1] == True and episode_success_list_10[0] == False:
idx = 0
while episode_success_list_10[idx] == False:
idx += 1
reachtime_list_10.append(idx)
if episode_success_list_5[-1] == True and episode_success_list_5[0] == False:
idx = 0
while episode_success_list_5[idx] == False:
idx += 1
reachtime_list_5.append(idx)
if log_bool:
# output_df.to_csv(log_path+"/res_episode_"+str(episode)+".csv", index=False) # slow
output_df.to_pickle(log_path+"/res_episode_"+str(episode)+".pkl")
# reset for new episode
episode_reward = 0.0
ep_len = 0
episode_success_list_50 = []
episode_success_list_20 = []
episode_success_list_10 = []
episode_success_list_5 = []
episode += 1
# Reset also when the goal is achieved when using HER
if done or infos[0].get('is_success', False):
if args.algo == 'her' and args.verbose > 1:
print("Success?", infos[0].get('is_success', False))
# Alternatively, you can add a check to wait for the end of the episode
# if done:
obs = env.reset()
if args.algo == 'her':
successes.append(infos[0].get('is_success', False))
episode_reward, ep_len = 0.0, 0
if args.verbose > 0 and len(successes) > 0:
print("Success rate: {:.2f}%".format(100 * np.mean(successes)))
if args.verbose > 0 and len(episode_rewards) > 0:
print("Mean reward: {:.2f} +/- {:.2f}".format(np.mean(episode_rewards), np.std(episode_rewards)))
print("success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}".format(success_threshold_50, np.mean(success_list_50), np.mean(reachtime_list_50)))
print("success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}".format(success_threshold_20, np.mean(success_list_20), np.mean(reachtime_list_20)))
print("success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}".format(success_threshold_10, np.mean(success_list_10), np.mean(reachtime_list_10)))
print("success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}".format(success_threshold_5, np.mean(success_list_5), np.mean(reachtime_list_5)))
# added by Pierre
print("path:", log_path)
d = {
"Eval mean reward": np.mean(episode_rewards),
"Eval std": np.std(episode_rewards),
"success ratio 50mm": np.mean(success_list_50),
"Average reach time 50mm": np.mean(reachtime_list_50),
"success ratio 20mm": np.mean(success_list_20),
"Average reach time 20mm": np.mean(reachtime_list_20),
"success ratio 10mm": np.mean(success_list_10),
"Average reach time 10mm": np.mean(reachtime_list_10),
"success ratio 5mm": np.mean(success_list_5),
"Average reach time 5mm": np.mean(reachtime_list_5),
}
df = pd.DataFrame(d, index=[0])
if args.random_pol:
df.to_csv("logs/random_policy_0.2M/"+env_id+"/stats.csv", index=False) # make path naming more robust
else:
df.to_csv(log_path+"/stats.csv", index=False)
if args.verbose > 0 and len(episode_lengths) > 0:
print("Mean episode length: {:.2f} +/- {:.2f}".format(np.mean(episode_lengths), np.std(episode_lengths)))
# Workaround for https://github.com/openai/gym/issues/893
if not args.no_render:
if args.n_envs == 1 and 'Bullet' not in env_id and not is_atari and isinstance(env, VecEnv):
# DummyVecEnv
# Unwrap env
while isinstance(env, VecNormalize) or isinstance(env, VecFrameStack):
env = env.venv
env.envs[0].env.close()
else:
# SubprocVecEnv
env.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env',
help='environment ID',
type=str,
default='CartPole-v1')
parser.add_argument('-f',
'--folder',
help='Log folder',
type=str,
default='trained_agents')
parser.add_argument('--algo',
help='RL Algorithm',
default='ppo2',
type=str,
required=False,
choices=list(ALGOS.keys()))
# parser.add_argument('-n', '--n-timesteps', help='number of timesteps', default=1000,
# type=int)
parser.add_argument('-n',
'--n-episodes',
help='number of episodes to collect',
default=20,
type=int)
parser.add_argument('--n-envs',
help='number of environments',
default=1,
type=int)
parser.add_argument(
'--exp-id',
help='Experiment ID (default: -1, no exp folder, 0: latest)',
default=-1,
type=int)
parser.add_argument('--verbose',
help='Verbose mode (0: no output, 1: INFO)',
default=1,
type=int)
parser.add_argument(
'--no-render',
action='store_true',
default=False,
help='Do not render the environment (useful for tests)')
# for deterministic (bool type)
parser.add_argument('--deterministic',
dest='deterministic',
action='store_true')
parser.add_argument('--no-deterministic',
dest='deterministic',
action='store_false')
parser.set_defaults(deterministic=True) # true by default
# parser.add_argument('--deterministic', action='store_true', default=False,
# help='Use deterministic actions')
# parser.add_argument('--stochastic', action='store_true', default=False,
# help='Use stochastic actions (for DDPG/DQN/SAC)')
parser.add_argument(
'--norm-reward',
action='store_true',
default=False,
help='Normalize reward if applicable (trained with VecNormalize)')
parser.add_argument('--seed',
help='Random generator seed',
type=int,
default=0)
parser.add_argument('--reward-log',
help='Where to log reward',
default='',
type=str)
parser.add_argument(
'--gym-packages',
type=str,
nargs='+',
default=[],
help=
'Additional external Gym environemnt package modules to import (e.g. gym_minigrid)'
)
args = parser.parse_args()
# Going through custom gym packages to let them register in the global registory
for env_module in args.gym_packages:
importlib.import_module(env_module)
env_id = args.env
algo = args.algo
folder = args.folder
if args.exp_id == 0:
args.exp_id = get_latest_run_id(os.path.join(folder, algo), env_id)
print('Loading latest experiment, id={}'.format(args.exp_id))
# Sanity checks
if args.exp_id > 0:
log_path = os.path.join(folder, algo,
'{}_{}'.format(env_id, args.exp_id))
else:
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(
log_path)
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(
stats_path, norm_reward=args.norm_reward, test_mode=True)
log_dir = args.reward_log if args.reward_log != '' else None
if algo in ['dqn', 'ddpg', 'sac', 'td3']:
args.n_envs = 1
set_global_seeds(args.seed)
is_atari = 'NoFrameskip' in env_id
env = create_test_env(env_id,
n_envs=args.n_envs,
is_atari=is_atari,
stats_path=stats_path,
seed=args.seed,
log_dir=log_dir,
should_render=not args.no_render,
hyperparams=hyperparams)
model_path = find_saved_model(algo, log_path, env_id)
model = ALGOS[algo].load(model_path, env=env)
# Force deterministic for DQN, DDPG, SAC and HER (that is a wrapper around)
# deterministic = args.deterministic or algo in ['dqn', 'ddpg', 'sac', 'her', 'td3'] and not args.stochastic
deterministic = args.deterministic
save_dir = os.path.join("expert_trajs_by_info_deterministic_with_std",
algo)
if not os.path.isdir(save_dir):
os.makedirs(save_dir)
runner(env,
env_id,
model,
args.n_episodes,
deterministic,
save=True,
save_dir=save_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env',
help='environment ID',
type=str,
default='CartPole-v1')
parser.add_argument('-f',
'--folder',
help='Log folder',
type=str,
default='trained_agents')
parser.add_argument('--algo',
help='RL Algorithm',
default='ppo2',
type=str,
required=False,
choices=list(ALGOS.keys()))
parser.add_argument('-n',
'--n-timesteps',
help='number of timesteps',
default=1000,
type=int)
parser.add_argument('--n-envs',
help='number of environments',
default=1,
type=int)
parser.add_argument(
'--exp-id',
help='Experiment ID (default: -1, no exp folder, 0: latest)',
default=-1,
type=int)
parser.add_argument('--verbose',
help='Verbose mode (0: no output, 1: INFO)',
default=1,
type=int)
parser.add_argument(
'--no-render',
action='store_true',
default=False,
help='Do not render the environment (useful for tests)')
parser.add_argument('--deterministic',
action='store_true',
default=False,
help='Use deterministic actions')
parser.add_argument('--stochastic',
action='store_true',
default=False,
help='Use stochastic actions (for DDPG/DQN/SAC)')
parser.add_argument(
'--norm-reward',
action='store_true',
default=False,
help='Normalize reward if applicable (trained with VecNormalize)')
parser.add_argument('--seed',
help='Random generator seed',
type=int,
default=0)
parser.add_argument('--reward-log',
help='Where to log reward',
default='',
type=str)
parser.add_argument(
'--gym-packages',
type=str,
nargs='+',
default=[],
help=
'Additional external Gym environemnt package modules to import (e.g. gym_minigrid)'
)
args = parser.parse_args()
# Going through custom gym packages to let them register in the global registory
for env_module in args.gym_packages:
importlib.import_module(env_module)
env_id = args.env
algo = args.algo
folder = args.folder
if args.exp_id == 0:
args.exp_id = get_latest_run_id(os.path.join(folder, algo), env_id)
print('Loading latest experiment, id={}'.format(args.exp_id))
# Sanity checks
if args.exp_id > 0:
log_path = os.path.join(folder, algo,
'{}_{}'.format(env_id, args.exp_id))
else:
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(
log_path)
model_path = find_saved_model(algo, log_path, env_id)
if algo in ['dqn', 'ddpg', 'sac', 'td3']:
args.n_envs = 1
set_global_seeds(args.seed)
is_atari = 'NoFrameskip' in env_id
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(
stats_path, norm_reward=args.norm_reward, test_mode=True)
log_dir = args.reward_log if args.reward_log != '' else None
env = create_test_env(env_id,
n_envs=args.n_envs,
is_atari=is_atari,
stats_path=stats_path,
seed=args.seed,
log_dir=log_dir,
should_render=not args.no_render,
hyperparams=hyperparams)
# ACER raises errors because the environment passed must have
# the same number of environments as the model was trained on.
load_env = None if algo == 'acer' else env
model = ALGOS[algo].load(model_path, env=load_env)
obs = env.reset()
# Force deterministic for DQN, DDPG, SAC and HER (that is a wrapper around)
deterministic = args.deterministic or algo in [
'dqn', 'ddpg', 'sac', 'her', 'td3'
] and not args.stochastic
episode_reward = 0.0
episode_rewards = []
ep_len = 0
# For HER, monitor success rate
successes = []
for _ in range(args.n_timesteps):
action, _ = model.predict(obs, deterministic=deterministic)
# Random Agent
# action = [env.action_space.sample()]
# Clip Action to avoid out of bound errors
if isinstance(env.action_space, gym.spaces.Box):
action = np.clip(action, env.action_space.low,
env.action_space.high)
obs, reward, done, infos = env.step(action)
if not args.no_render:
env.render('human')
episode_reward += reward[0]
ep_len += 1
if args.n_envs == 1:
# For atari the return reward is not the atari score
# so we have to get it from the infos dict
if is_atari and infos is not None and args.verbose >= 1:
episode_infos = infos[0].get('episode')
if episode_infos is not None:
print("Atari Episode Score: {:.2f}".format(
episode_infos['r']))
print("Atari Episode Length", episode_infos['l'])
if done and not is_atari and args.verbose > 0:
# NOTE: for env using VecNormalize, the mean reward
# is a normalized reward when `--norm_reward` flag is passed
print("Episode Reward: {:.2f}".format(episode_reward))
print("Episode Length", ep_len)
episode_rewards.append(episode_reward)
episode_reward = 0.0
ep_len = 0
# Reset also when the goal is achieved when using HER
if done or infos[0].get('is_success', False):
if args.algo == 'her' and args.verbose > 1:
print("Success?", infos[0].get('is_success', False))
# Alternatively, you can add a check to wait for the end of the episode
# if done:
obs = env.reset()
if args.algo == 'her':
successes.append(infos[0].get('is_success', False))
episode_reward, ep_len = 0.0, 0
if args.verbose > 0 and len(successes) > 0:
print("Success rate: {:.2f}%".format(100 * np.mean(successes)))
if args.verbose > 0 and len(episode_rewards) > 0:
print("Mean reward: {:.2f}".format(np.mean(episode_rewards)))
# Workaround for https://github.com/openai/gym/issues/893
if not args.no_render:
if args.n_envs == 1 and 'Bullet' not in env_id and not is_atari and isinstance(
env, VecEnv):
# DummyVecEnv
# Unwrap env
while isinstance(env, VecNormalize) or isinstance(
env, VecFrameStack):
env = env.venv
env.envs[0].env.close()
else:
# SubprocVecEnv
env.close()
def main():
seed = 0
num_samples = 20
parser = argparse.ArgumentParser()
parser.add_argument('--env',
help='environment ID',
type=str,
default='CartPole-v1')
parser.add_argument('-f',
'--folder',
help='Log folder',
type=str,
default='rl-baselines-zoo/trained_agents')
parser.add_argument('--algo',
help='RL Algorithm',
default='dqn',
type=str,
required=False,
choices=list(ALGOS.keys()))
parser.add_argument('-n',
'--n-timesteps',
help='number of timesteps',
default=2000,
type=int)
parser.add_argument('--n-envs',
help='number of environments',
default=1,
type=int)
parser.add_argument(
'--exp-id',
help='Experiment ID (default: -1, no exp folder, 0: latest)',
default=-1,
type=int)
parser.add_argument('--verbose',
help='Verbose mode (0: no output, 1: INFO)',
default=1,
type=int)
parser.add_argument(
'--no-render',
action='store_true',
default=False,
help='Do not render the environment (useful for tests)')
parser.add_argument('--deterministic',
action='store_true',
default=False,
help='Use deterministic actions')
parser.add_argument('--stochastic',
action='store_true',
default=False,
help='Use stochastic actions (for DDPG/DQN/SAC)')
parser.add_argument(
'--load-best',
action='store_true',
default=False,
help='Load best model instead of last model if available')
parser.add_argument(
'--norm-reward',
action='store_true',
default=False,
help='Normalize reward if applicable (trained with VecNormalize)')
args = parser.parse_args()
env_id = args.env
algo = args.algo
folder = args.folder
if args.exp_id == 0:
args.exp_id = get_latest_run_id(os.path.join(folder, algo), env_id)
print('Loading latest experiment, id={}'.format(args.exp_id))
# Sanity checks
if args.exp_id > 0:
log_path = os.path.join(folder, algo,
'{}_{}'.format(env_id, args.exp_id))
else:
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(
log_path)
model_path = find_saved_model(algo,
log_path,
env_id,
load_best=args.load_best)
if algo in ['dqn', 'ddpg', 'sac', 'td3']:
args.n_envs = 1
set_global_seeds(seed)
is_atari = 'NoFrameskip' in env_id
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(
stats_path, norm_reward=args.norm_reward, test_mode=True)
log_dir = None
env_kwargs = {}
env = create_test_env(env_id,
n_envs=args.n_envs,
is_atari=is_atari,
stats_path=stats_path,
seed=seed,
log_dir=log_dir,
should_render=not args.no_render,
hyperparams=hyperparams,
env_kwargs=env_kwargs)
# ACER raises errors because the environment passed must have
# the same number of environments as the model was trained on.
load_env = None if algo == 'acer' else env
model = ALGOS[algo].load(model_path, env=load_env)
env = gym.make('CartPole-v1')
obs = env.reset()
# Force deterministic for DQN, DDPG, SAC and HER (that is a wrapper around)
deterministic = args.deterministic or algo in [
'dqn', 'ddpg', 'sac', 'her', 'td3'
] and not args.stochastic
episode_reward = 0.0
episode_rewards, episode_lengths = [], []
ep_len = 0
# For HER, monitor success rate
successes = []
state = None
embedder = indicator_feature
halfspaces = {}
for i in range(num_samples):
print("+" * 10)
#sample random state to start in
#TODO: maybe reset with random actions? How to make it realistic? Does it matter. Let's just try random for now to test weird edge cases.
obs = env.reset(uniform=True) #sample more uniformly than typical
print("start state", obs)
# input()
#obs = env.reset_state(env.observation_space.sample())
#rollout once for each action and compute feature counts
start_state = obs.copy()
fcount_vectors = []
init_actions = []
##rollout code:
for init_action in range(env.action_space.n):
print("ACTION", init_action)
obs = env.reset(start_state=start_state)
print("init state", obs)
env.render()
# input()
ep_ret = 0
fcounts = embedder(start_state)
#do initial action
obs, r, done, info = env.step(init_action) # take a random action
fcounts += embedder(obs) #TODO: discount??
ep_ret += r
#print(r, obs)
if done:
print("final state", obs)
print("return", ep_ret)
print("fcounts", fcounts)
fcount_vectors.append(fcounts)
init_actions.append(init_action)
continue
#run tester policy thereafter
while True:
#env.render()
#TODO: sample within allowable range of angle and position
action, state = model.predict(obs,
state=state,
deterministic=deterministic)
# Random Agent
# action = [env.action_space.sample()]
# Clip Action to avoid out of bound errors
if isinstance(env.action_space, gym.spaces.Box):
action = np.clip(action, env.action_space.low,
env.action_space.high)
#a = env.action_space.sample()
#print(obs, action)
obs, r, done, info = env.step(action) # take a random action
fcounts += embedder(obs)
#print(obs)
#print(done)
ep_ret += r
#print(r, obs)
if done:
print("final state", obs)
print("return", ep_ret)
print("fcounts", fcounts)
fcount_vectors.append(fcounts)
init_actions.append(init_action)
break
print("action {} over {} => fcount diff = {}".format(
init_actions[0], init_actions[1],
fcount_vectors[0] - fcount_vectors[1]))
halfspaces[state, init_actions[0],
init_actions[1]] = fcount_vectors[0] - fcount_vectors[1]
# input()
#TODO: put this inside one of the value alignment verification classes to get sa_fcount_diffs and hopefully reuse that code
#then visualize test cases
# input()
# for _ in range(args.n_timesteps):
# action, state = model.predict(obs, state=state, deterministic=deterministic)
# # Random Agent
# # action = [env.action_space.sample()]
# # Clip Action to avoid out of bound errors
# if isinstance(env.action_space, gym.spaces.Box):
# action = np.clip(action, env.action_space.low, env.action_space.high)
# obs, reward, done, infos = env.step(action)
# if not args.no_render:
# env.render('human')
# episode_reward += reward
# ep_len += 1
# if args.n_envs == 1:
# # For atari the return reward is not the atari score
# # so we have to get it from the infos dict
# if is_atari and infos is not None and args.verbose >= 1:
# episode_infos = infos.get('episode')
# if episode_infos is not None:
# print("Atari Episode Score: {:.2f}".format(episode_infos['r']))
# print("Atari Episode Length", episode_infos['l'])
# if done and not is_atari and args.verbose > 0:
# # NOTE: for env using VecNormalize, the mean reward
# # is a normalized reward when `--norm_reward` flag is passed
# print("Episode Reward: {:.2f}".format(episode_reward))
# print("Episode Length", ep_len)
# state = None
# episode_rewards.append(episode_reward)
# episode_lengths.append(ep_len)
# episode_reward = 0.0
# ep_len = 0
# # Reset also when the goal is achieved when using HER
# if done or infos.get('is_success', False):
# if args.algo == 'her' and args.verbose > 1:
# print("Success?", infos[0].get('is_success', False))
# # Alternatively, you can add a check to wait for the end of the episode
# # if done:
# obs = env.reset()
# if args.algo == 'her':
# successes.append(infos[0].get('is_success', False))
# episode_reward, ep_len = 0.0, 0
# if args.verbose > 0 and len(successes) > 0:
# print("Success rate: {:.2f}%".format(100 * np.mean(successes)))
# if args.verbose > 0 and len(episode_rewards) > 0:
# print("Mean reward: {:.2f} +/- {:.2f}".format(np.mean(episode_rewards), np.std(episode_rewards)))
# if args.verbose > 0 and len(episode_lengths) > 0:
# print("Mean episode length: {:.2f} +/- {:.2f}".format(np.mean(episode_lengths), np.std(episode_lengths)))
# Workaround for https://github.com/openai/gym/issues/893
if not args.no_render:
if args.n_envs == 1 and 'Bullet' not in env_id and not is_atari and isinstance(
env, VecEnv):
# DummyVecEnv
# Unwrap env
while isinstance(env, VecNormalize) or isinstance(
env, VecFrameStack):
env = env.venv
env.envs[0].env.close()
else:
# SubprocVecEnv
env.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--env',
help='environment ID',
type=str,
default='CartPole-v1')
parser.add_argument('-f',
'--folder',
help='Log folder',
type=str,
default='trained_agents')
parser.add_argument('--algo',
help='RL Algorithm',
default='ppo2',
type=str,
required=False,
choices=list(ALGOS.keys()))
parser.add_argument('-n',
'--n-timesteps',
help='number of timesteps',
default=1000,
type=int)
parser.add_argument('--n-envs',
help='number of environments',
default=1,
type=int)
parser.add_argument(
'--exp-id',
help='Experiment ID (default: -1, no exp folder, 0: latest)',
default=-1,
type=int)
parser.add_argument('--verbose',
help='Verbose mode (0: no output, 1: INFO)',
default=1,
type=int)
parser.add_argument(
'--no-render',
action='store_true',
default=False,
help='Do not render the environment (useful for tests)')
parser.add_argument('--deterministic',
action='store_true',
default=False,
help='Use deterministic actions')
parser.add_argument('--stochastic',
action='store_true',
default=False,
help='Use stochastic actions (for DDPG/DQN/SAC)')
parser.add_argument(
'--load-best',
action='store_true',
default=False,
help='Load best model instead of last model if available')
parser.add_argument(
'--norm-reward',
action='store_true',
default=False,
help='Normalize reward if applicable (trained with VecNormalize)')
parser.add_argument('--seed',
help='Random generator seed',
type=int,
default=0)
parser.add_argument('--reward-log',
help='Where to log reward',
default='',
type=str)
parser.add_argument(
'--gym-packages',
type=str,
nargs='+',
default=[],
help=
'Additional external Gym environemnt package modules to import (e.g. gym_minigrid)'
)
parser.add_argument(
'--env-kwargs',
type=str,
nargs='+',
action=StoreDict,
help='Optional keyword argument to pass to the env constructor')
parser.add_argument('--render-pybullet',
help='Slow down Pybullet simulation to render',
default=False) # added by Pierre
parser.add_argument('--random-pol', help='Random policy',
default=False) # added by Pierre
parser.add_argument(
'--log-dir-random',
help='Log directory of the random policy') # added by Pierre
args = parser.parse_args()
# Going through custom gym packages to let them register in the global registory
for env_module in args.gym_packages:
importlib.import_module(env_module)
env_id = args.env
algo = args.algo
folder = args.folder
if args.exp_id == 0:
args.exp_id = get_latest_run_id(os.path.join(folder, algo), env_id)
print('Loading latest experiment, id={}'.format(args.exp_id))
# Sanity checks
if args.exp_id > 0:
log_path = os.path.join(folder, algo,
'{}_{}'.format(env_id, args.exp_id))
else:
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(
log_path)
if not args.random_pol: # added by Pierre
model_path = find_saved_model(algo,
log_path,
env_id,
load_best=args.load_best)
if algo in ['dqn', 'ddpg', 'sac', 'td3']:
args.n_envs = 1
set_global_seeds(args.seed)
is_atari = 'NoFrameskip' in env_id
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(
stats_path, norm_reward=args.norm_reward, test_mode=True)
log_dir = args.reward_log if args.reward_log != '' else None
env_kwargs = {} if args.env_kwargs is None else args.env_kwargs
env = create_test_env(env_id,
n_envs=args.n_envs,
is_atari=is_atari,
stats_path=stats_path,
seed=args.seed,
log_dir=log_dir,
should_render=not args.no_render,
hyperparams=hyperparams,
env_kwargs=env_kwargs)
# ACER raises errors because the environment passed must have
# the same number of environments as the model was trained on.
load_env = None if algo == 'acer' else env
if not args.random_pol: # added by Pierre
model = ALGOS[algo].load(model_path, env=load_env)
obs = env.reset()
# Force deterministic for DQN, DDPG, SAC and HER (that is a wrapper around)
deterministic = args.deterministic or algo in [
'dqn', 'ddpg', 'sac', 'her', 'td3'
] and not args.stochastic
# INITIALISE METRICS
episode_reward = 0.0
episode_rewards, episode_lengths = [], []
ep_len = 0
success_threshold_50 = 0.05
success_list_50, reachtime_list_50, episode_success_list_50 = [], [], []
success_threshold_20 = 0.02
success_list_20, reachtime_list_20, episode_success_list_20 = [], [], []
success_threshold_10 = 0.01
success_list_10, reachtime_list_10, episode_success_list_10 = [], [], []
success_threshold_5 = 0.005
success_list_5, reachtime_list_5, episode_success_list_5 = [], [], []
# For HER, monitor success rate
successes = []
state = None
for _ in range(args.n_timesteps):
# Added by Pierre
if args.random_pol:
action = [env.action_space.sample()] # Random Agent
else:
action, state = model.predict(obs,
state=state,
deterministic=deterministic)
# Clip Action to avoid out of bound errors
if isinstance(env.action_space, gym.spaces.Box):
action = np.clip(action, env.action_space.low,
env.action_space.high)
obs, reward, done, infos = env.step(action)
# if args.render_pybullet:
# time.sleep(1./30.) # added by Pierre (slow down Pybullet for rendering)
# added by Pierre
if infos[0]['dist_ft_t'] <= success_threshold_50:
episode_success_list_50.append(1)
else:
episode_success_list_50.append(0)
if infos[0]['dist_ft_t'] <= success_threshold_20:
episode_success_list_20.append(1)
else:
episode_success_list_20.append(0)
if infos[0]['dist_ft_t'] <= success_threshold_10:
episode_success_list_10.append(1)
else:
episode_success_list_10.append(0)
if infos[0]['dist_ft_t'] <= success_threshold_5:
episode_success_list_5.append(1)
else:
episode_success_list_5.append(0)
if not args.no_render:
env.render('human')
# env.render(mode="human")
episode_reward += reward[0]
ep_len += 1
if args.n_envs == 1:
# For atari the return reward is not the atari score
# so we have to get it from the infos dict
if is_atari and infos is not None and args.verbose >= 1:
episode_infos = infos[0].get('episode')
if episode_infos is not None:
print("Atari Episode Score: {:.2f}".format(
episode_infos['r']))
print("Atari Episode Length", episode_infos['l'])
if done and not is_atari and args.verbose > 0:
# NOTE: for env using VecNormalize, the mean reward
# is a normalized reward when `--norm_reward` flag is passed
print("Episode Reward: {:.2f}".format(episode_reward))
print("Episode Length", ep_len)
episode_rewards.append(episode_reward)
episode_lengths.append(ep_len)
# Pierre: append the last element of the episode success list when episode is done
success_list_50.append(episode_success_list_50[-1])
success_list_20.append(episode_success_list_20[-1])
success_list_10.append(episode_success_list_10[-1])
success_list_5.append(episode_success_list_5[-1])
# if the episode is successful and it starts from an unsucessful step, calculate reach time
if episode_success_list_50[
-1] == True and episode_success_list_50[0] == False:
idx = 0
while episode_success_list_50[idx] == False:
idx += 1
reachtime_list_50.append(idx)
if episode_success_list_20[
-1] == True and episode_success_list_20[0] == False:
idx = 0
while episode_success_list_20[idx] == False:
idx += 1
reachtime_list_20.append(idx)
if episode_success_list_10[
-1] == True and episode_success_list_10[0] == False:
idx = 0
while episode_success_list_10[idx] == False:
idx += 1
reachtime_list_10.append(idx)
if episode_success_list_5[
-1] == True and episode_success_list_5[0] == False:
idx = 0
while episode_success_list_5[idx] == False:
idx += 1
reachtime_list_5.append(idx)
# RESET FOR NEW EPISODE
state = None
episode_reward = 0.0
ep_len = 0
episode_success_list_50 = []
episode_success_list_20 = []
episode_success_list_10 = []
episode_success_list_5 = []
# Reset also when the goal is achieved when using HER
if done or infos[0].get('is_success', False):
if args.algo == 'her' and args.verbose > 1:
print("Success?", infos[0].get('is_success', False))
# Alternatively, you can add a check to wait for the end of the episode
# if done:
obs = env.reset()
if args.algo == 'her':
successes.append(infos[0].get('is_success', False))
episode_reward, ep_len = 0.0, 0
if args.verbose > 0 and len(successes) > 0:
print("Success rate: {:.2f}%".format(100 * np.mean(successes)))
if args.verbose > 0 and len(episode_rewards) > 0:
print("Mean reward: {:.2f} +/- {:.2f}".format(np.mean(episode_rewards),
np.std(episode_rewards)))
print(
"success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}"
.format(success_threshold_50, np.mean(success_list_50),
np.mean(reachtime_list_50)))
print(
"success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}"
.format(success_threshold_20, np.mean(success_list_20),
np.mean(reachtime_list_20)))
print(
"success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}"
.format(success_threshold_10, np.mean(success_list_10),
np.mean(reachtime_list_10)))
print(
"success threshold: {} | success ratio: {:.2f} | Average reach time: {:.2f}"
.format(success_threshold_5, np.mean(success_list_5),
np.mean(reachtime_list_5)))
if args.verbose > 0 and len(episode_lengths) > 0:
print("Mean episode length: {:.2f} +/- {:.2f}".format(
np.mean(episode_lengths), np.std(episode_lengths)))
# added by Pierre
print("path:", log_path)
d = {
"Eval mean reward": np.mean(episode_rewards),
"Eval std": np.std(episode_rewards),
"success ratio 50mm": np.mean(success_list_50),
"Average reach time 50mm": np.mean(reachtime_list_50),
"success ratio 20mm": np.mean(success_list_20),
"Average reach time 20mm": np.mean(reachtime_list_20),
"success ratio 10mm": np.mean(success_list_10),
"Average reach time 10mm": np.mean(reachtime_list_10),
"success ratio 5mm": np.mean(success_list_5),
"Average reach time 5mm": np.mean(reachtime_list_5),
}
df = pd.DataFrame(d, index=[0])
if args.random_pol:
log_rand = args.log_dir_random
df.to_csv(log_rand + "/stats.csv", index=False)
else:
df.to_csv(log_path + "/stats.csv", index=False)
# Workaround for https://github.com/openai/gym/issues/893
if not args.no_render:
if args.n_envs == 1 and 'Bullet' not in env_id and not is_atari and isinstance(
env, VecEnv):
# DummyVecEnv
# Unwrap env
while isinstance(env, VecNormalize) or isinstance(
env, VecFrameStack):
env = env.venv
env.envs[0].env.close()
else:
# SubprocVecEnv
env.close()
def rollout_halfspaces(env_id='CartPole-v1',algo='dqn',num_samples=20, precision=0.0001, render=False):
seed = 0
folder = 'rl-baselines-zoo/trained_agents'
n_envs = 1
no_render = False
deterministic = True
stochastic = False
norm_reward=False
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(log_path)
model_path = find_saved_model(algo, log_path, env_id, load_best=False)
set_global_seeds(seed)
is_atari = 'NoFrameskip' in env_id
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(stats_path, norm_reward=norm_reward, test_mode=True)
log_dir = None
env_kwargs = {}
env = create_test_env(env_id, n_envs=n_envs, is_atari=is_atari,
stats_path=stats_path, seed=seed, log_dir=log_dir,
should_render=not no_render,
hyperparams=hyperparams, env_kwargs=env_kwargs)
# ACER raises errors because the environment passed must have
# the same number of environments as the model was trained on.
load_env = None if algo == 'acer' else env
model = ALGOS[algo].load(model_path, env=load_env)
env = gym.make('CartPole-v1')
obs = env.reset()
# Force deterministic for DQN, DDPG, SAC and HER (that is a wrapper around)
deterministic = deterministic or algo in ['dqn', 'ddpg', 'sac', 'her', 'td3'] and not stochastic
episode_reward = 0.0
episode_rewards, episode_lengths = [], []
ep_len = 0
# For HER, monitor success rate
successes = []
state = None
embedder = indicator_feature
halfspaces = {}
for i in range(num_samples):
print("+"*10)
#sample random state to start in
#TODO: maybe reset with random actions? How to make it realistic? Does it matter. Let's just try random for now to test weird edge cases.
obs = env.reset(uniform=True) #sample more uniformly than typical
start_state = obs.copy()
print("start state", obs)
#find out the "near optimal" action for this state to compare other actions to
opt_action, _ = model.predict(obs, state=state, deterministic=deterministic)
#take this action
print("TEACHER ACTION", opt_action)
obs = env.reset(start_state=start_state)
print("init state", obs)
if render:
env.render()
# input()
ep_ret = 0
fcounts = embedder(start_state)
#do initial action
obs, r, done, info = env.step(opt_action) # take a random action
fcounts += embedder(obs) #TODO: discount??
ep_ret += r
#print(r, obs)
if done:
#sample again, since started with terminal state
continue
#run tester policy thereafter
while True:
#env.render()
#TODO: sample within allowable range of angle and position
action, state = model.predict(obs, state=state, deterministic=deterministic)
# Random Agent
# action = [env.action_space.sample()]
# Clip Action to avoid out of bound errors
if isinstance(env.action_space, gym.spaces.Box):
action = np.clip(action, env.action_space.low, env.action_space.high)
#a = env.action_space.sample()
#print(obs, action)
obs, r, done, info = env.step(action) # take a random action
fcounts += embedder(obs)
#print(obs)
#print(done)
ep_ret += r
#print(r, obs)
if done:
print("final state", obs)
print("return", ep_ret)
print("fcounts", fcounts)
opt_fcounts = fcounts
break
# input()
#obs = env.reset_state(env.observation_space.sample())
#rollout once for each action and compute feature counts
fcount_vectors = []
init_actions = []
##rollout code:
for init_action in range(env.action_space.n):
if init_action == opt_action:
#don't need to roll this out since we already did
continue
print("ACTION", init_action)
obs = env.reset(start_state=start_state)
print("init state", obs)
if render:
env.render()
# input()
ep_ret = 0
fcounts = embedder(start_state)
#do initial action
obs, r, done, info = env.step(init_action) # take a random action
fcounts += embedder(obs) #TODO: discount??
ep_ret += r
#print(r, obs)
if done:
print("final state", obs)
print("return", ep_ret)
print("fcounts", fcounts)
fcount_vectors.append(fcounts)
init_actions.append(init_action)
continue
#run tester policy thereafter
while True:
#env.render()
#TODO: sample within allowable range of angle and position
action, state = model.predict(obs, state=state, deterministic=deterministic)
# Random Agent
# action = [env.action_space.sample()]
# Clip Action to avoid out of bound errors
if isinstance(env.action_space, gym.spaces.Box):
action = np.clip(action, env.action_space.low, env.action_space.high)
#a = env.action_space.sample()
#print(obs, action)
obs, r, done, info = env.step(action) # take a random action
fcounts += embedder(obs)
#print(obs)
#print(done)
ep_ret += r
#print(r, obs)
if done:
print("final state", obs)
print("return", ep_ret)
print("fcounts", fcounts)
break
normal_vector = opt_fcounts - fcounts
print("action {} over {} => fcount diff = {}".format(opt_fcounts, init_action, normal_vector))
if np.linalg.norm(normal_vector) > precision:
halfspaces[tuple(start_state), init_action, opt_action] = normal_vector
input()
#TODO: put this inside one of the value alignment verification classes to get sa_fcount_diffs and hopefully reuse that code
#then visualize test cases
# input()
# for _ in range(args.n_timesteps):
# action, state = model.predict(obs, state=state, deterministic=deterministic)
# # Random Agent
# # action = [env.action_space.sample()]
# # Clip Action to avoid out of bound errors
# if isinstance(env.action_space, gym.spaces.Box):
# action = np.clip(action, env.action_space.low, env.action_space.high)
# obs, reward, done, infos = env.step(action)
# if not args.no_render:
# env.render('human')
# episode_reward += reward
# ep_len += 1
# if args.n_envs == 1:
# # For atari the return reward is not the atari score
# # so we have to get it from the infos dict
# if is_atari and infos is not None and args.verbose >= 1:
# episode_infos = infos.get('episode')
# if episode_infos is not None:
# print("Atari Episode Score: {:.2f}".format(episode_infos['r']))
# print("Atari Episode Length", episode_infos['l'])
# if done and not is_atari and args.verbose > 0:
# # NOTE: for env using VecNormalize, the mean reward
# # is a normalized reward when `--norm_reward` flag is passed
# print("Episode Reward: {:.2f}".format(episode_reward))
# print("Episode Length", ep_len)
# state = None
# episode_rewards.append(episode_reward)
# episode_lengths.append(ep_len)
# episode_reward = 0.0
# ep_len = 0
# # Reset also when the goal is achieved when using HER
# if done or infos.get('is_success', False):
# if args.algo == 'her' and args.verbose > 1:
# print("Success?", infos[0].get('is_success', False))
# # Alternatively, you can add a check to wait for the end of the episode
# # if done:
# obs = env.reset()
# if args.algo == 'her':
# successes.append(infos[0].get('is_success', False))
# episode_reward, ep_len = 0.0, 0
# if args.verbose > 0 and len(successes) > 0:
# print("Success rate: {:.2f}%".format(100 * np.mean(successes)))
# if args.verbose > 0 and len(episode_rewards) > 0:
# print("Mean reward: {:.2f} +/- {:.2f}".format(np.mean(episode_rewards), np.std(episode_rewards)))
# if args.verbose > 0 and len(episode_lengths) > 0:
# print("Mean episode length: {:.2f} +/- {:.2f}".format(np.mean(episode_lengths), np.std(episode_lengths)))
# Workaround for https://github.com/openai/gym/issues/893
if not no_render:
if n_envs == 1 and 'Bullet' not in env_id and not is_atari and isinstance(env, VecEnv):
# DummyVecEnv
# Unwrap env
while isinstance(env, VecNormalize) or isinstance(env, VecFrameStack):
env = env.venv
env.envs[0].env.close()
else:
# SubprocVecEnv
env.close()
return halfspaces
folder = args.folder
if args.exp_id == 0:
args.exp_id = get_latest_run_id(os.path.join(folder, algo), env_id)
print('Loading latest experiment, id={}'.format(args.exp_id))
# Sanity checks
if args.exp_id > 0:
log_path = os.path.join(folder, algo, '{}_{}'.format(env_id, args.exp_id))
else:
log_path = os.path.join(folder, algo)
assert os.path.isdir(log_path), "The {} folder was not found".format(log_path)
model_path = find_saved_model(algo, log_path, env_id, load_best=args.load_best)
if algo in ['dqn', 'ddpg', 'sac', 'td3']:
args.n_envs = 1
set_global_seeds(args.seed)
is_atari = 'NoFrameskip' in env_id
stats_path = os.path.join(log_path, env_id)
hyperparams, stats_path = get_saved_hyperparams(stats_path, norm_reward=args.norm_reward, test_mode=True)
log_dir = args.reward_log if args.reward_log != '' else None
env_kwargs = {} if args.env_kwargs is None else args.env_kwargs
