def main():
#Load parse parameters
#parser = otc_arg_parser()
#args = parser.parse_args()
#Challenge environment
# if args.env == 'ObtRetro-v6':
# env = ObstacleTowerEnv(
# '/home/home/Data/Carmen/py_workspace/ObstacleTower_v3/ObstacleTower-v3.1/obstacletower.x86_64',
# timeout_wait=6000,
# retro=args.retro,
# realtime_mode=args.test)
# env = RetroWrapper(env, args.sample_normal)
# env = OTCPreprocessing(env, args.action_reduction)
# # if show_obs:
# # env = RenderObservations(env)
# # env = KeyboardControlWrapper(env)
# else:
env = ObstacleTowerEnv(
'/home/home/Data/Carmen/py_workspace/ObstacleTower_v3/ObstacleTower-v3.1/obstacletower.x86_64',
retro=args.retro,
realtime_mode=args.test,
timeout_wait=6000)
#env = ObstacleTowerEnv('OBSTACLE_TOWER_PATH', retro=args.retro, realtime_mode=args.test, timeout_wait=6000)
#Dict of actions created by the ObstacleTowerEnv Class of obstacle_tower_env library
#print("ACTIONS:", env._flattener.action_lookup)
print('FEATURES :', args.features)
#Preprocess the environment (Grey Scales and action space reduction)
env = OTCPreprocessing(env, args.action_reduction, args.features)
env = DummyVecEnv([lambda: env])
#env = VecEnv(1, env.observation_space, env.action_space)
print("ACTION SPACE ///////////:", env.action_space)
print("OBSERVATION SPACE ///////////////:", env.observation_space)
#env = make_vec_env(env, n_envs=4)
########Training########
#Study of the impact of different values of the PPO params
if args.study:
params_test(MlpPolicy, env)
#If no Study Mode
else:
#If no Test Mode
if not args.test:
seed = random.seed(0)
if args.pretrained_model:
t = 300000
model = PPO2.load(args.pretrained_model,
env=env,
tensorboard_log=args.tensorboard_logdir)
else:
t = 0
#If Generalized Advantage Estimator is used
if args.use_gae:
model = PPO2(MlpPolicy,
env,
n_steps=args.num_steps,
verbose=1,
tensorboard_log=args.tensorboard_logdir,
cliprange=args.clip_param,
learning_rate=args.lr,
ent_coef=args.entropy_coef,
vf_coef=args.value_loss_coef,
max_grad_norm=args.max_grad_norm,
gamma=args.gamma,
lam=args.gae_lambda,
noptepochs=args.ppo_epoch,
seed=seed)
#If Generalized Advantage Estimator is not used
else:
model = PPO2(MlpPolicy,
env,
n_steps=args.num_steps,
verbose=1,
tensorboard_log=args.tensorboard_logdir,
cliprange=args.clip_param,
learning_rate=args.lr,
ent_coef=args.entropy_coef,
vf_coef=args.value_loss_coef,
max_grad_norm=args.max_grad_norm,
gamma=args.gamma,
noptepochs=args.ppo_epoch,
seed=seed)
else:
model = PPO2.load(args.pretrained_model, env=env)
#model.learn(total_timesteps=50000)
#model.save("ObstacleTower_prueba")
filename = 'argsparams.txt'
os.makedirs(args.results_dir, exist_ok=True)
myfile = open(args.results_dir + filename, 'a')
myfile.write(
'clip range: %f \n learning rate: %f \n coeficiente de entropía: %f \n coeficiente de pérdida: %f \n '
'máximo gradiente: %f \n gamma: %f \n ppo epoch: %f \n' %
(args.clip_param, args.lr, args.entropy_coef, args.value_loss_coef,
args.max_grad_norm, args.gamma, args.ppo_epoch))
myfile.close()
if not args.test:
while t < args.num_env_steps:
#TRAIN MODEL
if t == 0:
model.learn(total_timesteps=args.eval_interval)
else:
model.learn(total_timesteps=args.eval_interval,
reset_num_timesteps=False)
os.makedirs(GLOBAL_PATH, exist_ok=True)
print("Saving in '" + GLOBAL_PATH + "'")
model.save(GLOBAL_PATH + args.training_name + "_" +
str(int(t)).zfill(10))
avg_reward, avg_floor = test(
t, model, env=env, global_path=args.results_dir) # Test
log('T = ' + str(t) + ' / ' + str(args.num_env_steps) +
' | Avg. reward: ' + str(avg_reward) + ' | Avg. floor: ' +
str(avg_floor))
t += args.eval_interval
else:
obs = env.reset()
t = 0
while t < args.num_env_steps:
action, _states = model.predict(obs)
obs, rewards, done, info = env.step(action)
#print('action :', info)
env.render('rgb_array')
obs, reward, done, info = env.step(action)
action = 18
obs, reward, done, info = env.step(action)
obs, reward, done, info = env.step(action)
obs, reward, done, info = env.step(action)
obs, reward, done, info = env.step(action)
s, reward, done, info = env.step(action)
s = rgb2gray(s)
s = np.expand_dims(s, axis=2)
t = 0
track_r = []
track_a = []
track_s = []
while True:
env.render()
a = actor.choose_action(
s
) # state of dim (1, 84, 84) fed to actor to choose the next action based on the current state
s_, r, done, info = env.step(a)
s_ = rgb2gray(s_)
s_ = np.expand_dims(s_, axis=2)
if done:
r = -20
track_r.append(r)
track_a.append(a)
track_s.append(s)
handcrafted_step(18, steps[2])
s, reward, done, info = env.step(18)
s = rgb2gray(s)
s = np.expand_dims(s, axis=2)
t = 0
track_score = []
track_r = []
track_a = []
track_s = []
bad_seq = 0
score = 0
steps_after_key = 0
while True:
if RENDER: env.render()
a = actor.choose_action(
s
) # state of dim (1, 84, 84) fed to actor to choose the next action based on the current state
s_, r, done, info = env.step(a)
t += 1
if i_episode % 100 == 0 and i_episode != 0:
test_phase = True
test_counter += 1
if test_phase:
if test_counter > 5:
performance.append(keys_in_test / test_counter)
if keys_in_test > 2:
steps[-1] -= 1
class StableA2C():
def __init__(self,
env_path,
train,
evaluate,
policy_name='CnnPolicy',
save_dir='./model_files/',
eval_seeds=[]):
self.save_dir = save_dir
if not os.path.exists(save_dir):
os.makedirs(save_dir)
self.model_path = os.path.join(self.save_dir, 'model_stable_a2c')
self.log_dir = './logs/stable_a2c'
self.policy_name = policy_name
self.evaluate = evaluate
if train:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=True,
realtime_mode=False,
config=train_env_reset_config)
else:
if evaluate:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=True,
realtime_mode=False,
config=eval_env_reset_config)
self.env = ObstacleTowerEvaluation(self.env, eval_seeds)
else:
self.env = ObstacleTowerEnv(env_path,
worker_id=0,
retro=True,
realtime_mode=True,
config=eval_env_reset_config)
def load_model(self):
print('Loading model from: {}'.format(self.model_path))
model = A2C.load(self.model_path)
model.set_env(self.env)
model.tensorboard_log = self.log_dir
return model
def train(self, timesteps=10000, continue_training=False):
start_time = time.time()
if not continue_training:
print("Initializing from scratch")
model = A2C(self.policy_name,
self.env,
verbose=1,
tensorboard_log=self.log_dir)
else:
model = self.load_model()
print("Restored from {}".format(self.model_path))
model.learn(total_timesteps=timesteps)
print('\nTraining complete. Time taken = {} secs'.format(time.time() -
start_time))
model.save(self.model_path)
def play_single_episode(self):
""" have the trained agent play a single game """
action_space = ActionSpace()
done = False
reward_sum = 0
step_counter = 0
model = self.load_model()
obs = self.env.reset()
try:
print("Playing single episode...")
while not done:
action, _states = model.predict(obs)
obs, reward, done, info = self.env.step(action)
print("{}. Reward: {}, action: {}".format(
step_counter, reward_sum,
action_space.get_full_action_meaning(action)))
self.env.render()
step_counter += 1
reward_sum += reward
except KeyboardInterrupt:
print("Received Keyboard Interrupt. Shutting down.")
finally:
if not self.evaluate:
self.env.close()
print("Environment closed.")
print("Game play completed.")
return reward_sum
def evaluate(self):
""" run episodes until evaluation is complete """
while not self.env.evaluation_complete:
episode_reward = self.play_single_episode()
pprint(self.env.results)
self.env.close()