def get_env_type(args):
logger = logging.getLogger()
coloredlogs.install(
level='DEBUG',
fmt=
'%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s'
)
logger.setLevel(logging.DEBUG)
env_id = args.env
if args.env_type is not None:
return args.env_type, env_id
# Re-parse the gym registry, since we could have new envs since last time.
for env in gym.envs.registry.all():
env_type = env.entry_point.split(':')[0].split('.')[-1]
_game_envs[env_type].add(env.id) # This is a set so add is idempotent
if env_id in _game_envs.keys():
env_type = env_id
env_id = [g for g in _game_envs[env_type]][0]
else:
env_type = None
for g, e in _game_envs.items():
if env_id in e:
env_type = g
break
if ':' in env_id:
env_type = re.sub(r':.*', '', env_id)
assert env_type is not None, 'env_id {} is not recognized in env types'.format(
env_id, _game_envs.keys())
return env_type, env_id
def setup_logger(name, log_file, level=logging.INFO):
"""Function setup as many loggers as you want"""
handler = logging.FileHandler(log_file)
handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.setLevel(level)
logger.addHandler(handler)
return logger
def build_env(args):
logger = logging.getLogger()
coloredlogs.install(level='DEBUG', fmt='%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s')
logger.setLevel(logging.DEBUG)
ncpu = multiprocessing.cpu_count()
if sys.platform == 'darwin': ncpu //= 2
nenv = args.num_env or ncpu
alg = args.alg
seed = args.seed
env_type, env_id = get_env_type(args)
if env_type in {'atari', 'retro'}:
if alg == 'deepq':
env = make_env(env_id, env_type, seed=seed, wrapper_kwargs={'frame_stack': True})
elif alg == 'trpo_mpi':
env = make_env(env_id, env_type, seed=seed)
else:
frame_stack_size = 4
env = make_vec_env(env_id, env_type, nenv, seed, gamestate=args.gamestate, reward_scale=args.reward_scale)
env = VecFrameStack(env, frame_stack_size)
else:
# TODO: Ensure willuse GPU when sent to SLURM (Add as a command-line argument)
config = tf.ConfigProto(allow_soft_placement=True,
intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
config.gpu_options.allow_growth = True
get_session(config=config)
flatten_dict_observations = alg not in {'her'}
env = make_vec_env(env_id, env_type, args.num_env or 1, seed, reward_scale=args.reward_scale, flatten_dict_observations=flatten_dict_observations)
if env_type == 'mujoco':
env = VecNormalize(env, use_tf=True)
if env_id == "MsPacmanNoFrameskip-v4":
env = super_simple_dqn_wrapper.PacmanClearTheBoardRewardsWrapper(env)
env = super_simple_dqn_wrapper.FearDeathWrapper(env)
elif env_id == "FreewayNoFrameskip-v4":
env = super_simple_dqn_wrapper.AltFreewayRewardsWrapper(env)
env = super_simple_dqn_wrapper.FreewayUpRewarded(env)
env.ale.setDifficulty(1)
elif env_id == "JamesbondNoFrameskip-v4":
env = super_simple_dqn_wrapper.FearDeathWrapper(env)
return env
def train(args, extra_args):
logger = logging.getLogger()
coloredlogs.install(
level='DEBUG',
fmt=
'%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s'
)
logger.setLevel(logging.DEBUG)
env_type, env_id = get_env_type(args)
#logger.info('env_type: {}'.format(env_type))
total_timesteps = int(args.num_timesteps)
seed = args.seed
learn = get_learn_function(args.alg)
alg_kwargs = get_learn_function_defaults(args.alg, env_type)
alg_kwargs.update(extra_args)
env = build_highlights_env(args)
if args.save_video_interval != 0:
env = VecVideoRecorder(
env,
osp.join(logger.get_dir(), "videos"),
record_video_trigger=lambda x: x % args.save_video_interval == 0,
video_length=args.save_video_length)
if args.network:
alg_kwargs['network'] = args.network
else:
if alg_kwargs.get('network') is None:
alg_kwargs['network'] = get_default_network(env_type)
#logger.info('Training {} on {}:{} with arguments \n{}'.format(args.alg, env_type, env_id, alg_kwargs))
model = learn(env=env,
seed=seed,
total_timesteps=total_timesteps,
load_path=args.load_path,
save_path=args.save_path,
**alg_kwargs)
#print("Returned act: ")
#print(model)
return model, env
def make_logger(name,
filename,
stream_level=logging.INFO,
file_level=logging.DEBUG):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler(filename)
fh.setLevel(file_level)
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(stream_level)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
def main(args):
# configure logger, disable logging in child MPI processes (with rank > 0)
logger = logging.getLogger()
coloredlogs.install(
level='DEBUG',
fmt=
'%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s'
)
logger.setLevel(logging.DEBUG)
# Use parser for specifying agent reward structure
arg_parser = highlights_arg_parser()
args, unknown_args = arg_parser.parse_known_args(args)
extra_args = parse_cmdline_kwargs(unknown_args)
if MPI is None or MPI.COMM_WORLD.Get_rank() == 0:
rank = 0
# configure_logger(args.log_path)
else:
rank = MPI.COMM_WORLD.Get_rank()
configure_logger(args.log_path, format_strs=[])
# Set up saving to a single, logical location on folder above the code base to avoid
# Swelling the size of the code base with test outputs
# get current date and time for default data output folder name
if args.save_path is None:
# datetime object containing current date and time
now = datetime.now()
#logger.info("now =" + str(now))
# month_day_YY_H_M_S
dt_string = now.strftime("%b%d_%Y_%H_%M_")
dt_string = dt_string + str(args.training_wrapper)
#logger.info("date and time =" + str(dt_string))
args.save_path = dt_string
# get current directory
path = os.getcwd()
# use parent dir to save data, so we can keep the current folder small and portable
directory = os.path.abspath(os.path.join(path, os.pardir))
directory = os.path.abspath(os.path.join(directory, os.pardir))
directory = os.path.join(directory, 'train_agent_data')
directory = os.path.join(directory, args.save_path)
os.mkdir(directory)
directory2 = os.path.join(directory, args.save_path)
args.save_path = directory2
#logger.info("Now save path is: ")
#logger.info(args.save_path)
args.log_path = os.path.join(directory, 'logs')
os.mkdir(args.log_path)
#logger.info("Now log path is: ")
#logger.info(args.log_path)
if args.save_path is not None and rank == 0:
#logger.info("Inside custom run file and about to save model")
save_path = osp.expanduser(args.save_path)
args.log_path = os.path.join(directory, 'logs')
model, env = train(args, extra_args)
# Now save Model
#logger.info("Model is: ")
#logger.info(model)
model.save(save_path)
env.close()
return model
def build_highlights_env(args):
logger = logging.getLogger()
coloredlogs.install(
level='DEBUG',
fmt=
'%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s'
)
logger.setLevel(logging.DEBUG)
ncpu = multiprocessing.cpu_count()
if sys.platform == 'darwin': ncpu //= 2
nenv = args.num_env or ncpu
alg = args.alg
seed = args.seed
env_type = 'atari'
env_id = args.env
# Default alg is dqn, so make initial normal dqn environment
env = make_env(env_id,
env_type,
seed=seed,
wrapper_kwargs={'frame_stack': True})
#logger.info("About to check for training wrapper")
# Now switch on the training-based args to add wrappers ass needed
if args.training_wrapper == 'pacman_fear_only':
env = super_simple_dqn_wrapper.fear_only(env)
#logger.info("Training wrapper: " + str(args.training_wrapper))
if args.training_wrapper == 'pacman_power_pill_only':
env = super_simple_dqn_wrapper.pacman_power_pill_only(env)
#logger.info("Training wrapper: " + str(args.training_wrapper))
if args.training_wrapper == 'pacman_normal_pill_only':
env = super_simple_dqn_wrapper.pacman_normal_pill_only(env)
if args.training_wrapper == 'pacman_normal_pill_power_pill_only':
env = super_simple_dqn_wrapper.pacman_normal_pill_power_pill_only(env)
if args.training_wrapper == 'pacman_normal_pill_fear_only':
env = super_simple_dqn_wrapper.pacman_normal_pill_fear_only(env)
if args.training_wrapper == 'pacman_normal_pill_in_game':
env = super_simple_dqn_wrapper.pacman_normal_pill_in_game(env)
if args.training_wrapper == 'pacman_power_pill_fear_only':
env = super_simple_dqn_wrapper.pacman_power_pill_fear_only(env)
if args.training_wrapper == 'pacman_power_pill_in_game':
env = super_simple_dqn_wrapper.pacman_power_pill_in_game(env)
if args.training_wrapper == 'pacman_fear_in_game':
env = super_simple_dqn_wrapper.pacman_fear_in_game(env)
# training options for freeway (also specifies the environment)
if args.training_wrapper == 'freeway_up_only':
env = super_simple_dqn_wrapper.freeway_up_only(env)
if args.training_wrapper == 'freeway_down_only':
env = super_simple_dqn_wrapper.freeway_down_only(env)
if args.training_wrapper == 'freeway_up_down':
env = super_simple_dqn_wrapper.freeway_up_down(env)
# training options for asterix (also specifies the environment)
if args.training_wrapper == 'asterix_fear_only':
env = super_simple_dqn_wrapper.fear_only(env)
if args.training_wrapper == 'asterix_bonus_life_in_game':
env = super_simple_dqn_wrapper.asterix_bonus_life_in_game(env)
if args.training_wrapper == 'asterix_fear_in_game':
env = super_simple_dqn_wrapper.asterix_fear_in_game(env)
# training options for alien (also specifies the environment)
if args.training_wrapper == 'alien_fear_only':
env = super_simple_dqn_wrapper.fear_only(env)
if args.training_wrapper == 'alien_pulsar_only':
env = super_simple_dqn_wrapper.alien_pulsar_only(env)
if args.training_wrapper == 'alien_eggs_only':
env = super_simple_dqn_wrapper.alien_eggs_only(env)
if args.training_wrapper == 'alien_eggs_pulsar_only':
env = super_simple_dqn_wrapper.alien_eggs_pulsar_only(env)
if args.training_wrapper == 'alien_eggs_fear_only':
env = super_simple_dqn_wrapper.alien_eggs_fear_only(env)
if args.training_wrapper == 'alien_eggs_in_game':
env = super_simple_dqn_wrapper.alien_eggs_in_game(env)
if args.training_wrapper == 'alien_pulsar_fear_only':
env = super_simple_dqn_wrapper.alien_pulsar_fear_only(env)
if args.training_wrapper == 'alien_pulsar_in_game':
env = super_simple_dqn_wrapper.alien_pulsar_in_game(env)
if args.training_wrapper == 'alien_fear_in_game':
env = super_simple_dqn_wrapper.alien_fear_in_game(env)
return env
def learn(env,
network,
seed=None,
lr=5e-4,
total_timesteps=1000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
save_path=None,
**network_kwargs):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
batch_size: int
size of a batch sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
logger = logging.getLogger()
coloredlogs.install(
level='DEBUG',
fmt=
'%(asctime)s,%(msecs)03d %(filename)s[%(process)d] %(levelname)s %(message)s'
)
logger.setLevel(logging.DEBUG)
# DATAVAULT: Set up list of action meanings and two lists to store episode
# and total sums for each possible action in the list.
action_names = env.unwrapped.get_action_meanings()
action_episode_sums = []
action_total_sums = []
for x in range(len(action_names)):
action_episode_sums.append(0)
action_total_sums.append(0)
# And obviously, you need a datavault item
dv = DataVault()
# Create all the functions necessary to train the model
sess = get_session()
set_global_seeds(seed)
q_func = build_q_func(network, **network_kwargs)
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space = env.observation_space
def make_obs_ph(name):
return ObservationInput(observation_space, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
td = checkpoint_path or td
model_file = os.path.join(td, "model")
model_saved = False
if tf.train.latest_checkpoint(td) is not None:
load_variables(model_file)
logger.log('Loaded model from {}'.format(model_file))
model_saved = True
elif load_path is not None:
load_variables(load_path)
logger.log('Loaded model from {}'.format(load_path))
#DATAVAULT: This is where you usually want to scrape data - in the timestep loop
for t in range(total_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
# if environment is pacman, limit moves to four directions
name = env.unwrapped.spec.id
if name == "MsPacmanNoFrameskip-v4":
while True:
step_return = act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)
action = step_return[0][0]
env_action = action
q_values = np.squeeze(step_return[1])
# test for break condition
if 1 <= action <= 4:
break
else:
step_return = act(np.array(obs)[None],
update_eps=update_eps,
**kwargs)
action = step_return[0][0]
q_values = np.squeeze(step_return[1])
env_action = action
reset = False
new_obs, rew, done, info = env.step(env_action)
# DATAVAULT: after each step, we push the information out to the datavault
lives = env.ale.lives()
#store_data(self, action, action_name, action_episode_sums, action_total_sums, reward, done, info, lives, q_values, observation, mean_reward):
action_episode_sums, action_total_sums = dv.store_data(
action, action_names[action], action_episode_sums,
action_total_sums, rew, done, info, lives, q_values, new_obs,
saved_mean_reward)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
if (len(episode_rewards[-101:-1]) > 0):
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
else:
mean_100ep_reward = 0
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_variables(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_variables(model_file)
dv.make_dataframes()
print("Save path is: ")
print(save_path)
# use parent dir to save data, so we can keep the current folder small and portable
directory = os.path.abspath(os.path.join(save_path, os.pardir))
csv_path = os.path.join(directory, 'CSVs')
os.mkdir(csv_path)
dv.df_to_csv(csv_path)
return act