def main():
args = get_args()
args.noisy = True
args.double = True
args.dueling = True
args.prioritized_replay = True
args.c51 = True
args.multi_step = 3
args.load_agents = True
args.num_agents = 12
args.read_model = None
args.evaluate = False
print_args(args)
log_dir = create_log_dir(args)
if not args.evaluate:
writer = SummaryWriter(log_dir)
env = PanicEnv(num_agents=args.num_agents,
scenario_=Scenario.Two_Exits,
load_agents=True,
read_agents=False)
set_global_seeds(args.seed)
if args.evaluate:
test(env, args)
return
train(env, args, writer)
writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json"))
writer.close()
def main():
args = get_args()
print_args(args)
log_dir = create_log_dir(args)
if not args.evaluate:
writer = SummaryWriter(log_dir)
SEED = 721
env = make_env(args) # "LaserTag-small2-v0" "SlimeVolleyPixel-v0"
print(env.observation_space, env.action_space)
set_global_seeds(args.seed)
env.seed(args.seed)
if args.evaluate:
test(env, args)
env.close()
return
train(env, args, writer)
writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json"))
writer.close()
env.close()
def make_vec_env(env_id, env_type, num_env, seed,
wrapper_kwargs=None,
start_index=0,
reward_scale=1.0,
flatten_dict_observations=True,
gamestate=None):
"""
Create a wrapped, monitored SubprocVecEnv for Atari and MuJoCo.
"""
wrapper_kwargs = wrapper_kwargs or {}
mpi_rank = MPI.COMM_WORLD.Get_rank() if MPI else 0
seed = seed + 10000 * mpi_rank if seed is not None else None
logger_dir = logger.get_dir()
def make_thunk(rank):
return lambda: make_env(
env_id=env_id,
env_type=env_type,
mpi_rank=mpi_rank,
subrank=rank,
seed=seed,
reward_scale=reward_scale,
gamestate=gamestate,
flatten_dict_observations=flatten_dict_observations,
wrapper_kwargs=wrapper_kwargs,
logger_dir=logger_dir
)
set_global_seeds(seed)
if num_env > 1:
return SubprocVecEnv([make_thunk(i + start_index) for i in range(num_env)])
else:
return DummyVecEnv([make_thunk(start_index)])
def train():
tf.config.run_functions_eagerly(True)
config = Basic_DQN_Conf()
set_global_seeds(config.seed)
# init env
env = init_env(config, 'train')
config.num_actions = env.action_space.n
config.obs_shape = env.observation_space.shape
agent = init_agent(config, env)
agent.learn()
# # train()
# tf.config.run_functions_eagerly(True)
# config = Basic_DQN_Conf()
# # init env
# env = init_env(config, 'train')
#
# print(f'env.observation_space.shape {env.observation_space.shape}')
#
# obs = env.reset()
# print(obs.shape)
# print(obs.dtype)
# import matplotlib.pyplot as plt
#
# plt.imshow(obs[1, :, :, 0])
# plt.show()
def main():
args = get_args()
print_args(args)
model_path = f'models/bilateral_dqn/{args.env}'
os.makedirs(model_path, exist_ok=True)
log_dir = create_log_dir(args)
if not args.evaluate:
writer = SummaryWriter(log_dir)
SEED = 721
if args.num_envs == 1 or args.evaluate:
env = make_env(
args) # "SlimeVolley-v0", "SlimeVolleyPixel-v0" 'Pong-ram-v0'
else:
VectorEnv = [
DummyVectorEnv, SubprocVectorEnv
][1] # https://github.com/thu-ml/tianshou/blob/master/tianshou/env/venvs.py
env = VectorEnv([lambda: make_env(args) for _ in range(args.num_envs)])
print(env.observation_space, env.action_space)
set_global_seeds(args.seed)
env.seed(args.seed)
if args.evaluate:
test(env, args, model_path)
env.close()
return
train(env, args, writer, model_path)
# writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json"))
writer.close()
env.close()
def train():
tf.config.run_functions_eagerly(True)
config = Basic_DQN_FP_RNN_2_Conf()
set_global_seeds(config.seed)
# init env
env = init_env(config, 'train')
config.num_actions = env.action_space.n
config.obs_shape = env.observation_space.shape
agent = init_agent(config, env)
agent.learn()
def main():
args = get_args()
print_args(args)
if args.evaluate:
if args.env == "1DStatic":
env = Env1DStatic(args)
elif args.env == "1DDynamic":
env = Env1DDynamic_Validation(args)
elif args.env == "2DStatic":
env = Env2DStatic(args)
elif args.env == "2DDynamic":
env = Env2DDynamic_Validation(args)
elif args.env == "3DStatic":
env = Env3DStatic(args)
elif args.env == "3DDynamic":
env = Env3DDynamic_Validation(args)
else:
if args.env == "1DStatic":
env = Env1DStatic(args)
elif args.env == "1DDynamic":
env = Env1DDynamic(args)
elif args.env == "2DStatic":
env = Env2DStatic(args)
elif args.env == "2DDynamic":
env = Env2DDynamic(args)
elif args.env == "3DStatic":
env = Env3DStatic(args)
elif args.env == "3DDynamic":
env = Env3DDynamic(args)
datetime = time.time()
save_hyperparameters(args, datetime)
log_dir = create_log_dir(args)
writer = SummaryWriter(log_dir)
set_global_seeds(args.seed)
env.seed(args.seed)
if args.evaluate:
validate(env, args)
else:
train(env, args, writer, datetime)
writer.flush()
writer.close()
env.close()
def main():
args = get_args()
log_dir = create_log_dir(args)
if not args.evaluate:
writer = SummaryWriter(log_dir)
env = make_atari(args.env)
env = wrap_atari_dqn(env, args)
set_global_seeds(args.seed)
env.seed(args.seed)
if args.evaluate:
test(env, args)
env.close()
return
train(env, args, writer)
writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json"))
writer.close()
env.close()
def main():
args = get_args()
print_args(args)
log_dir = create_log_dir(args)
print("Log dir is:", log_dir)
if not args.evaluate:
writer = SummaryWriter(log_dir)
env = gym.make(args.env)
set_global_seeds(args.seed)
env.seed(args.seed)
if args.evaluate:
test(env, args)
env.close()
return
train(env, args, writer)
writer.export_scalars_to_json(os.path.join(log_dir, "all_scalars.json"))
writer.close()
env.close()
def main():
args = get_args()
print_args(args)
log_dir = create_log_dir(args)
wandb.init(project=args.wandb_project,
name=args.wandb_name,
notes=args.wandb_notes,
config=args)
env = make_atari(args.env)
env = wrap_atari_dqn(env, args)
set_global_seeds(args.seed)
env.seed(args.seed)
if args.evaluate:
test(env, args)
env.close()
return
train(env, args)
env.close()
def main():
Exploiter = 'DQN'
EvaluatedModel = 'NashDQN'
args = get_args()
# args.against_baseline=False
print_args(args)
env = make_env(
args) # "SlimeVolley-v0", "SlimeVolleyPixel-v0" 'Pong-ram-v0'
print(env.observation_space, env.action_space)
model_prefix = model_metadata[args.env]
exploiter = load_exploiter(env, Exploiter, args)
evaluated_model = load_evaluated_model(env, EvaluatedModel, args)
model_dir = "models/nash_dqn/{}/{}/".format(args.env, model_prefix)
exploiter_dir = "models/nash_dqn/{}/{}/exploiter/".format(
args.env, model_prefix)
os.makedirs(model_dir, exist_ok=True)
os.makedirs(exploiter_dir, exist_ok=True)
log_dir = create_log_dir(args)
if not args.evaluate:
writer = SummaryWriter(log_dir)
set_global_seeds(args.seed)
env.seed(args.seed)
# Parse all models saved during training in order
filelist, epi_list = [], []
for filename in os.listdir(model_dir):
if filename.endswith("dqn"):
filelist.append(filename.split('_')[0] +
'_') # remove '_dqn' at end
epi_list.append(int(filename.split('_')[0]))
sort_idx = np.argsort(epi_list).tolist()
filelist = [x for _, x in sorted(zip(epi_list, filelist))
] # sort filelist according to the sorting of epi_list
epi_list.sort() # filelist.sort() will not give correct answer
print(epi_list)
# Evaluate/exploit all models saved during training in order
eval_data = {}
for f, i in zip(filelist, epi_list):
print('load model: ', i, model_dir, f)
# if i>17000:
evaluated_model.load_model(model_dir + f,
eval=True,
map_location='cuda:0')
exploiter_path = exploiter_dir + f
r, l = exploit(env,
evaluated_model,
exploiter,
args,
exploiter_path=exploiter_path)
eval_data[str(i)] = [r, l]
save_dir = 'data/{}/'.format(args.env)
os.makedirs(save_dir, exist_ok=True)
if args.fictitious:
save_dir += '/fictitious_eval_data.npy'
else:
save_dir += '/eval_data.npy'
np.save(save_dir, eval_data)
writer.close()
env.close()
def learn(env,
total_timesteps,
seed=None,
replay_strategy='future',
policy_save_interval=5,
clip_return=True,
override_params=None,
load_path=None,
save_path=None,
**kwargs):
# env = gym.make(env_name)
override_params = override_params or {}
if MPI is not None:
rank = MPI.COMM_WORLD.Get_rank()
num_cpu = MPI.COMM_WORLD.Get_size()
# Seed everything.
rank_seed = seed + 1000000 * rank if seed is not None else None
set_global_seeds(rank_seed)
# prepare params
logger.info("preparing parameters for NN models")
params = config.DEFAULT_AGENT_PARAMS
env_name = env.spec.id
params['env_name'] = env_name
params['replay_strategy'] = replay_strategy
if env_name in config.DEFAULT_ENV_PARAMS:
params.update(config.DEFAULT_ENV_PARAMS[env_name])
params.update(**override_params)
params['rollout_per_worker'] = env.num_envs
params['rollout_batch_size'] = params['rollout_per_worker']
params['num_timesteps'] = total_timesteps
logger.save_params(params=params, filename='ddpg_params.json')
# initialize session
# tf_config = tf.ConfigProto(inter_op_parallelism_threads=1, intra_op_parallelism_threads=1)
# tf_config.gpu_options.allow_growth = True # may need if using GPU
# sess = tf.Session(config=tf_config)
# sess.__enter__()
# get policy given params
policy = config.config_params_get_policy(params=params,
clip_return=clip_return)
# get planner
planner = config.config_params_get_planner(params=params)
if load_path is not None:
U.load_variables(load_path +
'_pi') # pi and planner are seperately stored.
if load_path is not None:
U.load_variables(load_path + '_pln')
rollout_params = {
'exploit': False,
'act_rdm_dec': params['act_rdm_dec'],
'use_target_net': False,
'use_demo_states': True,
'compute_Q': False,
'T': params['T'],
'reward_fun': params['reward_fun'],
'goal_delta': params['goal_delta'],
'subgoal_strategy': params['subgoal_strategy'],
'subgoal_num': params['seq_len'] + 1,
'subgoal_norm': env_name.startswith('Hand')
}
eval_params = {
'exploit': True,
'act_rdm_dec': params['act_rdm_dec'],
'use_target_net': params['test_with_polyak'],
'use_demo_states': False,
'compute_Q': True,
'T': params['T'],
'reward_fun': params['reward_fun'],
'subgoal_strategy': params['subgoal_strategy'],
'goal_delta': params['goal_delta'],
'subgoal_num': params['seq_len'] + 1,
'subgoal_norm': env_name.startswith('Hand')
}
for name in [
'T', 'rollout_per_worker', 'gamma', 'noise_eps', 'random_eps'
]:
rollout_params[name] = params[name]
eval_params[name] = params[name]
eval_env = env
rollout_worker = RolloutWorker(env,
policy,
params['dims'],
logger,
planner=planner,
monitor=True,
**rollout_params)
evaluator = RolloutWorker(eval_env,
policy,
params['dims'],
logger,
planner=planner,
**eval_params)
n_cycles = params['n_cycles']
n_epochs = total_timesteps // n_cycles // rollout_worker.T // rollout_worker.rollout_per_worker
return train(save_path=save_path,
policy=policy,
planner=planner,
rollout_worker=rollout_worker,
evaluator=evaluator,
n_epochs=n_epochs,
n_test_rollouts=params['n_test_rollouts'],
n_cycles=params['n_cycles'],
n_batches=params['n_batches'],
policy_save_interval=policy_save_interval)
def learn(env,
seed=None,
num_agents = 2,
lr=0.00008,
total_timesteps=100000,
buffer_size=2000,
exploration_fraction=0.2,
exploration_final_eps=0.01,
train_freq=1,
batch_size=16,
print_freq=100,
checkpoint_freq=10000,
checkpoint_path=None,
learning_starts=2000,
gamma=0.99,
target_network_update_freq=1000,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None,
load_path=None,
**network_kwargs
):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
network: string or a function
neural network to use as a q function approximator. If string, has to be one of the names of registered models in baselines.common.models
(mlp, cnn, conv_only). If a function, should take an observation tensor and return a latent variable tensor, which
will be mapped to the Q function heads (see build_q_func in baselines.deepq.models for details on that)
seed: int or None
prng seed. The runs with the same seed "should" give the same results. If None, no seeding is used.
lr: float
learning rate for adam optimizer
total_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to total_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
param_noise: bool
whether or not to use parameter space noise (https://arxiv.org/abs/1706.01905)
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
load_path: str
path to load the model from. (default: None)
**network_kwargs
additional keyword arguments to pass to the network builder.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
set_global_seeds(seed)
double_q = True
grad_norm_clipping = True
shared_weights = True
play_test = 1000
nsteps = 16
agent_ids = env.agent_ids()
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size, alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = total_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction * total_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
print(f'agent_ids {agent_ids}')
num_actions = env.action_space.n
print(f'num_actions {num_actions}')
dqn_agent = MAgent(env, agent_ids, nsteps, lr, replay_buffer, shared_weights, double_q, num_actions,
gamma, grad_norm_clipping, param_noise)
if load_path is not None:
load_path = osp.expanduser(load_path)
ckpt = tf.train.Checkpoint(model=dqn_agent.q_network)
manager = tf.train.CheckpointManager(ckpt, load_path, max_to_keep=None)
ckpt.restore(manager.latest_checkpoint)
print("Restoring from {}".format(manager.latest_checkpoint))
dqn_agent.update_target()
episode_rewards = [0.0 for i in range(101)]
saved_mean_reward = None
obs_all = env.reset()
obs_shape = obs_all
reset = True
done = False
# Start total timer
tstart = time.time()
for t in range(total_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
kwargs = {}
if not param_noise:
update_eps = tf.constant(exploration.value(t))
update_param_noise_threshold = 0.
else:
update_eps = tf.constant(0.)
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(
1. - exploration.value(t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs['update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
if t % print_freq == 0:
time_1000_step = time.time()
nseconds = time_1000_step - tstart
tstart = time_1000_step
print(f'time spend to perform {t-print_freq} to {t} steps is {nseconds} ')
print('eps update', exploration.value(t))
mb_obs, mb_rewards, mb_actions, mb_values, mb_dones = [], [], [], [], []
# mb_states = states
epinfos = []
for _ in range(nsteps):
# Given observations, take action and value (V(s))
obs_ = tf.constant(obs_all)
# print(f'obs_.shape is {obs_.shape}')
# obs_ = tf.expand_dims(obs_, axis=1)
# print(f'obs_.shape is {obs_.shape}')
actions_list, fps_ = dqn_agent.choose_action(obs_, update_eps=update_eps, **kwargs)
fps = [[] for _ in agent_ids]
# print(f'fps_.shape is {np.asarray(fps_).shape}')
for a in agent_ids:
fps[a] = np.delete(fps_, a, axis=0)
# print(fps)
# print(f'actions_list is {actions_list}')
# print(f'values_list is {values_list}')
# Append the experiences
mb_obs.append(obs_all.copy())
mb_actions.append(actions_list)
mb_values.append(fps)
mb_dones.append([float(done) for _ in range(num_agents)])
# Take actions in env and look the results
obs1_all, rews, done, info = env.step(actions_list)
rews = [np.max(rews) for _ in range(len(rews))] # for cooperative purpose same reward for every one
# print(rews)
mb_rewards.append(rews)
obs_all = obs1_all
# print(rewards, done, info)
maybeepinfo = info[0].get('episode')
if maybeepinfo: epinfos.append(maybeepinfo)
episode_rewards[-1] += np.max(rews)
if done:
episode_rewards.append(0.0)
obs_all = env.reset()
reset = True
mb_dones.append([float(done) for _ in range(num_agents)])
# print(f'mb_actions is {mb_actions}')
# print(f'mb_rewards is {mb_rewards}')
# print(f'mb_values is {mb_values}')
# print(f'mb_dones is {mb_dones}')
mb_obs = np.asarray(mb_obs, dtype=obs_all[0].dtype)
mb_actions = np.asarray(mb_actions, dtype=actions_list[0].dtype)
mb_rewards = np.asarray(mb_rewards, dtype=np.float32)
mb_values = np.asarray(mb_values, dtype=np.float32)
# print(f'mb_values.shape is {mb_values.shape}')
mb_dones = np.asarray(mb_dones, dtype=np.bool)
mb_masks = mb_dones[:-1]
mb_dones = mb_dones[1:]
# print(f'mb_actions is {mb_actions}')
# print(f'mb_rewards is {mb_rewards}')
# print(f'mb_values is {mb_values}')
# print(f'mb_dones is {mb_dones}')
# print(f'mb_masks is {mb_masks}')
# print(f'mb_masks.shape is {mb_masks.shape}')
if gamma > 0.0:
# Discount/bootstrap off value fn
last_values = dqn_agent.value(tf.constant(obs_all))
# print(f'last_values is {last_values}')
if mb_dones[-1][0] == 0:
# print('================ hey ================ mb_dones[-1][0] == 0')
mb_rewards = discount_with_dones(np.concatenate((mb_rewards, [last_values])),
np.concatenate((mb_dones, [[float(False) for _ in range(num_agents)]]))
, gamma)[:-1]
else:
mb_rewards = discount_with_dones(mb_rewards, mb_dones, gamma)
# print(f'after discount mb_rewards is {mb_rewards}')
if replay_buffer is not None:
replay_buffer.add(mb_obs, mb_actions, mb_rewards, obs1_all, mb_masks[:,0],
mb_values, np.tile([exploration.value(t), t], (nsteps, num_agents, 1)))
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights, batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones, fps, extra_datas = replay_buffer.sample(batch_size)
weights, batch_idxes = np.ones_like(rewards), None
obses_t, obses_tp1 = tf.constant(obses_t), None
actions, rewards, dones = tf.constant(actions), tf.constant(rewards, dtype=tf.float32), tf.constant(dones)
weights, fps, extra_datas = tf.constant(weights), tf.constant(fps), tf.constant(extra_datas)
s = obses_t.shape
# print(f'obses_t.shape is {s}')
obses_t = tf.reshape(obses_t, (s[0] * s[1], *s[2:]))
s = actions.shape
# print(f'actions.shape is {s}')
actions = tf.reshape(actions, (s[0] * s[1], *s[2:]))
s = rewards.shape
# print(f'rewards.shape is {s}')
rewards = tf.reshape(rewards, (s[0] * s[1], *s[2:]))
s = weights.shape
# print(f'weights.shape is {s}')
weights = tf.reshape(weights, (s[0] * s[1], *s[2:]))
s = fps.shape
# print(f'fps.shape is {s}')
fps = tf.reshape(fps, (s[0] * s[1], *s[2:]))
# print(f'fps.shape is {fps.shape}')
s = extra_datas.shape
# print(f'extra_datas.shape is {s}')
extra_datas = tf.reshape(extra_datas, (s[0] * s[1], *s[2:]))
s = dones.shape
# print(f'dones.shape is {s}')
dones = tf.reshape(dones, (s[0], s[1], *s[2:]))
# print(f'dones.shape is {s}')
td_errors = dqn_agent.nstep_train(obses_t, actions, rewards, obses_tp1, dones, weights, fps, extra_datas)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
dqn_agent.update_target()
if t % play_test == 0 and t != 0:
play_test_games(dqn_agent)
mean_100ep_reward = np.mean(episode_rewards[-101:-1])
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(episode_rewards) % print_freq == 0:
print(f'last 100 episode mean reward {mean_100ep_reward} in {num_episodes} playing')
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward", mean_100ep_reward)
logger.record_tabular("% time spent exploring", int(100 * exploration.value(t)))
logger.dump_tabular()