def test_eps_greedy():
np.random.seed(88)
eps = Parameter(0.1)
pi = EpsGreedy(eps)
Q = Table((10, 3))
Q.table = np.random.randn(10, 3)
pi.set_q(Q)
s = np.array([2])
a = np.array([1])
p_s = pi(s)
p_s_test = np.array([0.03333333, 0.93333333, 0.03333333])
assert np.allclose(p_s, p_s_test)
p_sa = pi(s, a)
p_sa_test = np.array([0.93333333])
assert np.allclose(p_sa, p_sa_test)
a = pi.draw_action(s)
a_test = 1
assert a.item() == a_test
eps_2 = LinearParameter(0.2, 0.1, 2)
pi.set_epsilon(eps_2)
p_sa_2 = pi(s, a)
assert p_sa_2 < p_sa
pi.update(s, a)
pi.update(s, a)
p_sa_3 = pi(s, a)
print(eps_2.get_value())
assert p_sa_3 == p_sa
def test_prioritized_dqn_save(tmpdir):
agent_path = tmpdir / 'agent_{}'.format(
datetime.now().strftime("%H%M%S%f"))
replay_memory = PrioritizedReplayMemory(50,
500,
alpha=.6,
beta=LinearParameter(
.4,
threshold_value=1,
n=500 // 5))
params = dict(batch_size=50,
initial_replay_size=50,
max_replay_size=500,
target_update_frequency=50,
replay_memory=replay_memory)
agent_save = learn(DQN, params)
agent_save.save(agent_path, full_save=True)
agent_load = Agent.load(agent_path)
for att, method in vars(agent_save).items():
save_attr = getattr(agent_save, att)
load_attr = getattr(agent_load, att)
tu.assert_eq(save_attr, load_attr)
def test_prioritized_dqn_save():
replay_memory = PrioritizedReplayMemory(50,
500,
alpha=.6,
beta=LinearParameter(
.4,
threshold_value=1,
n=500 // 5))
params = dict(batch_size=50,
n_approximators=1,
initial_replay_size=50,
max_replay_size=500,
target_update_frequency=50,
replay_memory=replay_memory)
agent_save = learn(DQN, params)
agent_path = './agentdir{}/'.format(datetime.now().strftime("%H%M%S%f"))
agent_save.save(agent_path)
agent_load = Agent.load(agent_path)
shutil.rmtree(agent_path)
for att, method in agent_save.__dict__.items():
save_attr = getattr(agent_save, att)
load_attr = getattr(agent_load, att)
#print('{}: {}'.format(att, type(save_attr)))
tu.assert_eq(save_attr, load_attr)
def test_boltzmann():
np.random.seed(88)
beta = Parameter(0.1)
pi = Boltzmann(beta)
Q = Table((10, 3))
Q.table = np.random.randn(10, 3)
pi.set_q(Q)
s = np.array([2])
a = np.array([1])
p_s = pi(s)
p_s_test = np.array([0.30676679, 0.36223227, 0.33100094])
assert np.allclose(p_s, p_s_test)
p_sa = pi(s, a)
p_sa_test = np.array([0.36223227])
assert np.allclose(p_sa, p_sa_test)
a = pi.draw_action(s)
a_test = 2
assert a.item() == a_test
beta_2 = LinearParameter(0.2, 0.1, 2)
pi.set_beta(beta_2)
p_sa_2 = pi(s, a)
assert p_sa_2 < p_sa
pi.update(s, a)
p_sa_3 = pi(s, a)
p_sa_3_test = np.array([0.33100094])
assert np.allclose(p_sa_3, p_sa_3_test)
def build(self, mdp_info):
self.approximator_params[
'input_shape'] = mdp_info.observation_space.shape
self.approximator_params['output_shape'] = (mdp_info.action_space.n, )
self.approximator_params['n_actions'] = mdp_info.action_space.n
replay_memory = PrioritizedReplayMemory(
self.alg_params['initial_replay_size'],
self.alg_params['max_replay_size'],
alpha=.6,
beta=LinearParameter(.4, threshold_value=1, n=50000000 // 4))
self.alg_params['replay_memory'] = replay_memory
self.epsilon = LinearParameter(value=1, threshold_value=.05, n=1000000)
self.epsilon_test = Parameter(value=.01)
return DQN(mdp_info, self.policy, self.approximator,
self.approximator_params, **self.alg_params)
def build(self, mdp_info):
self.approximator_params[
'input_shape'] = mdp_info.observation_space.shape
self.approximator_params['output_shape'] = (mdp_info.action_space.n, )
self.approximator_params['n_actions'] = mdp_info.action_space.n
self.epsilon = LinearParameter(value=1, threshold_value=.05, n=1000000)
self.epsilon_test = Parameter(value=.01)
return DoubleDQN(mdp_info, self.policy, self.approximator,
self.approximator_params, **self.alg_params)
def test_prioritized_dqn():
replay_memory = PrioritizedReplayMemory(
50, 500, alpha=.6,
beta=LinearParameter(.4, threshold_value=1, n=500 // 5)
)
params = dict(batch_size=50, initial_replay_size=50,
max_replay_size=500, target_update_frequency=50,
replay_memory=replay_memory)
approximator = learn(DQN, params).approximator
w = approximator.get_weights()
w_test = np.array([-0.2410347, 0.39138362, 0.12457055, 0.60612524, -0.54973847,
-0.06486652, -0.07349031, 0.4376623, 0.14254288])
assert np.allclose(w, w_test)
def test_prioritized_dqn():
replay_memory = PrioritizedReplayMemory(
50, 500, alpha=.6,
beta=LinearParameter(.4, threshold_value=1, n=500 // 5)
)
params = dict(batch_size=50, n_approximators=1, initial_replay_size=50,
max_replay_size=500, target_update_frequency=50,
replay_memory=replay_memory)
approximator = learn(DQN, params)
w = approximator.get_weights()
w_test = np.array([-0.1384063, 0.48957556, 0.02254359, 0.50994426,
-0.56277484, -0.075808, -0.06829552, 0.3642576,
0.15519235])
assert np.allclose(w, w_test)
def test_rainbow_save(tmpdir):
agent_path = tmpdir / 'agent_{}'.format(datetime.now().strftime("%H%M%S%f"))
params = dict(batch_size=50, initial_replay_size=50,
max_replay_size=500, target_update_frequency=50, n_steps_return=1,
alpha_coeff=.6, beta=LinearParameter(.4, threshold_value=1, n=500 // 5))
agent_save = learn(Rainbow, params)
agent_save.save(agent_path, full_save=True)
agent_load = Agent.load(agent_path)
for att, method in vars(agent_save).items():
save_attr = getattr(agent_save, att)
load_attr = getattr(agent_load, att)
tu.assert_eq(save_attr, load_attr)
def test_rainbow():
params = dict(batch_size=50, initial_replay_size=50,
max_replay_size=500, target_update_frequency=50, n_steps_return=3,
alpha_coeff=.6, beta=LinearParameter(.4, threshold_value=1, n=500 // 5))
approximator = learn(Rainbow, params).approximator
w = approximator.get_weights()
w_test = np.array([0.41471523, -0.24608319, -0.18744999, 0.26587564, 0.39882535, 0.412821,
0.30898723, 0.29745516, -0.5973996, 0.35576734, 0.41858765, 0.2911771,
-0.09666843, 0.32220146, 0.04949852, -0.04904625, 0.3972141, 0.32487455,
0.3105287, 0.38326296, 0.15647355, 0.07453305, 0.31577617, 0.38884395,
0.30908346, -0.20951316, -0.1023823, -0.12970605, 0.40118366, 0.41426662,
0.30691648, 0.2924496, 0.08292492, 0.01674112, 0.33560023, 0.3732411,
0.5594649, 0.17095159, -0.20466673, -0.37797216, 0.29877642, 0.3118145,
0.40977645, 0.39796302, -0.0712048, -0.35232118, 0.40097338, 0.3074576])
assert np.allclose(w, w_test, rtol=1e-4)
def experiment():
np.random.seed()
# Argument parser
parser = argparse.ArgumentParser()
arg_game = parser.add_argument_group('Game')
arg_game.add_argument("--name",
type=str,
default='BreakoutDeterministic-v4',
help='Gym ID of the Atari game.')
arg_game.add_argument("--screen-width",
type=int,
default=84,
help='Width of the game screen.')
arg_game.add_argument("--screen-height",
type=int,
default=84,
help='Height of the game screen.')
arg_mem = parser.add_argument_group('Replay Memory')
arg_mem.add_argument("--initial-replay-size",
type=int,
default=50000,
help='Initial size of the replay memory.')
arg_mem.add_argument("--max-replay-size",
type=int,
default=500000,
help='Max size of the replay memory.')
arg_mem.add_argument("--prioritized",
action='store_true',
help='Whether to use prioritized memory or not.')
arg_net = parser.add_argument_group('Deep Q-Network')
arg_net.add_argument(
"--optimizer",
choices=['adadelta', 'adam', 'rmsprop', 'rmspropcentered'],
default='rmsprop',
help='Name of the optimizer to use.')
arg_net.add_argument("--learning-rate",
type=float,
default=.00025,
help='Learning rate value of the optimizer.')
arg_net.add_argument("--decay",
type=float,
default=.95,
help='Discount factor for the history coming from the'
'gradient momentum in rmspropcentered and'
'rmsprop')
arg_net.add_argument("--epsilon",
type=float,
default=1e-8,
help='Epsilon term used in rmspropcentered and'
'rmsprop')
arg_alg = parser.add_argument_group('Algorithm')
arg_alg.add_argument("--algorithm",
choices=['dqn', 'ddqn', 'adqn', 'mmdqn', 'cdqn'],
default='dqn',
help='Name of the algorithm. dqn is for standard'
'DQN, ddqn is for Double DQN and adqn is for'
'Averaged DQN.')
arg_alg.add_argument(
"--n-approximators",
type=int,
default=1,
help="Number of approximators used in the ensemble for"
"AveragedDQN or MaxminDQN.")
arg_alg.add_argument("--batch-size",
type=int,
default=32,
help='Batch size for each fit of the network.')
arg_alg.add_argument("--history-length",
type=int,
default=4,
help='Number of frames composing a state.')
arg_alg.add_argument("--target-update-frequency",
type=int,
default=10000,
help='Number of collected samples before each update'
'of the target network.')
arg_alg.add_argument("--evaluation-frequency",
type=int,
default=250000,
help='Number of collected samples before each'
'evaluation. An epoch ends after this number of'
'steps')
arg_alg.add_argument("--train-frequency",
type=int,
default=4,
help='Number of collected samples before each fit of'
'the neural network.')
arg_alg.add_argument("--max-steps",
type=int,
default=50000000,
help='Total number of collected samples.')
arg_alg.add_argument(
"--final-exploration-frame",
type=int,
default=1000000,
help='Number of collected samples until the exploration'
'rate stops decreasing.')
arg_alg.add_argument("--initial-exploration-rate",
type=float,
default=1.,
help='Initial value of the exploration rate.')
arg_alg.add_argument("--final-exploration-rate",
type=float,
default=.1,
help='Final value of the exploration rate. When it'
'reaches this values, it stays constant.')
arg_alg.add_argument("--test-exploration-rate",
type=float,
default=.05,
help='Exploration rate used during evaluation.')
arg_alg.add_argument("--test-samples",
type=int,
default=125000,
help='Number of collected samples for each'
'evaluation.')
arg_alg.add_argument(
"--max-no-op-actions",
type=int,
default=30,
help='Maximum number of no-op actions performed at the'
'beginning of the episodes.')
arg_alg.add_argument("--n-atoms",
type=int,
default=51,
help='Number of atoms for Categorical DQN.')
arg_alg.add_argument("--v-min",
type=int,
default=-10,
help='Minimum action-value for Categorical DQN.')
arg_alg.add_argument("--v-max",
type=int,
default=10,
help='Maximum action-value for Categorical DQN.')
arg_utils = parser.add_argument_group('Utils')
arg_utils.add_argument('--use-cuda',
action='store_true',
help='Flag specifying whether to use the GPU.')
arg_utils.add_argument('--save',
action='store_true',
help='Flag specifying whether to save the model.')
arg_utils.add_argument('--load-path',
type=str,
help='Path of the model to be loaded.')
arg_utils.add_argument('--render',
action='store_true',
help='Flag specifying whether to render the game.')
arg_utils.add_argument('--quiet',
action='store_true',
help='Flag specifying whether to hide the progress'
'bar.')
arg_utils.add_argument('--debug',
action='store_true',
help='Flag specifying whether the script has to be'
'run in debug mode.')
args = parser.parse_args()
scores = list()
optimizer = dict()
if args.optimizer == 'adam':
optimizer['class'] = optim.Adam
optimizer['params'] = dict(lr=args.learning_rate, eps=args.epsilon)
elif args.optimizer == 'adadelta':
optimizer['class'] = optim.Adadelta
optimizer['params'] = dict(lr=args.learning_rate, eps=args.epsilon)
elif args.optimizer == 'rmsprop':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=args.learning_rate,
alpha=args.decay,
eps=args.epsilon)
elif args.optimizer == 'rmspropcentered':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=args.learning_rate,
alpha=args.decay,
eps=args.epsilon,
centered=True)
else:
raise ValueError
# Summary folder
folder_name = './logs/atari_' + args.algorithm + '_' + args.name +\
'_' + datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
pathlib.Path(folder_name).mkdir(parents=True)
# Settings
if args.debug:
initial_replay_size = 50
max_replay_size = 500
train_frequency = 5
target_update_frequency = 10
test_samples = 20
evaluation_frequency = 50
max_steps = 1000
else:
initial_replay_size = args.initial_replay_size
max_replay_size = args.max_replay_size
train_frequency = args.train_frequency
target_update_frequency = args.target_update_frequency
test_samples = args.test_samples
evaluation_frequency = args.evaluation_frequency
max_steps = args.max_steps
# MDP
mdp = Atari(args.name,
args.screen_width,
args.screen_height,
ends_at_life=True,
history_length=args.history_length,
max_no_op_actions=args.max_no_op_actions)
if args.load_path:
# Agent
agent = DQN.load(args.load_path)
epsilon_test = Parameter(value=args.test_exploration_rate)
agent.policy.set_epsilon(epsilon_test)
# Algorithm
core_test = Core(agent, mdp)
# Evaluate model
dataset = core_test.evaluate(n_steps=args.test_samples,
render=args.render,
quiet=args.quiet)
get_stats(dataset)
else:
# Policy
epsilon = LinearParameter(value=args.initial_exploration_rate,
threshold_value=args.final_exploration_rate,
n=args.final_exploration_frame)
epsilon_test = Parameter(value=args.test_exploration_rate)
epsilon_random = Parameter(value=1)
pi = EpsGreedy(epsilon=epsilon_random)
class CategoricalLoss(nn.Module):
def forward(self, input, target):
input = input.clamp(1e-5)
return -torch.sum(target * torch.log(input))
# Approximator
approximator_params = dict(
network=Network if args.algorithm != 'cdqn' else FeatureNetwork,
input_shape=mdp.info.observation_space.shape,
output_shape=(mdp.info.action_space.n, ),
n_actions=mdp.info.action_space.n,
n_features=Network.n_features,
optimizer=optimizer,
loss=F.smooth_l1_loss
if args.algorithm != 'cdqn' else CategoricalLoss(),
use_cuda=args.use_cuda)
approximator = TorchApproximator
if args.prioritized:
replay_memory = PrioritizedReplayMemory(
initial_replay_size,
max_replay_size,
alpha=.6,
beta=LinearParameter(.4,
threshold_value=1,
n=max_steps // train_frequency))
else:
replay_memory = None
# Agent
algorithm_params = dict(
batch_size=args.batch_size,
target_update_frequency=target_update_frequency // train_frequency,
replay_memory=replay_memory,
initial_replay_size=initial_replay_size,
max_replay_size=max_replay_size)
if args.algorithm == 'dqn':
agent = DQN(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
**algorithm_params)
elif args.algorithm == 'ddqn':
agent = DoubleDQN(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
**algorithm_params)
elif args.algorithm == 'adqn':
agent = AveragedDQN(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
n_approximators=args.n_approximators,
**algorithm_params)
elif args.algorithm == 'mmdqn':
agent = MaxminDQN(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
n_approximators=args.n_approximators,
**algorithm_params)
elif args.algorithm == 'cdqn':
agent = CategoricalDQN(mdp.info,
pi,
approximator_params=approximator_params,
n_atoms=args.n_atoms,
v_min=args.v_min,
v_max=args.v_max,
**algorithm_params)
# Algorithm
core = Core(agent, mdp)
# RUN
# Fill replay memory with random dataset
print_epoch(0)
core.learn(n_steps=initial_replay_size,
n_steps_per_fit=initial_replay_size,
quiet=args.quiet)
if args.save:
agent.save(folder_name + '/agent_0.msh')
# Evaluate initial policy
pi.set_epsilon(epsilon_test)
mdp.set_episode_end(False)
dataset = core.evaluate(n_steps=test_samples,
render=args.render,
quiet=args.quiet)
scores.append(get_stats(dataset))
np.save(folder_name + '/scores.npy', scores)
for n_epoch in range(1, max_steps // evaluation_frequency + 1):
print_epoch(n_epoch)
print('- Learning:')
# learning step
pi.set_epsilon(epsilon)
mdp.set_episode_end(True)
core.learn(n_steps=evaluation_frequency,
n_steps_per_fit=train_frequency,
quiet=args.quiet)
if args.save:
agent.save(folder_name + '/agent_' + str(n_epoch) + '.msh')
print('- Evaluation:')
# evaluation step
pi.set_epsilon(epsilon_test)
mdp.set_episode_end(False)
dataset = core.evaluate(n_steps=test_samples,
render=args.render,
quiet=args.quiet)
scores.append(get_stats(dataset))
np.save(folder_name + '/scores.npy', scores)
return scores
train_frequency = 4
evaluation_frequency = 5000
target_update_frequency = 500
initial_replay_size = 5000
max_replay_size = 50000
test_samples = 100
max_steps = 200000
test_episodes = 50
# PO-MDP
mdp = ShowdownEnvironment()
# Policy
epsilon = LinearParameter(value=1.,
threshold_value=.1,
n=1000000)
epsilon_test = Parameter(value=.05)
epsilon_random = Parameter(value=1)
pi = EpsGreedy(epsilon=epsilon_random)
# Approximator
approximator_params = dict(
network=Network,
input_shape=(110,),
output_shape=(13,),
n_actions=13,
optimizer=optimizer,
loss=F.mse_loss
)
def experiment(args, idx):
np.random.seed()
args.games = [''.join(g) for g in args.games]
# MDP
mdp = list()
for i, g in enumerate(args.games):
mdp.append(Atari(g))
n_actions_per_head = [(m.info.action_space.n,) for m in mdp]
max_obs_dim = 0
max_act_n = 0
for i in range(len(args.games)):
n = mdp[i].info.observation_space.shape[0]
m = mdp[i].info.action_space.n
if n > max_obs_dim:
max_obs_dim = n
max_obs_idx = i
if m > max_act_n:
max_act_n = m
max_act_idx = i
gammas = [m.info.gamma for m in mdp]
horizons = [m.info.horizon for m in mdp]
mdp_info = MDPInfo(mdp[max_obs_idx].info.observation_space,
mdp[max_act_idx].info.action_space, gammas, horizons)
scores = list()
for _ in range(len(args.games)):
scores.append(list())
optimizer = dict()
if args.optimizer == 'adam':
optimizer['class'] = optim.Adam
optimizer['params'] = dict(lr=args.learning_rate,
eps=args.epsilon)
elif args.optimizer == 'adadelta':
optimizer['class'] = optim.Adadelta
optimizer['params'] = dict(lr=args.learning_rate,
eps=args.epsilon)
elif args.optimizer == 'rmsprop':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=args.learning_rate,
alpha=args.decay,
eps=args.epsilon)
elif args.optimizer == 'rmspropcentered':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=args.learning_rate,
alpha=args.decay,
eps=args.epsilon,
centered=True)
else:
raise ValueError
# DQN learning run
# Settings
if args.debug:
initial_replay_size = args.batch_size
max_replay_size = 500
train_frequency = 5
target_update_frequency = 10
test_samples = 20
evaluation_frequency = 50
max_steps = 1000
else:
initial_replay_size = args.initial_replay_size
max_replay_size = args.max_replay_size
train_frequency = args.train_frequency
target_update_frequency = args.target_update_frequency
test_samples = args.test_samples
evaluation_frequency = args.evaluation_frequency
max_steps = args.max_steps
# Policy
epsilon = LinearParameter(value=args.initial_exploration_rate,
threshold_value=args.final_exploration_rate,
n=args.final_exploration_frame)
epsilon_test = Parameter(value=args.test_exploration_rate)
epsilon_random = Parameter(value=1)
pi = EpsGreedyMultiple(parameter=epsilon,
n_actions_per_head=n_actions_per_head)
# Approximator
n_games = len(args.games)
loss = LossFunction(n_games, args.batch_size,
args.evaluation_frequency)
input_shape = (args.history_length, args.screen_height,
args.screen_width)
approximator_params = dict(
network=AtariNetwork,
input_shape=input_shape,
output_shape=(max(n_actions_per_head)[0],),
n_actions=max(n_actions_per_head)[0],
n_actions_per_head=n_actions_per_head,
n_games=len(args.games),
optimizer=optimizer,
loss=loss,
use_cuda=args.use_cuda,
features=args.features
)
approximator = TorchApproximator
if args.prioritized:
replay_memory = [PrioritizedReplayMemory(
initial_replay_size, max_replay_size, alpha=.6,
beta=LinearParameter(.4, threshold_value=1,
n=max_steps // train_frequency)
) for _ in range(n_games)]
else:
replay_memory = None
# Agent
algorithm_params = dict(
batch_size=args.batch_size,
n_games=len(args.games),
initial_replay_size=initial_replay_size,
max_replay_size=max_replay_size,
target_update_frequency=target_update_frequency // train_frequency,
replay_memory=replay_memory,
n_actions_per_head=n_actions_per_head,
clip_reward=True,
history_length=args.history_length
)
if args.algorithm == 'dqn':
agent = DQN(approximator, pi, mdp_info,
approximator_params=approximator_params,
**algorithm_params)
elif args.algorithm == 'ddqn':
agent = DoubleDQN(approximator, pi, mdp_info,
approximator_params=approximator_params,
**algorithm_params)
# Algorithm
core = Core(agent, mdp)
# RUN
# Fill replay memory with random dataset
print_epoch(0)
pi.set_parameter(epsilon_random)
core.learn(n_steps=initial_replay_size,
n_steps_per_fit=initial_replay_size, quiet=args.quiet)
if args.transfer:
weights = pickle.load(open(args.transfer, 'rb'))
agent.set_shared_weights(weights)
if args.load:
weights = np.load(args.load)
agent.approximator.set_weights(weights)
# Evaluate initial policy
pi.set_parameter(epsilon_test)
dataset = core.evaluate(n_steps=test_samples, render=args.render,
quiet=args.quiet)
for i in range(len(mdp)):
d = dataset[i::len(mdp)]
scores[i].append(get_stats(d, i, args.games)[2])
if args.unfreeze_epoch > 0:
agent.freeze_shared_weights()
best_score_sum = -np.inf
best_weights = None
np.save(folder_name + 'scores-exp-%d.npy' % idx, scores)
np.save(folder_name + 'loss-exp-%d.npy' % idx,
agent.approximator.model._loss.get_losses())
for n_epoch in range(1, max_steps // evaluation_frequency + 1):
if n_epoch >= args.unfreeze_epoch > 0:
agent.unfreeze_shared_weights()
print_epoch(n_epoch)
print('- Learning:')
# learning step
pi.set_parameter(None)
core.learn(n_steps=evaluation_frequency,
n_steps_per_fit=train_frequency, quiet=args.quiet)
print('- Evaluation:')
# evaluation step
pi.set_parameter(epsilon_test)
dataset = core.evaluate(n_steps=test_samples,
render=args.render, quiet=args.quiet)
current_score_sum = 0
for i in range(len(mdp)):
d = dataset[i::len(mdp)]
current_score = get_stats(d, i, args.games)[2]
scores[i].append(current_score)
current_score_sum += current_score
# Save shared weights if best score
if args.save_shared and current_score_sum >= best_score_sum:
best_score_sum = current_score_sum
best_weights = agent.get_shared_weights()
if args.save:
np.save(folder_name + 'weights-exp-%d-%d.npy' % (idx, n_epoch),
agent.approximator.get_weights())
np.save(folder_name + 'scores-exp-%d.npy' % idx, scores)
np.save(folder_name + 'loss-exp-%d.npy' % idx,
agent.approximator.model._loss.get_losses())
if args.save_shared:
pickle.dump(best_weights, open(args.save_shared, 'wb'))
return scores, agent.approximator.model._loss.get_losses()
def experiment():
np.random.seed()
# Argument parser
parser = argparse.ArgumentParser()
arg_mem = parser.add_argument_group('Replay Memory')
arg_mem.add_argument("--initial-replay-size",
type=int,
default=50000,
help='Initial size of the replay memory.')
arg_mem.add_argument("--max-replay-size",
type=int,
default=500000,
help='Max size of the replay memory.')
arg_mem.add_argument("--prioritized",
action='store_true',
help='Whether to use prioritized memory or not.')
arg_net = parser.add_argument_group('Deep Q-Network')
arg_net.add_argument(
"--optimizer",
choices=['adadelta', 'adam', 'rmsprop', 'rmspropcentered'],
default='adam',
help='Name of the optimizer to use.')
arg_net.add_argument("--learning-rate",
type=float,
default=.0001,
help='Learning rate value of the optimizer.')
arg_net.add_argument("--decay",
type=float,
default=.95,
help='Discount factor for the history coming from the'
'gradient momentum in rmspropcentered and'
'rmsprop')
arg_net.add_argument("--epsilon",
type=float,
default=1e-8,
help='Epsilon term used in rmspropcentered and'
'rmsprop')
arg_alg = parser.add_argument_group('Algorithm')
arg_alg.add_argument("--algorithm",
choices=[
'dqn', 'ddqn', 'adqn', 'mmdqn', 'cdqn', 'dueldqn',
'ndqn', 'rainbow'
],
default='dqn',
help='Name of the algorithm. dqn is for standard'
'DQN, ddqn is for Double DQN and adqn is for'
'Averaged DQN.')
arg_alg.add_argument(
"--n-approximators",
type=int,
default=1,
help="Number of approximators used in the ensemble for"
"AveragedDQN or MaxminDQN.")
arg_alg.add_argument("--batch-size",
type=int,
default=32,
help='Batch size for each fit of the network.')
arg_alg.add_argument("--history-length",
type=int,
default=4,
help='Number of frames composing a state.')
arg_alg.add_argument("--target-update-frequency",
type=int,
default=10000,
help='Number of collected samples before each update'
'of the target network.')
arg_alg.add_argument("--evaluation-frequency",
type=int,
default=250000,
help='Number of collected samples before each'
'evaluation. An epoch ends after this number of'
'steps')
arg_alg.add_argument("--train-frequency",
type=int,
default=4,
help='Number of collected samples before each fit of'
'the neural network.')
arg_alg.add_argument("--max-steps",
type=int,
default=5000000,
help='Total number of collected samples.')
arg_alg.add_argument(
"--final-exploration-frame",
type=int,
default=10000000,
help='Number of collected samples until the exploration'
'rate stops decreasing.')
arg_alg.add_argument("--initial-exploration-rate",
type=float,
default=1.,
help='Initial value of the exploration rate.')
arg_alg.add_argument("--final-exploration-rate",
type=float,
default=.1,
help='Final value of the exploration rate. When it'
'reaches this values, it stays constant.')
arg_alg.add_argument("--test-exploration-rate",
type=float,
default=.05,
help='Exploration rate used during evaluation.')
arg_alg.add_argument("--test-episodes",
type=int,
default=5,
help='Number of episodes for each evaluation.')
arg_alg.add_argument(
"--alpha-coeff",
type=float,
default=.6,
help='Prioritization exponent for prioritized experience replay.')
arg_alg.add_argument("--n-atoms",
type=int,
default=51,
help='Number of atoms for Categorical DQN.')
arg_alg.add_argument("--v-min",
type=int,
default=-10,
help='Minimum action-value for Categorical DQN.')
arg_alg.add_argument("--v-max",
type=int,
default=10,
help='Maximum action-value for Categorical DQN.')
arg_alg.add_argument("--n-steps-return",
type=int,
default=3,
help='Number of steps for n-step return for Rainbow.')
arg_alg.add_argument("--sigma-coeff",
type=float,
default=.5,
help='Sigma0 coefficient for noise initialization in'
'NoisyDQN and Rainbow.')
arg_utils = parser.add_argument_group('Utils')
arg_utils.add_argument('--use-cuda',
action='store_true',
help='Flag specifying whether to use the GPU.')
arg_utils.add_argument('--save',
action='store_true',
help='Flag specifying whether to save the model.')
arg_utils.add_argument('--load-path',
type=str,
help='Path of the model to be loaded.')
arg_utils.add_argument('--render',
action='store_true',
help='Flag specifying whether to render the grid.')
arg_utils.add_argument('--quiet',
action='store_true',
help='Flag specifying whether to hide the progress'
'bar.')
arg_utils.add_argument('--debug',
action='store_true',
help='Flag specifying whether the script has to be'
'run in debug mode.')
args = parser.parse_args()
scores = list()
optimizer = dict()
if args.optimizer == 'adam':
optimizer['class'] = optim.Adam
optimizer['params'] = dict(lr=args.learning_rate, eps=args.epsilon)
elif args.optimizer == 'adadelta':
optimizer['class'] = optim.Adadelta
optimizer['params'] = dict(lr=args.learning_rate, eps=args.epsilon)
elif args.optimizer == 'rmsprop':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=args.learning_rate,
alpha=args.decay,
eps=args.epsilon)
elif args.optimizer == 'rmspropcentered':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=args.learning_rate,
alpha=args.decay,
eps=args.epsilon,
centered=True)
else:
raise ValueError
# Summary folder
folder_name = './logs/habitat_nav_' + args.algorithm +\
'_' + datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
pathlib.Path(folder_name).mkdir(parents=True)
# Settings
if args.debug:
initial_replay_size = 50
max_replay_size = 500
train_frequency = 5
target_update_frequency = 10
test_episodes = 20
evaluation_frequency = 50
max_steps = 1000
else:
initial_replay_size = args.initial_replay_size
max_replay_size = args.max_replay_size
train_frequency = args.train_frequency
target_update_frequency = args.target_update_frequency
test_episodes = args.test_episodes
evaluation_frequency = args.evaluation_frequency
max_steps = args.max_steps
# MDP
config_file = os.path.join(
pathlib.Path(__file__).parent.resolve(),
'pointnav_apartment-0.yaml') # Custom task for Replica scenes
wrapper = 'HabitatNavigationWrapper'
mdp = Habitat(wrapper, config_file)
opt_return = mdp.env.get_optimal_policy_return()
if args.load_path:
logger = Logger(DQN.__name__, results_dir=None)
logger.strong_line()
logger.info('Optimal Policy Undiscounted Return: ' + str(opt_return))
logger.info('Experiment Algorithm: ' + DQN.__name__)
# Agent
agent = DQN.load(args.load_path)
epsilon_test = Parameter(value=args.test_exploration_rate)
agent.policy.set_epsilon(epsilon_test)
# Algorithm
core_test = Core(agent, mdp)
# Evaluate model
dataset = core_test.evaluate(n_episodes=args.test_episodes,
render=args.render,
quiet=args.quiet)
get_stats(dataset, logger)
else:
# Policy
epsilon = LinearParameter(value=args.initial_exploration_rate,
threshold_value=args.final_exploration_rate,
n=args.final_exploration_frame)
epsilon_test = Parameter(value=args.test_exploration_rate)
epsilon_random = Parameter(value=1)
pi = EpsGreedy(epsilon=epsilon_random)
# Approximator
approximator_params = dict(
network=Network if args.algorithm
not in ['dueldqn', 'cdqn', 'ndqn', 'rainbow'] else FeatureNetwork,
input_shape=mdp.info.observation_space.shape,
output_shape=(mdp.info.action_space.n, ),
n_actions=mdp.info.action_space.n,
n_features=Network.n_features,
optimizer=optimizer,
use_cuda=args.use_cuda)
if args.algorithm not in ['cdqn', 'rainbow']:
approximator_params['loss'] = F.smooth_l1_loss
approximator = TorchApproximator
if args.prioritized:
replay_memory = PrioritizedReplayMemory(
initial_replay_size,
max_replay_size,
alpha=args.alpha_coeff,
beta=LinearParameter(.4,
threshold_value=1,
n=max_steps // train_frequency))
else:
replay_memory = None
# Agent
algorithm_params = dict(
batch_size=args.batch_size,
target_update_frequency=target_update_frequency // train_frequency,
replay_memory=replay_memory,
initial_replay_size=initial_replay_size,
max_replay_size=max_replay_size)
if args.algorithm == 'dqn':
alg = DQN
agent = alg(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
**algorithm_params)
elif args.algorithm == 'ddqn':
alg = DoubleDQN
agent = alg(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
**algorithm_params)
elif args.algorithm == 'adqn':
alg = AveragedDQN
agent = alg(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
n_approximators=args.n_approximators,
**algorithm_params)
elif args.algorithm == 'mmdqn':
alg = MaxminDQN
agent = alg(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
n_approximators=args.n_approximators,
**algorithm_params)
elif args.algorithm == 'dueldqn':
alg = DuelingDQN
agent = alg(mdp.info,
pi,
approximator_params=approximator_params,
**algorithm_params)
elif args.algorithm == 'cdqn':
alg = CategoricalDQN
agent = alg(mdp.info,
pi,
approximator_params=approximator_params,
n_atoms=args.n_atoms,
v_min=args.v_min,
v_max=args.v_max,
**algorithm_params)
elif args.algorithm == 'ndqn':
alg = NoisyDQN
agent = alg(mdp.info,
pi,
approximator_params=approximator_params,
sigma_coeff=args.sigma_coeff,
**algorithm_params)
elif args.algorithm == 'rainbow':
alg = Rainbow
beta = LinearParameter(.4,
threshold_value=1,
n=max_steps // train_frequency)
agent = alg(mdp.info,
pi,
approximator_params=approximator_params,
n_atoms=args.n_atoms,
v_min=args.v_min,
v_max=args.v_max,
n_steps_return=args.n_steps_return,
alpha_coeff=args.alpha_coeff,
beta=beta,
sigma_coeff=args.sigma_coeff,
**algorithm_params)
logger = Logger(alg.__name__, results_dir=None)
logger.strong_line()
logger.info('Optimal Policy Undiscounted Return: ' + str(opt_return))
logger.info('Experiment Algorithm: ' + alg.__name__)
# Algorithm
core = Core(agent, mdp)
# RUN
# Fill replay memory with random dataset
print_epoch(0, logger)
core.learn(n_steps=initial_replay_size,
n_steps_per_fit=initial_replay_size,
quiet=args.quiet)
if args.save:
agent.save(folder_name + '/agent_0.msh')
# Evaluate initial policy
pi.set_epsilon(epsilon_test)
dataset = core.evaluate(n_episodes=test_episodes,
render=args.render,
quiet=args.quiet)
scores.append(get_stats(dataset, logger))
np.save(folder_name + '/scores.npy', scores)
for n_epoch in range(1, max_steps // evaluation_frequency + 1):
print_epoch(n_epoch, logger)
logger.info('- Learning:')
# learning step
pi.set_epsilon(epsilon)
core.learn(n_steps=evaluation_frequency,
n_steps_per_fit=train_frequency,
quiet=args.quiet)
if args.save:
agent.save(folder_name + '/agent_' + str(n_epoch) + '.msh')
logger.info('- Evaluation:')
# evaluation step
pi.set_epsilon(epsilon_test)
dataset = core.evaluate(n_episodes=test_episodes,
render=args.render,
quiet=args.quiet)
scores.append(get_stats(dataset, logger))
np.save(folder_name + '/scores.npy', scores)
return scores
def experiment(n_epochs, n_steps, n_steps_test):
np.random.seed()
# MDP
horizon = 1000
gamma = 0.99
gamma_eval = 1.
mdp = Gym('Acrobot-v1', horizon, gamma)
# Policy
epsilon = LinearParameter(value=1., threshold_value=.01, n=5000)
epsilon_test = Parameter(value=0.)
epsilon_random = Parameter(value=1.)
pi = EpsGreedy(epsilon=epsilon_random)
# Settings
initial_replay_size = 500
max_replay_size = 5000
target_update_frequency = 100
batch_size = 200
n_features = 80
train_frequency = 1
# Approximator
input_shape = mdp.info.observation_space.shape
approximator_params = dict(network=Network,
optimizer={
'class': optim.Adam,
'params': {
'lr': .001
}
},
loss=F.smooth_l1_loss,
n_features=n_features,
input_shape=input_shape,
output_shape=mdp.info.action_space.size,
n_actions=mdp.info.action_space.n)
# Agent
agent = DQN(mdp.info,
pi,
TorchApproximator,
approximator_params=approximator_params,
batch_size=batch_size,
n_approximators=1,
initial_replay_size=initial_replay_size,
max_replay_size=max_replay_size,
target_update_frequency=target_update_frequency)
# Algorithm
core = Core(agent, mdp)
core.learn(n_steps=initial_replay_size,
n_steps_per_fit=initial_replay_size)
# RUN
pi.set_epsilon(epsilon_test)
dataset = core.evaluate(n_steps=n_steps_test, render=False)
J = compute_J(dataset, gamma_eval)
print('J: ', np.mean(J))
for n in trange(n_epochs):
tqdm.write('Epoch: ' + str(n))
pi.set_epsilon(epsilon)
core.learn(n_steps=n_steps, n_steps_per_fit=train_frequency)
pi.set_epsilon(epsilon_test)
dataset = core.evaluate(n_steps=n_steps_test, render=False)
J = compute_J(dataset, gamma_eval)
tqdm.write('J: ' + str(np.mean(J)))
print('Press a button to visualize acrobot')
input()
core.evaluate(n_episodes=5, render=True)
def experiment(mdp, params, prob=None):
# Argument parser
# parser = argparse.ArgumentParser()
#
# args = parser.parse_args()
scores = list()
optimizer = dict()
if params['optimizer'] == 'adam':
optimizer['class'] = optim.Adam
optimizer['params'] = dict(lr=params['learning_rate'])
elif params['optimizer'] == 'adadelta':
optimizer['class'] = optim.Adadelta
optimizer['params'] = dict(lr=params['learning_rate'])
elif params['optimizer'] == 'rmsprop':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=params['learning_rate'],
alpha=params['decay'])
elif params['optimizer'] == 'rmspropcentered':
optimizer['class'] = optim.RMSprop
optimizer['params'] = dict(lr=params['learning_rate'],
alpha=params['decay'],
centered=True)
else:
raise ValueError
# DQN learning run
# Summary folder
folder_name = os.path.join(PROJECT_DIR, 'logs', params['name'])
if params['save']:
pathlib.Path(folder_name).mkdir(parents=True)
# Policy
epsilon = ExponentialParameter(value=params['initial_exploration_rate'],
exp=params['exploration_rate'],
min_value=params['final_exploration_rate'],
size=(1, ))
epsilon_random = Parameter(value=1)
epsilon_test = Parameter(value=0.01)
pi = EpsGreedy(epsilon=epsilon_random)
class CategoricalLoss(nn.Module):
def forward(self, input, target):
input = input.clamp(1e-5)
return -torch.sum(target * torch.log(input))
# Approximator
input_shape = mdp.observation.shape
resources = [[
(mdp.north - mdp.devices[device][0]) / (mdp.north - mdp.south),
(mdp.east - mdp.devices[device][1]) / (mdp.east - mdp.west)
] for device in mdp.device_ordering]
edges = [[
(mdp.north - mdp.graph.nodes[e[0]]['y']) / (mdp.north - mdp.south),
(mdp.east - mdp.graph.nodes[e[0]]['x']) / (mdp.east - mdp.west),
(mdp.north - mdp.graph.nodes[e[1]]['y']) / (mdp.north - mdp.south),
(mdp.east - mdp.graph.nodes[e[1]]['x']) / (mdp.east - mdp.west)
] for e in mdp.graph.edges]
N = {
'SimpleResourceNetwork': SimpleResourceNetwork,
'GraphConvolutionResourceNetwork': GraphConvolutionResourceNetwork,
}[params['network']]
N.n_features = params['hidden']
approximator_params = dict(
network=N,
input_shape=input_shape,
edges=edges,
resources=resources,
graph=mdp.graph,
allow_wait=params['allow_wait'],
long_term_q=params['long_term_q'],
resource_embeddings=params['resource_embeddings'],
edge_ordering=mdp.edge_ordering,
device_ordering=mdp.device_ordering,
resource_edges=mdp.resource_edges,
output_shape=(mdp.info.action_space.n, ),
n_actions=mdp.info.action_space.n,
n_features=params['hidden'],
optimizer=optimizer,
loss=F.smooth_l1_loss,
nn_scaling=params['nn_scaling'],
# quiet=False,
use_cuda=params['cuda'],
load_path=params.get('load_path', None))
approximator = TorchApproximator
replay_memory = PrioritizedReplayMemory(
params['initial_replay_size'],
params['max_replay_size'],
alpha=.6,
beta=LinearParameter(.4,
threshold_value=1,
n=params['max_steps'] //
params['train_frequency']))
# Agent
algorithm_params = dict(
batch_size=params['batch_size'],
n_approximators=1,
target_update_frequency=params['target_update_frequency'] //
params['train_frequency'],
replay_memory=replay_memory,
initial_replay_size=params['initial_replay_size'],
max_replay_size=params['max_replay_size'])
clz = DoubleDQN if mdp.info.gamma >= 1 else SMDPDQN
agent = clz(mdp.info,
pi,
approximator,
approximator_params=approximator_params,
**algorithm_params)
lr_scheduler = torch.optim.lr_scheduler.StepLR(
agent.approximator._impl.model._optimizer,
step_size=1,
gamma=params['lr_decay'],
last_epoch=-1) # params['max_steps'] // params['train_frequency']
# Algorithm
core = Core(agent, mdp)
if 'weights' in params:
best_weights = np.load(params['weights'])
agent.approximator.set_weights(best_weights)
agent.target_approximator.set_weights(best_weights)
else:
best_weights = agent.approximator.get_weights()
# RUN
pi.set_epsilon(epsilon_test)
eval_days = [i for i in range(1, 356) if i % 13 == 1]
ds = core.evaluate(initial_states=eval_days,
quiet=tuning,
render=params['save'])
test_result = np.mean(compute_J(ds))
test_result_discounted = np.mean(compute_J(ds, params['gamma']))
print("discounted validation result", test_result_discounted)
print("validation result", test_result)
results = [(0, 0, test_result_discounted, test_result)]
if params['save']:
mdp.save_rendered(folder_name + "/epoch_init.mp4")
# Fill replay memory with random dataset
print_epoch(0)
start = time()
core.learn(n_steps=params['initial_replay_size'],
n_steps_per_fit=params['initial_replay_size'],
quiet=tuning)
runtime = time() - start
steps = 0
if params['save']:
with open(folder_name + "/params.json", "w") as f:
json.dump(params, f, indent=4)
if isinstance(agent, DQN):
np.save(folder_name + '/weights-exp-0-0.npy',
agent.approximator.get_weights())
best_score = -np.inf
no_improvement = 0
patience = 6
if params['save']:
np.save(folder_name + '/scores.npy', scores)
for n_epoch in range(
1, int(params['max_steps'] // params['evaluation_frequency'] + 1)):
print_epoch(n_epoch)
print('- Learning:')
# learning step
pi.set_epsilon(epsilon)
# mdp.set_episode_end(True)
start = time()
core.learn(n_steps=params['evaluation_frequency'],
n_steps_per_fit=params['train_frequency'],
quiet=tuning)
runtime += time() - start
steps += params['evaluation_frequency']
lr_scheduler.step()
if params['save']:
if isinstance(agent, DQN):
np.save(
folder_name + '/weights-exp-0-' + str(n_epoch) + '.npy',
agent.approximator.get_weights())
print('- Evaluation:')
# evaluation step
pi.set_epsilon(epsilon_test)
ds = core.evaluate(initial_states=eval_days,
render=params['save'],
quiet=tuning)
test_result_discounted = np.mean(compute_J(ds, params['gamma']))
test_result = np.mean(compute_J(ds))
print("discounted validation result", test_result_discounted)
print("validation result", test_result)
if params['save']:
mdp.save_rendered(folder_name + ("/epoch%04d.mp4" % n_epoch))
results.append((runtime, steps, test_result_discounted, test_result))
if params['save']:
np.savetxt(folder_name + '/scores.csv',
np.asarray(results),
delimiter=',')
if test_result > best_score:
no_improvement = 0
best_score = test_result
best_weights = agent.approximator.get_weights().copy()
with open(folder_name + "/best_val.txt", "w") as f:
f.write("%f" % test_result)
else:
no_improvement += 1
if no_improvement >= patience:
break
print('---------- FINAL EVALUATION ---------')
agent.approximator.set_weights(best_weights)
agent.target_approximator.set_weights(best_weights)
pi.set_epsilon(epsilon_test)
eval_days = [i for i in range(1, 356) if i % 13 == 0]
ds = core.evaluate(initial_states=eval_days,
render=params['save'],
quiet=tuning)
test_result_discounted = np.mean(compute_J(ds, params['gamma']))
test_result = np.mean(compute_J(ds))
print("discounted test result", test_result_discounted)
print("test result", test_result)
with open(folder_name + "/test_result.txt", "w") as f:
f.write("%f" % test_result)
if params['save']:
mdp.save_rendered(folder_name + "/epoch_test.mp4", 10000)
return scores