from rl_coach.agents.ddqn_agent import DDQNAgentParameters
from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters
from rl_coach.environments.environment import SingleLevelSelection
from rl_coach.environments.gym_environment import Atari, atari_deterministic_v4, atari_schedule
from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager
from rl_coach.memories.non_episodic.prioritized_experience_replay import PrioritizedExperienceReplayParameters
from rl_coach.schedules import LinearSchedule
#########
# Agent #
#########
agent_params = DDQNAgentParameters()
agent_params.network_wrappers['main'].learning_rate = 0.00025 / 4
agent_params.memory = PrioritizedExperienceReplayParameters()
agent_params.memory.beta = LinearSchedule(
0.4, 1, 12500000) # 12.5M training iterations = 50M steps = 200M frames
###############
# Environment #
###############
env_params = Atari(level=SingleLevelSelection(atari_deterministic_v4))
########
# Test #
########
preset_validation_params = PresetValidationParameters()
preset_validation_params.trace_test_levels = [
'breakout', 'pong', 'space_invaders'
]
graph_manager = BasicRLGraphManager(
agent_params.network_wrappers['main'].learning_rate = 0.0001
agent_params.network_wrappers['main'].replace_mse_with_huber_loss = False
agent_params.network_wrappers['main'].l2_regularization = 0.0001
agent_params.network_wrappers['main'].softmax_temperature = 0.2
# agent_params.network_wrappers['main'].learning_rate_decay_rate = 0.95
# agent_params.network_wrappers['main'].learning_rate_decay_steps = int(DATASET_SIZE /
# agent_params.network_wrappers['main'].batch_size)
# reward model params
agent_params.network_wrappers['reward_model'] = deepcopy(
agent_params.network_wrappers['main'])
agent_params.network_wrappers['reward_model'].learning_rate = 0.0001
agent_params.network_wrappers['reward_model'].l2_regularization = 0
# ER size
agent_params.memory = EpisodicExperienceReplayParameters()
agent_params.memory.max_size = (MemoryGranularity.Transitions, DATASET_SIZE)
# E-Greedy schedule
agent_params.exploration.epsilon_schedule = LinearSchedule(0, 0, 10000)
agent_params.exploration.evaluation_epsilon = 0
agent_params.input_filter = InputFilter()
agent_params.input_filter.add_reward_filter('rescale',
RewardRescaleFilter(1 / 200.))
################
# Environment #
################
env_params = GymVectorEnvironment(level='CartPole-v0')
experience_generating_schedule_params.evaluation_steps = EnvironmentEpisodes(1)
# DQN params
experience_generating_agent_params.algorithm.num_steps_between_copying_online_weights_to_target = EnvironmentSteps(100)
experience_generating_agent_params.algorithm.discount = 0.99
experience_generating_agent_params.algorithm.num_consecutive_playing_steps = EnvironmentSteps(1)
# NN configuration
experience_generating_agent_params.network_wrappers['main'].learning_rate = 0.0001
experience_generating_agent_params.network_wrappers['main'].batch_size = 128
experience_generating_agent_params.network_wrappers['main'].replace_mse_with_huber_loss = False
experience_generating_agent_params.network_wrappers['main'].heads_parameters = \
[QHeadParameters(output_bias_initializer=tf.constant_initializer(-100))]
# ER size
experience_generating_agent_params.memory = EpisodicExperienceReplayParameters()
experience_generating_agent_params.memory.max_size = \
(MemoryGranularity.Transitions,
experience_generating_schedule_params.heatup_steps.num_steps +
experience_generating_schedule_params.improve_steps.num_steps + 1)
# E-Greedy schedule
experience_generating_agent_params.exploration.epsilon_schedule = LinearSchedule(1.0, 0.01, DATASET_SIZE)
experience_generating_agent_params.exploration.evaluation_epsilon = 0
################
# Environment #
################
env_params = GymVectorEnvironment(level='MountainCar-v0')
def train_using_experience_agent(env_params, n_epochs, dataset_size):
tf.reset_default_graph(
) # just to clean things up; only needed for the tutorial
# Experience Generating Agent parameters
experience_generating_agent_params = DDQNAgentParameters()
# schedule parameters
experience_generating_schedule_params = ScheduleParameters()
experience_generating_schedule_params.heatup_steps = EnvironmentSteps(1000)
experience_generating_schedule_params.improve_steps = TrainingSteps(
dataset_size -
experience_generating_schedule_params.heatup_steps.num_steps)
experience_generating_schedule_params.steps_between_evaluation_periods = EnvironmentEpisodes(
10)
experience_generating_schedule_params.evaluation_steps = EnvironmentEpisodes(
1)
# DQN params
experience_generating_agent_params.algorithm.num_steps_between_copying_online_weights_to_target = EnvironmentSteps(
100)
experience_generating_agent_params.algorithm.discount = 0.99
experience_generating_agent_params.algorithm.num_consecutive_playing_steps = EnvironmentSteps(
1)
# NN configuration
experience_generating_agent_params.network_wrappers[
'main'].learning_rate = 0.0001
experience_generating_agent_params.network_wrappers[
'main'].batch_size = 128
experience_generating_agent_params.network_wrappers[
'main'].replace_mse_with_huber_loss = False
experience_generating_agent_params.network_wrappers['main'].heads_parameters = \
[QHeadParameters(output_bias_initializer=tf.constant_initializer(-100))]
# experience_generating_agent_params.network_wrappers['main'].heads_parameters = \
# [QHeadParameters(output_bias_initializer=tf.constant_initializer(0))]
# ER size
experience_generating_agent_params.memory = EpisodicExperienceReplayParameters(
)
experience_generating_agent_params.memory.max_size = \
(MemoryGranularity.Transitions,
experience_generating_schedule_params.heatup_steps.num_steps +
experience_generating_schedule_params.improve_steps.num_steps)
# E-Greedy schedule
experience_generating_agent_params.exploration.epsilon_schedule = LinearSchedule(
1.0, 0.01, DATASET_SIZE)
experience_generating_agent_params.exploration.evaluation_epsilon = 0
schedule_params = set_schedule_params(n_epochs, dataset_size)
# set the agent params as before
# agent_params = set_agent_params(DDQNAgentParameters)
agent_params = set_agent_params(DDQNBCQAgentParameters)
agent_params.algorithm.action_drop_method_parameters = NNImitationModelParameters(
)
# 50 epochs of training (the entire dataset is used each epoch)
# schedule_params.improve_steps = TrainingSteps(50)
graph_manager = BatchRLGraphManager(
agent_params=agent_params,
experience_generating_agent_params=experience_generating_agent_params,
experience_generating_schedule_params=
experience_generating_schedule_params,
env_params=env_params,
schedule_params=schedule_params,
vis_params=VisualizationParameters(
dump_signals_to_csv_every_x_episodes=1),
reward_model_num_epochs=30,
train_to_eval_ratio=0.5)
graph_manager.create_graph(task_parameters)
graph_manager.improve()
return
