from rl_coach.agents.ddqn_agent import DDQNAgentParameters from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters from rl_coach.environments.environment import SingleLevelSelection from rl_coach.environments.gym_environment import Atari, atari_deterministic_v4, atari_schedule from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.memories.non_episodic.prioritized_experience_replay import PrioritizedExperienceReplayParameters from rl_coach.schedules import LinearSchedule ######### # Agent # ######### agent_params = DDQNAgentParameters() agent_params.network_wrappers['main'].learning_rate = 0.00025 / 4 agent_params.memory = PrioritizedExperienceReplayParameters() agent_params.memory.beta = LinearSchedule( 0.4, 1, 12500000) # 12.5M training iterations = 50M steps = 200M frames ############### # Environment # ############### env_params = Atari(level=SingleLevelSelection(atari_deterministic_v4)) ######## # Test # ######## preset_validation_params = PresetValidationParameters() preset_validation_params.trace_test_levels = [ 'breakout', 'pong', 'space_invaders' ] graph_manager = BasicRLGraphManager(
agent_params.network_wrappers['main'].learning_rate = 0.0001 agent_params.network_wrappers['main'].replace_mse_with_huber_loss = False agent_params.network_wrappers['main'].l2_regularization = 0.0001 agent_params.network_wrappers['main'].softmax_temperature = 0.2 # agent_params.network_wrappers['main'].learning_rate_decay_rate = 0.95 # agent_params.network_wrappers['main'].learning_rate_decay_steps = int(DATASET_SIZE / # agent_params.network_wrappers['main'].batch_size) # reward model params agent_params.network_wrappers['reward_model'] = deepcopy( agent_params.network_wrappers['main']) agent_params.network_wrappers['reward_model'].learning_rate = 0.0001 agent_params.network_wrappers['reward_model'].l2_regularization = 0 # ER size agent_params.memory = EpisodicExperienceReplayParameters() agent_params.memory.max_size = (MemoryGranularity.Transitions, DATASET_SIZE) # E-Greedy schedule agent_params.exploration.epsilon_schedule = LinearSchedule(0, 0, 10000) agent_params.exploration.evaluation_epsilon = 0 agent_params.input_filter = InputFilter() agent_params.input_filter.add_reward_filter('rescale', RewardRescaleFilter(1 / 200.)) ################ # Environment # ################ env_params = GymVectorEnvironment(level='CartPole-v0')
experience_generating_schedule_params.evaluation_steps = EnvironmentEpisodes(1) # DQN params experience_generating_agent_params.algorithm.num_steps_between_copying_online_weights_to_target = EnvironmentSteps(100) experience_generating_agent_params.algorithm.discount = 0.99 experience_generating_agent_params.algorithm.num_consecutive_playing_steps = EnvironmentSteps(1) # NN configuration experience_generating_agent_params.network_wrappers['main'].learning_rate = 0.0001 experience_generating_agent_params.network_wrappers['main'].batch_size = 128 experience_generating_agent_params.network_wrappers['main'].replace_mse_with_huber_loss = False experience_generating_agent_params.network_wrappers['main'].heads_parameters = \ [QHeadParameters(output_bias_initializer=tf.constant_initializer(-100))] # ER size experience_generating_agent_params.memory = EpisodicExperienceReplayParameters() experience_generating_agent_params.memory.max_size = \ (MemoryGranularity.Transitions, experience_generating_schedule_params.heatup_steps.num_steps + experience_generating_schedule_params.improve_steps.num_steps + 1) # E-Greedy schedule experience_generating_agent_params.exploration.epsilon_schedule = LinearSchedule(1.0, 0.01, DATASET_SIZE) experience_generating_agent_params.exploration.evaluation_epsilon = 0 ################ # Environment # ################ env_params = GymVectorEnvironment(level='MountainCar-v0')
def train_using_experience_agent(env_params, n_epochs, dataset_size): tf.reset_default_graph( ) # just to clean things up; only needed for the tutorial # Experience Generating Agent parameters experience_generating_agent_params = DDQNAgentParameters() # schedule parameters experience_generating_schedule_params = ScheduleParameters() experience_generating_schedule_params.heatup_steps = EnvironmentSteps(1000) experience_generating_schedule_params.improve_steps = TrainingSteps( dataset_size - experience_generating_schedule_params.heatup_steps.num_steps) experience_generating_schedule_params.steps_between_evaluation_periods = EnvironmentEpisodes( 10) experience_generating_schedule_params.evaluation_steps = EnvironmentEpisodes( 1) # DQN params experience_generating_agent_params.algorithm.num_steps_between_copying_online_weights_to_target = EnvironmentSteps( 100) experience_generating_agent_params.algorithm.discount = 0.99 experience_generating_agent_params.algorithm.num_consecutive_playing_steps = EnvironmentSteps( 1) # NN configuration experience_generating_agent_params.network_wrappers[ 'main'].learning_rate = 0.0001 experience_generating_agent_params.network_wrappers[ 'main'].batch_size = 128 experience_generating_agent_params.network_wrappers[ 'main'].replace_mse_with_huber_loss = False experience_generating_agent_params.network_wrappers['main'].heads_parameters = \ [QHeadParameters(output_bias_initializer=tf.constant_initializer(-100))] # experience_generating_agent_params.network_wrappers['main'].heads_parameters = \ # [QHeadParameters(output_bias_initializer=tf.constant_initializer(0))] # ER size experience_generating_agent_params.memory = EpisodicExperienceReplayParameters( ) experience_generating_agent_params.memory.max_size = \ (MemoryGranularity.Transitions, experience_generating_schedule_params.heatup_steps.num_steps + experience_generating_schedule_params.improve_steps.num_steps) # E-Greedy schedule experience_generating_agent_params.exploration.epsilon_schedule = LinearSchedule( 1.0, 0.01, DATASET_SIZE) experience_generating_agent_params.exploration.evaluation_epsilon = 0 schedule_params = set_schedule_params(n_epochs, dataset_size) # set the agent params as before # agent_params = set_agent_params(DDQNAgentParameters) agent_params = set_agent_params(DDQNBCQAgentParameters) agent_params.algorithm.action_drop_method_parameters = NNImitationModelParameters( ) # 50 epochs of training (the entire dataset is used each epoch) # schedule_params.improve_steps = TrainingSteps(50) graph_manager = BatchRLGraphManager( agent_params=agent_params, experience_generating_agent_params=experience_generating_agent_params, experience_generating_schedule_params= experience_generating_schedule_params, env_params=env_params, schedule_params=schedule_params, vis_params=VisualizationParameters( dump_signals_to_csv_every_x_episodes=1), reward_model_num_epochs=30, train_to_eval_ratio=0.5) graph_manager.create_graph(task_parameters) graph_manager.improve() return