def testMaze(): """ No comments here. Look at single_maze_learning_agent.py for more details! """ ValueLearning.DBG_LVL = 0 nx = 6 ny = 6 # Set the number of cells to be used per "place field" - Same for all the environments Hippocampus.N_CELLS_PER_FIELD = 1 n_fields = round(1.0 * (nx + 3) * (ny + 3)) n_cells = Hippocampus.N_CELLS_PER_FIELD * n_fields move_distance = 0.99 n_training_trials = 100 n_single_env_episodes = 2 n_alternations = 1 max_train_steps = 1000 # First Environment: Has its own place cells and place fields env_E1 = Environment.RandomGoalOpenField(nx, ny, move_distance) canvas_E1 = Graphics.WallMazeCanvas(env_E1) place_fields_E1 = Hippocampus.setupPlaceFields(env_E1, n_fields) place_cells_E1 = Hippocampus.assignPlaceCells(n_cells, place_fields_E1) # Train a critic on the first environment print('Training Critic solely on Env A') critic_E1 = None weights_E1 = np.empty((n_cells, n_single_env_episodes), dtype=float) for episode in range(n_single_env_episodes): (_, critic_E1, _) = ValueLearning.learnValueFunction(n_training_trials, env_E1, place_cells_E1, critic=critic_E1, max_steps=max_train_steps) weights_E1[:, episode] = critic_E1.getWeights() # Get a trajectory in the environment and plot the value function canvas_E1.plotValueFunction(place_cells_E1, critic_E1, continuous=True) input('Press return to run next environment...') components_E1 = Graphics.showDecomposition(weights_E1, title='Environment 01') # Create empty actors and critics actor = Agents.RandomAgent(env_E1.getActions(), n_cells) critic = Agents.Critic(n_cells) # Second Environment: This has a different set (but the same number) of # place fields and place cells (also has a bunch of walls) nx = 6 ny = 6 lp_wall = Environment.Wall((0, 3), (3, 3)) rp_wall = Environment.Wall((4, 3), (6, 3)) env_E2 = Environment.MazeWithWalls(nx, ny, [lp_wall, rp_wall], move_distance=move_distance) canvas_E2 = Graphics.WallMazeCanvas(env_E2) place_fields_E2 = Hippocampus.setupPlaceFields(env_E2, n_fields) place_cells_E2 = Hippocampus.assignPlaceCells(n_cells, place_fields_E2) # Train another critic on the second environment print() print('Training Critic solely on Env B') critic_E2 = None weights_E2 = np.empty((n_cells, n_single_env_episodes), dtype=float) for episode in range(n_single_env_episodes): (_, critic_E2, _) = ValueLearning.learnValueFunction(n_training_trials, env_E2, place_cells_E2, critic=critic_E2, max_steps=max_train_steps) weights_E2[:, episode] = critic_E2.getWeights() components_E2 = Graphics.showDecomposition(weights_E2, title='Environment 02') canvas_E2.plotValueFunction(place_cells_E2, critic_E2, continuous=True) # Look at the projection of one environment's weights on the other's principal components Graphics.showDecomposition(weights_E1, components=components_E2, title='E2 on E1') Graphics.showDecomposition(weights_E2, components=components_E1, title='E1 on E2') input('Press any key to start Alternation.') # This can be used to just reinforce the fact that the agent is indeed # random! The steps taken to goal would not change over time because of the # way the agent behaves. learning_steps_E1 = np.zeros((n_alternations, 1), dtype=float) learning_steps_E2 = np.zeros((n_alternations, 1), dtype=float) # keep track of weights for PCA weights = np.empty((n_cells, n_alternations * 2), dtype=float) for alt in range(n_alternations): n_alternation_trials = n_single_env_episodes * n_training_trials # n_alternation_trials = n_training_trials print('Alternation: %d' % alt) # First look at the performance of the agent in the task before it is # allowed to learn anything. Then allow learning print('Learning Environment A') (actor, critic, steps_E1) = ValueLearning.learnValueFunction( n_alternation_trials, env_E1, place_cells_E1, actor, critic, max_train_steps) learning_steps_E1[alt] = np.mean(steps_E1) weights[:, 2 * alt] = critic.getWeights() # Repeat for environment 1 print('Learning Environment B') (actor, critic, steps_E2) = ValueLearning.learnValueFunction( n_alternation_trials, env_E2, place_cells_E2, actor, critic, max_train_steps) learning_steps_E2[alt] = np.mean(steps_E2) weights[:, 2 * alt + 1] = critic.getWeights() # Show the alternation weights in the two basis Graphics.showDecomposition(weights, components=components_E1, title='Alternation weights in E1') Graphics.showDecomposition(weights, components=components_E2, title='Alternation weights in E2') # Show the value functions for both the environments input('Press return for Value Function of E1') canvas_E1.plotValueFunction(place_cells_E1, critic, continuous=True) canvas_E1.plotValueFunction(place_cells_E1, critic_E1, continuous=True) canvas_E1.plotValueFunction(place_cells_E1, critic_E2, continuous=True) # Plot the ideal value function ideal_critic = Agents.IdealValueAgent(env_E1, place_cells_E1) optimal_value_function = ideal_critic.getValueFunction() scaling_factor = 1.0 / (1 - critic_E1.getDiscountFactor()) # Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), range=(maze.NON_GOAL_STATE_REWARD, scaling_factor * maze.GOAL_STATE_REWARD)) Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), \ range=(env_E1.NON_GOAL_STATE_REWARD, scaling_factor * env_E1.GOAL_STATE_REWARD)) input('Press return for Value Function of E2') canvas_E2.plotValueFunction(place_cells_E2, critic, continuous=True) canvas_E2.plotValueFunction(place_cells_E2, critic_E2, continuous=True) canvas_E2.plotValueFunction(place_cells_E2, critic_E1, continuous=True) # Plot the ideal value function ideal_critic = Agents.IdealValueAgent(env_E2, place_cells_E2) optimal_value_function = ideal_critic.getValueFunction() scaling_factor = 1.0 / (1 - critic_E2.getDiscountFactor()) # Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), range=(maze.NON_GOAL_STATE_REWARD, scaling_factor * maze.GOAL_STATE_REWARD)) Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), \ range=(env_E2.NON_GOAL_STATE_REWARD, scaling_factor * env_E2.GOAL_STATE_REWARD)) input('Press any key to exit!')
def testMaze(n_trials, dbg_lvl=1): ValueLearning.DBG_LVL = dbg_lvl move_distance = 0.29 # Open field - Rather boring # maze = Environment.RandomGoalOpenField(nx, ny) # Maze with partition - 6 x 6 environment # ----------------- (6,6) # | | # | (2,3) (4,3) | # |----- -----| (6,3) # | | # | | # (0,0) ----------------- """ nx = 6 ny = 6 maze = Environment.RandomGoalOpenField(nx, ny, move_distance) use_limits = True """ # Adding walls and constructing the environment """ nx = 6 ny = 6 lp_wall = Environment.Wall((0,3), (2,3)) rp_wall = Environment.Wall((4,3), (6,3)) maze = Environment.MazeWithWalls(nx, ny, [lp_wall, rp_wall], move_distance) use_limits = False """ # Maze with walls - 10 x 10 environment # (2,10) (8,10) # --------------------- (10,10) # | | | # | | (4, 6) | | (10, 8) # | | ------| | # | | (6,4) | | # (0,4) | |------ | | # | | | | # | (2,2) | | # (0,0) --------------------- nx = 10 ny = 10 # Adding walls and constructing the environment lh_wall = Environment.Wall((2,4), (6,4)) lv_wall = Environment.Wall((2,2), (2,10)) rh_wall = Environment.Wall((4,6), (8,6)) rv_wall = Environment.Wall((8,0), (8,8)) maze = Environment.MazeWithWalls(nx, ny, [lh_wall, lv_wall, rh_wall, rv_wall]) use_limits = False n_fields = round(1.0 * (nx+3) * (ny+3)) Hippocampus.N_CELLS_PER_FIELD = 1 n_cells = n_fields * Hippocampus.N_CELLS_PER_FIELD place_fields = Hippocampus.setupPlaceFields(maze, n_fields) place_cells = Hippocampus.assignPlaceCells(n_cells, place_fields) # Learn the value function amateur_critic = None n_episodes = 25 canvas = Graphics.MazeCanvas(maze) weights = np.empty((n_cells, n_episodes), dtype=float) for episode in range(n_episodes): (_, amateur_critic, _) = ValueLearning.learnValueFunction(n_trials, maze, place_cells, critic=amateur_critic, max_steps=1000) weights[:, episode] = amateur_critic.getWeights() print('Ended Episode %d'% episode) # canvas.plotValueFunction(place_cells, amateur_critic, continuous=True) # input() # Draw the final value funciton canvas.plotValueFunction(place_cells, amateur_critic, continuous=True, limits=use_limits) # canvas.plotValueFunction(place_cells, amateur_critic) """ DEBUG print(components.explained_variance_ratio_) print(components.singular_values_) """ # Graphics.showDecomposition(weights) # Evaluate the theoritical value function for a random policy ideal_critic = Agents.IdealValueAgent(maze, place_cells) optimal_value_function = ideal_critic.getValueFunction() scaling_factor = 1.0/(1 - amateur_critic.getDiscountFactor()) # Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), range=(maze.NON_GOAL_STATE_REWARD, scaling_factor * maze.GOAL_STATE_REWARD)) Graphics.showImage(optimal_value_function, xticks=range(1,nx), yticks=range(1,ny), range=(maze.NON_GOAL_STATE_REWARD, scaling_factor * maze.GOAL_STATE_REWARD)) input('Press any key to Exit!')