def __init__(self, n_states, n_actions, hidden_dim=90, device="cpu", critic_lr=5e-3, actor_lr=5e-4, gamma=0.99, soft_tau=1e-2, memory_capacity=100000, batch_size=128): self.device = device self.critic_lr = critic_lr self.actor_lr = actor_lr self.critic = Critic(n_states, n_actions, hidden_dim).to(device) self.actor = Actor(n_states, n_actions, hidden_dim).to(device) self.target_critic = Critic(n_states, n_actions, hidden_dim).to(device) self.target_actor = Actor(n_states, n_actions, hidden_dim).to(device) for target_param, param in zip(self.target_critic.parameters(), self.critic.parameters()): target_param.data.copy_(param.data) for target_param, param in zip(self.target_actor.parameters(), self.actor.parameters()): target_param.data.copy_(param.data) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr) self.memory = ReplayBuffer(memory_capacity) self.batch_size = batch_size self.soft_tau = soft_tau self.gamma = gamma
def __init__(self, env): self.env = env self.num_robots = env.num_robots self.learning_rate = 0.0001 self.epsilon = .9 self.epsilon_decay = .99995 self.eps_counter = 0 self.gamma = .90 self.tau = .01 self.buffer_size = 1000000 self.batch_size = 512 self.hyper_parameters_lambda3 = 0.2 self.hyper_parameters_eps = 0.2 self.hyper_parameters_eps_d = 0.4 self.demo_size = 1000 self.time_str = time.strftime("%Y%m%d-%H%M%S") self.parent_dir = HOME + "/catkin_ws/src/Turtlebot3_Pheromone/src/DRLbasedController/weights" self.save_dir = HOME + "/catkin_ws/src/Turtlebot3_Pheromone/src/results/trained_weights/exp2/HLERnoisy/" self.path = os.path.join(self.parent_dir, self.time_str) os.mkdir(self.path) # Replay buffer self.memory = deque(maxlen=1000000) # Replay Buffer self.replay_buffer = ExperienceReplayBuffer(total_timesteps=5000*256, type_buffer="HER") # File name self.file_name = "reward_{}_{}_{}".format(self.time_str, self.num_robots, self.replay_buffer.type_buffer) # Hidden Layer list self.hid_list = [512, 512, 512] # ===================================================================== # # Actor Model # # Chain rule: find the gradient of chaging the actor network params in # # getting closest to the final value network predictions, i.e. de/dA # # Calculate de/dA as = de/dC * dC/dA, where e is error, C critic, A act # # ===================================================================== # self.actor_model = Actor(self.env.observation_space.shape, self.env.action_space.shape, self.hid_list) self.target_actor_model = Actor(self.env.observation_space.shape, self.env.action_space.shape, self.hid_list) self.actor_optim = optim.Adam(self.actor_model.parameters(), lr=self.learning_rate) # ===================================================================== # # Critic Model # # ===================================================================== # self.critic_model = Critic(self.env.observation_space.shape, self.env.action_space.shape, 1, self.hid_list) self.target_critic_model = Critic(self.env.observation_space.shape, self.env.action_space.shape, 1, self.hid_list) self.critic_optim = optim.Adam(self.critic_model.parameters(), lr=self.learning_rate) hard_update(self.target_actor_model, self.actor_model) # Make sure target is with the same weight hard_update(self.target_critic_model, self.critic_model) self.cuda()
def __init__(self, hparams): ''' Initializations ''' super().__init__() self.hparams = hparams # Position of human source_position = torch.tensor([[self.hparams.environment.position.end.x], [self.hparams.environment.position.end.y], [self.hparams.environment.position.end.z]]).float() # Position of agent agent_position = torch.tensor([[self.hparams.environment.position.start.x], [self.hparams.environment.position.start.y], [self.hparams.environment.position.start.z]]).float() # Initialize Replay buffer self.replay_buffer = ReplayBuffer(capacity = self.hparams.model.replay_buffer_size) # Initialize drone self.agent = Drone(start_position = agent_position, goal_position = source_position, velocity_factor = self.hparams.environment.agent.velocity_factor, hparams = self.hparams, buffer = self.replay_buffer) # Actor networks self.net = Actor(**self.hparams.model.actor) self.target_net = Actor(**self.hparams.model.actor) # Critic networks self.critic = Critic(**self.hparams.model.critic) self.target_critic = Critic(**self.hparams.model.critic) # Hard update self.target_net.load_state_dict(self.net.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) self.total_reward = -10000 self.episode_steps = 0.0 self.max_episode_steps = self.hparams.model.max_episode self.episode_reward = 0.0 self.populate(self.hparams.model.replay_buffer_size)
def __init__(self, env, env_obs, gamma=0.99, tau=0.001, lr_actor=1e-3, lr_critic=1e-3, weight_decay=0.1, batch_size=64, subpolicies=1, action_shape=2, replay_buffer_size=5000, replay_buffer_type="rb", noise=0.1, noise_decay=0.999, max_action=1, min_action=-1, teacher=False, alpha=0.1, bc=None): self.env = env self.subpolicies = subpolicies self.total_obs = np.sum(env_obs) self.weight_decay = weight_decay self.env_obs = env_obs self.max_action = max_action self.min_action = min_action self.action_shape = action_shape self.gamma = gamma self.tau = tau self.batch_size = batch_size self.replay_buffer_type = replay_buffer_type self.replay_buffer_size = replay_buffer_size self.init_noise = noise self.noise = noise self.noise_decay = noise_decay self.teacher = teacher self.bc = bc self.alpha = alpha self.mul = 1 if self.teacher is False else 2 self.actors = [[Actor(self.mul * env_obs[agent], action_shape) for i in range(self.subpolicies)] for agent in range(env.n)] self.actors_targets = [[Actor(self.mul * env_obs[agent], action_shape) for i in range(self.subpolicies)] for agent in range(env.n)] self.critics = [Critic(self.mul * self.total_obs + action_shape * len(env.agents)) for _ in env.agents] self.critics_targets = [Critic(self.mul * self.total_obs + action_shape * len(env.agents)) for _ in env.agents] self.actors_optimizers = [[torch.optim.RMSprop(self.actors[agent][i].parameters(), lr=lr_actor, weight_decay=weight_decay) for i in range(self.subpolicies)] for agent in range(len(env.agents))] self.critics_optimisers = [torch.optim.RMSprop(self.critics[agent].parameters(), lr=lr_critic ,weight_decay=weight_decay) for agent in range(len(env.agents))] if self.subpolicies > 1: if self.replay_buffer_type == "rb": self.replay_buffers = [[ReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(env.n)] else: self.replay_buffers = [[PrioritizedReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(env.n)] else: if self.replay_buffer_type == "rb": self.replay_buffers = ReplayBuffer(self.replay_buffer_size) else: self.replay_buffers = [[PrioritizedReplayBuffer(self.replay_buffer_size) for _ in range(self.subpolicies)] for _ in range(env.n)]
def __init__(self, task): self.task = task self.state_size = task.state_size self.action_size = task.action_size self.action_low = task.action_low self.action_high = task.action_high # Actor (Policy) Model self.actor_local = Actor(self.state_size, self.action_size, self.action_low, self.action_high) self.actor_target = Actor(self.state_size, self.action_size, self.action_low, self.action_high) # Critic (Value) Model self.critic_local = Critic(self.state_size, self.action_size) self.critic_target = Critic(self.state_size, self.action_size) # Initialize target model parameters with local model parameters self.critic_target.model.set_weights( self.critic_local.model.get_weights()) self.actor_target.model.set_weights( self.actor_local.model.get_weights()) # Noise process self.exploration_mu = 0 self.exploration_theta = 0.15 self.exploration_sigma = 0.2 self.noise = OUNoise(self.action_size, self.exploration_mu, self.exploration_theta, self.exploration_sigma) # Replay memory self.buffer_size = 100000 self.batch_size = 64 self.memory = ReplayBuffer(self.buffer_size, self.batch_size) # Algorithm parameters self.gamma = 0.99 # discount factor self.tau = 0.01 # for soft update of target parameters # initialize state self.last_state = self.task.reset()
def __init__(self, env, GAMMA=0.9): self.env = env print('obs space shape: {}'.format(self.env.observation_space.shape)) print('action space shape: {}'.format(self.env.action_space.shape)) self.states_dim = self.env.observation_space.shape[0] self.action_dim = self.env.action_space.shape[0] print('states dim: {}\t\t actions dim: {}'.format( self.states_dim, self.action_dim)) self.actor = Actor(self.states_dim, self.action_dim, lr=0.0001) self.critic = Critic(self.states_dim, self.action_dim, lr=0.0001) self.GAMMA = GAMMA self.RANDOM_PROB = 0.025 self.replay_buffer = ReplayBuffer(1280)
import gym import tensorflow as tf from ActorCritic import Actor from ActorCritic import Critic LR_A = 0.001 LR_C = 0.01 env = gym.make('MountainCar-v0') env = env.unwrapped sess = tf.Session() actor = Actor(sess, n_features=env.observation_space.shape[0], n_actions=env.action_space.n, learning_rate=LR_A) critic = Critic(sess, n_features=env.observation_space.shape[0], learning_rate=LR_C) sess.run(tf.global_variables_initializer()) for i_episode in range(1000): s = env.reset() t = 0 track_r = [] while True: # if RENDER: env.render() env.render()
import matplotlib.pyplot as plt DISPLAY_REWARD_THRESHOLD = -90 RENDER = False # rendering wastes time env = gym.make('MountainCar-v0') env.seed(1) # reproducible, general Policy gradient has high variance env = env.unwrapped print(env.action_space) print(env.observation_space) print(env.observation_space.high) print(env.observation_space.low) actor = Actor(epsilon=0) critic = Critic() Tmax = 1000 for i_episode in range(3000): observation = env.reset() action = actor.choose_action(observation) running_reward = 0 critic.reset() count = 0 while count < Tmax: count += 1 if RENDER: env.render() observation_, reward, done, info = env.step(