def __init__(self, conf, device): self.conf = conf self.state_dim = conf['state_dim'] self.action_dim = conf['action_dim'] self.device = device # create actor and critic network self.actor = Actor_RDPG(self.state_dim, self.action_dim).to(self.device) self.actor_target = Actor_RDPG(self.state_dim, self.action_dim).to(self.device) self.critic = Critic_RDPG(self.state_dim, self.action_dim).to(self.device) self.critic_target = Critic_RDPG(self.state_dim, self.action_dim).to(self.device) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) self.critic_optim = optim.Adam(self.critic.parameters(), lr=q_lr) self.actor_optim = optim.Adam(self.actor.parameters(), lr=policy_lr) #Create replay buffer self.random_process = OrnsteinUhlenbeckProcess(size=self.action_dim, theta=0.15, mu=0.0, sigma=0.2) # args.ou_theta:0.15 (noise theta), args.ou_sigma:0.2 (noise sigma), args.out_mu:0.0 (noise mu) self.epsilon = 1.0 self.depsilon = 1.0 / 50000 self.is_training = True self.tau = 0.001 # moving average for target network
def __init__(self, state_size, action_size, memory_size, batch_size=128, tan=0.001, actor_lr=0.001, critic_lr=0.001, epsilon=1.): self.state_size = state_size self.action_size = action_size self.batch_size = batch_size self.tan = tan self.warmup = WARM_UP self.epsilon = epsilon self.epsilon_decay = hyperparameters['D_EPSILON'] self.actor = Actor(state_size, action_size) self.actor_target = Actor(state_size, action_size) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr) self.critic = Critic(state_size, action_size) self.critic_target = Critic(state_size, action_size) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr) self.memory = Memory(memory_size) self.criterion = nn.MSELoss() self.random_process = OrnsteinUhlenbeckProcess(size=action_size, theta=0.15, mu=0., sigma=0.2) copy_parameter(self.actor, self.actor_target) copy_parameter(self.critic, self.critic_target)
def __init__(self, nb_states, nb_actions): self.critic = Critic(nb_states, nb_actions) # Q self.critic_target = Critic(nb_states, nb_actions) self.actor = Actor(nb_states, nb_actions) # policy mu self.actor_target = Actor(nb_states, nb_actions) hard_update(self.critic_target, self.critic) hard_update(self.actor_target, self.actor) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=0.001) self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=0.0001) self.criterion = nn.MSELoss() self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=0.15, mu=0, sigma=0.2) self.gamma = 0.99 self.batch_size = 64 if USE_CUDA: self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda()
def __init__(self, s_dim, a_dim, num_agent, **kwargs): self.s_dim = s_dim self.a_dim = a_dim self.config = kwargs['config'] self.num_agent = num_agent self.actor = Actor(s_dim, a_dim) self.actor_target = Actor(s_dim, a_dim) self.critic = Critic(s_dim, a_dim, num_agent) self.critic_target = Critic(s_dim, a_dim, num_agent) self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=self.config.a_lr) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=self.config.c_lr) self.a_loss = 0 self.c_loss = 0 if self.config.use_cuda: self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() hard_update(self.actor, self.actor_target) hard_update(self.critic, self.critic_target) self.random_process = OrnsteinUhlenbeckProcess( size=self.a_dim, theta=self.config.ou_theta, mu=self.config.ou_mu, sigma=self.config.ou_sigma)
def __init__(self, nb_state, nb_action): self.nb_state = nb_state self.nb_action = nb_action self.actor = Actor(self.nb_state, self.nb_action) self.actor_target = Actor(self.nb_state, self.nb_action) self.actor_optim = Adam(self.actor.parameters(), lr=LEARNING_RATE) self.critic = Critic(self.nb_state, self.nb_action) self.critic_target = Critic(self.nb_state, self.nb_action) self.critic_optim = Adam(self.critic.parameters(), lr=LEARNING_RATE) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=1) self.random_process = OrnsteinUhlenbeckProcess(size=nb_action, theta=OU_THETA, mu=OU_MU, sigma=OU_SIGMA) self.is_training = True self.epsilon = 1.0 self.a_t = None self.s_t = None if USE_CUDA: self.cuda()
def __init__(self, s_dim, a_dim, **kwargs): self.s_dim = s_dim self.a_dim = a_dim self.config = kwargs['config'] self.device = 'cuda' if self.config.use_cuda else 'cpu' self.actor = Actor(s_dim, a_dim) self.actor_target = Actor(s_dim, a_dim) self.critic = Critic(s_dim, a_dim, 1) self.critic_target = Critic(s_dim, a_dim, 1) self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=self.config.a_lr) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=self.config.c_lr) self.c_loss = 0 self.a_loss = 0 if self.config.use_cuda: self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() hard_update(self.actor, self.actor_target) hard_update(self.critic, self.critic_target) self.random_process = OrnsteinUhlenbeckProcess( size=self.a_dim, theta=self.config.ou_theta, mu=self.config.ou_mu, sigma=self.config.ou_sigma) self.replay_buffer = list() self.epsilon = 1. self.depsilon = self.epsilon / self.config.epsilon_decay
def __init__(self, args, nb_states, nb_actions): USE_CUDA = torch.cuda.is_available() if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions self.gpu_ids = [i for i in range(args.gpu_nums) ] if USE_CUDA and args.gpu_nums > 0 else [-1] self.gpu_used = True if self.gpu_ids[0] >= 0 else False net_cfg = { 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg).double() self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg).double() self.actor_optim = Adam(self.actor.parameters(), lr=args.p_lr, weight_decay=args.weight_decay) self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg).double() self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg).double() self.critic_optim = Adam(self.critic.parameters(), lr=args.c_lr, weight_decay=args.weight_decay) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau_update = args.tau_update self.gamma = args.gamma # Linear decay rate of exploration policy self.depsilon = 1.0 / args.epsilon # initial exploration rate self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True self.continious_action_space = False
def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions actor_net_cfg = { 'hidden1': 32, 'hidden2': 32, 'hidden3': 32, 'init_w': args.init_w } critic_net_cfg = { 'hidden1': 64, 'hidden2': 64, 'hidden3': 64, 'init_w': args.init_w } self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True self.best_reward = -10
def __init__(self, env, mem_size=7 * int(1e3), lr_critic=1e-3, lr_actor=1e-4, epsilon=1., max_epi=1500, epsilon_decay=1. / (1e5), gamma=.99, target_update_frequency=200, batch_size=64, random_process=True, max_step=None): self.CUDA = torch.cuda.is_available() self.orig_env = env #for recording if max_step is not None: self.orig_env._max_episode_steps = max_step self.env = self.orig_env self.N_S = self.env.observation_space.shape[0] self.N_A = self.env.action_space.shape[0] self.MAX_EPI = max_epi self.LOW = self.env.action_space.low self.HIGH = self.env.action_space.high self.actor = Actor(self.N_S, self.N_A) self.critic = Critic(self.N_S, self.N_A) self.target_actor = Actor(self.N_S, self.N_A) self.target_critic = Critic(self.N_S, self.N_A) self.target_actor.eval() self.target_critic.eval() self.target_actor.load_state_dict(self.actor.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) if self.CUDA: self.actor.cuda() self.critic.cuda() self.target_actor.cuda() self.target_critic.cuda() self.exp = Experience(mem_size) self.optim_critic = optim.Adam(self.critic.parameters(), lr=lr_critic) self.optim_actor = optim.Adam(self.actor.parameters(), lr=-lr_actor) self.random_process = OrnsteinUhlenbeckProcess(\ size=self.N_A, theta=.15, mu=0, sigma=.2) self.EPSILON = epsilon self.EPSILON_DECAY = epsilon_decay self.GAMMA = gamma self.TARGET_UPDATE_FREQUENCY = target_update_frequency self.BATCH_SIZE = batch_size title = {common.S_EPI: [], common.S_TOTAL_R: []} self.data = pd.DataFrame(title) self.RAND_PROC = random_process
def __init__(self, env, args): #(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.env = env self.nb_states = self.env.observation_space.shape[0] self.nb_actions = self.env.action_space.shape[0] # Create Actor and Critic Network net_cfg = { 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) self.load_weights(args.output) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon # self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True # if USE_CUDA: self.cuda()
def __init__(self, env, actor_model, critic_model, memory=10000, batch_size=64, gamma=0.99, tau=0.001, actor_lr=1e-4, critic_lr=1e-3, critic_decay=1e-2, ou_theta=0.15, ou_sigma=0.2, render=None, evaluate=None, save_path=None, save_every=10, render_every=10, train_per_step=True): self.env = env self.actor = actor_model self.actor_target = actor_model.clone() self.critic = critic_model self.critic_target = critic_model.clone() if use_cuda: for net in [ self.actor, self.actor_target, self.critic, self.critic_target ]: net.cuda() self.memory = ReplayMemory(memory) self.batch_size = batch_size self.gamma = gamma self.tau = tau self.random_process = OrnsteinUhlenbeckProcess( env.action_space.shape[0], theta=ou_theta, sigma=ou_sigma) self.optim_critic = optim.Adam(self.critic.parameters(), lr=critic_lr, weight_decay=critic_decay) self.optim_actor = optim.Adam(self.actor.parameters(), lr=actor_lr) self.render = render self.render_every = render_every self.evaluate = evaluate self.save_path = save_path self.save_every = save_every self.train_per_step = train_per_step
def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network self.actor = Actor(self.nb_states, self.nb_actions, args.init_w) self.actor_target = Actor(self.nb_states, self.nb_actions, args.init_w) self.critic = Critic(self.nb_states, self.nb_actions, args.init_w) self.critic_target = Critic(self.nb_states, self.nb_actions, args.init_w) self.reward_predictor = Critic(self.nb_states, self.nb_actions, args.init_w) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.trajectory_length = args.trajectory_length self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon # self.epsilon = 1.0 self.is_training = True # if USE_CUDA: self.cuda()
def __init__(self, s_dim, a_dim, n_agents, **kwargs): self.s_dim = s_dim self.a_dim = a_dim self.config = kwargs['config'] self.n_agents = n_agents self.device = 'cuda' if self.config.use_cuda else 'cpu' # Networks self.policy = Actor(s_dim, a_dim, n_agents) self.policy_target = Actor(s_dim, a_dim, n_agents) self.critic = Critic(s_dim, a_dim, n_agents) self.critic_target = Critic(s_dim, a_dim, n_agents) if self.config.use_cuda: self.policy.cuda() self.policy_target.cuda() self.critic.cuda() self.critic_target.cuda() self.policy_optimizer = torch.optim.Adam(self.policy.parameters(), lr=self.config.a_lr) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=self.config.c_lr) hard_update(self.policy, self.policy_target) hard_update(self.critic, self.critic_target) self.random_process = OrnsteinUhlenbeckProcess( size=self.a_dim, theta=self.config.ou_theta, mu=self.config.ou_mu, sigma=self.config.ou_sigma) self.replay_buffer = list() self.epsilon = 1. self.depsilon = self.epsilon / self.config.epsilon_decay self.c_loss = None self.a_loss = None self.action_log = list()
def __init__(self, nb_states, nb_actions): self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network self.actor = Actor(self.nb_states, self.nb_actions) self.actor_target = Actor(self.nb_states, self.nb_actions) self.actor_optim = Adam(self.actor.parameters(), lr=ACTOR_LR) self.critic = Critic(self.nb_states, self.nb_actions) self.critic_target = Critic(self.nb_states, self.nb_actions) self.critic_optim = Adam(self.critic.parameters(), lr=CRITIC_LR) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=HISTORY_LEN) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=OU_THETA, mu=OU_MU, sigma=OU_SIGMA) # Hyper-parameters self.batch_size = BATCH_SIZE self.tau = TAU self.discount = GAMMA self.depsilon = 1.0 / DEPSILON self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True if USE_CUDA: self.cuda()
def __init__(self, env, config): self.name = 'HierarchicalNet' self.save_folder = None self.test_record = {} self.train_record = {} self.config = config self.env = env self.epsilon = config.EPSILON self.commander_memory = Commander_Memory(config.MEMORY_SIZE,config.BATCH_SIZE) self.unit_memory = Unit_Memory(2*config.MEMORY_SIZE,config.UNIT_BATCH_SIZE) self.commander_actor = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE) self.commander_actor_target = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE) self.commander_critic = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE) self.commander_critic_target = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE) self.unit_actor = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM) self.unit_actor_target = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM) self.unit_critic = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE) self.unit_critic_target = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE) self.commander_actor_h0 = Variable(torch.zeros(2, 1, config.RNN_OUTSIZE),requires_grad=False) if config.GPU >= 0: self.commander_actor.cuda(device=config.GPU) self.commander_actor_target.cuda(device=config.GPU) self.commander_critic.cuda(device=config.GPU) self.commander_critic_target.cuda(device=config.GPU) self.unit_actor.cuda(device=config.GPU) self.unit_actor_target.cuda(device=config.GPU) self.unit_critic.cuda(device=config.GPU) self.unit_critic_target.cuda(device=config.GPU) self.commander_critic.h0 = self.commander_critic.h0.cuda(device=config.GPU) self.commander_critic_target.h0 = self.commander_critic_target.h0.cuda(device=config.GPU) self.commander_actor_h0 = self.commander_actor_h0.cuda(device=config.GPU) copy_parameter(self.commander_actor, self.commander_actor_target) copy_parameter(self.commander_critic, self.commander_critic_target) copy_parameter(self.unit_actor, self.unit_actor_target) copy_parameter(self.unit_critic, self.unit_critic_target) self.commander_actor_optimizer = optim.Adam(self.commander_actor.parameters(),lr=config.ACTOR_LR) self.unit_actor_optimizer = optim.Adam(self.unit_actor.parameters(),lr=config.ACTOR_LR) self.commander_critic_optimizer = optim.Adam(self.commander_critic.parameters(), lr=config.CRITIC_LR) self.unit_critic_optimizer = optim.Adam(self.unit_critic.parameters(), lr=config.CRITIC_LR) self.criterion = nn.MSELoss() self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=10, mu=0., sigma=2) self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=10, mu=0., sigma=2) # self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=30, mu=0., sigma=3) # self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=30, mu=0., sigma=3) # normalize state_normalization_myelf = [1,100,100,1,100,100,1] state_normalization_enemy = [1,100,100,100,100,10,100,100,1,1,1,10] self.state_normalization = state_normalization_myelf for i in range(config.K): self.state_normalization += state_normalization_enemy self.state_normalization = np.asarray(self.state_normalization,dtype=np.float32)
def __init__(self, gamma, tau, actor_hidden_size, critic_hidden_size, observation_space, action_space, args): self.num_inputs = observation_space.shape[0] self.action_space = action_space self.actor_hidden_size = actor_hidden_size self.critic_hidden_size = critic_hidden_size self.comm_hidden_size = actor_hidden_size // 2 self.gamma = gamma self.tau = tau self.args = args # replay for the update of attention unit self.queue = queue.Queue() # Define actor part 1 self.actor_p1 = ActorPart1(self.num_inputs, actor_hidden_size).to(device) self.actor_target_p1 = ActorPart1(self.num_inputs, actor_hidden_size).to(device) # attention unit is not end-to-end trained self.atten = AttentionUnit(actor_hidden_size, actor_hidden_size).to(device) self.atten_optim = Adam(self.atten.parameters(), lr=self.args.actor_lr) # Define Communication Channel self.comm = CommunicationChannel(actor_hidden_size, self.comm_hidden_size).to(device) self.comm_target = CommunicationChannel( actor_hidden_size, self.comm_hidden_size).to(device) self.comm_optim = Adam(self.comm.parameters(), lr=self.args.actor_lr) # Define actor part 2 # input -- [thoughts, intergrated thoughts] self.actor_p2 = ActorPart2( actor_hidden_size + self.comm_hidden_size * 2, self.action_space, actor_hidden_size).to(device) self.actor_target_p2 = ActorPart2( actor_hidden_size + self.comm_hidden_size * 2, self.action_space, actor_hidden_size).to(device) self.actor_optim = Adam([{ 'params': self.actor_p1.parameters(), 'lr': self.args.actor_lr }, { 'params': self.actor_p2.parameters(), 'lr': self.args.actor_lr }]) self.critic = Critic(self.num_inputs, self.action_space, critic_hidden_size).to(device) self.critic_target = Critic(self.num_inputs, self.action_space, critic_hidden_size).to(device) self.critic_optim = Adam(self.critic.parameters(), lr=self.args.critic_lr) # Make sure target is with the same weight hard_update(self.actor_target_p1, self.actor_p1) hard_update(self.comm_target, self.comm) hard_update(self.actor_target_p2, self.actor_p2) hard_update(self.critic_target, self.critic) # Create replay buffer self.memory = ReplayMemory(args.memory_size) self.random_process = OrnsteinUhlenbeckProcess(size=action_space.n, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma)
def run_agent(args, model_params, weights, data_queue, weights_queue, process, global_step, updates, best_reward, param_noise_prob, save_dir, max_steps=10000000): train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = build_model( **model_params) actor = Agent(actor_fn, params_actor, params_crit) actor.set_actor_weights(weights) env = RunEnv2(model=args.modeldim, prosthetic=args.prosthetic, difficulty=args.difficulty, skip_frame=config.skip_frames) env.spec.timestep_limit = 3000 # ndrw # random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.3, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6) sigma_rand = random.uniform(0.05, 0.5) dt_rand = random.uniform(0.002, 0.02) param_noise_prob = random.uniform(param_noise_prob * 0.25, min(param_noise_prob * 1.5, 1.)) random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=sigma_rand, dt=dt_rand, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6) print('OUProcess_sigma = ' + str(sigma_rand) + ' OUProcess_dt = ' + str(dt_rand) + ' param_noise_prob = ' + str(param_noise_prob)) # prepare buffers for data states = [] actions = [] rewards = [] terminals = [] total_episodes = 0 start = time() action_noise = True while global_step.value < max_steps: seed = random.randrange(2**32 - 2) state = env.reset(seed=seed, difficulty=args.difficulty) random_process.reset_states() total_reward = 0. total_reward_original = 0. terminal = False steps = 0 while not terminal: state = np.asarray(state, dtype='float32') action = actor.act(state) if action_noise: action += random_process.sample() next_state, reward, next_terminal, info = env._step(action) total_reward += reward total_reward_original += info['original_reward'] steps += 1 global_step.value += 1 # add data to buffers states.append(state) actions.append(action) rewards.append(reward) terminals.append(terminal) state = next_state terminal = next_terminal if terminal: break total_episodes += 1 # add data to buffers after episode end states.append(state) actions.append(np.zeros(env.noutput)) rewards.append(0) terminals.append(terminal) states_np = np.asarray(states).astype(np.float32) data = ( states_np, np.asarray(actions).astype(np.float32), np.asarray(rewards).astype(np.float32), np.asarray(terminals), ) weight_send = None if total_reward > best_reward.value: weight_send = actor.get_actor_weights() # send data for training data_queue.put((process, data, weight_send, total_reward)) # receive weights and set params to weights weights = weights_queue.get() # report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, original_reward {:.4f}, best reward: {:.2f}, noise: {}'. \ # format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params') # report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \ # format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, best_reward.value, 'actions' if action_noise else 'params') report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, pelvis_Z: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \ format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis'][0], info['pelvis'][2], total_reward, best_reward.value, 'actions' if action_noise else 'params') print(report_str) try: with open(os.path.join(save_dir, 'train_report.log'), 'a') as f: f.write(report_str + '\n') except: print('#############################################') print( 'except » with open(os.path.join(save_dir, train_report.log), a) as f:' ) print('#############################################') actor.set_actor_weights(weights) action_noise = np.random.rand() < 1 - param_noise_prob if not action_noise: set_params_noise(actor, states_np, random_process.current_sigma) # clear buffers del states[:] del actions[:] del rewards[:] del terminals[:] if total_episodes % 100 == 0: env = RunEnv2(model=args.modeldim, prosthetic=args.prosthetic, difficulty=args.difficulty, skip_frame=config.skip_frames)
def __init__(self, nb_states, nb_actions, now_date, now_time, args): print("UADDPG!!!!!!!!!!!!!!!!!!!!!!!!!") if args.seed > 0: self.seed(args.seed) self.total_training_step = 1 self.episode = 0 self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network net_cfg = { 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } # self.criterion = nn.MSELoss() self.critic_case = 'stochastic' self.actor = UAActor(self.nb_states, self.nb_actions, False, **net_cfg) self.actor_target = UAActor(self.nb_states, self.nb_actions, True, **net_cfg) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = UACritic(self.nb_states, self.nb_actions, False, **net_cfg) self.critic_target = UACritic(self.nb_states, self.nb_actions, True, **net_cfg) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) # Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon self.epsilon = 1.0 self.s_t = None # Most recent state self.s_t_noise = None # Most recent state self.a_t_mean = None # Most recent action self.a_t_var = None self.is_training = True if torch.cuda.is_available(): self.cuda() self.now_date = now_date self.now_time = now_time if os.path.exists('/mnt/sda2/DRL/UNIAC/model_' + self.now_date + '_' + self.now_time + '/') is False: os.mkdir('/mnt/sda2/DRL/UNIAC/model_' + self.now_date + '_' + self.now_time + '/')
def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions self.epistemic_actor = args.epistemic_actor # true / false self.epistemic_critic = args.epistemic_critic # true / false self.aleatoric_actor = args.aleatoric_actor # true / false self.aleatoric_critic = args.aleatoric_critic # true / false self.dropout_n_actor = args.dropout_n_actor self.dropout_n_critic = args.dropout_n_critic self.dropout_p_actor = args.dropout_p_actor self.dropout_p_critic = args.dropout_p_critic self.print_var_count = 0 self.action_std = np.array([]) self.save_dir = args.output self.episode = 0 # self.save_file = open(self.save_dir + '/std.txt', "a") # Create Actor and Critic Network net_cfg_actor = { 'dropout_n': args.dropout_n_actor, 'dropout_p': args.dropout_p_actor, 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } net_cfg_critic = { 'dropout_n': args.dropout_n_actor, 'dropout_p': args.dropout_p_critic, 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } self.actor = UAActor(self.nb_states, self.nb_actions, **net_cfg_actor) self.actor_target = UAActor(self.nb_states, self.nb_actions, **net_cfg_actor) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = UACritic(self.nb_states, self.nb_actions, **net_cfg_critic) self.critic_target = UACritic(self.nb_states, self.nb_actions, **net_cfg_critic) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update(self.actor_target, self.actor) hard_update(self.critic_target, self.critic) # Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon # self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True # if USE_CUDA: self.cuda()
def run_agent(model_params, weights, state_transform, data_queue, weights_queue, process, global_step, updates, best_reward, param_noise_prob, save_dir, max_steps=10000000): train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = \ build_model(**model_params) actor = Agent(actor_fn, params_actor, params_crit) actor.set_actor_weights(weights) env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames) random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.2, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6) # prepare buffers for data states = [] actions = [] rewards = [] terminals = [] total_episodes = 0 start = time() action_noise = True while global_step.value < max_steps: seed = random.randrange(2**32-2) state = env.reset(seed=seed, difficulty=2) random_process.reset_states() total_reward = 0. total_reward_original = 0. terminal = False steps = 0 while not terminal: state = np.asarray(state, dtype='float32') action = actor.act(state) if action_noise: action += random_process.sample() next_state, reward, next_terminal, info = env.step(action) total_reward += reward total_reward_original += info['original_reward'] steps += 1 global_step.value += 1 # add data to buffers states.append(state) actions.append(action) rewards.append(reward) terminals.append(terminal) state = next_state terminal = next_terminal if terminal: break total_episodes += 1 # add data to buffers after episode end states.append(state) actions.append(np.zeros(env.noutput)) rewards.append(0) terminals.append(terminal) states_np = np.asarray(states).astype(np.float32) data = (states_np, np.asarray(actions).astype(np.float32), np.asarray(rewards).astype(np.float32), np.asarray(terminals), ) weight_send = None if total_reward > best_reward.value: weight_send = actor.get_actor_weights() # send data for training data_queue.put((process, data, weight_send, total_reward)) # receive weights and set params to weights weights = weights_queue.get() report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len {}, ' \ 'reward: {:.2f}, original_reward {:.4f}; best reward: {:.2f} noise {}'. \ format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params') print(report_str) with open(os.path.join(save_dir, 'train_report.log'), 'a') as f: f.write(report_str + '\n') actor.set_actor_weights(weights) action_noise = np.random.rand() < 1 - param_noise_prob if not action_noise: set_params_noise(actor, states_np, random_process.current_sigma) # clear buffers del states[:] del actions[:] del rewards[:] del terminals[:] if total_episodes % 100 == 0: env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames)
def __init__( self, env, mem_size=int(1e6), lr_critic=1e-3, lr_actor=1e-4, max_epi=int(1e4), epsilon_decay=1. / (1e5), gamma=.99, target_update_frequency=200, batch_size=64, random_process_mode='default', max_step=None, actor_update_mode='default', popart=False, actor='standard', critic='43', epsilon_start=1., epsilon_end=.01, epsilon_rate=1. / 200, partition_num=100, env_log_freq=100, model_log_freq=500, target_update_mode='hard', tau=1e-3, grad_clip_mode=None, grad_clip_norm=5., critic_weight_decay=0., exp_trunc=[], exp_percent=[], exp_rebalance_freq=None, exp_type='rank', ): # configuration log frame = inspect.currentframe() args, _, _, values = inspect.getargvalues(frame) self.config = ['{}: {}'.format(arg, values[arg]) for arg in args] self.CUDA = torch.cuda.is_available() self.ENV_NORMALIZED = env.class_name() == 'NormalizedEnv' self.POPART = popart self.actor_update_mode = actor_update_mode self.orig_env = (env) #for recording if max_step is not None: tmp_env = env if isinstance(tmp_env, gym.Wrapper): while (tmp_env.class_name() != 'TimeLimit'): tmp_env = tmp_env.env tmp_env._max_episode_steps = max_step self.env = self.orig_env self.N = 1 if hasattr(self.env.unwrapped, 'N'): self.N = self.env.unwrapped.N self.N_S = self.env.observation_space.shape[0] self.N_A = self.env.action_space.shape[0] self.n_s = self.N_S / self.N self.n_a = self.N_A / self.N self.MAX_EPI = max_epi self.LOW = self.env.action_space.low self.HIGH = self.env.action_space.high self.actor = ActorRegistry[actor](self.n_s, self.n_a) self.critic = CriticRegistry[critic](self.N_S, self.N_A) self.target_actor = ActorRegistry[actor](self.n_s, self.n_a) self.target_critic = CriticRegistry[critic](self.N_S, self.N_A) self.target_actor.eval() self.target_critic.eval() self.target_actor.load_state_dict(self.actor.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) if self.CUDA: self.actor.cuda() self.critic.cuda() self.target_actor.cuda() self.target_critic.cuda() # pop-art self.update_counter = 0 self.beta = .1 self.y_mean = 0. self.y_square_mean = 0. self.target_y_mean = self.y_mean self.target_y_square_mean = self.y_square_mean # per self.total_step = 0 self.PARTITION_NUM = partition_num self.LEARN_START = mem_size / self.PARTITION_NUM + 1 self.exp_trunc = exp_trunc self.exp_percent = exp_percent self.exp_rebalance_freq = exp_rebalance_freq self.exp_batch_sizes = [] self.exp_type = exp_type #if len(self.exp_trunc)>0: if len(self.exp_trunc) != len(self.exp_percent): raise RuntimeError("different exp_trunc and exp_percent length") self.exp = [] for i in range(len(self.exp_trunc) + 1): tmp_batch_size = int( batch_size * (1 - sum(self.exp_percent))) if i == len( self.exp_trunc) else int(batch_size * self.exp_percent[i]) self.exp_batch_sizes.append(tmp_batch_size) exp_conf = { 'size': mem_size, 'learn_start': self.LEARN_START, 'partition_num': self.PARTITION_NUM, 'total_step': self.MAX_EPI * 50, 'batch_size': tmp_batch_size } self.exp.append(Experience( exp_conf)) if self.exp_type == 'rank' else self.exp.append( PrioritizedReplayBuffer(mem_size, alpha=.7)) #else: # exp_conf = { # 'size': mem_size, # 'learn_start': self.LEARN_START, # 'partition_num': self.PARTITION_NUM, # 'total_step': self.MAX_EPI * 50, # 'batch_size': batch_size, # } # self.exp = Experience(exp_conf) # uniform er #self.exp = Experience(mem_size) self.optim_critic = optim.Adam(self.critic.parameters(), lr=lr_critic, weight_decay=critic_weight_decay) self.optim_actor = optim.Adam(self.actor.parameters(), lr=-lr_actor) self.random_processes = [] for _ in xrange(self.N): random_process = OrnsteinUhlenbeckProcess(\ size=self.n_a, theta=.15, mu=0, sigma=.2) self.random_processes.append(random_process) self.EPSILON_START = epsilon_start self.EPSILON_END = epsilon_end # only default random process mode will use epsilon decay self.EPSILON_DECAY = epsilon_decay # other random process mode will use epsilon rate self.EPSILON_RATE = epsilon_rate self.GAMMA = gamma self.TARGET_UPDATE_FREQUENCY = target_update_frequency self.BATCH_SIZE = batch_size self.target_update_mode = target_update_mode self.tau = tau #title = {common.S_EPI:[], common.S_TOTAL_R:[]} #self.data = pd.DataFrame(title) self.RAND_PROC = random_process_mode self.grad_clip_mode = grad_clip_mode self.grad_clip_norm = grad_clip_norm # logger self.logger = None self.env_log_freq = env_log_freq self.model_log_freq = model_log_freq self.step = 0 # random seed self.seed = int(time.time()) random.seed(self.seed) np.random.seed(self.seed)
critic = Critic(state_dim, action_dim, max_action, args) critic_t = Critic(state_dim, action_dim, max_action, args) critic_t.load_state_dict(critic.state_dict()) print("OK 3") # actor actor = Actor(state_dim, action_dim, max_action, args) actor_t = Actor(state_dim, action_dim, max_action, args) actor_t.load_state_dict(actor.state_dict()) # action noise if not args.ou_noise: a_noise = GaussianNoise(action_dim, sigma=args.gauss_sigma) else: a_noise = OrnsteinUhlenbeckProcess(action_dim, mu=args.ou_mu, theta=args.ou_theta, sigma=args.ou_sigma) if USE_CUDA: critic.cuda() critic_t.cuda() actor.cuda() actor_t.cuda() print("OK 4") # CEM es = sepCEM(actor.get_size(), mu_init=actor.get_params(), sigma_init=args.sigma_init, damp=args.damp, damp_limit=args.damp_limit,
def __init__(self): # random seed for torch __seed = config.get(MODEL_SEED) self.policy_loss = [] self.critic_loss = [] if __seed > 0: self.seed(__seed) self.nb_states = config.get(MODEL_STATE_COUNT) self.nb_actions = config.get(MODEL_ACTION_COUNT) # Create Actor and Critic Network actor_net_cfg = { 'hidden1': config.get(MODEL_ACTOR_HIDDEN1), 'hidden2': config.get(MODEL_ACTOR_HIDDEN2), 'init_w': config.get(MODEL_INIT_WEIGHT) } critic_net_cfg = { 'hidden1': config.get(MODEL_CRITIC_HIDDEN1), 'hidden2': config.get(MODEL_CRITIC_HIDDEN2), 'init_w': config.get(MODEL_INIT_WEIGHT) } self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_optim = Adam( self.actor.parameters(), lr=config.get(MODEL_ACTOR_LR), weight_decay=config.get(MODEL_ACTOR_WEIGHT_DECAY)) self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_optim = Adam( self.critic.parameters(), lr=config.get(MODEL_CRITIC_LR), weight_decay=config.get(MODEL_CRITIC_WEIGHT_DECAY)) hard_update(self.actor_target, self.actor) hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = Memory() self.random_process = OrnsteinUhlenbeckProcess( size=self.nb_actions, theta=config.get(RANDOM_THETA), mu=config.get(RANDOM_MU), sigma=config.get(RANDOM_SIGMA)) # Hyper-parameters self.batch_size = config.get(MODEL_BATCH_SIZE) self.tau = config.get(MODEL_TARGET_TAU) self.discount = config.get(MODEL_DISCOUNT) self.depsilon = 1.0 / config.get(MODEL_EPSILON) self.model_path = config.get(MODEL_SAVE_PATH) # self.epsilon = 1.0 # init device self.device_init()