def __init__(self, conf, device):
self.conf = conf
self.state_dim = conf['state_dim']
self.action_dim = conf['action_dim']
self.device = device
# create actor and critic network
self.actor = Actor_RDPG(self.state_dim,
self.action_dim).to(self.device)
self.actor_target = Actor_RDPG(self.state_dim,
self.action_dim).to(self.device)
self.critic = Critic_RDPG(self.state_dim,
self.action_dim).to(self.device)
self.critic_target = Critic_RDPG(self.state_dim,
self.action_dim).to(self.device)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
self.critic_optim = optim.Adam(self.critic.parameters(), lr=q_lr)
self.actor_optim = optim.Adam(self.actor.parameters(), lr=policy_lr)
#Create replay buffer
self.random_process = OrnsteinUhlenbeckProcess(size=self.action_dim,
theta=0.15,
mu=0.0,
sigma=0.2)
# args.ou_theta:0.15 (noise theta), args.ou_sigma:0.2 (noise sigma), args.out_mu:0.0 (noise mu)
self.epsilon = 1.0
self.depsilon = 1.0 / 50000
self.is_training = True
self.tau = 0.001 # moving average for target network
def __init__(self,
state_size,
action_size,
memory_size,
batch_size=128,
tan=0.001,
actor_lr=0.001,
critic_lr=0.001,
epsilon=1.):
self.state_size = state_size
self.action_size = action_size
self.batch_size = batch_size
self.tan = tan
self.warmup = WARM_UP
self.epsilon = epsilon
self.epsilon_decay = hyperparameters['D_EPSILON']
self.actor = Actor(state_size, action_size)
self.actor_target = Actor(state_size, action_size)
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)
self.critic = Critic(state_size, action_size)
self.critic_target = Critic(state_size, action_size)
self.critic_optimizer = optim.Adam(self.critic.parameters(),
lr=critic_lr)
self.memory = Memory(memory_size)
self.criterion = nn.MSELoss()
self.random_process = OrnsteinUhlenbeckProcess(size=action_size,
theta=0.15,
mu=0.,
sigma=0.2)
copy_parameter(self.actor, self.actor_target)
copy_parameter(self.critic, self.critic_target)
def __init__(self, nb_states, nb_actions):
self.critic = Critic(nb_states, nb_actions) # Q
self.critic_target = Critic(nb_states, nb_actions)
self.actor = Actor(nb_states, nb_actions) # policy mu
self.actor_target = Actor(nb_states, nb_actions)
hard_update(self.critic_target, self.critic)
hard_update(self.actor_target, self.actor)
self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
lr=0.001)
self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),
lr=0.0001)
self.criterion = nn.MSELoss()
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=0.15,
mu=0,
sigma=0.2)
self.gamma = 0.99
self.batch_size = 64
if USE_CUDA:
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def __init__(self, s_dim, a_dim, num_agent, **kwargs):
self.s_dim = s_dim
self.a_dim = a_dim
self.config = kwargs['config']
self.num_agent = num_agent
self.actor = Actor(s_dim, a_dim)
self.actor_target = Actor(s_dim, a_dim)
self.critic = Critic(s_dim, a_dim, num_agent)
self.critic_target = Critic(s_dim, a_dim, num_agent)
self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),
lr=self.config.a_lr)
self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
lr=self.config.c_lr)
self.a_loss = 0
self.c_loss = 0
if self.config.use_cuda:
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
hard_update(self.actor, self.actor_target)
hard_update(self.critic, self.critic_target)
self.random_process = OrnsteinUhlenbeckProcess(
size=self.a_dim,
theta=self.config.ou_theta,
mu=self.config.ou_mu,
sigma=self.config.ou_sigma)
def __init__(self, nb_state, nb_action):
self.nb_state = nb_state
self.nb_action = nb_action
self.actor = Actor(self.nb_state, self.nb_action)
self.actor_target = Actor(self.nb_state, self.nb_action)
self.actor_optim = Adam(self.actor.parameters(), lr=LEARNING_RATE)
self.critic = Critic(self.nb_state, self.nb_action)
self.critic_target = Critic(self.nb_state, self.nb_action)
self.critic_optim = Adam(self.critic.parameters(), lr=LEARNING_RATE)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=1)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_action,
theta=OU_THETA,
mu=OU_MU,
sigma=OU_SIGMA)
self.is_training = True
self.epsilon = 1.0
self.a_t = None
self.s_t = None
if USE_CUDA: self.cuda()
def __init__(self, s_dim, a_dim, **kwargs):
self.s_dim = s_dim
self.a_dim = a_dim
self.config = kwargs['config']
self.device = 'cuda' if self.config.use_cuda else 'cpu'
self.actor = Actor(s_dim, a_dim)
self.actor_target = Actor(s_dim, a_dim)
self.critic = Critic(s_dim, a_dim, 1)
self.critic_target = Critic(s_dim, a_dim, 1)
self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),
lr=self.config.a_lr)
self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
lr=self.config.c_lr)
self.c_loss = 0
self.a_loss = 0
if self.config.use_cuda:
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
hard_update(self.actor, self.actor_target)
hard_update(self.critic, self.critic_target)
self.random_process = OrnsteinUhlenbeckProcess(
size=self.a_dim,
theta=self.config.ou_theta,
mu=self.config.ou_mu,
sigma=self.config.ou_sigma)
self.replay_buffer = list()
self.epsilon = 1.
self.depsilon = self.epsilon / self.config.epsilon_decay
def __init__(self, args, nb_states, nb_actions):
USE_CUDA = torch.cuda.is_available()
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
self.gpu_ids = [i for i in range(args.gpu_nums)
] if USE_CUDA and args.gpu_nums > 0 else [-1]
self.gpu_used = True if self.gpu_ids[0] >= 0 else False
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg).double()
self.actor_target = Actor(self.nb_states, self.nb_actions,
**net_cfg).double()
self.actor_optim = Adam(self.actor.parameters(),
lr=args.p_lr,
weight_decay=args.weight_decay)
self.critic = Critic(self.nb_states, self.nb_actions,
**net_cfg).double()
self.critic_target = Critic(self.nb_states, self.nb_actions,
**net_cfg).double()
self.critic_optim = Adam(self.critic.parameters(),
lr=args.c_lr,
weight_decay=args.weight_decay)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau_update = args.tau_update
self.gamma = args.gamma
# Linear decay rate of exploration policy
self.depsilon = 1.0 / args.epsilon
# initial exploration rate
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
self.continious_action_space = False
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
actor_net_cfg = {
'hidden1': 32,
'hidden2': 32,
'hidden3': 32,
'init_w': args.init_w
}
critic_net_cfg = {
'hidden1': 64,
'hidden2': 64,
'hidden3': 64,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions,
**actor_net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions,
**critic_net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
self.best_reward = -10
def __init__(self,
env,
mem_size=7 * int(1e3),
lr_critic=1e-3,
lr_actor=1e-4,
epsilon=1.,
max_epi=1500,
epsilon_decay=1. / (1e5),
gamma=.99,
target_update_frequency=200,
batch_size=64,
random_process=True,
max_step=None):
self.CUDA = torch.cuda.is_available()
self.orig_env = env #for recording
if max_step is not None:
self.orig_env._max_episode_steps = max_step
self.env = self.orig_env
self.N_S = self.env.observation_space.shape[0]
self.N_A = self.env.action_space.shape[0]
self.MAX_EPI = max_epi
self.LOW = self.env.action_space.low
self.HIGH = self.env.action_space.high
self.actor = Actor(self.N_S, self.N_A)
self.critic = Critic(self.N_S, self.N_A)
self.target_actor = Actor(self.N_S, self.N_A)
self.target_critic = Critic(self.N_S, self.N_A)
self.target_actor.eval()
self.target_critic.eval()
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
if self.CUDA:
self.actor.cuda()
self.critic.cuda()
self.target_actor.cuda()
self.target_critic.cuda()
self.exp = Experience(mem_size)
self.optim_critic = optim.Adam(self.critic.parameters(), lr=lr_critic)
self.optim_actor = optim.Adam(self.actor.parameters(), lr=-lr_actor)
self.random_process = OrnsteinUhlenbeckProcess(\
size=self.N_A, theta=.15, mu=0, sigma=.2)
self.EPSILON = epsilon
self.EPSILON_DECAY = epsilon_decay
self.GAMMA = gamma
self.TARGET_UPDATE_FREQUENCY = target_update_frequency
self.BATCH_SIZE = batch_size
title = {common.S_EPI: [], common.S_TOTAL_R: []}
self.data = pd.DataFrame(title)
self.RAND_PROC = random_process
def __init__(self, env, args): #(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.env = env
self.nb_states = self.env.observation_space.shape[0]
self.nb_actions = self.env.action_space.shape[0]
# Create Actor and Critic Network
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
self.load_weights(args.output)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
#
if USE_CUDA: self.cuda()
def __init__(self,
env,
actor_model,
critic_model,
memory=10000,
batch_size=64,
gamma=0.99,
tau=0.001,
actor_lr=1e-4,
critic_lr=1e-3,
critic_decay=1e-2,
ou_theta=0.15,
ou_sigma=0.2,
render=None,
evaluate=None,
save_path=None,
save_every=10,
render_every=10,
train_per_step=True):
self.env = env
self.actor = actor_model
self.actor_target = actor_model.clone()
self.critic = critic_model
self.critic_target = critic_model.clone()
if use_cuda:
for net in [
self.actor, self.actor_target, self.critic,
self.critic_target
]:
net.cuda()
self.memory = ReplayMemory(memory)
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.random_process = OrnsteinUhlenbeckProcess(
env.action_space.shape[0], theta=ou_theta, sigma=ou_sigma)
self.optim_critic = optim.Adam(self.critic.parameters(),
lr=critic_lr,
weight_decay=critic_decay)
self.optim_actor = optim.Adam(self.actor.parameters(), lr=actor_lr)
self.render = render
self.render_every = render_every
self.evaluate = evaluate
self.save_path = save_path
self.save_every = save_every
self.train_per_step = train_per_step
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
self.actor = Actor(self.nb_states, self.nb_actions, args.init_w)
self.actor_target = Actor(self.nb_states, self.nb_actions, args.init_w)
self.critic = Critic(self.nb_states, self.nb_actions, args.init_w)
self.critic_target = Critic(self.nb_states, self.nb_actions,
args.init_w)
self.reward_predictor = Critic(self.nb_states, self.nb_actions,
args.init_w)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.trajectory_length = args.trajectory_length
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.is_training = True
#
if USE_CUDA: self.cuda()
def __init__(self, s_dim, a_dim, n_agents, **kwargs):
self.s_dim = s_dim
self.a_dim = a_dim
self.config = kwargs['config']
self.n_agents = n_agents
self.device = 'cuda' if self.config.use_cuda else 'cpu'
# Networks
self.policy = Actor(s_dim, a_dim, n_agents)
self.policy_target = Actor(s_dim, a_dim, n_agents)
self.critic = Critic(s_dim, a_dim, n_agents)
self.critic_target = Critic(s_dim, a_dim, n_agents)
if self.config.use_cuda:
self.policy.cuda()
self.policy_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
self.policy_optimizer = torch.optim.Adam(self.policy.parameters(),
lr=self.config.a_lr)
self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
lr=self.config.c_lr)
hard_update(self.policy, self.policy_target)
hard_update(self.critic, self.critic_target)
self.random_process = OrnsteinUhlenbeckProcess(
size=self.a_dim,
theta=self.config.ou_theta,
mu=self.config.ou_mu,
sigma=self.config.ou_sigma)
self.replay_buffer = list()
self.epsilon = 1.
self.depsilon = self.epsilon / self.config.epsilon_decay
self.c_loss = None
self.a_loss = None
self.action_log = list()
def __init__(self, nb_states, nb_actions):
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
self.actor = Actor(self.nb_states, self.nb_actions)
self.actor_target = Actor(self.nb_states, self.nb_actions)
self.actor_optim = Adam(self.actor.parameters(), lr=ACTOR_LR)
self.critic = Critic(self.nb_states, self.nb_actions)
self.critic_target = Critic(self.nb_states, self.nb_actions)
self.critic_optim = Adam(self.critic.parameters(), lr=CRITIC_LR)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=MEMORY_SIZE,
window_length=HISTORY_LEN)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=OU_THETA,
mu=OU_MU,
sigma=OU_SIGMA)
# Hyper-parameters
self.batch_size = BATCH_SIZE
self.tau = TAU
self.discount = GAMMA
self.depsilon = 1.0 / DEPSILON
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
if USE_CUDA: self.cuda()
def __init__(self, env, config):
self.name = 'HierarchicalNet'
self.save_folder = None
self.test_record = {}
self.train_record = {}
self.config = config
self.env = env
self.epsilon = config.EPSILON
self.commander_memory = Commander_Memory(config.MEMORY_SIZE,config.BATCH_SIZE)
self.unit_memory = Unit_Memory(2*config.MEMORY_SIZE,config.UNIT_BATCH_SIZE)
self.commander_actor = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE)
self.commander_actor_target = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE)
self.commander_critic = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE)
self.commander_critic_target = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE)
self.unit_actor = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM)
self.unit_actor_target = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM)
self.unit_critic = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE)
self.unit_critic_target = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE)
self.commander_actor_h0 = Variable(torch.zeros(2, 1, config.RNN_OUTSIZE),requires_grad=False)
if config.GPU >= 0:
self.commander_actor.cuda(device=config.GPU)
self.commander_actor_target.cuda(device=config.GPU)
self.commander_critic.cuda(device=config.GPU)
self.commander_critic_target.cuda(device=config.GPU)
self.unit_actor.cuda(device=config.GPU)
self.unit_actor_target.cuda(device=config.GPU)
self.unit_critic.cuda(device=config.GPU)
self.unit_critic_target.cuda(device=config.GPU)
self.commander_critic.h0 = self.commander_critic.h0.cuda(device=config.GPU)
self.commander_critic_target.h0 = self.commander_critic_target.h0.cuda(device=config.GPU)
self.commander_actor_h0 = self.commander_actor_h0.cuda(device=config.GPU)
copy_parameter(self.commander_actor, self.commander_actor_target)
copy_parameter(self.commander_critic, self.commander_critic_target)
copy_parameter(self.unit_actor, self.unit_actor_target)
copy_parameter(self.unit_critic, self.unit_critic_target)
self.commander_actor_optimizer = optim.Adam(self.commander_actor.parameters(),lr=config.ACTOR_LR)
self.unit_actor_optimizer = optim.Adam(self.unit_actor.parameters(),lr=config.ACTOR_LR)
self.commander_critic_optimizer = optim.Adam(self.commander_critic.parameters(), lr=config.CRITIC_LR)
self.unit_critic_optimizer = optim.Adam(self.unit_critic.parameters(), lr=config.CRITIC_LR)
self.criterion = nn.MSELoss()
self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=10, mu=0., sigma=2)
self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=10, mu=0., sigma=2)
# self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=30, mu=0., sigma=3)
# self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=30, mu=0., sigma=3)
# normalize
state_normalization_myelf = [1,100,100,1,100,100,1]
state_normalization_enemy = [1,100,100,100,100,10,100,100,1,1,1,10]
self.state_normalization = state_normalization_myelf
for i in range(config.K):
self.state_normalization += state_normalization_enemy
self.state_normalization = np.asarray(self.state_normalization,dtype=np.float32)
def __init__(self, gamma, tau, actor_hidden_size, critic_hidden_size,
observation_space, action_space, args):
self.num_inputs = observation_space.shape[0]
self.action_space = action_space
self.actor_hidden_size = actor_hidden_size
self.critic_hidden_size = critic_hidden_size
self.comm_hidden_size = actor_hidden_size // 2
self.gamma = gamma
self.tau = tau
self.args = args
# replay for the update of attention unit
self.queue = queue.Queue()
# Define actor part 1
self.actor_p1 = ActorPart1(self.num_inputs,
actor_hidden_size).to(device)
self.actor_target_p1 = ActorPart1(self.num_inputs,
actor_hidden_size).to(device)
# attention unit is not end-to-end trained
self.atten = AttentionUnit(actor_hidden_size,
actor_hidden_size).to(device)
self.atten_optim = Adam(self.atten.parameters(), lr=self.args.actor_lr)
# Define Communication Channel
self.comm = CommunicationChannel(actor_hidden_size,
self.comm_hidden_size).to(device)
self.comm_target = CommunicationChannel(
actor_hidden_size, self.comm_hidden_size).to(device)
self.comm_optim = Adam(self.comm.parameters(), lr=self.args.actor_lr)
# Define actor part 2
# input -- [thoughts, intergrated thoughts]
self.actor_p2 = ActorPart2(
actor_hidden_size + self.comm_hidden_size * 2, self.action_space,
actor_hidden_size).to(device)
self.actor_target_p2 = ActorPart2(
actor_hidden_size + self.comm_hidden_size * 2, self.action_space,
actor_hidden_size).to(device)
self.actor_optim = Adam([{
'params': self.actor_p1.parameters(),
'lr': self.args.actor_lr
}, {
'params': self.actor_p2.parameters(),
'lr': self.args.actor_lr
}])
self.critic = Critic(self.num_inputs, self.action_space,
critic_hidden_size).to(device)
self.critic_target = Critic(self.num_inputs, self.action_space,
critic_hidden_size).to(device)
self.critic_optim = Adam(self.critic.parameters(),
lr=self.args.critic_lr)
# Make sure target is with the same weight
hard_update(self.actor_target_p1, self.actor_p1)
hard_update(self.comm_target, self.comm)
hard_update(self.actor_target_p2, self.actor_p2)
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = ReplayMemory(args.memory_size)
self.random_process = OrnsteinUhlenbeckProcess(size=action_space.n,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
def run_agent(args,
model_params,
weights,
data_queue,
weights_queue,
process,
global_step,
updates,
best_reward,
param_noise_prob,
save_dir,
max_steps=10000000):
train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = build_model(
**model_params)
actor = Agent(actor_fn, params_actor, params_crit)
actor.set_actor_weights(weights)
env = RunEnv2(model=args.modeldim,
prosthetic=args.prosthetic,
difficulty=args.difficulty,
skip_frame=config.skip_frames)
env.spec.timestep_limit = 3000 # ndrw
# random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.3, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6)
sigma_rand = random.uniform(0.05, 0.5)
dt_rand = random.uniform(0.002, 0.02)
param_noise_prob = random.uniform(param_noise_prob * 0.25,
min(param_noise_prob * 1.5, 1.))
random_process = OrnsteinUhlenbeckProcess(theta=.1,
mu=0.,
sigma=sigma_rand,
dt=dt_rand,
size=env.noutput,
sigma_min=0.05,
n_steps_annealing=1e6)
print('OUProcess_sigma = ' + str(sigma_rand) + ' OUProcess_dt = ' +
str(dt_rand) + ' param_noise_prob = ' + str(param_noise_prob))
# prepare buffers for data
states = []
actions = []
rewards = []
terminals = []
total_episodes = 0
start = time()
action_noise = True
while global_step.value < max_steps:
seed = random.randrange(2**32 - 2)
state = env.reset(seed=seed, difficulty=args.difficulty)
random_process.reset_states()
total_reward = 0.
total_reward_original = 0.
terminal = False
steps = 0
while not terminal:
state = np.asarray(state, dtype='float32')
action = actor.act(state)
if action_noise:
action += random_process.sample()
next_state, reward, next_terminal, info = env._step(action)
total_reward += reward
total_reward_original += info['original_reward']
steps += 1
global_step.value += 1
# add data to buffers
states.append(state)
actions.append(action)
rewards.append(reward)
terminals.append(terminal)
state = next_state
terminal = next_terminal
if terminal:
break
total_episodes += 1
# add data to buffers after episode end
states.append(state)
actions.append(np.zeros(env.noutput))
rewards.append(0)
terminals.append(terminal)
states_np = np.asarray(states).astype(np.float32)
data = (
states_np,
np.asarray(actions).astype(np.float32),
np.asarray(rewards).astype(np.float32),
np.asarray(terminals),
)
weight_send = None
if total_reward > best_reward.value:
weight_send = actor.get_actor_weights()
# send data for training
data_queue.put((process, data, weight_send, total_reward))
# receive weights and set params to weights
weights = weights_queue.get()
# report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, original_reward {:.4f}, best reward: {:.2f}, noise: {}'. \
# format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params')
# report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \
# format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, best_reward.value, 'actions' if action_noise else 'params')
report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, pelvis_Z: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \
format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis'][0], info['pelvis'][2], total_reward, best_reward.value, 'actions' if action_noise else 'params')
print(report_str)
try:
with open(os.path.join(save_dir, 'train_report.log'), 'a') as f:
f.write(report_str + '\n')
except:
print('#############################################')
print(
'except » with open(os.path.join(save_dir, train_report.log), a) as f:'
)
print('#############################################')
actor.set_actor_weights(weights)
action_noise = np.random.rand() < 1 - param_noise_prob
if not action_noise:
set_params_noise(actor, states_np, random_process.current_sigma)
# clear buffers
del states[:]
del actions[:]
del rewards[:]
del terminals[:]
if total_episodes % 100 == 0:
env = RunEnv2(model=args.modeldim,
prosthetic=args.prosthetic,
difficulty=args.difficulty,
skip_frame=config.skip_frames)
def __init__(self, nb_states, nb_actions, now_date, now_time, args):
print("UADDPG!!!!!!!!!!!!!!!!!!!!!!!!!")
if args.seed > 0:
self.seed(args.seed)
self.total_training_step = 1
self.episode = 0
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
# self.criterion = nn.MSELoss()
self.critic_case = 'stochastic'
self.actor = UAActor(self.nb_states, self.nb_actions, False, **net_cfg)
self.actor_target = UAActor(self.nb_states, self.nb_actions, True,
**net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = UACritic(self.nb_states, self.nb_actions, False,
**net_cfg)
self.critic_target = UACritic(self.nb_states, self.nb_actions, True,
**net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
self.epsilon = 1.0
self.s_t = None # Most recent state
self.s_t_noise = None # Most recent state
self.a_t_mean = None # Most recent action
self.a_t_var = None
self.is_training = True
if torch.cuda.is_available():
self.cuda()
self.now_date = now_date
self.now_time = now_time
if os.path.exists('/mnt/sda2/DRL/UNIAC/model_' + self.now_date + '_' +
self.now_time + '/') is False:
os.mkdir('/mnt/sda2/DRL/UNIAC/model_' + self.now_date + '_' +
self.now_time + '/')
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
self.epistemic_actor = args.epistemic_actor # true / false
self.epistemic_critic = args.epistemic_critic # true / false
self.aleatoric_actor = args.aleatoric_actor # true / false
self.aleatoric_critic = args.aleatoric_critic # true / false
self.dropout_n_actor = args.dropout_n_actor
self.dropout_n_critic = args.dropout_n_critic
self.dropout_p_actor = args.dropout_p_actor
self.dropout_p_critic = args.dropout_p_critic
self.print_var_count = 0
self.action_std = np.array([])
self.save_dir = args.output
self.episode = 0
# self.save_file = open(self.save_dir + '/std.txt', "a")
# Create Actor and Critic Network
net_cfg_actor = {
'dropout_n': args.dropout_n_actor,
'dropout_p': args.dropout_p_actor,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
net_cfg_critic = {
'dropout_n': args.dropout_n_actor,
'dropout_p': args.dropout_p_critic,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = UAActor(self.nb_states, self.nb_actions, **net_cfg_actor)
self.actor_target = UAActor(self.nb_states, self.nb_actions,
**net_cfg_actor)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = UACritic(self.nb_states, self.nb_actions,
**net_cfg_critic)
self.critic_target = UACritic(self.nb_states, self.nb_actions,
**net_cfg_critic)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor)
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
#
if USE_CUDA:
self.cuda()
def run_agent(model_params, weights, state_transform, data_queue, weights_queue,
process, global_step, updates, best_reward, param_noise_prob, save_dir,
max_steps=10000000):
train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = \
build_model(**model_params)
actor = Agent(actor_fn, params_actor, params_crit)
actor.set_actor_weights(weights)
env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames)
random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.2, size=env.noutput,
sigma_min=0.05, n_steps_annealing=1e6)
# prepare buffers for data
states = []
actions = []
rewards = []
terminals = []
total_episodes = 0
start = time()
action_noise = True
while global_step.value < max_steps:
seed = random.randrange(2**32-2)
state = env.reset(seed=seed, difficulty=2)
random_process.reset_states()
total_reward = 0.
total_reward_original = 0.
terminal = False
steps = 0
while not terminal:
state = np.asarray(state, dtype='float32')
action = actor.act(state)
if action_noise:
action += random_process.sample()
next_state, reward, next_terminal, info = env.step(action)
total_reward += reward
total_reward_original += info['original_reward']
steps += 1
global_step.value += 1
# add data to buffers
states.append(state)
actions.append(action)
rewards.append(reward)
terminals.append(terminal)
state = next_state
terminal = next_terminal
if terminal:
break
total_episodes += 1
# add data to buffers after episode end
states.append(state)
actions.append(np.zeros(env.noutput))
rewards.append(0)
terminals.append(terminal)
states_np = np.asarray(states).astype(np.float32)
data = (states_np,
np.asarray(actions).astype(np.float32),
np.asarray(rewards).astype(np.float32),
np.asarray(terminals),
)
weight_send = None
if total_reward > best_reward.value:
weight_send = actor.get_actor_weights()
# send data for training
data_queue.put((process, data, weight_send, total_reward))
# receive weights and set params to weights
weights = weights_queue.get()
report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len {}, ' \
'reward: {:.2f}, original_reward {:.4f}; best reward: {:.2f} noise {}'. \
format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps,
total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params')
print(report_str)
with open(os.path.join(save_dir, 'train_report.log'), 'a') as f:
f.write(report_str + '\n')
actor.set_actor_weights(weights)
action_noise = np.random.rand() < 1 - param_noise_prob
if not action_noise:
set_params_noise(actor, states_np, random_process.current_sigma)
# clear buffers
del states[:]
del actions[:]
del rewards[:]
del terminals[:]
if total_episodes % 100 == 0:
env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames)
def __init__(
self,
env,
mem_size=int(1e6),
lr_critic=1e-3,
lr_actor=1e-4,
max_epi=int(1e4),
epsilon_decay=1. / (1e5),
gamma=.99,
target_update_frequency=200,
batch_size=64,
random_process_mode='default',
max_step=None,
actor_update_mode='default',
popart=False,
actor='standard',
critic='43',
epsilon_start=1.,
epsilon_end=.01,
epsilon_rate=1. / 200,
partition_num=100,
env_log_freq=100,
model_log_freq=500,
target_update_mode='hard',
tau=1e-3,
grad_clip_mode=None,
grad_clip_norm=5.,
critic_weight_decay=0.,
exp_trunc=[],
exp_percent=[],
exp_rebalance_freq=None,
exp_type='rank',
):
# configuration log
frame = inspect.currentframe()
args, _, _, values = inspect.getargvalues(frame)
self.config = ['{}: {}'.format(arg, values[arg]) for arg in args]
self.CUDA = torch.cuda.is_available()
self.ENV_NORMALIZED = env.class_name() == 'NormalizedEnv'
self.POPART = popart
self.actor_update_mode = actor_update_mode
self.orig_env = (env) #for recording
if max_step is not None:
tmp_env = env
if isinstance(tmp_env, gym.Wrapper):
while (tmp_env.class_name() != 'TimeLimit'):
tmp_env = tmp_env.env
tmp_env._max_episode_steps = max_step
self.env = self.orig_env
self.N = 1
if hasattr(self.env.unwrapped, 'N'):
self.N = self.env.unwrapped.N
self.N_S = self.env.observation_space.shape[0]
self.N_A = self.env.action_space.shape[0]
self.n_s = self.N_S / self.N
self.n_a = self.N_A / self.N
self.MAX_EPI = max_epi
self.LOW = self.env.action_space.low
self.HIGH = self.env.action_space.high
self.actor = ActorRegistry[actor](self.n_s, self.n_a)
self.critic = CriticRegistry[critic](self.N_S, self.N_A)
self.target_actor = ActorRegistry[actor](self.n_s, self.n_a)
self.target_critic = CriticRegistry[critic](self.N_S, self.N_A)
self.target_actor.eval()
self.target_critic.eval()
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
if self.CUDA:
self.actor.cuda()
self.critic.cuda()
self.target_actor.cuda()
self.target_critic.cuda()
# pop-art
self.update_counter = 0
self.beta = .1
self.y_mean = 0.
self.y_square_mean = 0.
self.target_y_mean = self.y_mean
self.target_y_square_mean = self.y_square_mean
# per
self.total_step = 0
self.PARTITION_NUM = partition_num
self.LEARN_START = mem_size / self.PARTITION_NUM + 1
self.exp_trunc = exp_trunc
self.exp_percent = exp_percent
self.exp_rebalance_freq = exp_rebalance_freq
self.exp_batch_sizes = []
self.exp_type = exp_type
#if len(self.exp_trunc)>0:
if len(self.exp_trunc) != len(self.exp_percent):
raise RuntimeError("different exp_trunc and exp_percent length")
self.exp = []
for i in range(len(self.exp_trunc) + 1):
tmp_batch_size = int(
batch_size * (1 - sum(self.exp_percent))) if i == len(
self.exp_trunc) else int(batch_size * self.exp_percent[i])
self.exp_batch_sizes.append(tmp_batch_size)
exp_conf = {
'size': mem_size,
'learn_start': self.LEARN_START,
'partition_num': self.PARTITION_NUM,
'total_step': self.MAX_EPI * 50,
'batch_size': tmp_batch_size
}
self.exp.append(Experience(
exp_conf)) if self.exp_type == 'rank' else self.exp.append(
PrioritizedReplayBuffer(mem_size, alpha=.7))
#else:
# exp_conf = {
# 'size': mem_size,
# 'learn_start': self.LEARN_START,
# 'partition_num': self.PARTITION_NUM,
# 'total_step': self.MAX_EPI * 50,
# 'batch_size': batch_size,
# }
# self.exp = Experience(exp_conf)
# uniform er
#self.exp = Experience(mem_size)
self.optim_critic = optim.Adam(self.critic.parameters(),
lr=lr_critic,
weight_decay=critic_weight_decay)
self.optim_actor = optim.Adam(self.actor.parameters(), lr=-lr_actor)
self.random_processes = []
for _ in xrange(self.N):
random_process = OrnsteinUhlenbeckProcess(\
size=self.n_a, theta=.15, mu=0, sigma=.2)
self.random_processes.append(random_process)
self.EPSILON_START = epsilon_start
self.EPSILON_END = epsilon_end
# only default random process mode will use epsilon decay
self.EPSILON_DECAY = epsilon_decay
# other random process mode will use epsilon rate
self.EPSILON_RATE = epsilon_rate
self.GAMMA = gamma
self.TARGET_UPDATE_FREQUENCY = target_update_frequency
self.BATCH_SIZE = batch_size
self.target_update_mode = target_update_mode
self.tau = tau
#title = {common.S_EPI:[], common.S_TOTAL_R:[]}
#self.data = pd.DataFrame(title)
self.RAND_PROC = random_process_mode
self.grad_clip_mode = grad_clip_mode
self.grad_clip_norm = grad_clip_norm
# logger
self.logger = None
self.env_log_freq = env_log_freq
self.model_log_freq = model_log_freq
self.step = 0
# random seed
self.seed = int(time.time())
random.seed(self.seed)
np.random.seed(self.seed)
critic = Critic(state_dim, action_dim, max_action, args)
critic_t = Critic(state_dim, action_dim, max_action, args)
critic_t.load_state_dict(critic.state_dict())
print("OK 3")
# actor
actor = Actor(state_dim, action_dim, max_action, args)
actor_t = Actor(state_dim, action_dim, max_action, args)
actor_t.load_state_dict(actor.state_dict())
# action noise
if not args.ou_noise:
a_noise = GaussianNoise(action_dim, sigma=args.gauss_sigma)
else:
a_noise = OrnsteinUhlenbeckProcess(action_dim,
mu=args.ou_mu,
theta=args.ou_theta,
sigma=args.ou_sigma)
if USE_CUDA:
critic.cuda()
critic_t.cuda()
actor.cuda()
actor_t.cuda()
print("OK 4")
# CEM
es = sepCEM(actor.get_size(),
mu_init=actor.get_params(),
sigma_init=args.sigma_init,
damp=args.damp,
damp_limit=args.damp_limit,
def __init__(self):
# random seed for torch
__seed = config.get(MODEL_SEED)
self.policy_loss = []
self.critic_loss = []
if __seed > 0:
self.seed(__seed)
self.nb_states = config.get(MODEL_STATE_COUNT)
self.nb_actions = config.get(MODEL_ACTION_COUNT)
# Create Actor and Critic Network
actor_net_cfg = {
'hidden1': config.get(MODEL_ACTOR_HIDDEN1),
'hidden2': config.get(MODEL_ACTOR_HIDDEN2),
'init_w': config.get(MODEL_INIT_WEIGHT)
}
critic_net_cfg = {
'hidden1': config.get(MODEL_CRITIC_HIDDEN1),
'hidden2': config.get(MODEL_CRITIC_HIDDEN2),
'init_w': config.get(MODEL_INIT_WEIGHT)
}
self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions,
**actor_net_cfg)
self.actor_optim = Adam(
self.actor.parameters(),
lr=config.get(MODEL_ACTOR_LR),
weight_decay=config.get(MODEL_ACTOR_WEIGHT_DECAY))
self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions,
**critic_net_cfg)
self.critic_optim = Adam(
self.critic.parameters(),
lr=config.get(MODEL_CRITIC_LR),
weight_decay=config.get(MODEL_CRITIC_WEIGHT_DECAY))
hard_update(self.actor_target, self.actor)
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = Memory()
self.random_process = OrnsteinUhlenbeckProcess(
size=self.nb_actions,
theta=config.get(RANDOM_THETA),
mu=config.get(RANDOM_MU),
sigma=config.get(RANDOM_SIGMA))
# Hyper-parameters
self.batch_size = config.get(MODEL_BATCH_SIZE)
self.tau = config.get(MODEL_TARGET_TAU)
self.discount = config.get(MODEL_DISCOUNT)
self.depsilon = 1.0 / config.get(MODEL_EPSILON)
self.model_path = config.get(MODEL_SAVE_PATH)
#
self.epsilon = 1.0
# init device
self.device_init()