def __init__(self, env, epsilon_init, decay, epsilon_min, update_freq, sl_lr, rl_lr, sl_capa, rl_capa, n_step, gamma, eta, max_episode, negative, rl_start, sl_start, train_freq, rl_batch_size, sl_batch_size, render, device):
self.env = env
self.epsilon_init = epsilon_init
self.decay = decay
self.epsilon_min = epsilon_min
self.update_freq = update_freq
self.sl_lr = sl_lr
self.rl_lr = rl_lr
self.sl_capa = sl_capa
self.rl_capa = rl_capa
self.n_step = n_step
self.gamma = gamma
self.eta = eta
self.max_episode = max_episode
self.negative = negative
self.sl_start = sl_start
self.rl_start = rl_start
self.train_freq = train_freq
self.rl_batch_size = rl_batch_size
self.sl_batch_size = sl_batch_size
self.render = render
self.device = device
self.observation_dim = self.env.observation_space.shape
self.action_dim = self.env.action_space.n
self.sl_p1_buffer = reservoir_buffer(self.sl_capa)
self.sl_p2_buffer = reservoir_buffer(self.sl_capa)
if self.n_step > 1:
self.rl_p1_buffer = n_step_replay_buffer(self.rl_capa, self.n_step, self.gamma)
self.rl_p2_buffer = n_step_replay_buffer(self.rl_capa, self.n_step, self.gamma)
else:
self.rl_p1_buffer = replay_buffer(self.rl_capa)
self.rl_p2_buffer = replay_buffer(self.rl_capa)
self.p1_dqn_eval = dueling_ddqn(self.observation_dim, self.action_dim).to(self.device)
self.p1_dqn_target = dueling_ddqn(self.observation_dim, self.action_dim).to(self.device)
self.p2_dqn_eval = dueling_ddqn(self.observation_dim, self.action_dim).to(self.device)
self.p2_dqn_target = dueling_ddqn(self.observation_dim, self.action_dim).to(self.device)
self.p1_dqn_target.load_state_dict(self.p1_dqn_eval.state_dict())
self.p2_dqn_target.load_state_dict(self.p2_dqn_eval.state_dict())
self.p1_policy = policy(self.observation_dim, self.action_dim).to(self.device)
self.p2_policy = policy(self.observation_dim, self.action_dim).to(self.device)
self.epsilon = lambda x: self.epsilon_min + (self.epsilon_init - self.epsilon_min) * math.exp(-1. * x / self.decay)
self.p1_dqn_optimizer = torch.optim.Adam(self.p1_dqn_eval.parameters(), lr=self.rl_lr)
self.p2_dqn_optimizer = torch.optim.Adam(self.p2_dqn_eval.parameters(), lr=self.rl_lr)
self.p1_policy_optimizer = torch.optim.Adam(self.p1_policy.parameters(), lr=self.sl_lr)
self.p2_policy_optimizer = torch.optim.Adam(self.p2_policy.parameters(), lr=self.sl_lr)
def __init__(self):
self.sess = tf.Session()
self.memory = replay_buffer(max_length=1e5)
self.tau = 0.995
self.gamma = 0.99
self.state_size = 33
self.output_size = 4
self.action_limit = 1.0
self.hidden = [400, 300]
self.batch_size = 100
self.pi_lr = 1e-4
self.q_lr = 1e-4
self.noise = OU_noise(self.output_size, 1)
self.x_ph, self.a_ph, self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size, self.state_size, None, None)
with tf.variable_scope('main'):
self.pi, self.q, self.q_pi = cr.mlp_actor_critic(
self.x_ph,
self.a_ph,
self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit)
with tf.variable_scope('target'):
self.pi_targ, _, self.q_pi_targ = cr.mlp_actor_critic(self.x2_ph,\
self.a_ph, self.hidden, activation=tf.nn.relu, output_activation=tf.tanh,
output_size=self.output_size, action_limit=self.action_limit)
self.target = tf.stop_gradient(self.r_ph + self.gamma *
(1 - self.d_ph) * self.q_pi_targ)
self.pi_loss = -tf.reduce_mean(self.q_pi)
self.v_loss = tf.reduce_mean((self.q - self.target)**2) * 0.5
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
self.v_optimizer = tf.train.AdamOptimizer(self.q_lr)
self.pi_train = self.pi_optimizer.minimize(
self.pi_loss, var_list=cr.get_vars('main/pi'))
self.v_train = self.v_optimizer.minimize(
self.v_loss, var_list=cr.get_vars('main/q'))
self.target_update = tf.group([
tf.assign(v_targ, self.tau * v_targ + (1 - self.tau) * v_main) for
v_main, v_targ in zip(cr.get_vars('main'), cr.get_vars('target'))
])
self.target_init = tf.group([
tf.assign(v_targ, v_main) for v_main, v_targ in zip(
cr.get_vars('main'), cr.get_vars('target'))
])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
def __init__(self, observation_dim, action_dim, epsilon_init, decay,
epsilon_min, update_freq, sl_lr, rl_lr, sl_capa, rl_capa,
n_step, gamma, eta, rl_start, sl_start, train_freq,
rl_batch_size, sl_batch_size, device, eval_mode):
self.observation_dim = observation_dim
self.action_dim = action_dim
self.epsilon_init = epsilon_init
self.decay = decay
self.epsilon_min = epsilon_min
self.update_freq = update_freq
self.sl_lr = sl_lr
self.rl_lr = rl_lr
self.sl_capa = sl_capa
self.rl_capa = rl_capa
self.n_step = n_step
self.gamma = gamma
self.eta = eta
self.sl_start = sl_start
self.rl_start = rl_start
self.train_freq = train_freq
self.rl_batch_size = rl_batch_size
self.sl_batch_size = sl_batch_size
self.device = device
self.use_raw = False
self.eval_mode = eval_mode
self.sl_buffer = reservoir_buffer(self.sl_capa)
if self.n_step > 1:
self.rl_buffer = n_step_replay_buffer(self.rl_capa, self.n_step,
self.gamma)
else:
self.rl_buffer = replay_buffer(self.rl_capa)
self.dqn_eval = dueling_ddqn(self.observation_dim,
self.action_dim).to(self.device)
self.dqn_target = dueling_ddqn(self.observation_dim,
self.action_dim).to(self.device)
self.dqn_target.load_state_dict(self.dqn_eval.state_dict())
self.policy = policy(self.observation_dim,
self.action_dim).to(self.device)
self.epsilon = lambda x: self.epsilon_min + (
self.epsilon_init - self.epsilon_min) * math.exp(-1. * x / self.
decay)
self.dqn_optimizer = torch.optim.Adam(self.dqn_eval.parameters(),
lr=self.rl_lr)
self.policy_optimizer = torch.optim.Adam(self.policy.parameters(),
lr=self.sl_lr)
self.choose_policy_mode()
self.count = 0
def __init__(self):
self.sess = tf.Session()
self.state_size = env_set['state']
self.output_size = env_set['action']
self.worker_size = env_set['worker']
self.support_size = 64
self.tau = 0.995
self.gamma = env_set['gamma']
self.hidden = env_set['hidden']
self.batch_size = env_set['batch_size']
self.pi_lr = env_set['pi_lr']
self.q_lr = env_set['q_lr']
self.action_limit = 1.0
self.memory = replay_buffer(env_set['mem_size'])
self.kappa = 1.0
self.risk_factor = -1.0
self.random_risk = False
self.target_noise = 0.2
self.noise_clip = 0.5
tf.set_random_seed(10)
self.x_ph, self.a_ph,self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size,self.state_size, None, None)
self.risk_factor_ph = tf.placeholder(tf.float32)
with tf.variable_scope('main'):
self.pi, self.q1, self.q2, self.q1_pi, self.q2_pi = cr.dqpg_td3_actor_critic(
x=self.x_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit,
support_size=self.support_size)
with tf.variable_scope('target'):
_, _, _, self.q1_pi_targ, self.q2_pi_targ = cr.dqpg_td3_actor_critic(
x=self.x2_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit,
support_size=self.support_size,
pi_q_noise=self.target_noise,
noise_clip=self.noise_clip)
self.pi_params = cr.get_vars('main/pi')
self.q_params = cr.get_vars('main/q')
self.min_q_targ = tf.minimum(self.q1_pi_targ, self.q2_pi_targ)
self.backup = tf.stop_gradient(tf.expand_dims(self.r_ph,axis=1)\
+ self.gamma*tf.expand_dims(1-self.d_ph,axis=1)*self.min_q_targ)
self.quantile_weight = 1.0 - self.risk_factor_ph*\
(2.0*tf.reshape(tf.range(0.5/self.support_size, 1, 1 / self.support_size), [1, self.support_size]) - 1.0)
self.pi_loss = -tf.reduce_mean(
tf.reduce_mean(self.q1_pi * self.quantile_weight))
logit_valid_tile = tf.tile(tf.expand_dims(self.backup, axis=1),
[1, self.support_size, 1])
tau = tf.reshape(
tf.range(0.5 / self.support_size, 1, 1 / self.support_size),
[1, self.support_size])
tau = tf.tile(tf.expand_dims(tau, axis=2), [1, 1, self.support_size])
theta_loss_tile = tf.tile(tf.expand_dims(self.q1, axis=2),
[1, 1, self.support_size])
#Huber_loss = tf.compat.v1.losses.huber_loss(logit_valid_tile, theta_loss_tile, reduction=tf.losses.Reduction.NONE,delta=self.kappa)/self.kappa
bellman_errors = logit_valid_tile - theta_loss_tile
Logcosh = bellman_errors + tf.math.softplus(
-2. * bellman_errors) - tf.log(2.)
Loss = tf.abs(tau - tf.stop_gradient(tf.to_float(
bellman_errors < 0))) * Logcosh
self.v1_loss = tf.reduce_mean(
tf.reduce_sum(tf.reduce_mean(Loss, axis=1), axis=1))
theta_loss_tile = tf.tile(tf.expand_dims(self.q2, axis=2),
[1, 1, self.support_size])
#Huber_loss = tf.compat.v1.losses.huber_loss(logit_valid_tile, theta_loss_tile, reduction=tf.losses.Reduction.NONE,delta=self.kappa)/self.kappa
bellman_errors = logit_valid_tile - theta_loss_tile
Logcosh = bellman_errors + tf.math.softplus(
-2. * bellman_errors) - tf.log(2.)
Loss = tf.abs(tau - tf.stop_gradient(tf.to_float(
bellman_errors < 0))) * Logcosh
self.v2_loss = tf.reduce_mean(
tf.reduce_sum(tf.reduce_mean(Loss, axis=1), axis=1))
self.v_loss = self.v1_loss + self.v2_loss
self.value_optimizer = tf.train.AdamOptimizer(self.q_lr)
self.train_value_op = self.value_optimizer.minimize(
self.v_loss, var_list=self.q_params)
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
with tf.control_dependencies([self.train_value_op]):
self.train_pi_op = self.pi_optimizer.minimize(
self.pi_loss, var_list=self.pi_params)
with tf.control_dependencies([self.train_pi_op]):
self.target_update = tf.group([
tf.assign(v_targ, self.tau * v_targ + (1 - self.tau) * v_main)
for v_main, v_targ in zip(cr.get_vars('main'),
cr.get_vars('target'))
])
self.step_ops = [
self.pi_loss, self.v_loss, self.train_pi_op, self.train_value_op,
self.target_update
]
self.value_ops = [self.v_loss, self.train_value_op]
self.target_init = tf.group([
tf.assign(v_targ, v_main) for v_main, v_targ in zip(
cr.get_vars('main'), cr.get_vars('target'))
])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
print(
self.sess.run(self.quantile_weight,
feed_dict={self.risk_factor_ph: self.risk_factor}))
self.saver = tf.train.Saver()
def __init__(self):
self.sess = tf.Session()
self.state_size = 33
self.output_size = 4
self.tau = 0.995
self.gamma = 0.99
self.hidden = [400, 300]
self.batch_size = 64
self.pi_lr = 1e-3
self.q_lr = 1e-3
self.action_limit = 1.0
self.memory = replay_buffer(1e5)
self.target_noise = 0.2
self.noise_clip = 0.1
self.alpha = 1e-5
self.num_worker = 20
self.noise = OU_noise(self.output_size, self.num_worker)
self.x_ph, self.a_ph, self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size, self.state_size, None, None)
with tf.variable_scope('main'):
self.mu, self.pi, self.logp_pi, self.q1, self.q2, self.q1_pi, self.q2_pi, self.v = \
cr.sac_mlp_actor_critic(
x=self.x_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit
)
with tf.variable_scope('target'):
_, _, _, _, _, _, _, self.v_targ = \
cr.sac_mlp_actor_critic(
x=self.x2_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit
)
self.pi_params = cr.get_vars('main/pi')
self.value_params = cr.get_vars('main/q') + cr.get_vars('main/v')
self.min_q_pi = tf.minimum(self.q1_pi, self.q2_pi)
self.q_backup = tf.stop_gradient(self.r_ph + self.gamma *
(1 - self.d_ph) * self.v_targ)
self.v_backup = tf.stop_gradient(self.min_q_pi -
self.alpha * self.logp_pi)
self.pi_loss = tf.reduce_mean(self.alpha * self.logp_pi - self.q1_pi)
self.q1_loss = 0.5 * tf.reduce_mean((self.q_backup - self.q1)**2)
self.q2_loss = 0.5 * tf.reduce_mean((self.q_backup - self.q2)**2)
self.v_loss = 0.5 * tf.reduce_mean((self.v_backup - self.v)**2)
self.value_loss = self.q1_loss + self.q2_loss + self.v_loss
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
self.train_pi_op = self.pi_optimizer.minimize(self.pi_loss,
var_list=self.pi_params)
self.value_optimizer = tf.train.AdamOptimizer(self.q_lr)
with tf.control_dependencies([self.train_pi_op]):
self.train_value_op = self.value_optimizer.minimize(
self.value_loss, var_list=self.value_params)
with tf.control_dependencies([self.train_value_op]):
self.target_update = tf.group([
tf.assign(v_targ, self.tau * v_targ + (1 - self.tau) * v_main)
for v_main, v_targ in zip(cr.get_vars('main'),
cr.get_vars('target'))
])
self.step_ops = [
self.pi_loss, self.q1_loss, self.q2_loss, self.v_loss, self.q1,
self.q2, self.v, self.logp_pi, self.train_pi_op,
self.train_value_op, self.target_update
]
self.target_init = tf.group([
tf.assign(v_targ, v_main) for v_main, v_targ in zip(
cr.get_vars('main'), cr.get_vars('target'))
])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
def __init__(self):
self.sess = tf.Session()
self.state_size = env_set['state']
self.output_size = env_set['action']
self.worker_size = env_set['worker']
self.support_size = 8
self.target_update_tau = 0.995
self.gamma = 0.99
self.hidden = env_set['hidden']
self.batch_size = 64
self.pi_lr = 1e-4
self.q_lr = 1e-3
self.action_limit = 1.0
self.memory = replay_buffer(env_set['mem_size'])
self.target_noise = 0.2
self.noise_clip = 0.1
self.x_ph, self.a_ph, self.tau_ph,self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size, self.support_size,self.state_size, None, None)
with tf.variable_scope('main'):
self.pi, self.q, self.q_pi = cr.dipg_mlp_actor_critic(
x=self.x_ph,
a=self.a_ph,
tau=self.tau_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit)
with tf.variable_scope('target'):
_, _, self.q_pi_targ = cr.dipg_mlp_actor_critic(
x=self.x2_ph,
a=self.a_ph,
tau=self.tau_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit,
pi_q_noise=self.target_noise)
self.pi_params = cr.get_vars('main/pi')
self.q_params = cr.get_vars('main/q')
self.backup = tf.stop_gradient(tf.tile(tf.expand_dims(self.r_ph,axis=1),[1,self.support_size])\
+ self.gamma*tf.tile(tf.expand_dims(1-self.d_ph,axis=1),[1,self.support_size])*self.q_pi_targ)
self.pi_loss = -tf.reduce_mean(tf.reduce_mean(self.q_pi))
self.clip_tau = 5e-2
theta_loss_tile = tf.tile(tf.expand_dims(self.q, axis=2),
[1, 1, self.support_size])
logit_valid_tile = tf.tile(tf.expand_dims(self.backup, axis=1),
[1, self.support_size, 1])
Huber_loss = tf.losses.huber_loss(logit_valid_tile,
theta_loss_tile,
reduction=tf.losses.Reduction.NONE)
tau = tf.tile(tf.expand_dims(self.tau_ph, axis=2),
[1, 1, self.support_size])
bellman_errors = logit_valid_tile - theta_loss_tile
Loss = (
tf.abs(tau - tf.stop_gradient(tf.to_float(bellman_errors < 0))) *
Huber_loss)
self.v_loss = tf.reduce_mean(
tf.reduce_sum(tf.reduce_mean(Loss, axis=1)))
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
grad = self.pi_optimizer.compute_gradients(self.pi_loss,
var_list=self.pi_params)
grad = [(gr / self.support_size, var) for gr, var in grad]
self.train_pi_op = self.pi_optimizer.apply_gradients(grad)
self.value_optimizer = tf.train.AdamOptimizer(self.q_lr)
with tf.control_dependencies([self.train_pi_op]):
self.train_value_op = self.value_optimizer.minimize(
self.v_loss, var_list=self.q_params)
with tf.control_dependencies([self.train_value_op]):
self.target_update = tf.group([
tf.assign(
v_targ, self.target_update_tau * v_targ +
(1 - self.target_update_tau) * v_main)
for v_main, v_targ in zip(cr.get_vars('main'),
cr.get_vars('target'))
])
self.step_ops = [
self.pi_loss, self.v_loss, self.train_pi_op, self.train_value_op,
self.target_update
]
self.target_init = tf.group([
tf.assign(v_targ, v_main) for v_main, v_targ in zip(
cr.get_vars('main'), cr.get_vars('target'))
])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
def __init__(self):
self.sess = tf.Session()
self.state_size = 33
self.output_size = 4
self.tau = 0.995
self.gamma = 0.99
self.hidden = [400, 300]
self.batch_size = 64
self.pi_lr = 1e-3
self.q_lr = 1e-3
self.action_limit = 1.0
self.memory = replay_buffer(1e5)
self.target_noise = 0.2
self.noise = OU_noise(self.output_size, 1)
self.noise_clip = 0.1
self.x_ph, self.a_ph, self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size, self.state_size, None, None)
with tf.variable_scope('main'):
self.pi, self.q1, self.q2, self.q1_pi = cr.td3_mlp_actor_critic(
x=self.x_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit)
with tf.variable_scope('target'):
self.pi_targ, _, _, _ = cr.td3_mlp_actor_critic(
x=self.x2_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit)
with tf.variable_scope('target', reuse=True):
self.eps = tf.random_normal(tf.shape(self.pi_targ),
stddev=self.target_noise)
self.epsilon = tf.clip_by_value(self.eps, -self.noise_clip,
self.noise_clip)
self.a_prev = self.pi_targ + self.epsilon
self.a2 = tf.clip_by_value(self.a_prev, -self.action_limit,
self.action_limit)
_, self.q1_targ, self.q2_targ, self.q1_pi_targ = cr.td3_mlp_actor_critic(
x=self.x2_ph,
a=self.a2,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit)
self.pi_params = cr.get_vars('main/pi')
self.q_params = cr.get_vars('main/q')
self.min_q_targ = tf.minimum(self.q1_targ, self.q2_targ)
self.backup = tf.stop_gradient(self.r_ph + self.gamma *
(1 - self.d_ph) * self.min_q_targ)
self.pi_loss = -tf.reduce_mean(self.q1_pi)
self.q1_loss = tf.reduce_mean((self.q1 - self.backup)**2)
self.q2_loss = tf.reduce_mean((self.q2 - self.backup)**2)
self.v_loss = self.q1_loss + self.q2_loss
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
self.q_optimizer = tf.train.AdamOptimizer(self.q_lr)
self.pi_train = self.pi_optimizer.minimize(self.pi_loss,
var_list=self.pi_params)
self.v_train = self.pi_optimizer.minimize(self.v_loss,
var_list=self.q_params)
self.target_update = tf.group([
tf.assign(v_targ, self.tau * v_targ + (1 - self.tau) * v_main) for
v_main, v_targ in zip(cr.get_vars('main'), cr.get_vars('target'))
])
self.target_init = tf.group([
tf.assign(v_targ, v_main) for v_main, v_targ in zip(
cr.get_vars('main'), cr.get_vars('target'))
])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
def __init__(self):
self.sess = tf.Session()
self.state_size = env_set['state']
self.output_size = env_set['action']
self.worker_size = env_set['worker']
self.tau = 0.995
self.gamma = env_set['gamma']
self.hidden = env_set['hidden']
self.batch_size = 64
self.pi_lr = env_set['pi_lr']
self.q_lr = env_set['q_lr']
self.action_limit = 1.0
self.memory = replay_buffer(env_set['mem_size'])
self.target_noise = 0.2
self.noise_clip = 0.5
self.x_ph, self.a_ph, self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size, self.state_size, None, None)
with tf.variable_scope('main'):
self.pi, self.q1, self.q2, self.q1_pi, _ = cr.td3_mlp_actor_critic(
x=self.x_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit,
)
with tf.variable_scope('target'):
self.pi_targ, self.q1_double_targ, self.q2_double_targ, self.q1_pi_targ, self.q2_pi_targ = cr.td3_mlp_actor_critic(
x=self.x2_ph,
a=self.a_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size,
action_limit=self.action_limit,
pi_q_noise=self.target_noise,
noise_clip=self.noise_clip)
self.pi_params = cr.get_vars('main/pi')
self.q_params = cr.get_vars('main/q')
self.min_q_targ = tf.minimum(self.q1_pi_targ, self.q2_pi_targ)
#self.min_q_targ = tf.minimum(self.q1_double_targ,self.q2_double_targ)
self.backup = tf.stop_gradient(self.r_ph + self.gamma *
(1 - self.d_ph) * self.min_q_targ)
self.pi_loss = -tf.reduce_mean(self.q1_pi)
self.q1_loss = tf.reduce_mean((self.q1 - self.backup)**2)
self.q2_loss = tf.reduce_mean((self.q2 - self.backup)**2)
self.v_loss = self.q1_loss + self.q2_loss
self.value_optimizer = tf.train.AdamOptimizer(self.q_lr)
self.train_value_op = self.value_optimizer.minimize(
self.v_loss, var_list=self.q_params)
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
with tf.control_dependencies([self.train_value_op]):
self.train_pi_op = self.pi_optimizer.minimize(
self.pi_loss, var_list=self.pi_params)
with tf.control_dependencies([self.train_pi_op]):
self.target_update = tf.group([
tf.assign(v_targ, self.tau * v_targ + (1 - self.tau) * v_main)
for v_main, v_targ in zip(cr.get_vars('main'),
cr.get_vars('target'))
])
self.step_ops = [
self.pi_loss, self.v_loss, self.train_pi_op, self.train_value_op,
self.target_update
]
self.value_ops = [self.v_loss, self.train_value_op]
self.target_init = tf.group([
tf.assign(v_targ, v_main) for v_main, v_targ in zip(
cr.get_vars('main'), cr.get_vars('target'))
])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
self.saver = tf.train.Saver()
def __init__(self):
self.sess = tf.Session()
self.state_size = env_set['state']
self.output_size = env_set['action']
self.worker_size = env_set['worker']
self.support_size = 8
self.target_update_tau = 0.995
self.gamma = 0.99
self.hidden = env_set['hidden']
self.batch_size = 64
self.pi_lr = 1e-4
self.q_lr = 1e-3
self.action_limit = 1.0
self.memory = replay_buffer(env_set['mem_size'])
self.target_noise = 0.2
self.noise_clip = 0.1
self.alpha = 1e-5
self.x_ph, self.a_ph, self.tau_ph,self.x2_ph, self.r_ph, self.d_ph = \
cr.placeholders(self.state_size, self.output_size, self.support_size,self.state_size, None, None)
with tf.variable_scope('main'):
self.pi, self.logp_pi, self.q1, self.q2, self.q1_pi, self.q2_pi, self.v = cr.dipg_sac_mlp_actor_critic(
x=self.x_ph,
a=self.a_ph,
tau= self.tau_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size
)
with tf.variable_scope('target'):
_, _, _, _, _, _, self.v_targ = cr.dipg_sac_mlp_actor_critic(
x=self.x2_ph,
a=self.a_ph,
tau=self.tau_ph,
hidden=self.hidden,
activation=tf.nn.relu,
output_activation=tf.tanh,
output_size=self.output_size
)
self.pi_params = cr.get_vars('main/pi')
self.value_params = cr.get_vars('main/q') + cr.get_vars('main/v')
self.min_q = tf.where(tf.less(tf.reduce_mean(self.q1_pi),tf.reduce_mean(self.q2_pi)),self.q1_pi,self.q2_pi)
self.q_backup = tf.stop_gradient(tf.tile(tf.expand_dims(self.r_ph,axis=1),[1,self.support_size])\
+ self.gamma*tf.tile(tf.expand_dims(1-self.d_ph,axis=1),[1,self.support_size])*self.v_targ)
self.v_backup = tf.stop_gradient(self.min_q\
- self.alpha*tf.tile(tf.expand_dims(self.logp_pi,axis=1),[1,self.support_size]))
self.pi_loss = tf.reduce_mean(self.alpha * self.logp_pi - tf.reduce_mean(self.q1_pi*tf.square(self.tau_ph)))
tau = self.tau_ph
inv_tau = 1 - tau
tau = tf.tile(tf.expand_dims(tau, axis=1), [1, self.support_size, 1])
inv_tau = tf.tile(tf.expand_dims(inv_tau, axis=1), [1, self.support_size, 1])
logit_valid_tile = tf.tile(tf.expand_dims(self.q_backup, axis=1), [1, self.support_size, 1])
theta_loss_tile = tf.tile(tf.expand_dims(self.q1, axis=2), [1, 1, self.support_size])
Huber_loss = tf.losses.mean_squared_error(logit_valid_tile, theta_loss_tile, reduction=tf.losses.Reduction.NONE)
error_loss = logit_valid_tile - theta_loss_tile
Loss = tf.where(tf.less(error_loss, 0.0), Huber_loss, tau * Huber_loss)
self.q1_loss = 0.5*tf.reduce_mean(tf.reduce_sum(tf.reduce_mean(Loss, axis=2), axis=1))
theta_loss_tile = tf.tile(tf.expand_dims(self.q2, axis=2), [1, 1, self.support_size])
Huber_loss = tf.losses.mean_squared_error(logit_valid_tile, theta_loss_tile, reduction=tf.losses.Reduction.NONE)
error_loss = logit_valid_tile - theta_loss_tile
Loss = tf.where(tf.less(error_loss, 0.0), Huber_loss, tau * Huber_loss)
self.q2_loss = 0.5*tf.reduce_mean(tf.reduce_sum(tf.reduce_mean(Loss, axis=2), axis=1))
theta_loss_tile = tf.tile(tf.expand_dims(self.v, axis=2), [1, 1, self.support_size])
logit_valid_tile = tf.tile(tf.expand_dims(self.v_backup, axis=1), [1, self.support_size, 1])
Huber_loss = tf.losses.mean_squared_error(logit_valid_tile, theta_loss_tile, reduction=tf.losses.Reduction.NONE)
error_loss = logit_valid_tile - theta_loss_tile
Loss = tf.where(tf.less(error_loss, 0.0), Huber_loss, tau * Huber_loss)
self.v_loss = 0.5*tf.reduce_mean(tf.reduce_sum(tf.reduce_mean(Loss, axis=2), axis=1))
self.value_loss = self.q1_loss + self.q2_loss + self.v_loss
self.pi_optimizer = tf.train.AdamOptimizer(self.pi_lr)
self.train_pi_op = self.pi_optimizer.minimize(self.pi_loss, var_list=self.pi_params)
self.value_optimizer = tf.train.AdamOptimizer(self.q_lr)
with tf.control_dependencies([self.train_pi_op]):
self.train_value_op = self.value_optimizer.minimize(self.value_loss, var_list=self.value_params)
with tf.control_dependencies([self.train_value_op]):
self.target_update = tf.group([tf.assign(v_targ, self.target_update_tau * v_targ + (1 - self.target_update_tau) * v_main)
for v_main, v_targ in zip(cr.get_vars('main'), cr.get_vars('target'))])
self.step_ops = [self.pi_loss, self.value_loss, self.train_pi_op, self.train_value_op, self.target_update]
self.target_init = tf.group([tf.assign(v_targ, v_main)
for v_main, v_targ in zip(cr.get_vars('main/v'), cr.get_vars('target/v'))])
self.sess.run(tf.global_variables_initializer())
self.sess.run(self.target_init)
def __init__(self,
env_id,
episode,
learning_rate,
gamma,
capacity,
batch_size,
value_iter,
policy_iter,
rho,
episode_len,
render,
train_freq,
entropy_weight,
start_count=10000,
model_path=False):
self.env_id = env_id
self.env = make_env.make_env(self.env_id)
self.episode = episode
self.learning_rate = learning_rate
self.gamma = gamma
self.capacity = capacity
self.batch_size = batch_size
self.value_iter = value_iter
self.policy_iter = policy_iter
self.rho = rho
self.render = render
self.episode_len = episode_len
self.train_freq = train_freq
self.entropy_weight = entropy_weight
self.model_path = model_path
self.observation_dims = self.env.observation_space
self.action_dims = self.env.action_space
self.observation_total_dims = sum([
self.env.observation_space[i].shape[0] for i in range(self.env.n)
])
self.action_total_dims = sum([
self.env.action_space[i].n
if isinstance(self.env.action_space[i], Discrete) else
sum(self.env.action_space[i].high) + self.env.action_space[i].shape
for i in range(self.env.n)
])
self.policy_nets = [
policy_net(
self.observation_dims[i].shape[0], self.action_dims[i].n
if isinstance(self.env.action_space[i], Discrete) else
sum(self.env.action_space[i].high) +
self.env.action_space[i].shape) for i in range(self.env.n)
]
self.target_policy_nets = [
policy_net(
self.observation_dims[i].shape[0], self.action_dims[i].n
if isinstance(self.env.action_space[i], Discrete) else
sum(self.env.action_space[i].high) +
self.env.action_space[i].shape) for i in range(self.env.n)
]
self.value_nets = [
value_net(self.observation_total_dims, self.action_total_dims, 1)
for i in range(self.env.n)
]
self.target_value_nets = [
value_net(self.observation_total_dims, self.action_total_dims, 1)
for i in range(self.env.n)
]
self.policy_optimizers = [
torch.optim.Adam(policy_net.parameters(), lr=self.learning_rate)
for policy_net in self.policy_nets
]
self.value_optimizers = [
torch.optim.Adam(value_net.parameters(), lr=self.learning_rate)
for value_net in self.value_nets
]
if self.model_path:
for i in range(self.env.n):
self.policy_nets[i] = torch.load(
'./models/{}/policy_model{}.pkl'.format(self.env_id, i))
self.value_nets[i] = torch.load(
'./models/{}/value_model{}.pkl'.format(self.env_id, i))
[
target_policy_net.load_state_dict(policy_net.state_dict())
for target_policy_net, policy_net in zip(self.target_policy_nets,
self.policy_nets)
]
[
target_value_net.load_state_dict(value_net.state_dict())
for target_value_net, value_net in zip(self.target_value_nets,
self.value_nets)
]
self.buffer = replay_buffer(self.capacity)
self.count = 0
self.train_count = 0
self.start_count = start_count