class ConstrainedDDPG(CMDPAgent):
def init_parameters(self, sess):
if self.has_target_net:
super(CMDPAgent, self).init_parameters(sess)
sess.run(self.target_replace_op)
sess.run(self.a_target_replace_op)
def __init__(self, user_num, action_dim, action_bound, cvr_n_features, ddpg_n_features, init_roi, budget,
use_budget_control,
use_prioritized_experience_replay,
max_trajectory_length,
update_times_per_train=1, use_predict_cvr=False):
self.user_num = user_num
self.use_budget_control = use_budget_control
self.update_times_per_train = update_times_per_train
self.action_dim = action_dim
self.action_bound = action_bound
self.n_actions = 1
self.cvr_n_features = cvr_n_features
self.ddpg_n_features = ddpg_n_features
self.lr = 0.001
self.use_predict_cvr = use_predict_cvr
self.user_based_adjust_times = 40
self.epsilon = 0.9
self.epsilon_min = 0.05
self.epsilon_dec = 0.3
self.epsilon_dec_iter = 5000 // self.user_based_adjust_times
self.epsilon_dec_iter_min = 500 // self.user_based_adjust_times
self.replace_target_iter = 1
self.soft_update_iter = 1
self.softupdate = True
self.scope_name = "CDDPG-model"
self.epoch = 0
self.exploration_noise = OUNoise(self.action_dim)
self.cvr_buffer_size = 1000 * max_trajectory_length
self.cvr_batch_size = 512
self.cvr_replay_buffer = ReplayBuffer(self.cvr_buffer_size, save_return=False)
self.alpha = 0.6
self.beta = 0.4
self.use_prioritized_experience_replay = use_prioritized_experience_replay
self.ddpg_buffer_size = 1000 * max_trajectory_length
self.ddpg_batch_size = 256
if self.use_prioritized_experience_replay:
self.prioritized_replay_buffer = PrioritizedReplayBuffer(self.ddpg_buffer_size, alpha=self.alpha,
max_priority=20.)
else:
self.replay_buffer = ReplayBuffer(self.ddpg_buffer_size, save_return=True)
with tf.variable_scope(self.scope_name):
self._build_net()
self.build_model_saver(self.scope_name)
def _build_cvr_net(self, state, variable_scope, reuse=False):
with tf.variable_scope(variable_scope, reuse=reuse):
user_id_embedding_table = tf.get_variable(
name="user_id", shape=[self.user_num, 10], initializer=initializers.xavier_initializer(),
trainable=True, dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(state, units=n_features, activation=tf.nn.relu, name='fc1',
kernel_initializer=initializers.xavier_initializer())
fc2 = tf.layers.dense(fc1, units=n_features // 2, activation=tf.nn.relu, name='fc2',
kernel_initializer=initializers.xavier_initializer())
fc3 = tf.layers.dense(fc2, units=n_features // 2, activation=tf.nn.relu, name='fc3',
kernel_initializer=initializers.xavier_initializer())
cvr_out = tf.sigmoid(tf.layers.dense(fc3, units=1, name='cvr',
kernel_initializer=initializers.xavier_initializer()))
return cvr_out
def _build_q_net(self, state, action, variable_scope, reuse=False):
with tf.variable_scope(variable_scope, reuse=reuse):
user_id_embedding_table = tf.get_variable(
name="user_id", shape=[self.user_num, 10], initializer=initializers.xavier_initializer(),
trainable=True, dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
state = tf.concat([state, tf.expand_dims(action, axis=1, name="2d-action")], axis=1)
fc1 = tf.layers.dense(state, units=n_features, activation=tf.nn.relu, name='fc1')
fc2 = tf.layers.dense(fc1, units=n_features // 2, activation=tf.nn.relu, name='fc2')
q = tf.layers.dense(fc2, units=self.action_dim, name='q')
return q[:, 0]
def _build_action_net(self, state, variable_scope):
with tf.variable_scope(variable_scope):
user_id_embedding_table = tf.get_variable(
name="user_id", shape=[self.user_num, 10], initializer=initializers.xavier_initializer(),
trainable=True, dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(state, units=n_features, activation=tf.nn.relu, name='fc1')
fc2 = tf.layers.dense(fc1, units=n_features // 2, activation=tf.nn.relu, name='fc2')
actions = tf.layers.dense(fc2, self.action_dim, activation=tf.nn.sigmoid, name='a')
return actions[:, 0]
def __make_update_exp__(self, vals, target_vals):
polyak = 1.0 - 1e-2
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name), sorted(target_vals, key=lambda v: v.name)):
expression.append(var_target.assign(polyak * var_target + (1.0 - polyak) * var))
expression = tf.group(*expression)
return expression
def _build_net(self):
self.s_cvr = tf.placeholder(tf.float32, [None, self.cvr_n_features], name='s_cvr')
self.cvr = tf.placeholder(tf.float32, [None, ], name='r')
self.s = tf.placeholder(tf.float32, [None, self.ddpg_n_features], name='s')
self.s_ = tf.placeholder(tf.float32, [None, self.ddpg_n_features], name='s_')
self.r = tf.placeholder(tf.float32, [None, ], name='r')
self.a = tf.placeholder(tf.float32, [None, ], name='a')
self.gamma = 1.
self.done = tf.placeholder(tf.float32, [None, ], name='done')
self.return_value = tf.placeholder(tf.float32, [None, ], name='return')
self.important_sampling_weight_ph = tf.placeholder(tf.float32, [None], name="important_sampling_weight")
self.cvr_net = self._build_cvr_net(self.s_cvr, variable_scope="cvr_net")
self.predicted_cvr = self.cvr_net[:, 0]
self.a_eval = self._build_action_net(self.s, variable_scope="actor_eval_net")
self.a_target = self._build_action_net(self.s_, variable_scope="actor_target_net")
self.critic_eval = self._build_q_net(self.s, self.a, variable_scope="eval_q_net")
self.critic_eval_for_loss = self._build_q_net(self.s, self.a_eval, variable_scope="eval_q_net",
reuse=True)
self.critic_target = self._build_q_net(self.s_, self.a, variable_scope="target_q_net")
t_gmv_params = scope_vars(absolute_scope_name("target_q_net"))
e_gmv_params = scope_vars(absolute_scope_name("eval_q_net"))
ae_params = scope_vars(absolute_scope_name("actor_eval_net"))
at_params = scope_vars(absolute_scope_name("actor_target_net"))
cvr_params = scope_vars(absolute_scope_name("cvr_net"))
with tf.variable_scope('hard_replacement'):
self.a_target_replace_op = tf.group([tf.assign(t, e) for t, e in zip(at_params, ae_params)])
self.target_replace_op = tf.group([tf.assign(t, e) for t, e in zip(t_gmv_params, e_gmv_params)])
with tf.variable_scope('soft_update'):
self.a_update_target_q = self.__make_update_exp__(ae_params, at_params)
self.update_target_q = self.__make_update_exp__(e_gmv_params, t_gmv_params)
with tf.variable_scope('q_target'):
self.td0_q_target = tf.stop_gradient(self.r + self.gamma * (1. - self.done) * self.critic_target)
self.montecarlo_target = self.return_value
with tf.variable_scope('loss'):
self.cvr_loss = tf.reduce_mean(tf.squared_difference(self.predicted_cvr, self.cvr))
self._build_loss()
self._pick_loss()
with tf.variable_scope('train'):
self._train_cvr_op = tf.train.AdamOptimizer(self.lr).minimize(self.cvr_loss, var_list=cvr_params)
self._train_ddpg_critic_op = tf.train.AdamOptimizer(self.lr).minimize(self.loss, var_list=e_gmv_params)
self._train_ddpg_a_op = tf.train.AdamOptimizer(self.lr).minimize(self.actor_loss, var_list=ae_params)
def _pick_loss(self):
self.has_target_net = True
self.loss = self.ddpg_loss
self.priority_values = self.td0_error
self.actor_loss = self.a_loss
def _build_loss(self):
if self.use_prioritized_experience_replay:
self.ddpg_loss = tf.reduce_mean(
self.important_sampling_weight_ph * tf.squared_difference(self.td0_q_target, self.critic_eval,
name='TD0_loss'))
self.montecarlo_loss = tf.reduce_mean(self.important_sampling_weight_ph *
tf.squared_difference(self.montecarlo_target, self.critic_eval,
name='MonteCarlo_error'))
else:
self.ddpg_loss = tf.reduce_mean(tf.squared_difference(self.td0_q_target, self.critic_eval, name='TD0_loss'))
self.montecarlo_loss = tf.reduce_mean(tf.squared_difference(self.montecarlo_target, self.critic_eval,
name='MonteCarlo_error'))
self.a_loss = - tf.reduce_mean(self.critic_eval_for_loss)
self.td0_error = tf.abs(self.td0_q_target - self.critic_eval)
self.montecarlo_error = tf.abs(self.montecarlo_target - self.critic_eval)
def build_model_saver(self, var_scope):
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=var_scope)
self.model_saver = tf.train.Saver(var_list=var_list, max_to_keep=1)
def save(self, sess, path, step):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
self.model_saver.save(sess, save_path=path, global_step=step)
def restore(self, sess, path):
self.model_saver.restore(sess, save_path=path)
print('%s model reloaded from %s' % (self.scope_name, path))
def experience(self, new_trajectory, other_info=None):
cvr_trajectory = other_info["cvr"]
for ele in cvr_trajectory:
state, cvr = ele
self.cvr_replay_buffer.add(state, 0, cvr, state, 0, 0, 0)
def experience_cmdp(self, new_trajectory, other_info=None):
if self.use_prioritized_experience_replay:
add_episode(self.prioritized_replay_buffer, new_trajectory, gamma=self.gamma)
else:
add_episode(self.replay_buffer, new_trajectory, gamma=self.gamma)
def get_agent_name(self):
return self.scope_name
def get_action(self, sess, obs, is_test=False, other_info=None):
item_price = other_info["proxy_ad_price"]
ground_truth_cvr = other_info["cvr"]
user_alpha = other_info["user_alpha"]
roi_thr = other_info["roi_thr"]
observations = obs[np.newaxis, :]
cvr = sess.run(self.predicted_cvr, feed_dict={
self.s_cvr: observations
})[0]
if self.use_predict_cvr:
bid = cvr * item_price / roi_thr
else:
bid = ground_truth_cvr * item_price / roi_thr
return bid, {"cvr_over_estimate": [user_alpha, ground_truth_cvr, cvr]}
def get_cmdp_action(self, sess, obs, is_test=False, other_info=None):
if is_test:
discrete_action = self.__greedy__(sess, obs)
else:
discrete_action = self.__epsilon_greedy__(sess, obs)
return discrete_action
def __greedy__(self, sess, observation):
observation = observation[np.newaxis, :]
greedy_action = sess.run(self.a_eval, feed_dict={self.s: observation})
return greedy_action[0]
def __epsilon_greedy__(self, sess, observation):
if np.random.uniform() < self.epsilon:
observation = observation[np.newaxis, :]
actions_value = sess.run(self.a_eval, feed_dict={self.s: observation})
action_noise = self.exploration_noise.noise()
action = actions_value + action_noise
action = action[0]
else:
action = self.__greedy__(sess, observation)
return action
def _is_exploration_enough(self, buffer, min_pool_size):
return len(buffer) >= min_pool_size
def train_cvr(self, sess):
if not self._is_exploration_enough(self.cvr_replay_buffer, self.cvr_batch_size):
return False, [0, 0, 0]
cvr_loss, predicted_cvrs, cvr_targets = 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.cvr_replay_buffer.make_index(self.cvr_batch_size)
obs, act, cvr_targets, obs_next, done, dis_2_end, returns = self.cvr_replay_buffer.sample_index(
sample_indices)
_, cvr_loss, predicted_cvrs = sess.run(
[self._train_cvr_op, self.cvr_loss, self.predicted_cvr],
feed_dict={
self.s_cvr: obs,
self.cvr: cvr_targets
}
)
return True, [cvr_loss, np.average(predicted_cvrs), np.average(cvr_targets)]
def get_memory_returns(self):
if self.use_prioritized_experience_replay:
return self.prioritized_replay_buffer.current_mean_return
else:
return self.replay_buffer.current_mean_return
def update_target(self, sess):
if self.softupdate:
if self.epoch % self.soft_update_iter == 0:
sess.run(self.update_target_q)
sess.run(self.a_update_target_q)
else:
if self.epoch % self.replace_target_iter == 0:
sess.run(self.target_replace_op)
sess.run(self.a_target_replace_op)
def train(self, sess):
if self.has_target_net:
self.update_target(sess)
self.epoch += 1
buffer = self.prioritized_replay_buffer if self.use_prioritized_experience_replay else self.replay_buffer
if not self._is_exploration_enough(buffer, self.ddpg_batch_size):
return False, [0, 0, 0, 0], 0, 0
if self.use_prioritized_experience_replay:
loss, montecarlo_loss, q_eval, returns = self.train_prioritized(sess)
else:
loss, montecarlo_loss, q_eval, returns = self.train_normal(sess)
if self.epoch % self.epsilon_dec_iter == 0:
self.epsilon = max(self.epsilon - self.epsilon_dec, self.epsilon_min)
print("update epsilon:", self.epsilon)
return True, [loss, montecarlo_loss, q_eval, returns], self.get_memory_returns(), self.epsilon
def train_prioritized(self, sess):
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.prioritized_replay_buffer.make_index(self.ddpg_batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns, weights, ranges = self.prioritized_replay_buffer.sample_index(
sample_indices)
_, loss, montecarlo_loss, q_eval, \
priority_values = sess.run(
[self._train_ddpg_critic_op, self.loss, self.montecarlo_loss, self.critic_eval,
self.priority_values],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
self.important_sampling_weight_ph: weights,
})
priorities = priority_values + 1e-6
self.prioritized_replay_buffer.update_priorities(sample_indices, priorities)
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
def train_normal(self, sess):
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.replay_buffer.make_index(self.ddpg_batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns = self.replay_buffer.sample_index(
sample_indices)
_, loss, montecarlo_loss, q_eval = sess.run(
[self._train_ddpg_critic_op, self.loss, self.montecarlo_loss, self.critic_eval],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
})
_, actor_loss = sess.run(
[self._train_ddpg_a_op, self.actor_loss],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
})
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
class DQN_interface(LearningAgent):
def __init__(
self,
n_actions=11,
n_features=29,
use_prioritized_experience_replay=True,
max_trajectory_length=20,
):
self.n_actions = n_actions
self.n_features = n_features
self.gamma = 1.
self.lr = 0.001
self.epsilon = 0.5
self.epsilon_min = 0
self.epsilon_dec = 0.1
self.epsilon_dec_iter = 1000
self.replace_target_iter = 100
self.soft_update_iter = 1
self.softupdate = False
self.scope_name = "DQN-model"
self.epoch = 0
self.buffer_size = 5000 * max_trajectory_length
self.batch_size = 512
self.alpha = 0.6
self.beta = 0.4
self.use_prioritized_experience_replay = use_prioritized_experience_replay
if self.use_prioritized_experience_replay:
self.prioritized_replay_buffer = PrioritizedReplayBuffer(
self.buffer_size, alpha=self.alpha, max_priority=20.)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
self.margin_constant = 2
with tf.variable_scope(self.scope_name):
self._build_net()
self.build_model_saver(self.scope_name)
def _build_net(self):
self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s')
self.s_ = tf.placeholder(tf.float32, [None, self.n_features],
name='s_')
self.r = tf.placeholder(tf.float32, [
None,
], name='r')
self.a = tf.placeholder(tf.int32, [
None,
], name='a')
self.done = tf.placeholder(tf.float32, [
None,
], name='done')
self.return_value = tf.placeholder(tf.float32, [
None,
], name='return')
self.important_sampling_weight_ph = tf.placeholder(
tf.float32, [None], name="important_sampling_weight")
self.q_eval = self._build_q_net(self.s,
self.n_actions,
variable_scope="eval_net")
self.q_next = self._build_q_net(self.s_,
self.n_actions,
variable_scope="target_net")
t_params = scope_vars(absolute_scope_name("target_net"))
e_params = scope_vars(absolute_scope_name("eval_net"))
with tf.variable_scope('hard_replacement'):
self.target_replace_op = tf.group(
[tf.assign(t, e) for t, e in zip(t_params, e_params)])
with tf.variable_scope('soft_update'):
self.update_target_q = self.__make_update_exp__(e_params, t_params)
with tf.variable_scope('q_target'):
self.td0_q_target = tf.stop_gradient(
self.r + self.gamma * (1. - self.done) *
tf.reduce_max(self.q_next, axis=1, name='Qmax_s_'))
target_action = tf.argmax(self.q_eval,
axis=-1,
output_type=tf.int32)
target_a_indices = tf.stack(
[tf.range(tf.shape(self.a)[0], dtype=tf.int32), target_action],
axis=1)
target_q_sa = tf.gather_nd(params=self.q_next,
indices=target_a_indices)
self.double_dqn_target = tf.stop_gradient(self.r + self.gamma *
(1. - self.done) *
target_q_sa)
self.montecarlo_target = self.return_value
with tf.variable_scope('q_eval'):
a_indices = tf.stack(
[tf.range(tf.shape(self.a)[0], dtype=tf.int32), self.a],
axis=1)
self.q_eval_wrt_a = tf.gather_nd(params=self.q_eval,
indices=a_indices)
with tf.variable_scope('loss'):
self._build_loss()
self._pick_loss()
with tf.variable_scope('train'):
self._train_op = tf.train.AdamOptimizer(self.lr).minimize(
self.loss, var_list=e_params)
def _pick_loss(self):
self.loss = self.double_dqn_loss
self.priority_values = self.doubel_dqn_error
def _build_loss(self):
if self.use_prioritized_experience_replay:
self.dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph * tf.squared_difference(
self.td0_q_target, self.q_eval_wrt_a, name='TD0_loss'))
self.double_dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.double_dqn_target,
self.q_eval_wrt_a,
name='Double_DQN_error'))
else:
self.dqn_loss = tf.reduce_mean(
tf.squared_difference(self.td0_q_target,
self.q_eval_wrt_a,
name='TD0_loss'))
self.double_dqn_loss = tf.reduce_mean(
tf.squared_difference(self.double_dqn_target,
self.q_eval_wrt_a,
name='Double_DQN_error'))
self.montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_target,
self.q_eval_wrt_a,
name='MonteCarlo_error'))
self.td0_error = tf.abs(self.td0_q_target - self.q_eval_wrt_a)
self.doubel_dqn_error = tf.abs(self.double_dqn_target -
self.q_eval_wrt_a)
self.montecarlo_error = tf.abs(self.montecarlo_target -
self.q_eval_wrt_a)
margin_diff = tf.one_hot(self.a,
self.n_actions,
on_value=0.,
off_value=1.,
dtype=tf.float32) * self.margin_constant
self.margin_loss = tf.reduce_mean(
tf.reduce_max(self.q_eval + margin_diff, axis=1, keepdims=False) -
self.q_eval_wrt_a)
self.mse_margin_loss = tf.reduce_mean(
tf.squared_difference(
tf.reduce_max(self.q_eval + margin_diff,
axis=1,
keepdims=False), self.q_eval_wrt_a))
def _build_q_net(self, state, n_actions, variable_scope):
with tf.variable_scope(variable_scope):
fc1 = tf.layers.dense(state,
units=self.n_features,
activation=tf.nn.relu,
name='fc1')
q_out = tf.layers.dense(fc1, units=n_actions, name='q')
return q_out
def __make_update_exp__(self, vals, target_vals):
polyak = 1.0 - 1e-2
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name),
sorted(target_vals, key=lambda v: v.name)):
expression.append(
var_target.assign(polyak * var_target + (1.0 - polyak) * var))
expression = tf.group(*expression)
return expression
def __make_hardreplace_exp__(self, vals, target_vals):
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name),
sorted(target_vals, key=lambda v: v.name)):
expression.append(var_target.assign(var))
expression = tf.group(*expression)
return expression
def build_model_saver(self, var_scope):
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=var_scope)
self.model_saver = tf.train.Saver(var_list=var_list, max_to_keep=3)
def save(self, sess, path, step):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
self.model_saver.save(sess, save_path=path, global_step=step)
def restore(self, sess, path):
self.model_saver.restore(sess, save_path=path)
print('%s model reloaded from %s' % (self.scope_name, path))
def experience(self, new_trajectory, other_info=None):
if self.use_prioritized_experience_replay:
add_episode(self.prioritized_replay_buffer,
new_trajectory,
gamma=self.gamma)
else:
add_episode(self.replay_buffer, new_trajectory, gamma=self.gamma)
def get_action(self, sess, obs, is_test=False, other_info=None):
if is_test:
discrete_action = self.greedy_action(sess, obs)
else:
discrete_action = self.choose_action(sess, obs)
other_action_info = {"learning_action": discrete_action}
return 3 * discrete_action, other_action_info
def choose_action(self, sess, observation):
observation = observation[np.newaxis, :]
if np.random.uniform() < self.epsilon:
action = np.random.randint(0, self.n_actions)
else:
actions_value = sess.run(self.q_eval,
feed_dict={self.s: observation})
action = np.argmax(actions_value, axis=1)[0]
return action
def greedy_action(self, sess, single_observation):
observation = single_observation[np.newaxis, :]
actions_value = sess.run(self.q_eval, feed_dict={self.s: observation})
greedy_action = np.argmax(actions_value, axis=1)[0]
return greedy_action
def get_memory_returns(self):
if self.use_prioritized_experience_replay:
return self.prioritized_replay_buffer.current_mean_return
else:
return self.replay_buffer.current_mean_return
def _is_exploration_enough(self, min_pool_size):
if self.use_prioritized_experience_replay:
return len(self.prioritized_replay_buffer) >= min_pool_size
else:
return len(self.replay_buffer) >= min_pool_size
def update_target(self, sess):
if self.softupdate:
if self.epoch % self.soft_update_iter == 0:
sess.run(self.update_target_q)
else:
if self.epoch % self.replace_target_iter == 0:
sess.run(self.target_replace_op)
def train(self, sess):
self.update_target(sess)
self.epoch += 1
if not self._is_exploration_enough(self.batch_size):
return False, [0, 0, 0, 0], 0, 0
if self.use_prioritized_experience_replay:
loss, montecarlo_loss, q_eval, returns = self.train_prioritized(
sess)
else:
loss, montecarlo_loss, q_eval, returns = self.train_normal(sess)
if self.epoch % self.epsilon_dec_iter == 0:
self.epsilon = max(self.epsilon - self.epsilon_dec,
self.epsilon_min)
print("update epsilon:", self.epsilon)
return True, [loss, montecarlo_loss, q_eval,
returns], self.get_memory_returns(), self.epsilon
def train_prioritized(self, sess):
loss, q_eval, returns, montecarlo_loss = 0, 0, 0, 0
for idx in range(1):
sample_indices = self.prioritized_replay_buffer.make_index(
self.batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns, weights, ranges = self.prioritized_replay_buffer.sample_index(
sample_indices)
_, loss, q_eval, montecarlo_loss, priority_values = sess.run(
[
self._train_op, self.loss, self.q_eval_wrt_a,
self.montecarlo_loss, self.priority_values
],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
self.important_sampling_weight_ph: weights
})
priorities = priority_values + 1e-6
self.prioritized_replay_buffer.update_priorities(
sample_indices, priorities)
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
def train_normal(self, sess):
loss, q_eval, returns, montecarlo_loss = 0, 0, 0, 0
for idx in range(1):
sample_index = self.replay_buffer.make_index(self.batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns = self.replay_buffer.sample_index(
sample_index)
_, loss, q_eval, montecarlo_loss = sess.run(
[
self._train_op, self.loss, self.q_eval_wrt_a,
self.montecarlo_loss
],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
})
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
class DQN2Net_interface(LearningAgent, PIDAgent):
def __init__(
self,
user_num,
n_actions,
n_features,
init_roi,
budget,
use_budget_control,
use_prioritized_experience_replay,
max_trajectory_length,
update_times_per_train=1,
):
PIDAgent.__init__(self,
init_roi=init_roi,
default_alpha=1,
budget=budget,
integration=2)
self.user_num = user_num
self.use_budget_control = use_budget_control
self.update_times_per_train = update_times_per_train
self.n_actions = n_actions
self.n_features = n_features
self.gamma = 1.
self.lr = 0.001
self.user_based_adjust_times = 40
self.epsilon = 0.4
self.epsilon_min = 0.05
self.epsilon_dec = 0.1
self.epsilon_dec_iter = 5000 // self.user_based_adjust_times
self.epsilon_dec_iter_min = 500 // self.user_based_adjust_times
self.replace_target_iter = 1
self.soft_update_iter = 1
self.softupdate = True
self.scope_name = "DQN-model"
self.epoch = 0
self.buffer_size = 1000 * max_trajectory_length
self.batch_size = 512
self.alpha = 0.6
self.beta = 0.4
self.use_prioritized_experience_replay = use_prioritized_experience_replay
if self.use_prioritized_experience_replay:
self.prioritized_replay_buffer = PrioritizedReplayBuffer(
self.buffer_size, alpha=self.alpha, max_priority=20.)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
self.cost_replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
self.gmv_replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
self.margin_constant = 2
with tf.variable_scope(self.scope_name):
self._build_net()
self.build_model_saver(self.scope_name)
def _build_q_net(self, state, n_actions, variable_scope, reuse=False):
with tf.variable_scope(variable_scope, reuse=reuse):
user_id_embedding_table = tf.get_variable(
name="user_id",
shape=[self.user_num, 10],
initializer=initializers.xavier_initializer(),
trainable=True,
dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(
user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(
state,
units=n_features,
activation=tf.nn.relu,
name='fc1',
kernel_initializer=initializers.xavier_initializer())
fc2 = tf.layers.dense(
fc1,
units=n_features // 2,
activation=tf.nn.relu,
name='fc2',
kernel_initializer=initializers.xavier_initializer())
fc3 = tf.layers.dense(
fc2,
units=n_features // 2,
activation=tf.nn.relu,
name='fc3',
kernel_initializer=initializers.xavier_initializer())
q_out = tf.maximum(
tf.layers.dense(
fc3,
units=n_actions,
name='q',
kernel_initializer=initializers.xavier_initializer()), 0)
return q_out
def _build_net(self):
self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s')
self.s_ = tf.placeholder(tf.float32, [None, self.n_features],
name='s_')
self.r_gmv = tf.placeholder(tf.float32, [
None,
], name='r_gmv')
self.r_cost = tf.placeholder(tf.float32, [
None,
], name='r_cost')
self.roi_thr = tf.placeholder(tf.float32, [], name="roi_thr")
self.r = tf.placeholder(tf.float32, [
None,
], name='r')
self.a = tf.placeholder(tf.int32, [
None,
], name='a')
self.done = tf.placeholder(tf.float32, [
None,
], name='done')
self.return_gmv_value = tf.placeholder(tf.float32, [
None,
],
name='return_gmv')
self.return_cost_value = tf.placeholder(tf.float32, [
None,
],
name='return_cost')
self.return_value = tf.placeholder(tf.float32, [
None,
], name='return')
self.important_sampling_weight_ph = tf.placeholder(
tf.float32, [None], name="important_sampling_weight")
self.q_eval_gmv = self._build_q_net(self.s,
self.n_actions,
variable_scope="eval_gmv_net")
self.q_next_gmv = self._build_q_net(self.s_,
self.n_actions,
variable_scope="target_gmv_net")
self.q_eval_cost = self._build_q_net(self.s,
self.n_actions,
variable_scope="eval_cost_net")
self.q_next_cost = self._build_q_net(self.s_,
self.n_actions,
variable_scope="target_cost_net")
self.q_eval = self.q_eval_gmv - self.roi_thr * self.q_eval_cost
self.q_next = self.q_next_gmv - self.roi_thr * self.q_next_cost
t_gmv_params = scope_vars(absolute_scope_name("target_gmv_net"))
e_gmv_params = scope_vars(absolute_scope_name("eval_gmv_net"))
t_cost_params = scope_vars(absolute_scope_name("target_cost_net"))
e_cost_params = scope_vars(absolute_scope_name("eval_cost_net"))
with tf.variable_scope('hard_replacement'):
self.target_gmv_replace_op = tf.group(
[tf.assign(t, e) for t, e in zip(t_gmv_params, e_gmv_params)])
self.target_cost_replace_op = tf.group([
tf.assign(t, e) for t, e in zip(t_cost_params, e_cost_params)
])
with tf.variable_scope('soft_update'):
self.update_gmv_target_q = self.__make_update_exp__(
e_gmv_params, t_gmv_params)
self.update_cost_target_q = self.__make_update_exp__(
e_cost_params, t_cost_params)
with tf.variable_scope('q_target'):
greedy_action_s_ = tf.argmax(self.q_next,
axis=-1,
name="td0_argmax_action",
output_type=tf.int32)
greedy_a_indices = tf.stack([
tf.range(tf.cast(tf.shape(self.a)[0], dtype=tf.int32),
dtype=tf.int32), greedy_action_s_
],
axis=1)
target_q_gmv_sa = tf.gather_nd(params=self.q_next_gmv,
indices=greedy_a_indices)
target_q_cost_sa = tf.gather_nd(params=self.q_next_cost,
indices=greedy_a_indices)
target_q_sa = tf.gather_nd(params=self.q_next,
indices=greedy_a_indices)
self.td0_q_gmv_target = tf.stop_gradient(self.r_gmv + self.gamma *
(1. - self.done) *
target_q_gmv_sa)
self.td0_q_cost_target = tf.stop_gradient(self.r_cost +
self.gamma *
(1. - self.done) *
target_q_cost_sa)
self.td0_q_target = tf.stop_gradient(self.r + self.gamma *
(1. - self.done) *
target_q_sa)
target_action = tf.argmax(self.q_eval,
axis=-1,
name="doubeldqn_argmax_action",
output_type=tf.int32)
target_a_indices = tf.stack([
tf.range(tf.cast(tf.shape(self.a)[0], dtype=tf.int32),
dtype=tf.int32), target_action
],
axis=1)
ddqn_target_q_gmv_sa = tf.gather_nd(params=self.q_next_gmv,
indices=target_a_indices)
ddqn_target_q_cost_sa = tf.gather_nd(params=self.q_next_cost,
indices=target_a_indices)
ddqn_target_q_sa = tf.gather_nd(params=self.q_next,
indices=target_a_indices)
self.double_dqn_gmv_target = tf.stop_gradient(self.r_gmv +
self.gamma *
(1. - self.done) *
ddqn_target_q_gmv_sa)
self.double_dqn_cost_target = tf.stop_gradient(
self.r_cost + self.gamma *
(1. - self.done) * ddqn_target_q_cost_sa)
self.double_dqn_target = tf.stop_gradient(self.r + self.gamma *
(1. - self.done) *
ddqn_target_q_sa)
self.montecarlo_gmv_target = self.return_gmv_value
self.montecarlo_cost_target = self.return_cost_value
self.montecarlo_target = self.return_value
with tf.variable_scope('q_eval'):
a_indices = tf.stack([
tf.range(tf.cast(tf.shape(self.a)[0], dtype=tf.int32),
dtype=tf.int32), self.a
],
axis=1)
self.q_eval_gmv_wrt_a = tf.gather_nd(params=self.q_eval_gmv,
indices=a_indices)
self.q_eval_cost_wrt_a = tf.gather_nd(params=self.q_eval_cost,
indices=a_indices)
self.q_eval_wrt_a = tf.gather_nd(params=self.q_eval,
indices=a_indices)
with tf.variable_scope('loss'):
self._build_loss()
self._pick_loss()
with tf.variable_scope('train'):
self._train_op = tf.train.AdamOptimizer(self.lr).minimize(
self.loss, var_list=e_gmv_params + e_cost_params)
self._train_gmv_op = tf.train.AdamOptimizer(self.lr).minimize(
self.gmv_loss, var_list=e_gmv_params)
self._train_cost_op = tf.train.AdamOptimizer(self.lr).minimize(
self.cost_loss, var_list=e_cost_params)
with tf.variable_scope('roi'):
greedy_action_indices = tf.stack([
tf.range(tf.cast(tf.shape(self.a)[0], dtype=tf.int32),
dtype=tf.int32), self.a
],
axis=1)
self.plongterm_roi = tf.gather_nd(
params=self.q_eval_gmv, indices=greedy_action_indices) / (
tf.gather_nd(params=self.q_eval_cost,
indices=greedy_action_indices) + 1e-6)
def _pick_loss(self):
self.has_target_net = True
self.gmv_loss = self.gmv_double_dqn_loss
self.cost_loss = self.cost_double_dqn_loss
self.loss = self.double_dqn_loss
self.priority_values = self.gmv_doubel_dqn_error + self.cost_doubel_dqn_error + self.doubel_dqn_error
def _build_loss(self):
if self.use_prioritized_experience_replay:
self.gmv_dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.td0_q_gmv_target,
self.q_eval_gmv_wrt_a,
name='TD0_gmv_loss'))
self.cost_dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.td0_q_cost_target,
self.q_eval_cost_wrt_a,
name='TD0_cost_loss'))
self.dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph * tf.squared_difference(
self.td0_q_target, self.q_eval_wrt_a, name='TD0_loss'))
self.gmv_double_dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.double_dqn_gmv_target,
self.q_eval_gmv_wrt_a,
name='Double_DQN_gmv_loss'))
self.cost_double_dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.double_dqn_cost_target,
self.q_eval_cost_wrt_a,
name='Double_DQN_cost_loss'))
self.double_dqn_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.double_dqn_target,
self.q_eval_wrt_a,
name='Double_DQN_error'))
self.gmv_montecarlo_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.montecarlo_gmv_target,
self.q_eval_gmv_wrt_a,
name='GMV_error'))
self.cost_montecarlo_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.montecarlo_cost_target,
self.q_eval_cost_wrt_a,
name='COST_error'))
self.montecarlo_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.montecarlo_target,
self.q_eval_wrt_a,
name='MonteCarlo_error'))
else:
self.gmv_dqn_loss = tf.reduce_mean(
tf.squared_difference(self.td0_q_gmv_target,
self.q_eval_gmv_wrt_a,
name='TD0_gmv_loss'))
self.cost_dqn_loss = tf.reduce_mean(
tf.squared_difference(self.td0_q_cost_target,
self.q_eval_cost_wrt_a,
name='TD0_cost_loss'))
self.dqn_loss = tf.reduce_mean(
tf.squared_difference(self.td0_q_target,
self.q_eval_wrt_a,
name='TD0_loss'))
self.gmv_double_dqn_loss = tf.reduce_mean(
tf.squared_difference(self.double_dqn_gmv_target,
self.q_eval_gmv_wrt_a,
name='Double_DQN_gmv_loss'))
self.cost_double_dqn_loss = tf.reduce_mean(
tf.squared_difference(self.double_dqn_cost_target,
self.q_eval_cost_wrt_a,
name='Double_DQN_cost_loss'))
self.double_dqn_loss = tf.reduce_mean(
tf.squared_difference(self.double_dqn_target,
self.q_eval_wrt_a,
name='Double_DQN_error'))
self.gmv_montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_gmv_target,
self.q_eval_gmv_wrt_a,
name='MonteCarlo_gmv_loss'))
self.cost_montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_cost_target,
self.q_eval_cost_wrt_a,
name='MonteCarlo_cost_loss'))
self.montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_target,
self.q_eval_wrt_a,
name='MonteCarlo_error'))
self.gmv_td0_error = tf.abs(self.td0_q_gmv_target -
self.q_eval_gmv_wrt_a)
self.cost_td0_error = tf.abs(self.td0_q_cost_target -
self.q_eval_cost_wrt_a)
self.td0_error = tf.abs(self.td0_q_target - self.q_eval_wrt_a)
self.gmv_doubel_dqn_error = tf.abs(self.double_dqn_gmv_target -
self.q_eval_gmv_wrt_a)
self.cost_doubel_dqn_error = tf.abs(self.double_dqn_cost_target -
self.q_eval_cost_wrt_a)
self.doubel_dqn_error = tf.abs(self.double_dqn_target -
self.q_eval_wrt_a)
self.gmv_montecarlo_error = tf.abs(self.montecarlo_gmv_target -
self.q_eval_gmv_wrt_a)
self.cost_montecarlo_error = tf.abs(self.montecarlo_cost_target -
self.q_eval_cost_wrt_a)
self.montecarlo_error = tf.abs(self.montecarlo_target -
self.q_eval_wrt_a)
def __make_update_exp__(self, vals, target_vals):
polyak = 1.0 - 1e-2
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name),
sorted(target_vals, key=lambda v: v.name)):
expression.append(
var_target.assign(polyak * var_target + (1.0 - polyak) * var))
expression = tf.group(*expression)
return expression
def __make_hardreplace_exp__(self, vals, target_vals):
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name),
sorted(target_vals, key=lambda v: v.name)):
expression.append(var_target.assign(var))
expression = tf.group(*expression)
return expression
def build_model_saver(self, var_scope):
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=var_scope)
self.model_saver = tf.train.Saver(var_list=var_list, max_to_keep=1)
def save(self, sess, path, step):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
self.model_saver.save(sess, save_path=path, global_step=step)
def restore(self, sess, path):
self.model_saver.restore(sess, save_path=path)
print('%s model reloaded from %s' % (self.scope_name, path))
def experience(self, new_trajectory, other_info=None):
new_trajectory_gmv = other_info["gmv"]
new_trajectory_cost = other_info["cost"]
if self.use_prioritized_experience_replay:
add_episode(self.prioritized_replay_buffer,
new_trajectory,
gamma=self.gamma)
else:
add_episode(self.replay_buffer, new_trajectory, gamma=self.gamma)
add_episode(self.gmv_replay_buffer,
new_trajectory_gmv,
gamma=self.gamma)
add_episode(self.cost_replay_buffer,
new_trajectory_cost,
gamma=self.gamma)
def get_action(self, sess, obs, is_test=False, other_info=None):
if is_test:
discrete_action = self.greedy_action(sess, obs, other_info)
else:
discrete_action = self.choose_action(sess, obs, other_info)
bid_max = MultiUserEnv.bid_max
bid_min = MultiUserEnv.bid_min
other_action_info = {"learning_action": discrete_action}
return bid_min + (bid_max - bid_min) / (
self.n_actions - 1) * discrete_action, other_action_info
def __greedy__(self, sess, observation, roi_thr):
observations = observation[np.newaxis, :]
actions_value = sess.run(self.q_eval,
feed_dict={
self.s: observations,
self.roi_thr: roi_thr
})
greedy_action = np.argmax(actions_value, axis=1)[0]
return greedy_action
def __epsilon_greedy__(self, sess, observation, roi_thr):
if np.random.uniform() < self.epsilon:
action = np.random.randint(0, self.n_actions)
else:
action = self.__greedy__(sess, observation, roi_thr)
return action
def choose_action(self, sess, observation, other_info):
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
return self.__epsilon_greedy__(sess, observation, roi_thr)
def greedy_action(self, sess, observation, other_info):
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
greedy_action = self.__greedy__(sess, observation, roi_thr)
if self.use_budget_control:
user_idx = other_info["user_idx"]
request_idx = other_info["request_idx"]
roi_threshold = self.get_roi_threshold()
if request_idx == 0:
observations = np.expand_dims(observation, axis=0)
max_plongterm_roi = sess.run(self.plongterm_roi,
feed_dict={
self.s: observations,
self.a: [greedy_action],
})
if max_plongterm_roi >= roi_threshold:
self.explore_user(user_idx)
return greedy_action
else:
return 0
else:
if self.is_user_selected(user_idx):
return greedy_action
else:
return 0
else:
return greedy_action
def get_memory_returns(self):
if self.use_prioritized_experience_replay:
return self.prioritized_replay_buffer.current_mean_return
else:
return self.replay_buffer.current_mean_return
def _is_exploration_enough(self, min_pool_size):
if self.use_prioritized_experience_replay:
return len(self.prioritized_replay_buffer) >= min_pool_size
else:
return len(self.replay_buffer) >= min_pool_size
def update_target(self, sess):
if self.softupdate:
if self.epoch % self.soft_update_iter == 0:
sess.run(self.update_gmv_target_q)
sess.run(self.update_cost_target_q)
else:
if self.epoch % self.replace_target_iter == 0:
sess.run(self.target_gmv_replace_op)
sess.run(self.target_cost_replace_op)
def train(self, sess):
if self.has_target_net:
self.update_target(sess)
self.epoch += 1
if not self._is_exploration_enough(self.batch_size):
return False, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 0, 0
if self.use_prioritized_experience_replay:
loss, montecarlo_loss, q_eval, returns, \
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns, \
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = self.train_prioritized(sess)
else:
loss, montecarlo_loss, q_eval, returns, \
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns, \
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = self.train_normal(sess)
if self.epoch % self.epsilon_dec_iter == 0:
self.epsilon = max(self.epsilon - self.epsilon_dec,
self.epsilon_min)
self.epsilon_dec_iter //= 1.5
self.epsilon_dec_iter = max(self.epsilon_dec_iter,
self.epsilon_dec_iter_min)
print("update epsilon:", self.epsilon)
return True, [
0, 0, loss, montecarlo_loss, q_eval, returns, gmv_loss,
gmv_montecarlo_loss, gmv_q_eval, gmv_returns, cost_loss,
cost_montecarlo_loss, cost_q_eval, cost_returns
], self.get_memory_returns(), self.epsilon
def train_prioritized(self, sess):
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns = 0, 0, 0, 0
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = 0, 0, 0, 0
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
for idx in range(self.update_times_per_train):
sample_indices = self.prioritized_replay_buffer.make_index(
self.batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns, weights, ranges = self.prioritized_replay_buffer.sample_index(
sample_indices)
obs, act, rew_gmv, obs_next, done, dis_2_end, gmv_returns, weights, ranges = self.gmv_replay_buffer.sample_index(
sample_indices)
obs, act, rew_cost, obs_next, done, dis_2_end, cost_returns = self.cost_replay_buffer.sample_index(
sample_indices)
_, loss, montecarlo_loss, q_eval, \
_1, gmv_loss, gmv_montecarlo_loss, gmv_q_eval, \
_2, cost_loss, cost_montecarlo_loss, cost_q_eval, \
priority_values = sess.run(
[self._train_op, self.loss, self.montecarlo_loss, self.q_eval_wrt_a,
self._train_gmv_op, self.gmv_loss, self.gmv_montecarlo_loss, self.q_eval_gmv_wrt_a,
self._train_cost_op, self.cost_loss, self.cost_montecarlo_loss, self.q_eval_cost_wrt_a,
self.priority_values],
feed_dict={
self.s: obs,
self.a: act,
self.r_gmv: rew_gmv,
self.r_cost: rew_cost,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_gmv_value: gmv_returns,
self.return_cost_value: cost_returns,
self.return_value: returns,
self.important_sampling_weight_ph: weights,
self.roi_thr: roi_thr
})
priorities = priority_values + 1e-6
self.prioritized_replay_buffer.update_priorities(
sample_indices, priorities)
return loss, montecarlo_loss, np.average(q_eval), np.average(returns), \
gmv_loss, gmv_montecarlo_loss, np.average(gmv_q_eval), np.average(gmv_returns), \
cost_loss, cost_montecarlo_loss, np.average(cost_q_eval), np.average(cost_returns)
def train_normal(self, sess):
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns = 0, 0, 0, 0
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = 0, 0, 0, 0
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
for idx in range(self.update_times_per_train):
sample_indices = self.replay_buffer.make_index(self.batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns = self.replay_buffer.sample_index(
sample_indices)
obs, act, rew_gmv, obs_next, done, dis_2_end, gmv_returns = self.gmv_replay_buffer.sample_index(
sample_indices)
obs, act, rew_cost, obs_next, done, dis_2_end, cost_returns = self.cost_replay_buffer.sample_index(
sample_indices)
_, loss, montecarlo_loss, q_eval, \
_1, gmv_loss, gmv_montecarlo_loss, gmv_q_eval, \
_2, cost_loss, cost_montecarlo_loss, cost_q_eval \
= sess.run(
[self._train_op, self.loss, self.montecarlo_loss, self.q_eval_wrt_a,
self._train_gmv_op, self.gmv_loss, self.gmv_montecarlo_loss, self.q_eval_gmv_wrt_a,
self._train_cost_op, self.cost_loss, self.cost_montecarlo_loss, self.q_eval_cost_wrt_a],
feed_dict={
self.s: obs,
self.a: act,
self.r_gmv: rew_gmv,
self.r_cost: rew_cost,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_gmv_value: gmv_returns,
self.return_cost_value: cost_returns,
self.return_value: returns,
self.roi_thr: roi_thr
})
return loss, montecarlo_loss, np.average(q_eval), np.average(returns), \
gmv_loss, gmv_montecarlo_loss, np.average(gmv_q_eval), np.average(gmv_returns), \
cost_loss, cost_montecarlo_loss, np.average(cost_q_eval), np.average(cost_returns)
class ConstrainedPPO(CMDPAgent):
def init_parameters(self, sess):
if self.has_target_net:
super(CMDPAgent, self).init_parameters(sess)
sess.run(self.a_target_replace_op)
def __init__(self, user_num, n_actions, cvr_n_features, ppo_n_features, init_roi, budget, use_budget_control,
use_prioritized_experience_replay,
max_trajectory_length,
update_times_per_train=1, use_predict_cvr=False):
self.user_num = user_num
self.use_budget_control = use_budget_control
self.update_times_per_train = update_times_per_train
self.n_actions = n_actions
self.action_dim = 1
self.cvr_n_features = cvr_n_features
self.ppo_n_features = ppo_n_features
self.lr = 0.001
self.use_predict_cvr = use_predict_cvr
self.user_based_adjust_times = 40
self.epsilon = 0.4
self.epsilon_min = 0.05
self.epsilon_dec = 0.1
self.epsilon_dec_iter = 5000 // self.user_based_adjust_times
self.epsilon_dec_iter_min = 500 // self.user_based_adjust_times
self.epsilon_clip = 0.2
self.lam = 0.5
self.update_step = 1
self.kl_target = 0.01
self.gamma = 1.
self.method = 'clip'
self.policy_logvar = 1e-7
self.replace_target_iter = 1
self.soft_update_iter = 1
self.softupdate = False
self.scope_name = "CPPO-model"
self.epoch = 0
self.cvr_buffer_size = 1000 * max_trajectory_length
self.cvr_batch_size = 512
self.cvr_replay_buffer = ReplayBuffer(self.cvr_buffer_size, save_return=False)
self.alpha = 0.6
self.beta = 0.4
self.use_prioritized_experience_replay = use_prioritized_experience_replay
self.ppo_buffer_size = 1000 * max_trajectory_length
self.ppo_batch_size = 250
if self.use_prioritized_experience_replay:
self.prioritized_replay_buffer = PrioritizedReplayBuffer(self.ppo_buffer_size, alpha=self.alpha,
max_priority=20.)
else:
self.replay_buffer = ReplayBuffer(self.ppo_buffer_size, save_return=True)
with tf.variable_scope(self.scope_name):
self._build_net()
self.build_model_saver(self.scope_name)
def _build_cvr_net(self, state, variable_scope, reuse=False):
with tf.variable_scope(variable_scope, reuse=reuse):
user_id_embedding_table = tf.get_variable(
name="user_id", shape=[self.user_num, 10], initializer=initializers.xavier_initializer(),
trainable=True, dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(state, units=n_features, activation=tf.nn.relu, name='fc1',
kernel_initializer=initializers.xavier_initializer())
fc2 = tf.layers.dense(fc1, units=n_features // 2, activation=tf.nn.relu, name='fc2',
kernel_initializer=initializers.xavier_initializer())
fc3 = tf.layers.dense(fc2, units=n_features // 2, activation=tf.nn.relu, name='fc3',
kernel_initializer=initializers.xavier_initializer())
cvr_out = tf.sigmoid(tf.layers.dense(fc3, units=1, name='cvr',
kernel_initializer=initializers.xavier_initializer()))
return cvr_out
def _build_action_net(self, state, variable_scope):
with tf.variable_scope(variable_scope):
user_id_embedding_table = tf.get_variable(
name="user_id", shape=[self.user_num, 10], initializer=initializers.xavier_initializer(),
trainable=True, dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(state, units=n_features, activation=tf.nn.relu, name='fc1',
kernel_initializer=initializers.xavier_initializer())
fc2 = tf.layers.dense(fc1, units=n_features // 2, activation=tf.nn.relu, name='fc2',
kernel_initializer=initializers.xavier_initializer())
fc3 = tf.layers.dense(fc2, units=n_features // 4, activation=tf.nn.relu, name='fc3',
kernel_initializer=initializers.xavier_initializer())
a_prob = tf.layers.dense(fc3, self.n_actions, tf.nn.softmax,
kernel_initializer=initializers.xavier_initializer())
return a_prob
def _build_q_net(self, state, variable_scope, reuse=False):
with tf.variable_scope(variable_scope, reuse=reuse):
user_id_embedding_table = tf.get_variable(
name="user_id", shape=[self.user_num, 10], initializer=initializers.xavier_initializer(),
trainable=True, dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(state, units=n_features, activation=tf.nn.relu, name='fc1',
kernel_initializer=initializers.xavier_initializer())
fc2 = tf.layers.dense(fc1, units=n_features // 2, activation=tf.nn.relu, name='fc2',
kernel_initializer=initializers.xavier_initializer())
fc3 = tf.layers.dense(fc2, units=n_features // 4, activation=tf.nn.relu, name='fc3',
kernel_initializer=initializers.xavier_initializer())
v = tf.layers.dense(fc3, 1, kernel_initializer=initializers.xavier_initializer())
return v[:, 0]
def __make_update_exp__(self, vals, target_vals):
polyak = 1.0 - 1e-2
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name), sorted(target_vals, key=lambda v: v.name)):
expression.append(var_target.assign(polyak * var_target + (1.0 - polyak) * var))
expression = tf.group(*expression)
return expression
def _build_net(self):
self.s_cvr = tf.placeholder(tf.float32, [None, self.cvr_n_features], name='s_cvr')
self.cvr = tf.placeholder(tf.float32, [None, ], name='r')
self.s = tf.placeholder(tf.float32, [None, self.ppo_n_features], name='s')
self.s_ = tf.placeholder(tf.float32, [None, self.ppo_n_features], name='s_')
self.r = tf.placeholder(tf.float32, [None, ], name='r')
self.a = tf.placeholder(tf.int32, [None, ], name='a')
self.adv = tf.placeholder(tf.float32, [None, ], name='advantage')
self.gamma = 1.
self.done = tf.placeholder(tf.float32, [None, ], name='done')
self.return_value = tf.placeholder(tf.float32, [None, ], name='return')
self.important_sampling_weight_ph = tf.placeholder(tf.float32, [None], name="important_sampling_weight")
self.cvr_net = self._build_cvr_net(self.s_cvr, variable_scope="cvr_net")
self.predicted_cvr = self.cvr_net[:, 0]
self.a_eval = self._build_action_net(self.s, variable_scope="actor_eval_net")
self.a_target = self._build_action_net(self.s, variable_scope="actor_target_net")
self.critic = self._build_q_net(self.s, variable_scope="eval_q_net")
ae_params = scope_vars(absolute_scope_name("actor_eval_net"))
at_params = scope_vars(absolute_scope_name("actor_target_net"))
e_gmv_params = scope_vars(absolute_scope_name("eval_q_net"))
cvr_params = scope_vars(absolute_scope_name("cvr_net"))
with tf.variable_scope('hard_replacement'):
self.a_target_replace_op = tf.group([tf.assign(t, e) for t, e in zip(at_params, ae_params)])
with tf.variable_scope('loss'):
self.cvr_loss = tf.reduce_mean(tf.squared_difference(self.predicted_cvr, self.cvr))
self._build_loss()
self._pick_loss()
with tf.variable_scope('train'):
self._train_cvr_op = tf.train.AdamOptimizer(self.lr).minimize(self.cvr_loss, var_list=cvr_params)
self._train_ppo_critic_op = tf.train.AdamOptimizer(self.lr).minimize(self.critic_loss)
self._train_ppo_actor_op = tf.train.AdamOptimizer(self.lr).minimize(self.actor_loss)
def _pick_loss(self):
self.has_target_net = True
self.critic_loss = self.closs
self.actor_loss = self.aloss
def _build_loss(self):
with tf.variable_scope('critic'):
self.c_loss = self.return_value - self.critic
self.closs = tf.reduce_mean(tf.square(self.c_loss))
self.advantage = self.return_value - self.critic
with tf.variable_scope('surrogate'):
a_indices = tf.stack([tf.range(tf.shape(self.a)[0], dtype=tf.int32), self.a], axis=1)
pi_prob = tf.gather_nd(params=self.a_eval, indices=a_indices)
oldpi_prob = tf.gather_nd(params=self.a_target, indices=a_indices)
ratio = pi_prob / (oldpi_prob + 1e-8)
surr = ratio * self.adv
if self.method == 'kl_pen':
kl = tf.distributions.kl_divergence(self.a_target, self.a_eval)
self.kl_mean = tf.reduce_mean(kl)
self.aloss = -(tf.reduce_mean(surr - self.lam * kl))
else:
self.aloss = -tf.reduce_mean(tf.minimum(
surr,
tf.clip_by_value(ratio, 1. - self.epsilon_clip, 1. + self.epsilon_clip) * self.adv))
def build_model_saver(self, var_scope):
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope=var_scope)
self.model_saver = tf.train.Saver(var_list=var_list, max_to_keep=1)
def save(self, sess, path, step):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
self.model_saver.save(sess, save_path=path, global_step=step)
def restore(self, sess, path):
self.model_saver.restore(sess, save_path=path)
print('%s model reloaded from %s' % (self.scope_name, path))
def experience(self, new_trajectory, other_info=None):
cvr_trajectory = other_info["cvr"]
for ele in cvr_trajectory:
state, cvr = ele
self.cvr_replay_buffer.add(state, 0, cvr, state, 0, 0, 0)
def experience_cmdp(self, new_trajectory, other_info=None):
if self.use_prioritized_experience_replay:
add_episode(self.prioritized_replay_buffer, new_trajectory, gamma=self.gamma)
else:
add_episode(self.replay_buffer, new_trajectory, gamma=self.gamma)
def get_agent_name(self):
return self.scope_name
def get_action(self, sess, obs, is_test=False, other_info=None):
item_price = other_info["proxy_ad_price"]
ground_truth_cvr = other_info["cvr"]
user_alpha = other_info["user_alpha"]
roi_thr = other_info["roi_thr"]
observations = obs[np.newaxis, :]
cvr = sess.run(self.predicted_cvr, feed_dict={
self.s_cvr: observations
})[0]
if self.use_predict_cvr:
bid = cvr * item_price / roi_thr
else:
bid = ground_truth_cvr * item_price / roi_thr
return bid, {"cvr_over_estimate": [user_alpha, ground_truth_cvr, cvr]}
def get_cmdp_action(self, sess, obs, is_test=False, other_info=None):
if is_test:
discrete_action = self.__greedy__(sess, obs)
else:
discrete_action = self.__epsilon_greedy__(sess, obs)
return discrete_action
def __greedy__(self, sess, observation):
s = observation[np.newaxis, :]
prob_weights = sess.run(self.a_eval, feed_dict={self.s: s})
greedy_action = np.argmax(prob_weights, axis=1)[0]
return greedy_action
def __epsilon_greedy__(self, sess, observation):
if np.random.uniform() < self.epsilon:
action = np.random.randint(0, self.n_actions)
else:
action = self.__greedy__(sess, observation)
return action
def _is_exploration_enough(self, buffer, min_pool_size):
return len(buffer) >= min_pool_size
def train_cvr(self, sess):
if not self._is_exploration_enough(self.cvr_replay_buffer, self.cvr_batch_size):
return False, [0, 0, 0]
cvr_loss, predicted_cvrs, cvr_targets = 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.cvr_replay_buffer.make_index(self.cvr_batch_size)
obs, act, cvr_targets, obs_next, done, dis_2_end, returns = self.cvr_replay_buffer.sample_index(
sample_indices)
_, cvr_loss, predicted_cvrs = sess.run(
[self._train_cvr_op, self.cvr_loss, self.predicted_cvr],
feed_dict={
self.s_cvr: obs,
self.cvr: cvr_targets
}
)
return True, [cvr_loss, np.average(predicted_cvrs), np.average(cvr_targets)]
def get_memory_returns(self):
if self.use_prioritized_experience_replay:
return self.prioritized_replay_buffer.current_mean_return
else:
return self.replay_buffer.current_mean_return
def update_target(self, sess):
if self.epoch % self.replace_target_iter == 0:
sess.run(self.a_target_replace_op)
def train(self, sess):
if self.has_target_net:
self.update_target(sess)
self.epoch += 1
buffer = self.prioritized_replay_buffer if self.use_prioritized_experience_replay else self.replay_buffer
if not self._is_exploration_enough(buffer, self.ppo_batch_size):
return False, [0, 0, 0, 0], 0, 0
if self.use_prioritized_experience_replay:
loss, montecarlo_loss, q_eval, returns = self.train_prioritized(sess)
else:
loss, montecarlo_loss, q_eval, returns = self.train_normal(sess)
if self.epoch % self.epsilon_dec_iter == 0:
self.epsilon = max(self.epsilon - self.epsilon_dec, self.epsilon_min)
print("update epsilon:", self.epsilon)
return True, [loss, montecarlo_loss, q_eval, returns], self.get_memory_returns(), self.epsilon
def train_prioritized(self, sess):
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.prioritized_replay_buffer.make_index(self.ppo_batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns, weights, ranges = self.prioritized_replay_buffer.sample_index(
sample_indices)
_, loss, montecarlo_loss, q_eval, \
priority_values = sess.run(
[self._train_ppo_op, self.loss, self.montecarlo_loss, self.q_eval_wrt_a,
self.priority_values],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
self.important_sampling_weight_ph: weights,
})
priorities = priority_values + 1e-6
self.prioritized_replay_buffer.update_priorities(sample_indices, priorities)
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
def train_normal(self, sess):
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.replay_buffer.make_index(self.ppo_batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns = self.replay_buffer.sample_index(
sample_indices)
adv = sess.run(self.advantage, {self.s: obs, self.return_value: returns})
_, montecarlo_loss, q_eval = sess.run(
[self._train_ppo_critic_op, self.critic_loss, self.critic],
feed_dict={
self.s: obs,
self.a: act,
self.adv: adv,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
})
if self.method == 'kl_pen':
for _ in range(self.update_step):
_, kl, loss = sess.run(
[self._train_ppo_actor_op, self.kl_mean, self.actor_loss],
feed_dict={
self.adv: adv,
self.s: obs,
self.a: act,
self.r: rew,
self.done: done,
})
if kl > 4 * self.kl_target:
break
if kl < self.kl_target / 1.5:
self.lam /= 2
elif kl > self.kl_target * 1.5:
self.lam *= 2
self.lam = np.clip(self.lam, 1e-4, 10)
else:
for _ in range(self.update_step):
_, loss = sess.run(
[self._train_ppo_actor_op, self.actor_loss],
feed_dict={
self.adv: adv,
self.s: obs,
self.a: act,
self.r: rew,
self.done: done,
self.return_value: returns,
})
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
class DDPG_interface(LearningAgent, PIDAgent):
def __init__(
self,
user_num,
action_dim,
action_bound,
n_features,
init_roi,
budget,
use_budget_control,
use_prioritized_experience_replay,
max_trajectory_length,
update_times_per_train,
):
PIDAgent.__init__(self,
init_roi=init_roi,
default_alpha=1,
budget=budget,
integration=2)
self.use_budget_control = use_budget_control
self.user_num = user_num
self.action_bound = action_bound
self.action_dim = action_dim
self.n_actions = 1
self.n_features = n_features
self.gamma = 1.
self.update_times_per_train = update_times_per_train
self.lr = 0.001
self.epsilon = 0.9
self.epsilon_min = 0.1
self.epsilon_dec = 0.3
self.epsilon_dec_iter = 100
self.replace_target_iter = 300
self.soft_update_iter = 1
self.softupdate = True
self.scope_name = "DDPG-model"
self.epoch = 0
self.exploration_noise = OUNoise(self.action_dim)
self.noise_weight = 1
self.noise_descrement_per_sampling = 0.0001
self.buffer_size = 20000 * max_trajectory_length
self.batch_size = 512
self.alpha = 0.6
self.beta = 0.4
self.use_prioritized_experience_replay = use_prioritized_experience_replay
if self.use_prioritized_experience_replay:
self.prioritized_replay_buffer = PrioritizedReplayBuffer(
self.buffer_size, alpha=self.alpha, max_priority=20.)
else:
self.replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
self.cost_replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
self.gmv_replay_buffer = ReplayBuffer(self.buffer_size,
save_return=True)
with tf.variable_scope(self.scope_name):
self._build_net()
self.build_model_saver(self.scope_name)
def _build_net(self):
self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s')
self.s_ = tf.placeholder(tf.float32, [None, self.n_features],
name='s_')
self.r_gmv = tf.placeholder(tf.float32, [
None,
], name='r_gmv')
self.r_cost = tf.placeholder(tf.float32, [
None,
], name='r_cost')
self.r = tf.placeholder(tf.float32, [
None,
], name='r')
self.roi_thr = tf.placeholder(tf.float32, [], name="roi_thr")
self.a = tf.placeholder(tf.float32, [
None,
], name='a')
self.done = tf.placeholder(tf.float32, [
None,
], name='done')
self.gmv_return_value = tf.placeholder(tf.float32, [
None,
],
name='gmv_return')
self.cost_return_value = tf.placeholder(tf.float32, [
None,
],
name='cost_return')
self.return_value = tf.placeholder(tf.float32, [
None,
], name='return')
self.important_sampling_weight_ph = tf.placeholder(
tf.float32, [None], name="important_sampling_weight")
self.a_eval = self._build_action_net(self.s,
variable_scope="actor_eval_net")
self.a_target = self._build_action_net(
self.s_, variable_scope="actor_target_net")
self.gmv_critic_eval = self._build_q_net(
self.s, self.a, variable_scope="gmv_critic_eval_net")
self.gmv_critic_eval_for_loss = self._build_q_net(
self.s,
self.a_eval,
variable_scope="gmv_critic_eval_net",
reuse=True)
self.gmv_critic_target = self._build_q_net(
self.s_, self.a_target, variable_scope="gmv_critic_target_net")
self.cost_critic_eval = self._build_q_net(
self.s, self.a, variable_scope="cost_critic_eval_net")
self.cost_critic_eval_for_loss = self._build_q_net(
self.s,
self.a_eval,
variable_scope="cost_critic_eval_net",
reuse=True)
self.cost_critic_target = self._build_q_net(
self.s_, self.a_target, variable_scope="cost_critic_target_net")
self.critic_eval = self.gmv_critic_eval - self.roi_thr * self.cost_critic_eval
self.critic_eval_for_loss = self.gmv_critic_eval_for_loss - self.roi_thr * self.cost_critic_eval_for_loss
self.critic_target = self.gmv_critic_target - self.roi_thr * self.cost_critic_target
ae_params = scope_vars(absolute_scope_name("actor_eval_net"))
at_params = scope_vars(absolute_scope_name("actor_target_net"))
gmv_ce_params = scope_vars(absolute_scope_name("gmv_critic_eval_net"))
gmv_ct_params = scope_vars(
absolute_scope_name("gmv_critic_target_net"))
cost_ce_params = scope_vars(
absolute_scope_name("cost_critic_eval_net"))
cost_ct_params = scope_vars(
absolute_scope_name("cost_critic_target_net"))
print(ae_params)
print(at_params)
print(gmv_ce_params)
print(gmv_ct_params)
print(cost_ce_params)
print(cost_ct_params)
with tf.variable_scope('hard_replacement'):
self.a_target_replace_op = tf.group(
[tf.assign(t, e) for t, e in zip(at_params, ae_params)])
self.gmv_c_target_replace_op = tf.group([
tf.assign(t, e) for t, e in zip(gmv_ct_params, gmv_ce_params)
])
self.cost_c_target_replace_op = tf.group([
tf.assign(t, e) for t, e in zip(cost_ct_params, cost_ce_params)
])
with tf.variable_scope('soft_update'):
self.a_update_target_q = self.__make_update_exp__(
ae_params, at_params)
self.gmv_c_update_target_q = self.__make_update_exp__(
gmv_ce_params, gmv_ct_params)
self.cost_c_update_target_q = self.__make_update_exp__(
cost_ce_params, cost_ct_params)
with tf.variable_scope('q_target'):
self.td0_gmv_q_target = tf.stop_gradient(self.r_gmv + self.gamma *
(1. - self.done) *
self.gmv_critic_target)
self.td0_cost_q_target = tf.stop_gradient(self.r_cost +
self.gamma *
(1. - self.done) *
self.cost_critic_target)
self.td0_q_target = tf.stop_gradient(self.r + self.gamma *
(1. - self.done) *
self.critic_target)
self.montecarlo_gmv_target = self.gmv_return_value
self.montecarlo_cost_target = self.cost_return_value
self.montecarlo_target = self.return_value
with tf.variable_scope('loss'):
self._build_loss()
self._pick_loss()
with tf.variable_scope('train'):
self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(
self.loss, var_list=gmv_ce_params + cost_ce_params)
self._train_gmv_c_op = tf.train.AdamOptimizer(self.lr).minimize(
self.gmv_loss, var_list=gmv_ce_params)
self._train_cost_c_op = tf.train.AdamOptimizer(self.lr).minimize(
self.cost_loss, var_list=cost_ce_params)
self._train_a_op = tf.train.AdamOptimizer(self.lr).minimize(
self.actor_loss, var_list=ae_params)
with tf.variable_scope('roi'):
self.max_longterm_roi = self.gmv_critic_eval / (
self.cost_critic_eval + 1e-4)
def _pick_loss(self):
self.has_target_net = True
self.loss = self.td_loss
self.gmv_loss = self.gmv_td_loss
self.cost_loss = self.cost_td_loss
self.actor_loss = self.a_loss
self.priority_values = self.montecarlo_gmv_error + self.montecarlo_cost_error
def _build_loss(self):
if self.use_prioritized_experience_replay:
self.gmv_td_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.td0_gmv_q_target,
self.gmv_critic_eval,
name='TD0_gmv_loss'))
self.cost_td_loss = tf.reduce_mean(
self.important_sampling_weight_ph *
tf.squared_difference(self.td0_cost_q_target,
self.cost_critic_eval,
name='TD0_cost_loss'))
else:
self.gmv_td_loss = tf.reduce_mean(
tf.squared_difference(self.td0_gmv_q_target,
self.gmv_critic_eval,
name='TD0_gmv_loss'))
self.cost_td_loss = tf.reduce_mean(
tf.squared_difference(self.td0_cost_q_target,
self.cost_critic_eval,
name='TD0_cost_loss'))
self.td_loss = tf.reduce_mean(
tf.squared_difference(self.td0_q_target,
self.critic_eval,
name='TD0_loss'))
self.a_loss = -tf.reduce_mean(self.critic_eval_for_loss)
self.gmv_montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_gmv_target,
self.gmv_critic_eval,
name='MonteCarlo_gmv_error'))
self.cost_montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_cost_target,
self.cost_critic_eval,
name='MonteCarlo_cost_error'))
self.montecarlo_loss = tf.reduce_mean(
tf.squared_difference(self.montecarlo_target,
self.critic_eval,
name='MonteCarlo_error'))
self.td0_gmv_error = tf.abs(self.td0_gmv_q_target -
self.gmv_critic_eval)
self.td0_cost_error = tf.abs(self.td0_cost_q_target -
self.cost_critic_eval)
self.td0_error = tf.abs(self.td0_q_target - self.critic_eval)
self.montecarlo_gmv_error = tf.abs(self.montecarlo_gmv_target -
self.gmv_critic_eval)
self.montecarlo_cost_error = tf.abs(self.montecarlo_cost_target -
self.cost_critic_eval)
self.montecarlo_error = tf.abs(self.montecarlo_target -
self.critic_eval)
def _build_q_net(self, state, action, variable_scope, reuse=False):
with tf.variable_scope(variable_scope, reuse=reuse):
user_id_embedding_table = tf.get_variable(
name="user_id",
shape=[self.user_num, 20],
initializer=initializers.xavier_initializer(),
trainable=True,
dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(
user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
state = tf.concat(
[state,
tf.expand_dims(action, axis=1, name="2d-action")],
axis=1)
fc1 = tf.layers.dense(state,
units=n_features,
activation=tf.nn.relu,
name='fc1')
fc2 = tf.layers.dense(fc1,
units=n_features // 2,
activation=tf.nn.relu,
name='fc2')
q = tf.layers.dense(fc2, units=self.action_dim, name='q')
return q[:, 0]
def _build_action_net(self, state, variable_scope):
with tf.variable_scope(variable_scope):
user_id_embedding_table = tf.get_variable(
name="user_id",
shape=[self.user_num, 20],
initializer=initializers.xavier_initializer(),
trainable=True,
dtype=tf.float32)
user_id = tf.cast(state[:, 0], dtype=tf.int32)
user_id_embeddings = tf.nn.embedding_lookup(
user_id_embedding_table, ids=user_id, name="user_id_embedding")
state = tf.concat([user_id_embeddings, state[:, 1:]], axis=1)
n_features = state.get_shape()[1]
fc1 = tf.layers.dense(state,
units=n_features // 2,
activation=tf.nn.relu,
name='fc1')
actions = tf.layers.dense(fc1,
self.action_dim,
activation=tf.nn.sigmoid,
name='a')
scaled_a = tf.multiply(actions, 1, name='scaled_a')
return scaled_a[:, 0]
def __make_update_exp__(self, vals, target_vals):
polyak = 1.0 - 1e-2
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name),
sorted(target_vals, key=lambda v: v.name)):
expression.append(
var_target.assign(polyak * var_target + (1.0 - polyak) * var))
expression = tf.group(*expression)
return expression
def __make_hardreplace_exp__(self, vals, target_vals):
expression = []
for var, var_target in zip(sorted(vals, key=lambda v: v.name),
sorted(target_vals, key=lambda v: v.name)):
expression.append(var_target.assign(var))
expression = tf.group(*expression)
return expression
def build_model_saver(self, var_scope):
var_list = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope=var_scope)
self.model_saver = tf.train.Saver(var_list=var_list, max_to_keep=3)
def save(self, sess, path, step):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
self.model_saver.save(sess, save_path=path, global_step=step)
def restore(self, sess, path):
self.model_saver.restore(sess, save_path=path)
print('%s model reloaded from %s' % (self.scope_name, path))
def experience(self, new_trajectory, other_info=None):
new_trajectory_gmv = other_info["gmv"]
new_trajectory_cost = other_info["cost"]
if self.use_prioritized_experience_replay:
add_episode(self.prioritized_replay_buffer,
new_trajectory,
gamma=self.gamma)
else:
add_episode(self.replay_buffer, new_trajectory, gamma=self.gamma)
add_episode(self.gmv_replay_buffer,
new_trajectory_gmv,
gamma=self.gamma)
add_episode(self.cost_replay_buffer,
new_trajectory_cost,
gamma=self.gamma)
def __epsilon_greedy__(self, sess, observation, roi_thr):
if np.random.uniform() < self.epsilon:
observation = observation[np.newaxis, :]
actions_value = sess.run(self.a_eval,
feed_dict={
self.s: observation,
self.roi_thr: roi_thr
})
action_noise = self.exploration_noise.noise()
bid = actions_value + action_noise
bid = bid[0]
else:
bid = self.__greedy__(sess, observation, roi_thr)
return bid
def __greedy__(self, sess, observation, roi_thr):
observation = observation[np.newaxis, :]
bid = sess.run(self.a_eval,
feed_dict={
self.s: observation,
self.roi_thr: roi_thr
})
return bid[0]
def choose_action(self, sess, observation, other_info):
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
return self.__epsilon_greedy__(sess, observation, roi_thr)
def greedy_action(self, sess, observation, other_info):
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
bid = self.__greedy__(sess, observation, roi_thr)
if self.use_budget_control:
user_idx = other_info["user_idx"]
request_idx = other_info["request_idx"]
roi_threshold = self.get_roi_threshold()
if request_idx == 0:
observations = observation[np.newaxis, :]
max_plongterm_roi = sess.run(self.max_longterm_roi,
feed_dict={
self.s: observations,
self.a: [bid]
})
if max_plongterm_roi >= roi_threshold:
self.explore_user(user_idx)
return bid
else:
return 0.
else:
if self.is_user_selected(user_idx):
return bid
else:
return 0
else:
return bid
def get_action(self, sess, obs, is_test=False, other_info=None):
if is_test:
discrete_action = self.greedy_action(sess, obs, other_info)
else:
discrete_action = self.choose_action(sess, obs, other_info)
other_action_info = {"learning_action": discrete_action}
return self.action_bound * np.clip(discrete_action, 0,
1), other_action_info
def get_memory_returns(self):
if self.use_prioritized_experience_replay:
return self.prioritized_replay_buffer.current_mean_return
else:
return self.replay_buffer.current_mean_return
def _is_exploration_enough(self, min_pool_size):
if self.use_prioritized_experience_replay:
return len(self.prioritized_replay_buffer) >= min_pool_size
else:
return len(self.replay_buffer) >= min_pool_size
def update_target(self, sess):
if self.softupdate:
if self.epoch % self.soft_update_iter == 0:
sess.run(self.gmv_c_update_target_q)
sess.run(self.cost_c_update_target_q)
sess.run(self.a_update_target_q)
else:
if self.epoch % self.replace_target_iter == 0:
sess.run(self.gmv_c_update_target_q)
sess.run(self.cost_c_update_target_q)
sess.run(self.a_target_replace_op)
def train(self, sess):
if self.has_target_net:
self.update_target(sess)
self.epoch += 1
if not self._is_exploration_enough(self.batch_size):
return False, [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 0, 0
if self.use_prioritized_experience_replay:
policy_loss, policy_entropy, loss, montecarlo_loss, q_eval, returns, \
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns, \
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = self.train_prioritized(sess)
else:
policy_loss, policy_entropy, loss, montecarlo_loss, q_eval, returns, \
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns, \
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = self.train_normal(sess)
if self.epoch % self.epsilon_dec_iter == 0:
self.epsilon = max(self.epsilon - self.epsilon_dec,
self.epsilon_min)
print("update epsilon:", self.epsilon)
return True, [
policy_loss, policy_entropy, loss, montecarlo_loss, q_eval,
returns, gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns,
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns
], self.get_memory_returns(), self.epsilon
def train_prioritized(self, sess):
loss, q_eval, returns, montecarlo_loss = 0, 0, 0, 0
for idx in range(self.update_times_per_train):
sample_indices = self.prioritized_replay_buffer.make_index(
self.batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns, weights, ranges = self.prioritized_replay_buffer.sample_index(
sample_indices)
_, loss, q_eval, montecarlo_loss, priority_values = sess.run(
[
self._train_c_op, self.loss, self.critic_eval,
self.montecarlo_loss, self.priority_values
],
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
self.important_sampling_weight_ph: weights
})
sess.run(self._train_a_op,
feed_dict={
self.s: obs,
self.a: act,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.return_value: returns,
self.important_sampling_weight_ph: weights
})
priorities = priority_values + 1e-6
self.prioritized_replay_buffer.update_priorities(
sample_indices, priorities)
return loss, montecarlo_loss, np.average(q_eval), np.average(returns)
def train_normal(self, sess):
policy_loss, policy_entropy = 0, 0
loss, montecarlo_loss, q_eval, returns = 0, 0, 0, 0
gmv_loss, gmv_montecarlo_loss, gmv_q_eval, gmv_returns = 0, 0, 0, 0
cost_loss, cost_montecarlo_loss, cost_q_eval, cost_returns = 0, 0, 0, 0
if self.use_budget_control:
roi_thr = self.get_roi_threshold()
else:
roi_thr = self.init_roi
for idx in range(self.update_times_per_train):
sample_indices = self.replay_buffer.make_index(self.batch_size)
obs, act, rew, obs_next, done, dis_2_end, returns = self.replay_buffer.sample_index(
sample_indices)
obs, act, rew_gmv, obs_next, done, dis_2_end, gmv_returns = self.gmv_replay_buffer.sample_index(
sample_indices)
obs, act, rew_cost, obs_next, done, dis_2_end, cost_returns = self.cost_replay_buffer.sample_index(
sample_indices)
_, loss, montecarlo_loss, q_eval, \
_1, gmv_loss, gmv_montecarlo_loss, gmv_q_eval, \
_2, cost_loss, cost_montecarlo_loss, cost_q_eval \
= sess.run(
[self._train_op, self.loss, self.montecarlo_loss, self.critic_eval,
self._train_gmv_c_op, self.gmv_loss, self.gmv_montecarlo_loss, self.gmv_critic_eval,
self._train_cost_c_op, self.cost_loss, self.cost_montecarlo_loss, self.cost_critic_eval],
feed_dict={
self.s: obs,
self.a: act,
self.r_gmv: rew_gmv,
self.r_cost: rew_cost,
self.r: rew,
self.s_: obs_next,
self.done: done,
self.gmv_return_value: gmv_returns,
self.cost_return_value: cost_returns,
self.return_value: returns,
self.roi_thr: roi_thr
})
_, actor_loss = sess.run(
[self._train_a_op, self.actor_loss],
feed_dict={
self.roi_thr: roi_thr,
self.s: obs,
self.a: act,
self.r_gmv: rew_gmv,
self.r_cost: rew_cost,
self.s_: obs_next,
self.done: done,
self.gmv_return_value: gmv_returns,
self.cost_return_value: cost_returns,
})
return 0, 0, loss, montecarlo_loss, np.average(q_eval), np.average(returns), \
gmv_loss, gmv_montecarlo_loss, np.average(gmv_q_eval), np.average(gmv_returns), \
cost_loss, cost_montecarlo_loss, np.average(cost_q_eval), np.average(cost_returns)