def softmax_sampling(self, scores, mask, eta):
scores = scores * mask
scores_padded = layers.squeeze(
fluid_sequence_pad(scores, 0, maxlen=128),
[2]) # (b*s, 1) -> (b, s, 1) -> (b, s)
mask_padded = layers.squeeze(fluid_sequence_pad(mask, 0, maxlen=128),
[2])
seq_lens = fluid_sequence_get_seq_len(scores)
def normalize(scores_padded, mask_padded):
mean_S = layers.reduce_sum(scores_padded, dim=1,
keep_dim=True) / layers.reduce_sum(
mask_padded, dim=1, keep_dim=True)
S = scores_padded - mean_S
std_S = layers.sqrt(
layers.reduce_sum(layers.square(S * mask_padded),
dim=1,
keep_dim=True))
return S / (std_S + self.SAFE_EPS)
norm_S = normalize(scores_padded, mask_padded)
# set mask to large negative values
norm_S = norm_S * mask_padded - (mask_padded *
(-1) + 1) * self.BIG_VALUE
soft_prob = layers.softmax(norm_S / eta) * mask_padded
sampled_id = layers.reshape(layers.sampling_id(soft_prob), [-1, 1])
max_id = layers.cast(layers.cast(seq_lens, 'float32') - 1, 'int64')
sampled_id = layers.elementwise_min(sampled_id, max_id)
return layers.cast(sampled_id, 'int64')
def learn(self, obs, action, reward, next_obs, terminal):
""" 使用DQN算法更新self.model的value网络
"""
# 从target_model中获取 max Q' 的值，用于计算target_Q
next_pred_value = self.target_model.value(next_obs)
best_v = layers.reduce_max(next_pred_value, dim=1)
best_v.stop_gradient = True # 阻止梯度传递
terminal = layers.cast(terminal, dtype='float32')
target = reward + (1.0 - terminal) * self.gamma * best_v
pred_value = self.model.value(obs) # 获取Q预测值
# 将action转onehot向量，比如：3 => [0,0,0,1,0]，独热编码有好处
action_onehot = layers.one_hot(action, self.act_dim)
action_onehot = layers.cast(action_onehot, dtype='float32')
# 下面一行是逐元素相乘，拿到action对应的 Q(s,a)
# 比如：pred_value = [[2.3, 5.7, 1.2, 3.9, 1.4]], action_onehot = [[0,0,0,1,0]]
# ==> pred_action_value = [[3.9]]
pred_action_value = layers.reduce_sum(layers.elementwise_mul(
action_onehot, pred_value),
dim=1)
# 计算 Q(s,a) 与 target_Q的均方差，得到loss
cost = layers.square_error_cost(pred_action_value, target)
cost = layers.reduce_mean(cost)
optimizer = fluid.optimizer.Adam(learning_rate=self.lr) # 使用Adam优化器
optimizer.minimize(cost)
return cost
def define_learn(self, obs, action, reward, next_obs, terminal, weight):
#Q(s,a|θ)
pred_value = self.model.value(obs)
#Q(s',a'|θ')
targetQ_predict_value = self.target_model.value(next_obs)
#Q(s',a'|θ)
next_s_predcit_value = self.model.value(next_obs)
#argMax[Q(s',a'|θ)]
greedy_action = fluid_argmax(next_s_predcit_value)
predict_onehot = fluid.layers.one_hot(greedy_action, self.action_dim)
#Q(s',argMax[Q(s',a'|θ)]|θ')
best_v = fluid.layers.reduce_sum(fluid.layers.elementwise_mul(
predict_onehot, targetQ_predict_value),
dim=1)
best_v.stop_gradient = True
#TD目标: R+γ*Q(s',argMax[Q(s',a'|θ)]|θ')
target = reward + (
1.0 - layers.cast(terminal, dtype='float32')) * self.gamma * best_v
action_onehot = layers.one_hot(action, self.action_dim)
action_onehot = layers.cast(action_onehot, dtype='float32')
pred_action_value = layers.reduce_sum(layers.elementwise_mul(
action_onehot, pred_value),
dim=1)
#计算新的TD-Error
newTd = layers.abs(target - pred_action_value)
cost = layers.square_error_cost(pred_action_value, target)
#weight表示样本的权重，影响cost的更新幅度
cost = weight * cost
cost = layers.reduce_mean(cost)
optimizer = fluid.optimizer.Adam(self.lr, epsilon=1e-3)
optimizer.minimize(cost)
return cost, newTd
def eps_greedy_sampling(self, scores, mask, eps):
scores = scores * mask
scores_padded = layers.squeeze(
fluid_sequence_pad(scores, 0, maxlen=128),
[2]) # (b*s, 1) -> (b, s, 1) -> (b, s)
mask_padded = layers.squeeze(fluid_sequence_pad(mask, 0, maxlen=128),
[2])
seq_lens = fluid_sequence_get_seq_len(scores)
def get_greedy_prob(scores_padded, mask_padded):
s = scores_padded - (mask_padded * (-1) + 1) * self.BIG_VALUE
max_value = layers.reduce_max(s, dim=1, keep_dim=True)
greedy_prob = layers.cast(s >= max_value, 'float32')
return greedy_prob
greedy_prob = get_greedy_prob(scores_padded, mask_padded)
eps_prob = mask_padded * eps / layers.reduce_sum(
mask_padded, dim=1, keep_dim=True)
final_prob = (greedy_prob + eps_prob) * mask_padded
final_prob = final_prob / layers.reduce_sum(
final_prob, dim=1, keep_dim=True)
sampled_id = layers.reshape(layers.sampling_id(final_prob), [-1, 1])
max_id = layers.cast(layers.cast(seq_lens, 'float32') - 1, 'int64')
sampled_id = layers.elementwise_min(sampled_id, max_id)
return layers.cast(sampled_id, 'int64')
def _cut_by_decode_len(self, input, decode_len):
zeros = layers.fill_constant_batch_size_like(input,
shape=[-1, 1],
value=0,
dtype='int64')
output = layers.sequence_slice(layers.cast(input, 'float32'),
offset=zeros,
length=decode_len)
return layers.cast(output, input.dtype)
def dynamic_rnn(self, item_fc, h_0, output_type=None, double_type=None, double_id=None):
drnn = fluid.layers.DynamicRNN()
pos = fluid_sequence_get_pos(item_fc)
with drnn.block():
cur_item_fc = drnn.step_input(item_fc)
cur_h_0 = drnn.memory(init=h_0, need_reorder=True)
cur_item_fc = layers.lod_reset(cur_item_fc, cur_h_0)
next_h_0 = self.simple_step_rnn(cur_item_fc, h_0=cur_h_0)
if output_type == 'c_Q':
Q = self.out_Q_fc2_op(self.out_Q_fc1_op(next_h_0))
drnn.output(Q)
elif output_type in ['max_Q', 'double_Q']:
# batch_size = 2
# item_fc: lod = [0,4,7]
# cur_h_0: lod = [0,1,2]
item_fc = drnn.static_input(item_fc)
pos = drnn.static_input(pos)
cur_step = drnn.memory(shape=[1], dtype='int64', value=0)
expand_h_0 = layers.sequence_expand(cur_h_0, item_fc) # lod = [0,1,2,3,4,5,6,7]
new_item_fc = layers.lod_reset(item_fc, expand_h_0) # lod = [0,1,2,3,4,5,6,7]
next_expand_h_0 = self.simple_step_rnn(new_item_fc, expand_h_0) # lod = [0,1,2,3,4,5,6,7]
next_expand_h_0 = layers.lod_reset(next_expand_h_0, item_fc) # lod = [0,4,7]
expand_Q = self.out_Q_fc2_op(self.out_Q_fc1_op(next_expand_h_0))
cur_step_id = layers.slice(cur_step, axes=[0, 1], starts=[0, 0], ends=[1, 1])
mask = layers.cast(pos >= cur_step_id, 'float32')
expand_Q = expand_Q * mask
if output_type == 'max_Q':
max_Q = layers.sequence_pool(expand_Q, 'max') # lod = [0,1,2]
drnn.output(max_Q)
elif output_type == 'double_Q':
if double_type == 'max_id':
max_id = self.eps_greedy_sampling(expand_Q, mask, eps=0)
drnn.output(max_id)
elif double_type == 'double_Q':
cur_double_id = drnn.step_input(double_id)
double_Q = fluid_sequence_index(expand_Q, cur_double_id)
drnn.output(double_Q)
# update
next_step = cur_step + 1
drnn.update_memory(cur_step, next_step)
elif output_type == 'hidden':
drnn.output(next_h_0)
else:
raise NotImplementedError(output_type)
# update
drnn.update_memory(cur_h_0, next_h_0)
drnn_output = drnn()
return drnn_output
def _ensemble_predict(self, obs):
actor_outputs = []
for i in range(self.ensemble_num):
actor_outputs.append(self.actors[i].predict(obs))
batch_actions = layers.concat(actor_outputs, axis=0)
batch_obs = layers.expand(obs, expand_times=[self.ensemble_num, 1])
critic_outputs = []
for i in range(self.ensemble_num):
critic_output = self.critics[i].predict(batch_obs, batch_actions)
critic_output = layers.unsqueeze(critic_output, axes=[1])
critic_outputs.append(critic_output)
score_matrix = layers.concat(critic_outputs, axis=1)
# Normalize scores given by each critic
sum_critic_score = layers.reduce_sum(
score_matrix, dim=0, keep_dim=True)
sum_critic_score = layers.expand(
sum_critic_score, expand_times=[self.ensemble_num, 1])
norm_score_matrix = score_matrix / sum_critic_score
actions_mean_score = layers.reduce_mean(
norm_score_matrix, dim=1, keep_dim=True)
best_score_id = layers.argmax(actions_mean_score, axis=0)
best_score_id = layers.cast(best_score_id, dtype='int32')
ensemble_predict_action = layers.gather(batch_actions, best_score_id)
ensemble_predict_action = layers.squeeze(
ensemble_predict_action, axes=[0])
return ensemble_predict_action
def learn(self, obs, action, reward, next_obs, terminal):
next_pred_value = self.target_model.value(next_obs)
best_v = layers.reduce_max(next_pred_value, dim=-1)
best_v.stop_gradient = True
terminal = layers.cast(terminal, dtype="float32")
target = reward + (1.0 - terminal) * self.gamma * best_v
pred_value = self.model.value(obs)
action_onehot = layers.one_hot(action, self.act_dim)
action_onehot = layers.cast(action_onehot, dtype="float32")
pred_action_value = layers.reduce_sum(layers.elementwise_mul(
pred_value, action_onehot),
dim=-1)
cost = layers.square_error_cost(target, pred_action_value)
cost = layers.reduce_mean(cost)
optimizer = fluid.optimizer.Adam(learning_rate=self.lr)
optimizer.minimize(cost)
return cost
def learn(self, obs, action, reward, next_obs, terminal):
'''
:param obs: St
:param action: At
:param reward: Rt+1
:param next_obs: St+1
:param terminal: done, True代表episode结束
:return: 损失函数的值
'''
# 通过目标网络计算得到target_Q的值
target_Q_tensor = self.target_model.value(next_obs) # 计算St+1对应的价值向量
max_Q = layers.reduce_max(target_Q_tensor, dim=1) # 获取每行的最大值，按dim=1收缩
max_Q.stop_gradient = True # 停止梯度更新
# 由于terminal不是标量，所以不能直接用判断
terminal = layers.cast(terminal, dtype="float32")
target_Q = reward + (1.0 - terminal) * self.gamma * max_Q
# 通过主网络计算得到perdict_Q的值
predict_Q_tensor = self.model.value(obs)
# 将action转成one-hot向量，并将每一位都变成浮点数
action_onehot = layers.one_hot(action, self.act_dim)
action = layers.cast(action_onehot, dtype="float32")
# 进行elementwise计算并降低张量阶数
# 比如 predict_Q_tensor = [[2.3, 5.7, 1.2, 3.9, 1.4], action_onehot=[[0, 0, 0, 1, 0]
# [2.1, 3.7, 4.5, 6.7, 7.1]] [0, 1, 0, 0, 0]]
# 那么elementwise乘法运算后的结果是 [[0, 0, 0, 3.9, 0]
# [0, 3.7, 0, 0, 0]]
# 再进行dim=1的reduce_sum操作后的结果是 [3.9, 3.7]
predict_Q = layers.reduce_sum(layers.elementwise_mul(
action_onehot, predict_Q_tensor),
dim=1)
# 得到这个batch每条数据的损失函数值的平均值
cost = layers.square_error_cost(predict_Q, target_Q)
cost = layers.reduce_mean(cost)
# 申明优化器（使用Adam优化器）
optimizer = fluid.optimizer.Adam(learning_rate=self.lr)
optimizer.minimize(cost) # 指定优化目标
return cost
def train_rnn(self,
item_fc,
atten_item_fc,
h_0,
pos,
pos_embed,
output_type=''):
shifted_item_fc = fluid_sequence_advance(item_fc, OOV=0)
drnn = fluid.layers.DynamicRNN()
with drnn.block():
cur_item_fc = drnn.step_input(shifted_item_fc)
cur_pos_embed = drnn.step_input(pos_embed)
cur_h_0 = drnn.memory(init=h_0, need_reorder=True)
# step_input will remove lod info
cur_item_fc = layers.lod_reset(cur_item_fc, cur_h_0)
cur_pos_embed = layers.lod_reset(cur_pos_embed, cur_h_0)
next_h_0, hidden_fc = self.sampling_rnn_forward(
cur_item_fc, cur_h_0, cur_pos_embed)
if output_type == 'c_Q':
cur_atten_item_fc = drnn.step_input(atten_item_fc)
cur_atten_item_fc = layers.lod_reset(cur_atten_item_fc,
cur_h_0)
Q = layers.reduce_sum(hidden_fc * cur_atten_item_fc,
dim=1,
keep_dim=True)
drnn.output(Q)
elif output_type == 'max_Q':
cur_pos = drnn.step_input(pos)
pos = drnn.static_input(pos)
atten_item_fc = drnn.static_input(atten_item_fc)
expand_Q = self._dot_attention(hidden_fc, atten_item_fc)
cur_step_id = layers.slice(cur_pos,
axes=[0, 1],
starts=[0, 0],
ends=[1, 1])
mask = layers.cast(pos >= cur_step_id, 'float32')
expand_Q = expand_Q * mask
max_Q = layers.sequence_pool(expand_Q, 'max')
drnn.output(max_Q)
else:
raise NotImplementedError(output_type)
# update
drnn.update_memory(cur_h_0, next_h_0)
drnn_output = drnn()
return drnn_output
def train_rnn(self, item_fc, h_0, pos, pos_embed, output_type=''):
drnn = fluid.layers.DynamicRNN()
with drnn.block():
cur_item_fc = drnn.step_input(item_fc)
cur_pos_embed = drnn.step_input(pos_embed)
cur_h_0 = drnn.memory(init=h_0, need_reorder=True)
# step_input will remove lod info
cur_item_fc = layers.lod_reset(cur_item_fc, cur_h_0)
cur_pos_embed = layers.lod_reset(cur_pos_embed, cur_h_0)
next_h_0, Q = self.sampling_rnn_forward(cur_item_fc, cur_h_0,
cur_pos_embed)
if output_type == 'c_Q':
drnn.output(Q)
elif output_type == 'max_Q':
# e.g. batch_size = 2
# cur_h_0: lod = [0,1,2]
cur_pos = drnn.step_input(pos)
pos = drnn.static_input(pos) # lod = [0,4,7]
item_fc = drnn.static_input(item_fc) # lod = [0,4,7]
# expand
expand_h_0 = layers.sequence_expand(
cur_h_0, item_fc) # lod = [0,1,2,3,4,5,6,7]
expand_pos_embed = layers.sequence_expand(
cur_pos_embed, item_fc) # lod = [0,1,2,3,4,5,6,7]
expand_item_fc = layers.lod_reset(item_fc, expand_h_0)
# forward
_, expand_scores = self.sampling_rnn_forward(
expand_item_fc, expand_h_0, expand_pos_embed)
# reset result lod
expand_Q = layers.lod_reset(expand_scores,
item_fc) # lod = [0,4,7]
cur_step_id = layers.slice(cur_pos,
axes=[0, 1],
starts=[0, 0],
ends=[1, 1])
mask = layers.cast(pos >= cur_step_id, 'float32')
expand_Q = expand_Q * mask
max_Q = layers.sequence_pool(expand_Q, 'max') # lod = [0,1,2]
drnn.output(max_Q)
else:
raise NotImplementedError(output_type)
# update
drnn.update_memory(cur_h_0, next_h_0)
drnn_output = drnn()
return drnn_output
def _critic_learn(self, obs, action, reward, next_obs, terminal):
next_action = self.target_model.policy(next_obs)
next_Q = self.target_model.value(next_obs, next_action)
terminal = layers.cast(terminal, dtype='float32')
target_Q = reward + (1.0 - terminal) * self.gamma * next_Q
target_Q.stop_gradient = True
Q = self.model.value(obs, action)
cost = layers.square_error_cost(Q, target_Q)
cost = layers.reduce_mean(cost)
optimizer = fluid.optimizer.AdamOptimizer(self.critic_lr)
optimizer.minimize(cost)
return cost
def train(self):
"""train"""
inputs = self.model.create_inputs(mode='train')
output_dict = self.model.forward(inputs, mode='train')
total_loss = 0
if 'click' in self._output_type:
click_id = inputs['click_id']
click_prob = output_dict['click_prob']
click_loss = layers.reduce_mean(
layers.cross_entropy(input=click_prob, label=click_id))
total_loss += click_loss
if 'credit' in self._output_type:
credit = inputs['credit'] * self._credit_scale
credit_pred = output_dict['credit_pred']
credit_loss = layers.reduce_mean(
layers.square_error_cost(input=credit_pred, label=credit))
total_loss += credit_loss
if 'rate' in self._output_type:
rate = layers.cast(inputs['click_id'],
'float32') * self._rate_scale
rate_pred = output_dict['rate_pred']
rate_loss = layers.reduce_mean(
layers.square_error_cost(input=rate_pred, label=rate))
total_loss += rate_loss
if self.optimizer == 'Adam':
optimizer = fluid.optimizer.Adam(learning_rate=self.lr,
epsilon=1e-4)
elif self.optimizer == 'SGD':
optimizer = fluid.optimizer.SGD(learning_rate=self.lr)
optimizer.minimize(total_loss)
fetch_dict = OrderedDict()
fetch_dict[
'loss'] = total_loss # don't rename 'loss', which will be used in parallel exe in computational task
if 'click' in self._output_type:
fetch_dict['click_prob'] = click_prob
fetch_dict['click_id'] = click_id
fetch_dict['click_loss'] = click_loss
if 'credit' in self._output_type:
fetch_dict['credit_pred'] = credit_pred / self._credit_scale
fetch_dict['credit'] = credit / self._credit_scale
fetch_dict['credit_loss'] = credit_loss
if 'rate' in self._output_type:
fetch_dict['rate_pred'] = rate_pred / self._rate_scale
fetch_dict['rate'] = rate / self._rate_scale
fetch_dict['rate_loss'] = rate_loss
return {'fetch_dict': fetch_dict}
def test(self):
"""test"""
inputs = self.model.create_inputs(mode='train')
click_id = layers.cast(inputs['click_id'], 'float32') * self._reward_scale
output_dict = self.model.forward(inputs, output_type='c_Q')
c_Q = output_dict['Q']
target_Q = self.get_target_Q(inputs, click_id)
loss = layers.reduce_mean(layers.square_error_cost(c_Q, target_Q))
fetch_dict = OrderedDict()
fetch_dict['loss'] = loss
fetch_dict['c_Q'] = c_Q / self._reward_scale
fetch_dict['click_id'] = click_id / self._reward_scale
return {'fetch_dict': fetch_dict}
def test(self):
"""test"""
inputs = self.model.create_inputs(mode='train')
reward = layers.cast(inputs['reward'], 'float32')
c_Q = self.model.forward(inputs, output_type='c_Q')
max_Q = self.target_model.forward(inputs, output_type='max_Q')
target_Q = self.get_target_Q(max_Q, reward)
loss = layers.reduce_mean(layers.square_error_cost(c_Q, target_Q))
fetch_dict = OrderedDict()
fetch_dict['loss'] = loss
fetch_dict['c_Q'] = c_Q
fetch_dict['reward'] = reward
return {'fetch_dict': fetch_dict}
def _critic_learn(self, obs, action, reward, next_obs, terminal):
next_action = self.target_model.policy(next_obs)
next_Q = self.target_model.value(next_obs, next_action)
terminal = layers.cast(terminal, dtype='float32')
target_Q = reward + (1.0 - terminal) * self.gamma * next_Q
target_Q.stop_gradient = True
Q = self.model.value(obs, action)
cost = layers.square_error_cost(Q, target_Q)
cost = layers.reduce_mean(cost)
# optimizer = fluid.optimizer.AdamOptimizer(self.critic_lrvalue)
optimizer = fluid.optimizer.AdamOptimizer(
learning_rate=fluid.layers.piecewise_decay(
boundaries=self.boundaries, values=self.critic_lrvalue),
regularization=fluid.regularizer.L2Decay(1e-4))
optimizer.minimize(cost)
return cost
def train(self):
"""train"""
inputs = self.model.create_inputs(mode='train')
reward = layers.cast(inputs['reward'], 'float32')
c_Q = self.model.forward(inputs, output_type='c_Q')
max_Q = self.target_model.forward(inputs, output_type='max_Q')
target_Q = self.get_target_Q(max_Q, reward)
loss = layers.reduce_mean(layers.square_error_cost(c_Q, target_Q))
if self.optimizer == 'Adam':
optimizer = fluid.optimizer.Adam(learning_rate=self.lr, epsilon=1e-4)
elif self.optimizer == 'SGD':
optimizer = fluid.optimizer.SGD(learning_rate=self.lr)
optimizer.minimize(loss)
fetch_dict = OrderedDict()
fetch_dict['loss'] = loss # don't rename 'loss', which will be used in parallel exe in computational task
fetch_dict['c_Q'] = c_Q
fetch_dict['reward'] = reward
return {'fetch_dict': fetch_dict}
def train(self):
"""train"""
inputs = self.model.create_inputs(mode='train')
click_id = layers.cast(inputs['click_id'], 'float32') * self._reward_scale
def train_actor(inputs):
output_dict = self.model.forward(inputs, output_type='max_Q')
max_Q = output_dict['Q']
actor_loss = layers.reduce_mean(-1.0 * max_Q)
actor_lr = self.lr * 0.1 # actor lr should be smaller than critic lr, so critic can learn faster
if self.optimizer == 'Adam':
optimizer = fluid.optimizer.Adam(learning_rate=actor_lr, epsilon=1e-4)
elif self.optimizer == 'SGD':
optimizer = fluid.optimizer.SGD(learning_rate=actor_lr)
optimizer.minimize(actor_loss, parameter_list=self.model.actor_param_names)
return actor_loss
def train_critic(inputs, click_id):
output_dict = self.model.forward(inputs, output_type='c_Q')
c_Q = output_dict['Q']
target_Q = self.get_target_Q(inputs, click_id)
target_Q.stop_gradient = True
critic_loss = layers.reduce_mean(layers.square_error_cost(c_Q, target_Q))
if self.optimizer == 'Adam':
optimizer = fluid.optimizer.Adam(learning_rate=self.lr, epsilon=1e-4)
elif self.optimizer == 'SGD':
optimizer = fluid.optimizer.SGD(learning_rate=self.lr)
optimizer.minimize(critic_loss)
return critic_loss
actor_loss = train_actor(inputs)
critic_loss = train_critic(inputs, click_id)
loss = actor_loss + critic_loss
fetch_dict = OrderedDict()
fetch_dict['loss'] = loss # don't rename 'loss', which will be used in parallel exe in computational task
fetch_dict['actor_loss'] = actor_loss
fetch_dict['critic_loss'] = critic_loss
# fetch_dict['click_id'] = click_id / self._reward_scale
return {'fetch_dict': fetch_dict}
def train(self):
"""train"""
inputs = self.model.create_inputs(mode='train')
click_id = layers.cast(inputs['click_id'], 'float32') * self._reward_scale
output_dict = self.model.forward(inputs, output_type='c_Q')
c_Q = output_dict['Q']
target_Q = self.get_target_Q(inputs, click_id)
target_Q.stop_gradient = True
loss = layers.reduce_mean(layers.square_error_cost(c_Q, target_Q))
if self.optimizer == 'Adam':
optimizer = fluid.optimizer.Adam(learning_rate=self.lr, epsilon=1e-4)
elif self.optimizer == 'SGD':
optimizer = fluid.optimizer.SGD(learning_rate=self.lr)
optimizer.minimize(loss)
fetch_dict = OrderedDict()
fetch_dict['loss'] = loss # don't rename 'loss', which will be used in parallel exe in computational task
fetch_dict['c_Q'] = c_Q / self._reward_scale
fetch_dict['click_id'] = click_id / self._reward_scale
return {'fetch_dict': fetch_dict}
def test(self):
"""test"""
inputs = self.model.create_inputs(mode='test')
output_dict = self.model.forward(inputs, mode='test')
fetch_dict = OrderedDict()
if 'click' in self._output_type:
fetch_dict['click_prob'] = output_dict['click_prob']
fetch_dict['click_id'] = inputs['click_id'] + layers.reduce_mean(
output_dict['click_prob']
) * 0 # IMPORTANT!!! equals to label = label, otherwise parallel executor won't get this variable
if 'credit' in self._output_type:
fetch_dict['credit_pred'] = output_dict[
'credit_pred'] / self._credit_scale
fetch_dict['credit'] = inputs['credit'] + layers.reduce_mean(
output_dict['credit_pred']) * 0
if 'rate' in self._output_type:
fetch_dict[
'rate_pred'] = output_dict['rate_pred'] / self._rate_scale
fetch_dict['rate'] = layers.cast(inputs['click_id'], 'float32') \
+ layers.reduce_mean(output_dict['rate_pred']) * 0
return {'fetch_dict': fetch_dict}
def ensemble_predict(self, obs):
""" ensemble predict:
1. For actions of all actors, each critic will score them
and normalize its scores;
2. For each actor, will calculate its score by
average scores given by all critics
3. choose action of the actor whose score is best
"""
actor_outputs = []
for i in range(self.ensemble_num):
actor_outputs.append(self.models[i].policy(obs))
batch_actions = layers.concat(actor_outputs, axis=0)
batch_obs = layers.expand(obs, expand_times=[self.ensemble_num, 1])
critic_outputs = []
for i in range(self.ensemble_num):
critic_output = self.models[i].value(batch_obs, batch_actions)
critic_output = layers.unsqueeze(critic_output, axes=[1])
critic_outputs.append(critic_output)
score_matrix = layers.concat(critic_outputs, axis=1)
# Normalize scores given by each critic
sum_critic_score = layers.reduce_sum(score_matrix,
dim=0,
keep_dim=True)
sum_critic_score = layers.expand(sum_critic_score,
expand_times=[self.ensemble_num, 1])
norm_score_matrix = score_matrix / sum_critic_score
actions_mean_score = layers.reduce_mean(norm_score_matrix,
dim=1,
keep_dim=True)
best_score_id = layers.argmax(actions_mean_score, axis=0)
best_score_id = layers.cast(best_score_id, dtype='int32')
ensemble_predict_action = layers.gather(batch_actions, best_score_id)
return ensemble_predict_action
def test_param_sharing(self):
"""
Test case for parameter sharing between layers of the same type
"""
net = MyNetWork()
## we bind the paras of embedding to those of fc1
batch_size = 10
dict_size = 100
input_cx = np.random.uniform(0, 1, [batch_size, 100]).astype("float32")
input_x = np.random.randint(dict_size,
size=(batch_size, 1)).astype("int64")
#################################
main_program1 = fluid.Program()
with fluid.program_guard(main_program1):
x = layers.data(name='x', shape=[100], dtype="float32")
y1 = net.fc1(input=x)
y11 = net.fc1(input=x)
y2 = net.fc2(input=x)
y3 = net.fc3(input=x)
y4 = net.fc4(input=x)
main_program2 = fluid.Program()
with fluid.program_guard(main_program2):
x_ = layers.data(name='x', shape=[1], dtype="int64")
cx_ = layers.cast(x=layers.one_hot(input=x_, depth=dict_size),
dtype="float32")
y1_ = net.fc1(input=cx_)
y2_ = net.embedding(input=x_)
x1_ = layers.data(name='x1', shape=[100], dtype="float32")
y3_ = net.fc1(input=x1_)
#### we run the startup program only once to make sure
#### only one para init across the two programs
place = fluid.CPUPlace()
exe = fluid.Executor(place)
exe.run(fluid.default_startup_program())
######################################################
outputs = exe.run(main_program1,
feed={"x": input_cx},
fetch_list=[y1, y11, y2, y3, y4])
old_y1 = outputs[0]
self.assertEqual(np.sum(outputs[0].flatten()),
np.sum(outputs[1].flatten()))
self.assertNotEqual(np.sum(outputs[1].flatten()),
np.sum(outputs[2].flatten()))
self.assertNotEqual(np.sum(outputs[3].flatten()),
np.sum(outputs[4].flatten()))
outputs = exe.run(main_program2,
feed={
'x': input_x,
'x1': input_cx
},
fetch_list=[y1_, y2_, y3_])
### test two different layers sharing the same para matrix
self.assertEqual(np.sum(outputs[0].flatten()),
np.sum(outputs[1].flatten()))
### test if the same layer can have the same parameters across two different programs
self.assertEqual(np.sum(outputs[2].flatten()),
np.sum(old_y1.flatten()))
def get_greedy_prob(scores_padded, mask_padded):
s = scores_padded - (mask_padded * (-1) + 1) * self.BIG_VALUE
max_value = layers.reduce_max(s, dim=1, keep_dim=True)
greedy_prob = layers.cast(s >= max_value, 'float32')
return greedy_prob
