def evaluate_fairness(data_split, model, rank_weights, labels, num_samples):
cutoff = rank_weights.size
scores = model(data_split.feature_matrix).numpy()[:, 0]
result = 0.
squared_result = 0.
for qid in range(data_split.num_queries()):
q_scores = data_split.query_values_from_vector(qid, scores)
q_labels = data_split.query_values_from_vector(qid, labels)
if np.sum(q_labels) > 0 and q_labels.size > 1:
sampled_rankings = pl.gumbel_sample_rankings(q_scores,
num_samples,
cutoff=cutoff)[0]
q_n_docs = q_labels.shape[0]
q_cutoff = min(cutoff, q_n_docs)
doc_exposure = np.zeros(q_n_docs, dtype=np.float64)
np.add.at(doc_exposure, sampled_rankings, rank_weights[:q_cutoff])
doc_exposure /= num_samples
swap_reward = doc_exposure[:, None] * q_labels[None, :]
q_result = np.mean((swap_reward - swap_reward.T)**2.)
q_result *= q_n_docs / (q_n_docs - 1.)
q_squared = np.mean(np.abs(swap_reward - swap_reward.T))
q_squared *= q_n_docs / (q_n_docs - 1.)
result += q_result
squared_result += q_squared
result /= data_split.num_queries()
squared_result /= data_split.num_queries()
return result, squared_result
def evaluate_expected(data_split, model, rank_weights, labels, ideal_metrics,
num_samples):
cutoff = rank_weights.size
scores = model(data_split.feature_matrix).numpy()[:, 0]
result = 0.
query_normalized_result = 0.
for qid in range(data_split.num_queries()):
q_scores = data_split.query_values_from_vector(qid, scores)
q_labels = data_split.query_values_from_vector(qid, labels)
sampled_rankings = pl.gumbel_sample_rankings(q_scores,
num_samples,
cutoff=cutoff)[0]
q_result = np.mean(
np.sum(rank_weights[None, :sampled_rankings.shape[1]] *
q_labels[sampled_rankings],
axis=1),
axis=0)
result += q_result
if ideal_metrics[qid] != 0:
query_normalized_result += q_result / ideal_metrics[qid]
result /= data_split.num_queries()
query_normalized_result /= data_split.num_queries()
normalized_result = result / np.mean(ideal_metrics)
return result, normalized_result, query_normalized_result
def lambdarank(rank_weights, labels, scores, n_samples):
n_docs = scores.shape[0]
cutoff = min(rank_weights.shape[0], n_docs)
(_, sampled_inv_rankings, _, _,
gumbel_scores) = pl.gumbel_sample_rankings(scores,
n_samples,
cutoff=None,
inverted=True,
return_gumbel=True)
(greater_i,
lesser_i) = np.where(np.greater(labels[:, None], labels[None, :]))
delta_rank = np.abs(sampled_inv_rankings[:, greater_i] -
sampled_inv_rankings[:, lesser_i])
if n_docs > cutoff:
safe_rank_weights = np.zeros(n_docs)
safe_rank_weights[:cutoff] = rank_weights
else:
safe_rank_weights = rank_weights
delta_weight = np.mean(safe_rank_weights[delta_rank - 1] -
safe_rank_weights[delta_rank],
axis=0)
pair_weight = delta_weight * (labels[greater_i] - labels[lesser_i])
exp_score_diff = np.exp(
np.minimum(scores[greater_i] - scores[lesser_i], 100))
pair_deriv = pair_weight * exp_score_diff / (
(exp_score_diff + 1.) * np.log(2.))
doc_weights = np.zeros(n_docs, dtype=np.float64)
np.add.at(doc_weights, greater_i, pair_deriv)
np.add.at(doc_weights, lesser_i, -pair_deriv)
return doc_weights
def single_ranking_generation(qid,
data_split,
doc_weights,
alpha,
beta,
model=None,
all_policy_scores=None,
return_scores=False):
assert model is not None or policy_scores is not None
n_docs = data_split.query_size(qid)
cutoff = min(alpha.shape[0], n_docs)
if all_policy_scores is None:
q_feat = data_split.query_feat(qid)
policy_scores = model(q_feat)[:, 0].numpy()
else:
policy_scores = data_split.query_values_from_vector(
qid, all_policy_scores)
rankings = pl.gumbel_sample_rankings(policy_scores, 1, cutoff)[0]
q_doc_weights = data_split.query_values_from_vector(qid, doc_weights)
clicks = generate_clicks(rankings, q_doc_weights, alpha, beta)
if return_scores:
return rankings[0, :], clicks[0, :], policy_scores
else:
return rankings[0, :], clicks[0, :]
def PL_rank_2(rank_weights, labels, scores, n_samples=None, sampled_rankings=None):
n_docs = labels.shape[0]
result = np.zeros(n_docs, dtype=np.float64)
cutoff = min(rank_weights.shape[0], n_docs)
assert n_samples is not None or sampled_rankings is not None
if sampled_rankings is None:
sampled_rankings = pl.gumbel_sample_rankings(
scores,
n_samples,
cutoff=cutoff)[0]
else:
n_samples = sampled_rankings.shape[0]
srange = np.arange(n_samples)
crange = np.arange(cutoff)
relevant_docs = np.where(np.not_equal(labels, 0))[0]
n_relevant_docs = relevant_docs.size
weighted_labels = labels[sampled_rankings]*rank_weights[None,:cutoff]
cumsum_labels = np.cumsum(weighted_labels[:,::-1], axis=1)[:,::-1]
np.add.at(result, sampled_rankings[:,:-1], cumsum_labels[:,1:])
result /= n_samples
ninf_mask = np.zeros((n_samples, cutoff-1, n_docs), dtype=np.float64)
ninf_mask[srange[:,None],
crange[None,:-1],
sampled_rankings[:,:-1]] = np.NINF
ninf_mask[:,:] = np.cumsum(ninf_mask, axis=1)
tiled_scores = np.tile(scores[None,None,:], (n_samples, cutoff, 1))
tiled_scores[:,1:,:] += ninf_mask
max_per_rank = np.max(tiled_scores, axis=2)
tiled_scores -= max_per_rank[:,:,None]
denom_per_rank = np.log(np.sum(np.exp(tiled_scores), axis=2))
prob_per_rank = np.exp(tiled_scores - denom_per_rank[:,:,None])
result -= np.mean(
np.sum(prob_per_rank*cumsum_labels[:,:,None], axis=1)
, axis=0, dtype=np.float64)
result[relevant_docs] += np.mean(
np.sum(prob_per_rank[:,:,relevant_docs]*(
rank_weights[None,:cutoff,None]
*labels[None,None,relevant_docs]), axis=1)
, axis=0, dtype=np.float64)
return result
def single_query_generation(qid,
data_split,
n_samples,
doc_weights,
alpha,
beta,
model=None,
all_policy_scores=None,
return_display=False,
store_per_rank=False):
assert model is not None or policy_scores is not None
n_docs = data_split.query_size(qid)
cutoff = min(alpha.shape[0], n_docs)
if all_policy_scores is None:
q_feat = data_split.query_feat(qid)
policy_scores = model(q_feat)[:, 0].numpy()
else:
policy_scores = data_split.query_values_from_vector(
qid, all_policy_scores)
rankings = pl.gumbel_sample_rankings(policy_scores, n_samples, cutoff)[0]
q_doc_weights = data_split.query_values_from_vector(qid, doc_weights)
clicks = generate_clicks(rankings, q_doc_weights, alpha, beta)
if store_per_rank:
store_cutoff = alpha.shape[0]
clicks_per_doc = np.zeros((n_docs, store_cutoff), dtype=np.int32)
ind_tile = np.tile(np.arange(cutoff)[None, :], (n_samples, 1))
np.add.at(clicks_per_doc, (rankings[clicks], ind_tile[clicks]), 1)
else:
clicks_per_doc = np.zeros(n_docs, dtype=np.int32)
np.add.at(clicks_per_doc, rankings[clicks], 1)
if return_display:
if store_per_rank:
displays_per_doc = np.zeros((n_docs, store_cutoff), dtype=np.int32)
np.add.at(displays_per_doc, (rankings, ind_tile), 1)
else:
displays_per_doc = np.zeros(n_docs, dtype=np.int32)
np.add.at(displays_per_doc, rankings, 1)
else:
displays_per_doc = None
return clicks_per_doc, displays_per_doc
def placement_policy_gradient(rank_weights,
labels,
scores,
n_samples=None,
sampled_rankings=None):
n_docs = labels.shape[0]
result = np.zeros(n_docs, dtype=np.float64)
cutoff = min(rank_weights.shape[0], n_docs)
np_scores = scores.numpy()[:, 0]
assert n_samples is not None or sampled_rankings is not None
if sampled_rankings is None:
sampled_rankings = pl.gumbel_sample_rankings(np_scores,
n_samples,
cutoff=cutoff)[0]
else:
n_samples = sampled_rankings.shape[0]
srange = np.arange(n_samples)
crange = np.arange(cutoff)
ninf_mask = np.zeros((n_samples, cutoff, n_docs), dtype=bool)
ninf_mask[srange[:, None], crange[None, 1:],
sampled_rankings[:, :-1]] = True
ninf_mask[:, :] = np.cumsum(ninf_mask, axis=1)
sampled_scores = tf.gather(scores, sampled_rankings)[:, :, 0]
tiled_scores = tf.tile(scores[None, None, :, 0], (n_samples, cutoff, 1))
tiled_scores = tf.where(ninf_mask, np.NINF, tiled_scores)
max_per_rank = np.max(tiled_scores, axis=2)
tiled_scores -= max_per_rank[:, :, None]
sample_denom = tf.reduce_logsumexp(tiled_scores, axis=2)
sample_log_prob = sampled_scores - sample_denom
rewards = rank_weights[None, :cutoff] * labels[sampled_rankings]
cum_rewards = tf.cumsum(rewards, axis=1, reverse=True)
result = tf.reduce_sum(
tf.reduce_mean(sample_log_prob * cum_rewards, axis=0))
return -result
def prob_per_rank_query(n_samples, cutoff, policy_scores):
n_docs = policy_scores.size
q_cutoff = min(cutoff, policy_scores.size)
if n_docs <= 1:
return np.ones((n_docs, q_cutoff))
rankings = pl.gumbel_sample_rankings(policy_scores, n_samples, q_cutoff)[0]
freq_per_rank = np.zeros((n_docs, q_cutoff), dtype=np.int64)
np.add.at(freq_per_rank,
(rankings[:, :-1], np.arange(q_cutoff - 1)[None, :]), 1)
prob_per_rank = freq_per_rank.astype(np.float64) / n_samples
scores_per_ranking = np.tile(policy_scores,
(n_samples, 1)).astype(np.float64)
scores_per_ranking[np.arange(n_samples)[:, None],
rankings[:, :-1]] = np.NINF
scores_per_ranking -= np.amax(scores_per_ranking, axis=1)[:, None]
denom = np.log(np.sum(np.exp(scores_per_ranking), axis=1))[:, None]
prob_per_rank[:, -1] = np.mean(np.exp(scores_per_ranking - denom), axis=0)
return prob_per_rank
qid, train_labels)
q_feat = data.train.query_feat(qid)
if np.sum(q_labels) > 0 and q_labels.size > 1:
q_n_docs = q_labels.shape[0]
q_cutoff = min(cutoff, q_n_docs)
q_metric_weights = metric_weights[:q_cutoff] #/q_ideal_metric
with tf.GradientTape() as tape:
q_tf_scores = model(q_feat)
q_np_scores = q_tf_scores.numpy()[:,0]
if args.loss == 'lambdaloss':
(sampled_rankings, sampled_inv_rankings, _, _,
gumbel_scores) = pl.gumbel_sample_rankings(
q_np_scores,
num_exposure_samples,
cutoff=None,
inverted=True,
return_gumbel=True)
sampled_rankings = sampled_rankings[:,:cutoff]
else:
sampled_rankings = pl.gumbel_sample_rankings(
q_np_scores,
num_exposure_samples,
cutoff=q_cutoff)[0]
doc_exposure = np.zeros(q_n_docs, dtype=np.float64)
np.add.at(doc_exposure, sampled_rankings[:,1:], q_metric_weights[1:])
doc_exposure /= num_exposure_samples
max_score = np.amax(q_np_scores)
first_prob = np.exp(q_np_scores-max_score)/np.sum(np.exp(q_np_scores-max_score))
def optimize_policy(model,
optimizer,
data_train,
train_doc_weights,
train_alpha,
train_beta,
data_vali,
vali_doc_weights,
vali_alpha,
vali_beta,
n_grad_samples=100,
n_eval_samples=100,
max_epochs=50,
early_stop_diff=0.001,
early_stop_per_epochs=3,
print_updates=False):
early_stop_per_epochs = min(early_stop_per_epochs, max_epochs)
stacked_alphas = np.stack([train_alpha, vali_alpha], axis=-1)
stacked_betas = np.stack([train_beta, vali_beta], axis=-1)
cutoff = stacked_alphas.shape[0]
policy_vali_scores = model(data_vali.feature_matrix)[:, 0].numpy()
metrics = pl.datasplit_metrics(
data_vali,
policy_vali_scores,
stacked_alphas,
stacked_betas,
vali_doc_weights,
n_samples=n_eval_samples,
)
if print_updates:
print('epoch %d: train %0.04f vali %0.04f' %
(0, metrics[0], metrics[1]))
first_metric_value = metrics[1]
last_metric_value = metrics[1]
best_metric_value = metrics[1]
best_weights = model.get_weights()
cum_doc_weights = np.cumsum(np.abs(train_doc_weights))
start_weights = cum_doc_weights[data_train.doclist_ranges[:-1]]
end_weights = cum_doc_weights[data_train.doclist_ranges[1:] - 1]
qid_included = np.where(np.not_equal(start_weights, end_weights))[0]
qid_included = np.random.permutation(qid_included)
start_time = time.time()
n_queries = qid_included.shape[0]
for i in range(n_queries * max_epochs):
qid = qid_included[i % n_queries]
q_doc_weights = data_train.query_values_from_vector(
qid, train_doc_weights)
q_feat = data_train.query_feat(qid)
q_cutoff = min(cutoff, data_train.query_size(qid))
# print(q_doc_weights)
with tf.GradientTape() as tape:
tf_scores = model(q_feat)[:, 0]
scores = tf_scores.numpy()
sampled_rankings = pl.gumbel_sample_rankings(
scores,
n_grad_samples,
cutoff=q_cutoff,
)[0]
gradient = pl.fast_gradient_based_on_samples(sampled_rankings,
train_alpha,
q_doc_weights,
scores,
cutoff=q_cutoff)
# hybrid_gradient = pl.hybrid_gradient_based_on_samples(
# sampled_rankings,
# train_alpha,
# q_doc_weights,
# scores,
# cutoff=q_cutoff)
loss = -tf.reduce_sum(tf_scores * gradient)
gradients = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
# reshuffle queries every epoch
if (i + 1) % (n_queries) == 0:
qid_included = np.random.permutation(qid_included)
if (i + 1) % (n_queries * early_stop_per_epochs) == 0:
epoch_i = (i + 1) / n_queries
policy_vali_scores = model(data_vali.feature_matrix)[:, 0].numpy()
metrics = pl.datasplit_metrics(
data_vali,
policy_vali_scores,
stacked_alphas,
stacked_betas,
vali_doc_weights,
n_samples=n_eval_samples,
)
abs_improvement = metrics[1] - last_metric_value
if print_updates:
improvement = metrics[1] / last_metric_value - 1.
total_improvement = metrics[1] / first_metric_value - 1.
average_time = (time.time() - start_time) / (i +
1.) * n_queries
print('epoch %d: '
'train %0.04f '
'vali %0.04f '
'epoch-time %0.04f '
'abs-improvement %0.05f '
'improvement %0.05f '
'total-improvement %0.05f ' %
(epoch_i, metrics[0], metrics[1], average_time,
abs_improvement, improvement, total_improvement))
last_metric_value = metrics[1]
if best_metric_value < metrics[1]:
best_weights = model.get_weights()
if abs_improvement < early_stop_diff:
break
model.set_weights(best_weights)
return model, last_metric_value
