def __init__(self, n_candidates, GRAD_SIZE, EXP_SIZE, k_initial, k_increase, TB_QUEUE_SIZE=None, TB_WINDOW_SIZE=None, prev_qeury_len=None, *args, **kargs):
super(TD_NSGD_DSP, self).__init__(*args, **kargs)
self.model = LinearModel(n_features = self.n_features,
learning_rate = self.learning_rate,
n_candidates = n_candidates)
self.GRAD_SIZE = GRAD_SIZE
self.EXP_SIZE = EXP_SIZE
self.TB_QUEUE_SIZE = TB_QUEUE_SIZE
self.TB_WINDOW_SIZE = TB_WINDOW_SIZE
self.sample_basis = True
self.clicklist = np.empty([self.GRAD_SIZE,1], dtype=int) #click array
self.grad = np.zeros([self.GRAD_SIZE,self.n_features], dtype=float)
self.gradCol = 0
# DQ tie-break related lists
self.difficult_NDCG =[]
self.difficult_queries =[]
self.difficult_document =[]
self.difficult_time =[]
self.query_id = 0
self.k_initial = k_initial
self.k_increase = k_increase
# Secondary techniques
self.prev_qeury_len = prev_qeury_len
if prev_qeury_len:
self.prev_feat_list = []
def __init__(self, learning_rate, learning_rate_decay, *args, **kargs):
super(TD_DBGD, self).__init__(*args, **kargs)
self.learning_rate = learning_rate
self.model = LinearModel(n_features=self.n_features,
learning_rate=learning_rate,
n_candidates=1,
learning_rate_decay=learning_rate_decay)
self.multileaving = TeamDraftMultileave(n_results=self.n_results)
def __init__(self, learning_rate, learning_rate_decay, *args, **kargs):
super(PDGD, self).__init__(*args, **kargs)
self.learning_rate = learning_rate
self.learning_rate_decay = learning_rate_decay
self.model = LinearModel(n_features=self.n_features,
learning_rate=learning_rate,
learning_rate_decay=learning_rate_decay,
n_candidates=1)
def __init__(self, alpha, _lambda, refine, rank, update, learning_rate, learning_rate_decay, ind, *args, **kargs):
super(PairRank, self).__init__(*args, **kargs)
self.alpha = alpha
self._lambda = _lambda
self.refine = refine
self.rank = rank
self.update = update
self.learning_rate = learning_rate
self.learning_rate_decay = learning_rate_decay
self.ind = ind
self.A = self._lambda * np.identity(self.n_features)
self.InvA = np.linalg.pinv(self.A)
self.model = LinearModel(
n_features=self.n_features, learning_rate=learning_rate, learning_rate_decay=1, n_candidates=1,
)
self.history = {}
self.n_pairs = []
self.pair_index = []
self.log = {}
self.get_name()
def __init__(self,
k_initial,
k_increase,
n_candidates,
prev_qeury_len=None,
docspace=[False, 0],
*args,
**kargs):
super(P_MGD_DSP, self).__init__(*args, **kargs)
self.n_candidates = n_candidates
self.model = LinearModel(n_features=self.n_features,
learning_rate=self.learning_rate,
n_candidates=self.n_candidates)
self.k_initial = k_initial
self.k_increase = k_increase
self.prev_qeury_len = prev_qeury_len # queue size of features from previous queries
if prev_qeury_len:
self.prev_feat_list = []
# for document space length experiment
# docspace=[True,3] means use superset of document space with three additional documents to perfect DS user examined.
self.docspace = docspace
class TD_DBGD(BasicOnlineRanker):
def __init__(self, learning_rate, learning_rate_decay, *args, **kargs):
super(TD_DBGD, self).__init__(*args, **kargs)
self.learning_rate = learning_rate
self.model = LinearModel(n_features=self.n_features,
learning_rate=learning_rate,
n_candidates=1,
learning_rate_decay=learning_rate_decay)
self.multileaving = TeamDraftMultileave(n_results=self.n_results)
@staticmethod
def default_parameters():
parent_parameters = BasicOnlineRanker.default_parameters()
parent_parameters.update({
'learning_rate': 0.01,
'learning_rate_decay': 1.0,
})
return parent_parameters
def get_test_rankings(self, features, query_ranges, inverted=True):
scores = self.model.score(features)
return rnk.rank_multiple_queries(scores,
query_ranges,
inverted=inverted,
n_results=self.n_results)
def _create_train_ranking(self, query_id, query_feat, inverted):
assert inverted == False
self.model.sample_candidates()
scores = self.model.candidate_score(query_feat)
rankings = rnk.rank_single_query(scores,
inverted=False,
n_results=self.n_results)
multileaved_list = self.multileaving.make_multileaving(rankings)
return multileaved_list
def update_to_interaction(self, clicks):
winners = self.multileaving.winning_rankers(clicks)
self.model.update_to_mean_winners(winners)
class P_MGD_DSP(P_DBGD):
def __init__(self,
k_initial,
k_increase,
n_candidates,
prev_qeury_len=None,
docspace=[False, 0],
*args,
**kargs):
super(P_MGD_DSP, self).__init__(*args, **kargs)
self.n_candidates = n_candidates
self.model = LinearModel(n_features=self.n_features,
learning_rate=self.learning_rate,
n_candidates=self.n_candidates)
self.k_initial = k_initial
self.k_increase = k_increase
self.prev_qeury_len = prev_qeury_len # queue size of features from previous queries
if prev_qeury_len:
self.prev_feat_list = []
# for document space length experiment
# docspace=[True,3] means use superset of document space with three additional documents to perfect DS user examined.
self.docspace = docspace
@staticmethod
def default_parameters():
parent_parameters = P_DBGD.default_parameters()
parent_parameters.update({
'n_candidates': 49,
})
return parent_parameters
def _create_train_ranking(self, query_id, query_feat, inverted):
# Save query_id to get access to query_feat when updating
self.query_id = query_id
assert inverted == False
self.model.sample_candidates()
scores = self.model.candidate_score(query_feat)
inverted_rankings = rnk.rank_single_query(scores,
inverted=True,
n_results=None)
multileaved_list = self.multileaving.make_multileaving(
inverted_rankings)
return multileaved_list
def update_to_interaction(self, clicks, stop_index=None):
winners = self.multileaving.winning_rankers(clicks)
###############################################################
if True in clicks:
# For projection
# keep track of feature vectors of doc list
viewed_list = []
# index of last click
last_click = max(loc for loc, val in enumerate(clicks)
if val == True)
# prevent last_click+k from exceeding interleaved list length
k_current = self.k_initial
if self.k_increase:
# gradually increast k
k_current += int(self.n_interactions / 1000)
last_doc_index = min(last_click + k_current,
len(self._last_ranking))
if self.docspace[
0] and stop_index is not None: # for document space length experiment
# create sub/super set of perfect document space user examined.
# user examined documents coming from ccm, where user leaves.
last_doc_index = stop_index + self.docspace[
1] + 1 # 1 added for stopping document, which has been examined.
last_doc_index = max(last_doc_index, 1) # At least 1
last_doc_index = min(last_doc_index, len(
self._last_ranking)) # At most length of current list
query_feat = self.get_query_features(self.query_id,
self._train_features,
self._train_query_ranges)
for i in range(last_doc_index):
docid = self._last_ranking[i]
feature = query_feat[docid]
viewed_list.append(feature)
add_list = viewed_list
# Append feature vectors from previous queries
if self.prev_qeury_len:
if len(self.prev_feat_list) > 0:
viewed_list = np.append(viewed_list,
self.prev_feat_list,
axis=0)
# Add examined feature vectors of current query to be used in later iterations
for i in add_list:
if len(self.prev_feat_list) >= self.prev_qeury_len:
self.prev_feat_list.pop(
0) # Remove oldest document feature.
# if prev_feat_list is not filled up, add current list
self.prev_feat_list.append(i)
self.model.update_to_mean_winners(winners, viewed_list)
###############################################################
else:
self.model.update_to_mean_winners(winners)
class TD_NSGD_DSP(TD_DBGD):
def __init__(self, n_candidates, GRAD_SIZE, EXP_SIZE, k_initial, k_increase, TB_QUEUE_SIZE=None, TB_WINDOW_SIZE=None, prev_qeury_len=None, *args, **kargs):
super(TD_NSGD_DSP, self).__init__(*args, **kargs)
self.model = LinearModel(n_features = self.n_features,
learning_rate = self.learning_rate,
n_candidates = n_candidates)
self.GRAD_SIZE = GRAD_SIZE
self.EXP_SIZE = EXP_SIZE
self.TB_QUEUE_SIZE = TB_QUEUE_SIZE
self.TB_WINDOW_SIZE = TB_WINDOW_SIZE
self.sample_basis = True
self.clicklist = np.empty([self.GRAD_SIZE,1], dtype=int) #click array
self.grad = np.zeros([self.GRAD_SIZE,self.n_features], dtype=float)
self.gradCol = 0
# DQ tie-break related lists
self.difficult_NDCG =[]
self.difficult_queries =[]
self.difficult_document =[]
self.difficult_time =[]
self.query_id = 0
self.k_initial = k_initial
self.k_increase = k_increase
# Secondary techniques
self.prev_qeury_len = prev_qeury_len
if prev_qeury_len:
self.prev_feat_list = []
@staticmethod
def default_parameters():
parent_parameters = TD_DBGD.default_parameters()
parent_parameters.update({
'n_candidates': 9,
})
return parent_parameters
def update_to_interaction(self, clicks, stop_index=None):
winners, ranker_clicks = self.multileaving.winning_rankers_with_clicks(clicks)
# Fill out recent difficult query queues.
if self.TB_QUEUE_SIZE > 0:
self.fill_difficult_query(clicks)
# Trigger difficult-query tie-break strategy
if len(self.difficult_queries) < self.TB_QUEUE_SIZE and len(winners) > 1:
winners = self.tieBreak_difficultQuery(winners)
###############################################################
if True in clicks:
# For projection
# keep track of feature vectors of doc list
viewed_list = []
# index of last click
last_click = max(loc for loc, val in enumerate(clicks) if val == True)
# prevent last_click+k from exceeding interleaved list length
k_current = self.k_initial
if self.k_increase:
# gradually increast k
k_current += int(self.n_interactions/1000)
last_doc_index = min(last_click+k_current, len(self._last_ranking)-1)
query_feat = self.get_query_features(self.query_id,
self._train_features,
self._train_query_ranges)
for i in range(last_doc_index):
docid = self._last_ranking[i]
feature = query_feat[docid]
viewed_list.append(feature)
self.model.update_to_mean_winners(winners,viewed_list)
###############################################################
else:
self.model.update_to_mean_winners(winners)
cl_sorted = sorted(ranker_clicks) # in ascending order
for i in range(1, len(ranker_clicks)):
# only save subset of rankers (worst 4 ouf of 9 rankers)
# add if current cl is smaller than or equal to maximum form the set of candidates
if ranker_clicks[i] <= cl_sorted[3] and ranker_clicks[i]<ranker_clicks[0]:
self.clicklist[self.gradCol] = ranker_clicks[i] -ranker_clicks[0]
self.grad[self.gradCol] = self.model.gs[i-1]
self.gradCol = (self.gradCol + 1) % self.GRAD_SIZE # update to reflect next column to be updaed
def _create_train_ranking(self, query_id, query_feat, inverted):
self.query_id = query_id
assert inverted == False
# Get the worst gradients by click
nums = []
dif = self.GRAD_SIZE - self.EXP_SIZE
for i in range(0, dif):
max = -maxint-1
n = 0
# Choose
for j in range(0, self.GRAD_SIZE):
if self.clicklist[j] > max and j not in nums:
max = self.clicklist[j] # The better cl value to be excluded
n = j # index of it
nums.append(n)
# create subset of gradient matrix
grad_temp = np.zeros([self.EXP_SIZE, self.n_features], dtype=float)
c = 0
for i in range(0,self.GRAD_SIZE):
if i not in nums:
# The wrost 'EXP_SIZE' gradients from grad[] added to gr_temp
grad_temp[c] = copy.deepcopy(self.grad[i])
c = c + 1
self.model.sample_candidates_null_space(grad_temp, query_feat, self.sample_basis)
scores = self.model.candidate_score(query_feat)
rankings = rnk.rank_single_query(scores, inverted=False, n_results=self.n_results)
multileaved_list = self.multileaving.make_multileaving(rankings)
return multileaved_list
def fill_difficult_query(self, clicks):
# Set up for tie breaker- keep track of difficult queries
# Find the rank of first clicked document
ndcg_current = 0
clickedList = []
for count, elem in enumerate(clicks):
if elem == 1: # if clicked
ndcg_current += 1 / (count + 1.0)
# Keep track of clicked documents of current query
clickedList.append(self._last_ranking[count])
# If difficult queries for tie breaking is not filled up, add current query
if len(self.difficult_NDCG) < self.TB_QUEUE_SIZE and ndcg_current > 0:
self.difficult_NDCG.append(ndcg_current)
self.difficult_queries.append(self.query_id)
self.difficult_document.append(clickedList) # first clicked doc to follow
self.difficult_time.append(self.n_interactions)
else:
# If already filled up, check if current query is more difficult than any saved query.
if len(self.difficult_NDCG) > 0:
flag = False
for i in range(len(self.difficult_NDCG)):
if self.n_interactions - self.difficult_time[i] > self.TB_WINDOW_SIZE:
# Maintain queries winthin the window size
flag = True
index = i
break
if not flag and max(self.difficult_NDCG) > ndcg_current and ndcg_current > 0:
# Current query is more difficult than one of queued ones
flag = True
index = self.difficult_NDCG.index(max(self.difficult_NDCG))
if flag:
self.difficult_NDCG[index] = ndcg_current
self.difficult_queries[index] = self.query_id
self.difficult_document[index] = clickedList
self.difficult_time[index] = self.n_interactions
def tieBreak_difficultQuery(self, winners):
# ScoreList keeps track of ranks each tied candidate perform in tie breaking
scoreList = np.zeros(self.model.n_models)
# Iterate through 10 stored difficult queries
for count_q, diff_query in enumerate(self.difficult_queries):
query_feat = self.get_query_features(diff_query,
self._train_features,
self._train_query_ranges)
scores = self.model.candidate_score(query_feat)
rankings = rnk.rank_single_query(scores, inverted=False, n_results=self.n_results)
# Iterate through tied candidates
for winner in winners:
candidate_NDCG = 0.0
for count_d, doc in enumerate(self.difficult_document[count_q]):
# Calculate NDCG performance in current difficult query
diff_doc_rank = np.where(rankings[winner] == self.difficult_document[count_q][count_d])[0][0]
temp = 1 / (diff_doc_rank + 1.0)
candidate_NDCG += 1 / (diff_doc_rank + 1.0)
# Add the NDCG value of diff. query
scoreList[winner] += candidate_NDCG
# Ranker with the least sum of NDCGs is the winner
maxRank_score = np.max(scoreList[np.nonzero(scoreList)])
winner = scoreList.tolist().index(maxRank_score)
return [winner]
class PDGD(BasicOnlineRanker):
def __init__(self, learning_rate, learning_rate_decay, *args, **kargs):
super(PDGD, self).__init__(*args, **kargs)
self.learning_rate = learning_rate
self.learning_rate_decay = learning_rate_decay
self.model = LinearModel(n_features=self.n_features,
learning_rate=learning_rate,
learning_rate_decay=learning_rate_decay,
n_candidates=1)
@staticmethod
def default_parameters():
parent_parameters = BasicOnlineRanker.default_parameters()
parent_parameters.update({
'learning_rate': 0.1,
'learning_rate_decay': 1.0,
})
return parent_parameters
def get_test_rankings(self, features, query_ranges, inverted=True):
scores = -self.model.score(features)
return rnk.rank_multiple_queries(scores,
query_ranges,
inverted=inverted,
n_results=self.n_results)
def _create_train_ranking(self, query_id, query_feat, inverted):
assert inverted == False
n_docs = query_feat.shape[0]
k = np.minimum(self.n_results, n_docs)
self.doc_scores = self.model.score(query_feat)
self.doc_scores += 18 - np.amax(self.doc_scores)
self.ranking = self._recursive_choice(np.copy(self.doc_scores),
np.array([], dtype=np.int32), k)
self._last_query_feat = query_feat
return self.ranking
def _recursive_choice(self, scores, incomplete_ranking, k_left):
n_docs = scores.shape[0]
scores[incomplete_ranking] = np.amin(scores)
scores += 18 - np.amax(scores)
exp_scores = np.exp(scores)
exp_scores[incomplete_ranking] = 0
probs = exp_scores / np.sum(exp_scores)
safe_n = np.sum(probs > 10**(-4) / n_docs)
safe_k = np.minimum(safe_n, k_left)
next_ranking = np.random.choice(np.arange(n_docs),
replace=False,
p=probs,
size=safe_k)
ranking = np.concatenate((incomplete_ranking, next_ranking))
k_left = k_left - safe_k
if k_left > 0:
return self._recursive_choice(scores, ranking, k_left)
else:
return ranking
def update_to_interaction(self, clicks):
if np.any(clicks):
self._update_to_clicks(clicks)
def _update_to_clicks(self, clicks):
n_docs = self.ranking.shape[0]
cur_k = np.minimum(n_docs, self.n_results)
included = np.ones(cur_k, dtype=np.int32)
if not clicks[-1]:
included[1:] = np.cumsum(clicks[::-1])[:0:-1]
neg_ind = np.where(np.logical_xor(clicks, included))[0]
pos_ind = np.where(clicks)[0]
n_pos = pos_ind.shape[0]
n_neg = neg_ind.shape[0]
n_pairs = n_pos * n_neg
if n_pairs == 0:
return
pos_r_ind = self.ranking[pos_ind]
neg_r_ind = self.ranking[neg_ind]
pos_scores = self.doc_scores[pos_r_ind]
neg_scores = self.doc_scores[neg_r_ind]
log_pair_pos = np.tile(pos_scores, n_neg)
log_pair_neg = np.repeat(neg_scores, n_pos)
pair_trans = 18 - np.maximum(log_pair_pos, log_pair_neg)
exp_pair_pos = np.exp(log_pair_pos + pair_trans)
exp_pair_neg = np.exp(log_pair_neg + pair_trans)
pair_denom = (exp_pair_pos + exp_pair_neg)
pair_w = np.maximum(exp_pair_pos, exp_pair_neg)
pair_w /= pair_denom
pair_w /= pair_denom
pair_w *= np.minimum(exp_pair_pos, exp_pair_neg)
pair_w *= self._calculate_unbias_weights(pos_ind, neg_ind)
reshaped = np.reshape(pair_w, (n_neg, n_pos))
pos_w = np.sum(reshaped, axis=0)
neg_w = -np.sum(reshaped, axis=1)
all_w = np.concatenate([pos_w, neg_w])
all_ind = np.concatenate([pos_r_ind, neg_r_ind])
self.model.update_to_documents(all_ind, all_w)
def _calculate_unbias_weights(self, pos_ind, neg_ind):
ranking_prob = self._calculate_observed_prob(pos_ind, neg_ind,
self.doc_scores)
flipped_prob = self._calculate_flipped_prob(pos_ind, neg_ind,
self.doc_scores)
return flipped_prob / (ranking_prob + flipped_prob)
def _calculate_observed_prob(self, pos_ind, neg_ind, doc_scores):
n_pos = pos_ind.shape[0]
n_neg = neg_ind.shape[0]
n_pairs = n_pos * n_neg
n_results = self.ranking.shape[0]
n_docs = doc_scores.shape[0]
results_i = np.arange(n_results)
pair_i = np.arange(n_pairs)
doc_i = np.arange(n_docs)
pos_pair_i = np.tile(pos_ind, n_neg)
neg_pair_i = np.repeat(neg_ind, n_pos)
min_pair_i = np.minimum(pos_pair_i, neg_pair_i)
max_pair_i = np.maximum(pos_pair_i, neg_pair_i)
range_mask = np.logical_and(min_pair_i[:, None] <= results_i,
max_pair_i[:, None] >= results_i)
safe_log = np.tile(doc_scores[None, :], [n_results, 1])
mask = np.zeros((n_results, n_docs))
mask[results_i[1:], self.ranking[:-1]] = True
mask = np.cumsum(mask, axis=0).astype(bool)
safe_log[mask] = np.amin(safe_log)
safe_max = np.amax(safe_log, axis=1)
safe_log -= safe_max[:, None] - 18
safe_exp = np.exp(safe_log)
safe_exp[mask] = 0
ranking_log = doc_scores[self.ranking] - safe_max + 18
ranking_exp = np.exp(ranking_log)
safe_denom = np.sum(safe_exp, axis=1)
ranking_prob = ranking_exp / safe_denom
tiled_prob = np.tile(ranking_prob[None, :], [n_pairs, 1])
safe_prob = np.ones((n_pairs, n_results))
safe_prob[range_mask] = tiled_prob[range_mask]
safe_pair_prob = np.prod(safe_prob, axis=1)
return safe_pair_prob
def _calculate_flipped_prob(self, pos_ind, neg_ind, doc_scores):
n_pos = pos_ind.shape[0]
n_neg = neg_ind.shape[0]
n_pairs = n_pos * n_neg
n_results = self.ranking.shape[0]
n_docs = doc_scores.shape[0]
results_i = np.arange(n_results)
pair_i = np.arange(n_pairs)
doc_i = np.arange(n_docs)
pos_pair_i = np.tile(pos_ind, n_neg)
neg_pair_i = np.repeat(neg_ind, n_pos)
flipped_rankings = np.tile(self.ranking[None, :], [n_pairs, 1])
flipped_rankings[pair_i, pos_pair_i] = self.ranking[neg_pair_i]
flipped_rankings[pair_i, neg_pair_i] = self.ranking[pos_pair_i]
min_pair_i = np.minimum(pos_pair_i, neg_pair_i)
max_pair_i = np.maximum(pos_pair_i, neg_pair_i)
range_mask = np.logical_and(min_pair_i[:, None] <= results_i,
max_pair_i[:, None] >= results_i)
flipped_log = doc_scores[flipped_rankings]
safe_log = np.tile(doc_scores[None, None, :], [n_pairs, n_results, 1])
results_ij = np.tile(results_i[None, 1:], [n_pairs, 1])
pair_ij = np.tile(pair_i[:, None], [1, n_results - 1])
mask = np.zeros((n_pairs, n_results, n_docs))
mask[pair_ij, results_ij, flipped_rankings[:, :-1]] = True
mask = np.cumsum(mask, axis=1).astype(bool)
safe_log[mask] = np.amin(safe_log)
safe_max = np.amax(safe_log, axis=2)
safe_log -= safe_max[:, :, None] - 18
flipped_log -= safe_max - 18
flipped_exp = np.exp(flipped_log)
safe_exp = np.exp(safe_log)
safe_exp[mask] = 0
safe_denom = np.sum(safe_exp, axis=2)
safe_prob = np.ones((n_pairs, n_results))
safe_prob[range_mask] = (flipped_exp / safe_denom)[range_mask]
safe_pair_prob = np.prod(safe_prob, axis=1)
return safe_pair_prob
class PairRank(BasicOnlineRanker):
def __init__(self, alpha, _lambda, refine, rank, update, learning_rate, learning_rate_decay, ind, *args, **kargs):
super(PairRank, self).__init__(*args, **kargs)
self.alpha = alpha
self._lambda = _lambda
self.refine = refine
self.rank = rank
self.update = update
self.learning_rate = learning_rate
self.learning_rate_decay = learning_rate_decay
self.ind = ind
self.A = self._lambda * np.identity(self.n_features)
self.InvA = np.linalg.pinv(self.A)
self.model = LinearModel(
n_features=self.n_features, learning_rate=learning_rate, learning_rate_decay=1, n_candidates=1,
)
self.history = {}
self.n_pairs = []
self.pair_index = []
self.log = {}
self.get_name()
@staticmethod
def default_parameters():
parent_parameters = BasicOnlineRanker.default_parameters()
parent_parameters.update({"learning_rate": 0.1, "learning_rate_decay": 1.0})
return parent_parameters
def get_test_rankings(self, features, query_ranges, inverted=True):
scores = -self.model.score(features)
return rnk.rank_multiple_queries(scores, query_ranges, inverted=inverted, n_results=self.n_results)
def cost_func_reg(self, theta, x, y):
log_func_v = logistic_func(theta, x)
step1 = y * safe_ln(log_func_v)
step2 = (1 - y) * safe_ln(1 - log_func_v)
final = (-step1 - step2).mean()
final += self._lambda * theta.dot(theta)
return final
def log_gradient_reg(self, theta, x, y):
# n = len(y)
first_calc = logistic_func(theta, x) - y
final_calc = first_calc.T.dot(x) / len(y)
reg = 2 * self._lambda * theta
final_calc += reg
return final_calc
def get_name(self):
if self.update == "gd" or self.update == "gd_diag" or self.update == "gd_recent":
self.name = "PAIRRANK-None-None-{}-{}-{}-{}-{}-{}".format(
self.update, self._lambda, self.alpha, self.refine, self.rank, self.ind
)
else:
self.name = "PAIRRANK-{}-{}-{}-{}-{}-{}-{}-{}".format(
self.learning_rate,
self.learning_rate_decay,
self.update,
self._lambda,
self.alpha,
self.refine,
self.rank,
self.ind,
)
def get_lcb(self, query_feat):
if self.update == "gd_diag":
# InvA = np.diag(np.diag(self.InvA))
Id = np.identity(self.InvA.shape[0])
InvA = np.multiply(self.InvA, Id)
else:
InvA = self.InvA
pairwise_feat = (query_feat[:, np.newaxis] - query_feat).reshape(-1, self.n_features)
pairwise_estimation = self.model.score(pairwise_feat)
n_doc = len(query_feat)
prob_est = logist(pairwise_estimation).reshape(n_doc, n_doc)
for i in range(n_doc):
for j in range(i + 1, n_doc):
feat = query_feat[i] - query_feat[j]
uncertainty = self.alpha * np.sqrt(np.dot(np.dot(feat, InvA), feat.T))
prob_est[i, j] -= uncertainty
prob_est[j, i] -= uncertainty
lcb_matrix = prob_est
return lcb_matrix
def get_partitions(self, lcb_matrix):
n_nodes = len(lcb_matrix)
# find all the certain edges
certain_edges = set()
for i in range(n_nodes):
indices = [k for k, v in enumerate(lcb_matrix[i]) if v > 0.5]
for j in indices:
certain_edges.add((i, j))
# refine the certain edges: remove the cycles between partitions.
if self.refine:
nodes = np.array(range(n_nodes))
certainG = nx.DiGraph()
certainG.add_nodes_from(nodes)
certainG.add_edges_from(certain_edges)
for n in certainG.nodes():
a = nx.algorithms.dag.ancestors(certainG, n)
for k in a:
certain_edges.add((k, n))
# cut the complete graph by the certain edges
uncertainG = nx.complete_graph(n_nodes)
uncertainG.remove_edges_from(certain_edges)
# get all the connected component by the uncertain edges
sn_list = list(nx.connected_components(uncertainG))
n_sn = len(sn_list)
super_nodes = {}
for i in range(n_sn):
super_nodes[i] = sn_list[i]
# create inv_cp to store the cp_id for each node
inv_sn = {}
for i in range(n_sn):
for j in super_nodes[i]:
inv_sn[j] = i
super_edges = {}
for i in range(n_sn):
super_edges[i] = set([])
for i, e in enumerate(certain_edges):
start_node, end_node = e[0], e[1]
start_sn, end_sn = inv_sn[start_node], inv_sn[end_node]
if start_sn != end_sn:
super_edges[start_sn].add(end_sn)
SG = nx.DiGraph(super_edges)
flag = True
cycle = []
try:
cycle = nx.find_cycle(SG)
except Exception as e:
flag = False
while flag:
# get all candidate nodes
candidate_nodes = set()
for c in cycle:
n1, n2 = c
candidate_nodes.add(n1)
candidate_nodes.add(n2)
new_id = min(candidate_nodes)
# update the edges
super_edges = update_edges(super_edges, candidate_nodes, new_id)
super_nodes = update_nodes(super_nodes, candidate_nodes, new_id)
# print("=======After merge {}=======".format(cycle))
# print("super_edges: ", super_edges)
# print("super_nodes: ", super_nodes)
SG = nx.DiGraph(super_edges)
try:
cycle = nx.find_cycle(SG)
except Exception as e:
print(e)
flag = False
sorted_list = list(nx.topological_sort(SG))
self.log[self.n_interactions]["partition"] = super_nodes
self.log[self.n_interactions]["sorted_list"] = sorted_list
return super_nodes, sorted_list
def _create_train_ranking(self, query_id, query_feat, inverted):
# record the information
self.log[self.n_interactions] = {}
self.log[self.n_interactions]["qid"] = query_id
# t1 = datetime.datetime.now()
lcb_matrix = self.get_lcb(query_feat)
# t2 = datetime.datetime.now()
partition, sorted_list = self.get_partitions(lcb_matrix)
# t3 = datetime.datetime.now()
ranked_list = []
for i, k in enumerate(sorted_list):
cur_p = list(partition[k])
if self.rank == "random":
np.random.shuffle(cur_p)
elif self.rank == "mean":
feat = query_feat[cur_p]
score = self.model.score(feat)
ranked_idx = np.argsort(-score)
ranked_id = np.array(cur_p)[ranked_idx]
cur_p = ranked_id.tolist()
elif self.rank == "certain":
parent = {}
child = {}
for m in cur_p:
for n in cur_p:
if lcb_matrix[m][n] > 0.5:
if m not in child.keys():
child[m] = [n]
else:
child[m].append(n)
if n not in parent.keys():
parent[n] = [m]
else:
parent[n].append(m)
# topological sort
candidate = []
for m in cur_p:
if m not in parent.keys():
candidate.append(m)
ranked_id = []
while len(candidate) != 0:
node = np.random.choice(candidate)
ranked_id.append(node)
candidate.remove(node)
if node in child.keys():
children = child[node]
else:
children = []
for j in children:
parent[j].remove(node)
if len(parent[j]) == 0:
candidate.append(j)
cur_p = ranked_id
else:
print("Rank method is incorrect")
sys.exit()
ranked_list.extend(cur_p)
self.ranking = np.array(ranked_list)
self._last_query_feat = query_feat
self.log[self.n_interactions]["ranking"] = self.ranking
self.log[self.n_interactions]["model"] = self.model.weights[:, 0]
return self.ranking
def update_to_interaction(self, clicks):
if np.any(clicks):
self._update_to_clicks(clicks)
def generate_pairs(self, clicks):
n_docs = self.ranking.shape[0]
cur_k = np.minimum(n_docs, self.n_results)
included = np.ones(cur_k, dtype=np.int32)
if not clicks[-1]:
included[1:] = np.cumsum(clicks[::-1])[:0:-1]
neg_ind = np.where(np.logical_xor(clicks, included))[0]
pos_ind = np.where(clicks)[0]
pos_r_ind = self.ranking[pos_ind]
neg_r_ind = self.ranking[neg_ind]
if self.ind:
np.random.shuffle(pos_r_ind)
np.random.shuffle(neg_r_ind)
pairs = list(zip(pos_r_ind, neg_r_ind))
else:
pairs = list(itertools.product(pos_r_ind, neg_r_ind))
for p in pairs:
diff_feat = (self._last_query_feat[p[0]] - self._last_query_feat[p[1]]).reshape(1, -1)
self.InvA -= multi_dot([self.InvA, diff_feat.T, diff_feat, self.InvA]) / float(
1 + np.dot(np.dot(diff_feat, self.InvA), diff_feat.T)
)
return pairs
def update_history(self, pairs):
query_id = self._last_query_id
idx = len(self.history)
self.history[idx] = {}
self.history[idx]["qid"] = query_id
self.history[idx]["pairs"] = pairs
def generate_training_data(self):
train_x = []
train_y = []
if self.update == "gd_recent":
# only use the most recent observations to update the model
max_ind = max(self.history.keys())
for idx in range(max(max_ind - 500, 0), max_ind + 1):
qid = self.history[idx]["qid"]
feat = self.get_query_features(qid, self._train_features, self._train_query_ranges)
pairs = self.history[idx]["pairs"]
pos_ids = [pair[0] for pair in pairs]
neg_ids = [pair[1] for pair in pairs]
x = feat[pos_ids] - feat[neg_ids]
train_x.append(x)
y = np.ones(len(pairs))
train_y.append(y)
else:
for idx in self.history.keys():
qid = self.history[idx]["qid"]
feat = self.get_query_features(qid, self._train_features, self._train_query_ranges)
pairs = self.history[idx]["pairs"]
pos_ids = [pair[0] for pair in pairs]
neg_ids = [pair[1] for pair in pairs]
x = feat[pos_ids] - feat[neg_ids]
train_x.append(x)
y = np.ones(len(pairs))
train_y.append(y)
train_x = np.vstack(train_x)
train_y = np.hstack(train_y)
return train_x, train_y
def _update_to_clicks(self, clicks):
# generate all pairs from the clicks
pairs = self.generate_pairs(clicks)
n_pairs = len(pairs)
if n_pairs == 0:
return
pairs = np.array(pairs)
self.update_history(pairs)
self.n_pairs.append(n_pairs)
if len(self.n_pairs) == 1:
self.pair_index.append(n_pairs)
else:
self.pair_index.append(self.pair_index[-1] + n_pairs)
if self.update == "gd" or self.update == "gd_diag" or self.update == "gd_recent":
self.update_to_history()
elif self.update == "sgd":
self.update_sgd(pairs)
elif self.update == "batch_sgd":
self.update_batch_sgd()
else:
print("Wrong update mode")
def update_batch_sgd(self):
n_sample = self.pair_index[-1]
batch_size = min(n_sample, 128)
batch_index = random.sample(range(n_sample), batch_size)
data_id = []
data_index = []
for i in batch_index:
j = 0
while self.pair_index[j] <= i:
j += 1
if j == 0:
start = 0
else:
start = self.pair_index[j - 1]
data_id.append(j)
data_index.append(i - start)
# generate training data
batch_x = []
batch_y = []
for i in range(batch_size):
# print i, data_id[i]
idx = data_id[i]
qid = self.history[idx]["qid"]
feat = self.get_query_features(qid, self._train_features, self._train_query_ranges)
pairs = self.history[idx]["pairs"][data_index[i]]
feat = feat[pairs[0]] - feat[pairs[1]]
batch_x.append(feat)
batch_y.append(1)
batch_x = np.array(batch_x).reshape(-1, self.n_features)
batch_y = np.array(batch_y).reshape(-1, 1)
gradient = self.log_gradient_reg(self.model.weights[:, 0], batch_x, batch_y)
self.model.update_to_gradient(-gradient)
def update_sgd(self, pairs):
feat = self.get_query_features(self._last_query_id, self._train_features, self._train_query_ranges)
n_p = pairs.shape[0]
pos_feat = feat[pairs[:, 0]] - feat[pairs[:, 1]]
pos_label = np.ones(n_p)
gradient = self.log_gradient_reg(self.model.weights[:, 0], pos_feat, pos_label)
self.model.update_to_gradient(-gradient)
def update_to_history(self):
train_x, train_y = self.generate_training_data()
myargs = (train_x, train_y)
betas = np.random.rand(train_x.shape[1])
result = minimize(
self.cost_func_reg,
x0=betas,
args=myargs,
method="L-BFGS-B",
jac=self.log_gradient_reg,
options={"ftol": 1e-6},
)
self.model.update_weights(result.x)
def __init__(self, n_candidates, *args, **kargs):
super(TD_MGD, self).__init__(*args, **kargs)
self.model = LinearModel(n_features = self.n_features,
learning_rate = self.learning_rate,
n_candidates = n_candidates,
learning_rate_decay = self.model.learning_rate_decay)