def step2_InferOutcome(self):
print('Testing step 2: infer_outcome')
l = self.foundDocs
rankers = self.rankers
user_model = CascadeUserModel("--p_click 0:.0, 1:1.0"
" --p_stop 0:.0, 1:.0")
query = self.query
clicks = user_model.get_clicks(l, query.get_labels())
creds = self.multil.infer_outcome(l, rankers, clicks, query)
print "Clicks on list: ", clicks
print "Ranks: ", creds
assert(len(creds) == len(self.rankers))
# New asserts because ranks are returned, not creds
assert(1 in creds)
assert(0 not in creds)
assert(len(creds)+1 not in creds)
def step2_InferOutcome(self):
print('Testing step 2: infer_outcome')
l = self.foundDocs
rankers = self.rankers
user_model = CascadeUserModel("--p_click 0:.0, 1:1.0"
" --p_stop 0:.0, 1:.0")
query = self.query
clicks = user_model.get_clicks(l, query.get_labels())
creds = self.multil.infer_outcome(l, rankers, clicks, query)
print "Clicks on list: ", clicks
print "Ranks: ", creds
assert (len(creds) == len(self.rankers))
# New asserts because ranks are returned, not creds
assert (1 in creds)
assert (0 not in creds)
assert (len(creds) + 1 not in creds)
def __init__(self, feature_sets, n_features=64, cutoff=10, click_model="navigational", experiment_type=""):
self.experiment_type = experiment_type
self.n_rankers = len(feature_sets)
self.n_features = n_features
self.cutoff = cutoff
self.train_queries = qu.load_queries(PATH_TRAIN_QUERIES, self.n_features)
self.test_queries = qu.load_queries(PATH_TEST_QUERIES, self.n_features)
self.samplemultil100 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 100")
self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 1000")
self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 10000")
self.samplemultil100000 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 100000")
#self.samplemultil10 = sbml.SampleBasedProbabilisticMultileave("--n_samples 10")
#self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 100")
#self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 1000")
self.multil = ml.ProbabilisticMultileave()
self.multil_nonbin = ml.ProbabilisticMultileave("-c True")
self.interl = ProbabilisticInterleave('--aggregate binary')
self.TeamDraftMultileave = TeamDraftMultileave()
self.allrankers = [rnk("1", "random", self.n_features)
for _ in range(self.n_rankers)]
for feature_ids,ranker in zip(feature_sets,self.allrankers):
weights = np.zeros(self.n_features)
for fid in feature_ids:
weights[fid] = 1
ranker.update_weights(weights)
if experiment_type == "sensitivity":
ndcg = NdcgEval()
average_ndcgs = np.zeros((self.n_rankers))
for query in self.test_queries:
for i, ranker in enumerate(self.allrankers):
ranker.init_ranking(query)
average_ndcgs[i] += ndcg.get_value(ranker.get_ranking(),
query.get_labels().tolist(),
None, self.cutoff)
average_ndcgs /= len(self.test_queries)
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[i, j] = 0.5 * (average_ndcgs[i] -
average_ndcgs[j]) + 0.5
elif experiment_type == "bias":
click_model = "random"
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[i, j] = 0.5
else:
raise Exception("Set experiment type")
if click_model=="navigational":
click_str="--p_click 0:.05, 1:0.95 --p_stop 0:.2, 1:.5"
elif click_model=="perfect":
click_str="--p_click 0:.0, 1:1. --p_stop 0:.0, 1:.0"
elif click_model=="informational":
click_str="--p_click 0:.4, 1:.9 --p_stop 0:.1, 1:.5"
elif click_model=="random":
click_str="--p_click 0:.5, 1:.5 --p_stop 0:.0, 1:.0"
self.user_model = CascadeUserModel(click_str)
class Experiment(object):
# 64 features as in NP2003
# k = ranking length
def __init__(self, feature_sets, n_features=64, cutoff=10, click_model="navigational", experiment_type=""):
self.experiment_type = experiment_type
self.n_rankers = len(feature_sets)
self.n_features = n_features
self.cutoff = cutoff
self.train_queries = qu.load_queries(PATH_TRAIN_QUERIES, self.n_features)
self.test_queries = qu.load_queries(PATH_TEST_QUERIES, self.n_features)
self.samplemultil100 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 100")
self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 1000")
self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 10000")
self.samplemultil100000 = sbml.SampleBasedProbabilisticMultileaveAS("--n_samples 100000")
#self.samplemultil10 = sbml.SampleBasedProbabilisticMultileave("--n_samples 10")
#self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 100")
#self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 1000")
self.multil = ml.ProbabilisticMultileave()
self.multil_nonbin = ml.ProbabilisticMultileave("-c True")
self.interl = ProbabilisticInterleave('--aggregate binary')
self.TeamDraftMultileave = TeamDraftMultileave()
self.allrankers = [rnk("1", "random", self.n_features)
for _ in range(self.n_rankers)]
for feature_ids,ranker in zip(feature_sets,self.allrankers):
weights = np.zeros(self.n_features)
for fid in feature_ids:
weights[fid] = 1
ranker.update_weights(weights)
if experiment_type == "sensitivity":
ndcg = NdcgEval()
average_ndcgs = np.zeros((self.n_rankers))
for query in self.test_queries:
for i, ranker in enumerate(self.allrankers):
ranker.init_ranking(query)
average_ndcgs[i] += ndcg.get_value(ranker.get_ranking(),
query.get_labels().tolist(),
None, self.cutoff)
average_ndcgs /= len(self.test_queries)
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[i, j] = 0.5 * (average_ndcgs[i] -
average_ndcgs[j]) + 0.5
elif experiment_type == "bias":
click_model = "random"
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[i, j] = 0.5
else:
raise Exception("Set experiment type")
if click_model=="navigational":
click_str="--p_click 0:.05, 1:0.95 --p_stop 0:.2, 1:.5"
elif click_model=="perfect":
click_str="--p_click 0:.0, 1:1. --p_stop 0:.0, 1:.0"
elif click_model=="informational":
click_str="--p_click 0:.4, 1:.9 --p_stop 0:.1, 1:.5"
elif click_model=="random":
click_str="--p_click 0:.5, 1:.5 --p_stop 0:.0, 1:.0"
self.user_model = CascadeUserModel(click_str)
def run(self, n_impressions, n_rankers):
self.n_rankers = n_rankers
self.rankerids = sorted(random.sample(range(len(self.allrankers)), self.n_rankers))
self.rankers = [self.allrankers[i] for i in self.rankerids]
self.true_pref = np.zeros((self.n_rankers, self.n_rankers))
for inew, i in enumerate(self.rankerids):
for jnew, j in enumerate(self.rankerids):
self.true_pref[inew, jnew] = self.all_true_pref[i, j]
error = np.zeros(len(self.rankers))
#total_pm = np.zeros((self.n_rankers, self.n_rankers))
total_spm100 = np.zeros((self.n_rankers, self.n_rankers))
total_spm1000 = np.zeros((self.n_rankers, self.n_rankers))
total_spm10000 = np.zeros((self.n_rankers, self.n_rankers))
total_spm100000 = np.zeros((self.n_rankers, self.n_rankers))
total_td = np.zeros((self.n_rankers, self.n_rankers))
total_pi = np.zeros((self.n_rankers, self.n_rankers))
count_pi = np.zeros((self.n_rankers, self.n_rankers))
# total_pm_nb = np.zeros((self.n_rankers, self.n_rankers))
ave_nb_cred = np.zeros((self.n_rankers))
for i in range(1,n_impressions+1):
td_preferences, ((pi_r1, pi_r2), pi_creds), sbpm_pref100, sbpm_pref1000, sbpm_pref10000, sbpm_pref100000 = self.impression()
total_spm100 += sbpm_pref100
total_spm1000 += sbpm_pref1000
total_spm10000 += sbpm_pref10000
total_spm100000 += sbpm_pref100000
#total_pm += pm_preferences
total_td += td_preferences
total_pi[pi_r1][pi_r2] += 1-pi_creds
total_pi[pi_r2][pi_r1] += pi_creds
count_pi[pi_r1][pi_r2] += 1
count_pi[pi_r2][pi_r1] += 1
# ave_nb_cred += pm_nonbin_creds
# total_pm_nb = self.preferencesFromCredits((1-ave_nb_cred/i))
# may be usefull for later debugging
# print
# print pm_nonbin_creds
# print ave_nb_cred/i
# print
# print self.preferencesFromCredits(ave_nb_cred/i)
# print
# print self.true_pref
# print
print i,
for matrix in [total_td/i, total_pi/count_pi, 1-total_spm100/i, 1-total_spm1000/i, 1-total_spm10000/i, 1-total_spm100000/i]:
score = self.preference_error(matrix)
print score,
print
total_spm100 /= n_impressions
total_spm1000 /= n_impressions
total_spm10000 /= n_impressions
total_spm100000 /= n_impressions
#total_pm /= n_impressions
total_td /= n_impressions
total_pi /= count_pi
return [ self.preference_error(matrix) for matrix in [total_td, total_pi, 1-total_spm100,1-total_spm1000, 1-total_spm10000,1-total_spm100000]]
def impression(self):
'''
TODO: docstring
RETURN:
- probabilistic multileaving: preference matrix
- team draft: preference matrix
- probabilisticInterleaving: tuple of (tuple with index of rankers),
credits)
'''
query = self.train_queries[random.choice(self.train_queries.keys())]
#pm_creds = self.impression_probabilisticMultileave(query)
pi_creds, (pi_r1, pi_r2) = \
self.impression_probabilisticInterleave(query)
td_creds = self.impression_teamDraftMultileave(query)
# pm_nonbin_creds = self.impression_probabilisticMultileave(query,False)
#pm_preferences = self.preferencesFromCredits(pm_creds)
td_preferences = self.preferencesFromCredits(td_creds)
sbpm_pref100 = self.impression_sampleProbabilisticMultileave(query, self.samplemultil100)
sbpm_pref1000 = self.impression_sampleProbabilisticMultileave(query, self.samplemultil1000)
sbpm_pref10000 = self.impression_sampleProbabilisticMultileave(query, self.samplemultil10000)
sbpm_pref100000 = self.impression_sampleProbabilisticMultileave(query, self.samplemultil100000)
return td_preferences, ((pi_r1, pi_r2), pi_creds), sbpm_pref100, sbpm_pref1000, sbpm_pref10000, sbpm_pref100000
def impression_sampleProbabilisticMultileave(self, query, multileaver):
ranking, _ = multileaver.multileave(self.rankers, query, self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
creds = multileaver.infer_outcome(ranking, self.rankers, clicks,
query)
return 1-creds
def impression_probabilisticMultileave(self, query, binary=True):
if binary:
ranking, _ = self.multil.multileave(self.rankers, query, self.cutoff)
else:
ranking, _ = self.multil_nonbin.multileave(self.rankers, query, self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
if binary:
creds = self.multil.infer_outcome(ranking, self.rankers, clicks,
query)
else:
creds = self.multil_nonbin.infer_outcome(ranking, self.rankers, clicks,
query)
return creds
def impression_probabilisticInterleave(self, query):
[r1, r2] = random.sample(self.rankers, 2)
ranking, _ = self.interl.interleave(r1, r2, query, self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
creds = self.interl.infer_outcome(ranking, (_, r1, r2),
clicks, query)
return creds/2.+0.5, (self.rankers.index(r1), self.rankers.index(r2))
def impression_teamDraftMultileave(self, query):
ranking, a = self.TeamDraftMultileave.interleave(self.rankers, query,
self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
creds = self.TeamDraftMultileave.infer_outcome(ranking, a, clicks,
query)
return creds
def sgn(self, v):
if v < (0 - sys.float_info.epsilon):
return -1
elif v > (0 + sys.float_info.epsilon):
return 1
else:
return 0
def preference_error(self, preference_matrix):
error = 0.0
counter = 0
for i in range(self.n_rankers):
for j in range(self.n_rankers):
if j != i:
if math.isnan(preference_matrix[i, j]):
continue
counter += 1
if self.experiment_type == "bias":
error += abs(preference_matrix[i, j] - 0.5)
else:
if self.sgn(preference_matrix[i, j] - 0.5) != \
self.sgn(self.true_pref[i, j] - 0.5):
error += 1.
return error / counter
def preferencesFromCredits(self, creds):
'''
ARGS:
- creds: a list with credits for each of the rankers
RETURNS:
- preferences: list of list containing 1 if ranker_row is better than
ranker_collumn, 0 if he is worse, 0.5 if he is equal
'''
preferences = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(i+1):
if creds[i] > creds[j]:
pref = 1
elif creds[i] < creds[j]:
pref = 0
else:
pref = 0.5
preferences[i][j] = pref
preferences[j][i] = 1 - pref
return preferences
def __init__(self,
feature_sets,
n_features=64,
cutoff=10,
click_model="navigational",
experiment_type=""):
self.experiment_type = experiment_type
self.n_rankers = len(feature_sets)
self.n_features = n_features
self.cutoff = cutoff
self.train_queries = qu.load_queries(PATH_TRAIN_QUERIES,
self.n_features)
self.test_queries = qu.load_queries(PATH_TEST_QUERIES, self.n_features)
self.samplemultil100 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 100")
self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 1000")
self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 10000")
self.samplemultil100000 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 100000")
#self.samplemultil10 = sbml.SampleBasedProbabilisticMultileave("--n_samples 10")
#self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 100")
#self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 1000")
self.multil = ml.ProbabilisticMultileave()
self.multil_nonbin = ml.ProbabilisticMultileave("-c True")
self.interl = ProbabilisticInterleave('--aggregate binary')
self.TeamDraftMultileave = TeamDraftMultileave()
self.allrankers = [
rnk("1", "random", self.n_features) for _ in range(self.n_rankers)
]
for feature_ids, ranker in zip(feature_sets, self.allrankers):
weights = np.zeros(self.n_features)
for fid in feature_ids:
weights[fid] = 1
ranker.update_weights(weights)
if experiment_type == "sensitivity":
ndcg = NdcgEval()
average_ndcgs = np.zeros((self.n_rankers))
for query in self.test_queries:
for i, ranker in enumerate(self.allrankers):
ranker.init_ranking(query)
average_ndcgs[i] += ndcg.get_value(
ranker.get_ranking(),
query.get_labels().tolist(), None, self.cutoff)
average_ndcgs /= len(self.test_queries)
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[
i,
j] = 0.5 * (average_ndcgs[i] - average_ndcgs[j]) + 0.5
elif experiment_type == "bias":
click_model = "random"
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[i, j] = 0.5
else:
raise Exception("Set experiment type")
if click_model == "navigational":
click_str = "--p_click 0:.05, 1:0.95 --p_stop 0:.2, 1:.5"
elif click_model == "perfect":
click_str = "--p_click 0:.0, 1:1. --p_stop 0:.0, 1:.0"
elif click_model == "informational":
click_str = "--p_click 0:.4, 1:.9 --p_stop 0:.1, 1:.5"
elif click_model == "random":
click_str = "--p_click 0:.5, 1:.5 --p_stop 0:.0, 1:.0"
self.user_model = CascadeUserModel(click_str)
class Experiment(object):
# 64 features as in NP2003
# k = ranking length
def __init__(self,
feature_sets,
n_features=64,
cutoff=10,
click_model="navigational",
experiment_type=""):
self.experiment_type = experiment_type
self.n_rankers = len(feature_sets)
self.n_features = n_features
self.cutoff = cutoff
self.train_queries = qu.load_queries(PATH_TRAIN_QUERIES,
self.n_features)
self.test_queries = qu.load_queries(PATH_TEST_QUERIES, self.n_features)
self.samplemultil100 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 100")
self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 1000")
self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 10000")
self.samplemultil100000 = sbml.SampleBasedProbabilisticMultileaveAS(
"--n_samples 100000")
#self.samplemultil10 = sbml.SampleBasedProbabilisticMultileave("--n_samples 10")
#self.samplemultil1000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 100")
#self.samplemultil10000 = sbml.SampleBasedProbabilisticMultileave("--n_samples 1000")
self.multil = ml.ProbabilisticMultileave()
self.multil_nonbin = ml.ProbabilisticMultileave("-c True")
self.interl = ProbabilisticInterleave('--aggregate binary')
self.TeamDraftMultileave = TeamDraftMultileave()
self.allrankers = [
rnk("1", "random", self.n_features) for _ in range(self.n_rankers)
]
for feature_ids, ranker in zip(feature_sets, self.allrankers):
weights = np.zeros(self.n_features)
for fid in feature_ids:
weights[fid] = 1
ranker.update_weights(weights)
if experiment_type == "sensitivity":
ndcg = NdcgEval()
average_ndcgs = np.zeros((self.n_rankers))
for query in self.test_queries:
for i, ranker in enumerate(self.allrankers):
ranker.init_ranking(query)
average_ndcgs[i] += ndcg.get_value(
ranker.get_ranking(),
query.get_labels().tolist(), None, self.cutoff)
average_ndcgs /= len(self.test_queries)
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[
i,
j] = 0.5 * (average_ndcgs[i] - average_ndcgs[j]) + 0.5
elif experiment_type == "bias":
click_model = "random"
self.all_true_pref = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(self.n_rankers):
self.all_true_pref[i, j] = 0.5
else:
raise Exception("Set experiment type")
if click_model == "navigational":
click_str = "--p_click 0:.05, 1:0.95 --p_stop 0:.2, 1:.5"
elif click_model == "perfect":
click_str = "--p_click 0:.0, 1:1. --p_stop 0:.0, 1:.0"
elif click_model == "informational":
click_str = "--p_click 0:.4, 1:.9 --p_stop 0:.1, 1:.5"
elif click_model == "random":
click_str = "--p_click 0:.5, 1:.5 --p_stop 0:.0, 1:.0"
self.user_model = CascadeUserModel(click_str)
def run(self, n_impressions, n_rankers):
self.n_rankers = n_rankers
self.rankerids = sorted(
random.sample(range(len(self.allrankers)), self.n_rankers))
self.rankers = [self.allrankers[i] for i in self.rankerids]
self.true_pref = np.zeros((self.n_rankers, self.n_rankers))
for inew, i in enumerate(self.rankerids):
for jnew, j in enumerate(self.rankerids):
self.true_pref[inew, jnew] = self.all_true_pref[i, j]
error = np.zeros(len(self.rankers))
#total_pm = np.zeros((self.n_rankers, self.n_rankers))
total_spm100 = np.zeros((self.n_rankers, self.n_rankers))
total_spm1000 = np.zeros((self.n_rankers, self.n_rankers))
total_spm10000 = np.zeros((self.n_rankers, self.n_rankers))
total_spm100000 = np.zeros((self.n_rankers, self.n_rankers))
total_td = np.zeros((self.n_rankers, self.n_rankers))
total_pi = np.zeros((self.n_rankers, self.n_rankers))
count_pi = np.zeros((self.n_rankers, self.n_rankers))
# total_pm_nb = np.zeros((self.n_rankers, self.n_rankers))
ave_nb_cred = np.zeros((self.n_rankers))
for i in range(1, n_impressions + 1):
td_preferences, (
(pi_r1, pi_r2), pi_creds
), sbpm_pref100, sbpm_pref1000, sbpm_pref10000, sbpm_pref100000 = self.impression(
)
total_spm100 += sbpm_pref100
total_spm1000 += sbpm_pref1000
total_spm10000 += sbpm_pref10000
total_spm100000 += sbpm_pref100000
#total_pm += pm_preferences
total_td += td_preferences
total_pi[pi_r1][pi_r2] += 1 - pi_creds
total_pi[pi_r2][pi_r1] += pi_creds
count_pi[pi_r1][pi_r2] += 1
count_pi[pi_r2][pi_r1] += 1
# ave_nb_cred += pm_nonbin_creds
# total_pm_nb = self.preferencesFromCredits((1-ave_nb_cred/i))
# may be usefull for later debugging
# print
# print pm_nonbin_creds
# print ave_nb_cred/i
# print
# print self.preferencesFromCredits(ave_nb_cred/i)
# print
# print self.true_pref
# print
print i,
for matrix in [
total_td / i, total_pi / count_pi, 1 - total_spm100 / i,
1 - total_spm1000 / i, 1 - total_spm10000 / i,
1 - total_spm100000 / i
]:
score = self.preference_error(matrix)
print score,
print
total_spm100 /= n_impressions
total_spm1000 /= n_impressions
total_spm10000 /= n_impressions
total_spm100000 /= n_impressions
#total_pm /= n_impressions
total_td /= n_impressions
total_pi /= count_pi
return [
self.preference_error(matrix) for matrix in [
total_td, total_pi, 1 - total_spm100, 1 - total_spm1000, 1 -
total_spm10000, 1 - total_spm100000
]
]
def impression(self):
'''
TODO: docstring
RETURN:
- probabilistic multileaving: preference matrix
- team draft: preference matrix
- probabilisticInterleaving: tuple of (tuple with index of rankers),
credits)
'''
query = self.train_queries[random.choice(self.train_queries.keys())]
#pm_creds = self.impression_probabilisticMultileave(query)
pi_creds, (pi_r1, pi_r2) = \
self.impression_probabilisticInterleave(query)
td_creds = self.impression_teamDraftMultileave(query)
# pm_nonbin_creds = self.impression_probabilisticMultileave(query,False)
#pm_preferences = self.preferencesFromCredits(pm_creds)
td_preferences = self.preferencesFromCredits(td_creds)
sbpm_pref100 = self.impression_sampleProbabilisticMultileave(
query, self.samplemultil100)
sbpm_pref1000 = self.impression_sampleProbabilisticMultileave(
query, self.samplemultil1000)
sbpm_pref10000 = self.impression_sampleProbabilisticMultileave(
query, self.samplemultil10000)
sbpm_pref100000 = self.impression_sampleProbabilisticMultileave(
query, self.samplemultil100000)
return td_preferences, (
(pi_r1, pi_r2), pi_creds
), sbpm_pref100, sbpm_pref1000, sbpm_pref10000, sbpm_pref100000
def impression_sampleProbabilisticMultileave(self, query, multileaver):
ranking, _ = multileaver.multileave(self.rankers, query, self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
creds = multileaver.infer_outcome(ranking, self.rankers, clicks, query)
return 1 - creds
def impression_probabilisticMultileave(self, query, binary=True):
if binary:
ranking, _ = self.multil.multileave(self.rankers, query,
self.cutoff)
else:
ranking, _ = self.multil_nonbin.multileave(self.rankers, query,
self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
if binary:
creds = self.multil.infer_outcome(ranking, self.rankers, clicks,
query)
else:
creds = self.multil_nonbin.infer_outcome(ranking, self.rankers,
clicks, query)
return creds
def impression_probabilisticInterleave(self, query):
[r1, r2] = random.sample(self.rankers, 2)
ranking, _ = self.interl.interleave(r1, r2, query, self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
creds = self.interl.infer_outcome(ranking, (_, r1, r2), clicks, query)
return creds / 2. + 0.5, (self.rankers.index(r1),
self.rankers.index(r2))
def impression_teamDraftMultileave(self, query):
ranking, a = self.TeamDraftMultileave.interleave(
self.rankers, query, self.cutoff)
clicks = self.user_model.get_clicks(ranking, query.get_labels())
creds = self.TeamDraftMultileave.infer_outcome(ranking, a, clicks,
query)
return creds
def sgn(self, v):
if v < (0 - sys.float_info.epsilon):
return -1
elif v > (0 + sys.float_info.epsilon):
return 1
else:
return 0
def preference_error(self, preference_matrix):
error = 0.0
counter = 0
for i in range(self.n_rankers):
for j in range(self.n_rankers):
if j != i:
if math.isnan(preference_matrix[i, j]):
continue
counter += 1
if self.experiment_type == "bias":
error += abs(preference_matrix[i, j] - 0.5)
else:
if self.sgn(preference_matrix[i, j] - 0.5) != \
self.sgn(self.true_pref[i, j] - 0.5):
error += 1.
return error / counter
def preferencesFromCredits(self, creds):
'''
ARGS:
- creds: a list with credits for each of the rankers
RETURNS:
- preferences: list of list containing 1 if ranker_row is better than
ranker_collumn, 0 if he is worse, 0.5 if he is equal
'''
preferences = np.zeros((self.n_rankers, self.n_rankers))
for i in range(self.n_rankers):
for j in range(i + 1):
if creds[i] > creds[j]:
pref = 1
elif creds[i] < creds[j]:
pref = 0
else:
pref = 0.5
preferences[i][j] = pref
preferences[j][i] = 1 - pref
return preferences