def main():
ks = [3, 5, 10, 20]
mapk = 200
train, test = load_data()
train, test = train.as_matrix(), test.as_matrix()
x = train.T
res = np.zeros(9)
number_of_user = train.shape[0]
print int(number_of_user * 0.01)
pca = PCA(n_components=int(number_of_user * 0.01))
new_x = pca.fit_transform(x)
for u in xrange(train.shape[0]):
y = x[:, u]
truth = test[u]
clf = LogisticRegression(random_state=42, C=0.001, solver='lbfgs')
clf.fit(new_x, y)
#print u, classification.accuracy_score(clf.predict(x), y)
pred_buy_proba = clf.predict_proba(new_x)[:, 1].ravel()
pruned_buy_proba = pred_buy_proba - y.ravel()
pred_order = pruned_buy_proba.argsort()[::-1]
actual_bought = truth.nonzero()[0]
score = apk(actual_bought, pred_order, mapk)
tmp = [score]
for k in ks:
tmp.append(prec(actual_bought, pred_order, k))
tmp.append(recall(actual_bought, pred_order, k))
res += np.array(tmp)
if u % 50 == 0:
print res / (u + 1)
return res / (u + 1)
def main():
ks = [3, 5, 10, 20]
mapk = 200
train, test = load_data()
train, test = train.as_matrix(), test.as_matrix()
x = train
pca = PCA(n_components=int(train.shape[1] * 0.01))
pca.fit(train)
new_x = pca.transform(train)
res = []
for i in xrange(train.shape[1]):
y = x[:, i]
clf = LogisticRegression(random_state=42, C=0.001, solver='lbfgs')
clf.fit(new_x, y)
pred_buy_proba = clf.predict_proba(new_x)[:, 1].ravel()
res.append(pred_buy_proba)
res = np.array(res).T
pred = (res - train).argsort(axis=1)[::-1]
res = np.zeros(9)
for u in xrange(train.shape[0]):
truth = test[u]
pred_order = pred[u]
actual_bought = truth.nonzero()[0]
score = apk(actual_bought, pred_order, mapk)
tmp = [score]
for k in ks:
tmp.append(prec(actual_bought, pred_order, k))
tmp.append(recall(actual_bought, pred_order, k))
res += np.array(tmp)
if u % 50 == 0:
print res / (u + 1)
return res / (u + 1)
def main():
ks = [3, 5, 10, 20]
mapk = 200
train, test = load_data()
train, test = train.as_matrix(), test.as_matrix()
pred = train.sum(axis=0)
res = np.zeros(9)
x = train.T
for u in xrange(train.shape[0]):
y = x[:, u]
truth = test[u]
pred_buy_proba = pred
y[y > 0] = float('-inf')
pruned_buy_proba = pred_buy_proba + y.ravel()
pred_order = pruned_buy_proba.argsort()[::-1]
actual_bought = truth.nonzero()[0]
score = apk(actual_bought, pred_order, mapk)
tmp = [score]
for k in ks:
tmp.append(prec(actual_bought, pred_order, k))
tmp.append(recall(actual_bought, pred_order, k))
res += np.array(tmp)
if u % 50 == 0:
print res / (u + 1)
return res / (u + 1)
def evaluate_results(qids_rs, Y, k):
values = defaultdict(list)
for qid, r in qids_rs:
gold = harvest(Y, qid)
gold_topk = gold[argtopk(gold, k)]
R = np.count_nonzero(gold_topk)
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(r, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
return values
def evaluate_results(qids_rs, Y, k):
values = defaultdict(list)
for qid, r in qids_rs:
gold = harvest(Y, qid)
gold_topk = gold[argtopk(gold, k)]
R = np.count_nonzero(gold_topk)
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(r, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
return values
def main():
ks = [3, 5, 10, 20]
mapk = 200
epoch = 1
res = np.zeros(9)
train, test = load_data()
train, test = train.as_matrix(), test.as_matrix()
x = train.T
clf = SGDClassifier(random_state=42, loss='log')
for i in xrange(epoch):
for u in xrange(train.shape[0]):
y = x[:, u]
new_x = np.append(x,
np.repeat(y[:, np.newaxis], x.shape[1], axis=1),
axis=1)
clf.partial_fit(new_x, y, classes=[0, 1])
for u in xrange(train.shape[0]):
y = x[:, u]
truth = test[u]
new_x = np.append(x,
np.repeat(y[:, np.newaxis], x.shape[1], axis=1),
axis=1)
clf.predict(new_x)
pred_buy_proba = clf.predict_proba(new_x)[:, 1].ravel()
pruned_buy_proba = pred_buy_proba - y.ravel()
pred_order = pruned_buy_proba.argsort()[::-1]
actual_bought = truth.nonzero()[0]
score = apk(actual_bought, pred_order, mapk)
tmp = [score]
for k in ks:
tmp.append(prec(actual_bought, pred_order, k))
tmp.append(recall(actual_bought, pred_order, k))
res += np.array(tmp)
if u % 50 == 0:
print res / (u + 1)
return res / (u + 1)
def evaluate(self, X, Y, k=20, verbose=0, replacement=0, n_jobs=1):
"""
:X: [(qid, str)] query id, query string pairs
:Y: pandas dataseries with qid,docid index or [dict]
:k: Limit the result for all metrics to this value, the models are also
given a hint of how many they should return.
:replacement: 0 means that (query, doc) pairs not prevalent in Y will
not be considered relevant, None means that those are not considered
(skipped).
"""
# rs = []
# if n_jobs > 1:
# return process_and_evaluate(self, X, Y, k, n_jobs)
values = defaultdict(list)
for qid, query in X:
# execute query
if verbose > 0:
print(qid, ":", query)
t0 = timer()
# if replacement is None, we need to drop after querying
result = self.query(query, k=(None if replacement is None else k))
values["time_per_query"].append(timer() - t0)
# if verbose > 0:
# print(result[:k])
# result = result[:k] # TRIM HERE
# soak the generator
scored_result = [
harvest(Y, qid, docid, replacement) for docid in result
]
if replacement is None:
scored_result, notfound = filter_none(scored_result)
values["gold_not_found"].append(notfound)
if k is not None:
# dont let the models cheat by returning more than k
r = scored_result[:k]
else:
# if k is None, consider all
r = scored_result
# if verbose > 0:
# print(r)
# gold = np.array(list(Y[qid].values()))
gold = harvest(Y, qid)
import sys
# print(gold, file=sys.stderr)
topk_indices = argtopk(gold, k)
print(topk_indices, file=sys.stderr)
gold_topk = gold[topk_indices]
# print('Top k in gold standard:', gold_topk, file=sys.stderr)
R = np.count_nonzero(gold_topk)
if verbose > 0:
print("Retrieved {} relevant out of {} possible.".format(
np.count_nonzero(r), R))
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(scored_result, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
# rs.append(r)
if verbose > 0:
# print("Precision: {:.4f}".format(precision))
# print("Recall: {:.4f}".format(recall))
# print("F1-Score: {:.4f}".format(f1))
print("AP: {:.4f}".format(ap))
print("RR: {:.4f}".format(mrr))
print("NDCG: {:.4f}".format(ndcg))
return values
def evaluate(self, X, Y, k=20, verbose=0, replacement=0, n_jobs=1):
"""
:X: [(qid, str)] query id, query string pairs
:Y: pandas dataseries with qid,docid index or [dict]
:k: Limit the result for all metrics to this value, the models are also
given a hint of how many they should return.
:replacement: 0 means that (query, doc) pairs not prevalent in Y will
not be considered relevant, None means that those are not considered
(skipped).
"""
# rs = []
# if n_jobs > 1:
# return process_and_evaluate(self, X, Y, k, n_jobs)
values = defaultdict(list)
for qid, query in X:
# execute query
if verbose > 0:
print(qid, ":", query)
t0 = timer()
# if replacement is None, we need to drop after querying
result = self.query(query, k=(None if replacement is None else k))
values["time_per_query"].append(timer() - t0)
# if verbose > 0:
# print(result[:k])
# result = result[:k] # TRIM HERE
# soak the generator
scored_result = [harvest(Y, qid, docid, replacement)
for docid in result]
if replacement is None:
scored_result, notfound = filter_none(scored_result)
values["gold_not_found"].append(notfound)
if k is not None:
# dont let the models cheat by returning more than k
r = scored_result[:k]
else:
# if k is None, consider all
r = scored_result
# if verbose > 0:
# print(r)
# gold = np.array(list(Y[qid].values()))
gold = harvest(Y, qid)
import sys
# print(gold, file=sys.stderr)
topk_indices = argtopk(gold, k)
print(topk_indices, file=sys.stderr)
gold_topk = gold[topk_indices]
# print('Top k in gold standard:', gold_topk, file=sys.stderr)
R = np.count_nonzero(gold_topk)
if verbose > 0:
print("Retrieved {} relevant out of {} possible."
.format(np.count_nonzero(r), R))
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(scored_result, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
# rs.append(r)
if verbose > 0:
# print("Precision: {:.4f}".format(precision))
# print("Recall: {:.4f}".format(recall))
# print("F1-Score: {:.4f}".format(f1))
print("AP: {:.4f}".format(ap))
print("RR: {:.4f}".format(mrr))
print("NDCG: {:.4f}".format(ndcg))
return values
