def compute_differences(d1, d2, test_words):
ups_max = []
ups_n = []
rhos_max = []
rhos_n = []
n_var = parameters["evaluation n value"]
for w in sorted(test_words):
n_max = min(len(d1[w]), len(d2[w]))
tw1_sorted = sorted(d1[w].items(), key=lambda x: (-x[1], x[0]))
tw2_sorted = sorted(d2[w].items(), key=lambda x: (-x[1], x[0]))
lw1 = [e[0] for e in tw1_sorted]
lw2 = [e[0] for e in tw2_sorted]
dw1_max = normalize_dict(dict(tw1_sorted[:n_max]))
dw2_max = normalize_dict(dict(tw2_sorted[:n_max]))
dw1_n = normalize_dict(dict(tw1_sorted[:n_var]))
dw2_n = normalize_dict(dict(tw2_sorted[:n_var]))
upsilon_max = 0.5 * sum(abs((dw1_max.get(resp) or 0) - (dw2_max.get(resp) or 0)) for resp in sorted(set(dw1_max) | set(dw2_max)))
upsilon_n = 0.5 * sum(abs((dw1_n.get(resp) or 0) - (dw2_n.get(resp) or 0)) for resp in sorted(set(dw1_n) | set(dw2_n)))
sl, ll = sorted([(len(lw1), lw1), (len(lw2), lw2)])
s, S = sl
l, L = ll
rho_max = 1 - metrics.apk(S, L, s)
rho_n = 1 - metrics.apk(S[:n_var], L, n_var)
# ups_max.append((w, upsilon_max))
# ups_n.append((w, upsilon_n))
# rhos_max.append((w, rho_max))
# rhos_n.append((w, rho_n))
ups_max.append(upsilon_max)
ups_n.append(upsilon_n)
rhos_max.append(rho_max)
rhos_n.append(rho_n)
return ups_max, ups_n, rhos_max, rhos_n
def test_apk(self):
print metrics.apk(range(1, 6), [6, 4, 7, 1, 2], 2)
self.assertAlmostEqual(metrics.apk(range(1, 6), [1, 1, 1, 1, 1], 5),
0.2)
predicted = range(1, 21)
predicted.extend(range(200, 600))
self.assertAlmostEqual(metrics.apk(range(1, 100), predicted, 20), 1.0)
def mapk(truth, predict, k=5):
count = 0
sum = 0
for i, v in enumerate(truth):
sum = sum + ml_metrics.apk([v], predict[i], k)
count = count + 1
return float(sum)/count
def average_precision(sorted_predictions, holdout_items, k=5):
"""compute average precision for a single user"""
predicted_items = [pair[0] for pair in sorted_predictions]
apk = ml_metrics.apk(holdout_items, predicted_items, k=k)
frame = pd.DataFrame({
'user_id': [user_recommendations['user_id'].values[0]],
'average_precision': [apk]
})
return frame
def apk_via_inds_orders(pos_inds, order, k=10):
# Ranking for apk
# Note: Usually we would use
# ```
# topk_part = np.argpartition(-y_p, k)[:k] # topk partition
# topk = topk_part[np.argsort(-y_p[topk_part])] # recompute actual
# ```
# which is faster than `np.argsort(-y_p)[:k]`
# however, we want `order` which can be reused for other metrics (MRR)
# No need to truncate `order[:k]`
score_apk = apk(list(pos_inds), order, k=k)
return score_apk
def compute_mapk(gt, hypo, k_val):
if list_depth(hypo) == 2:
hypo = add_list_level(hypo.copy())
apk_list = []
for ii, query in enumerate(gt):
for jj, sq in enumerate(query):
apk_val = 0.0
if len(hypo[ii]) > jj:
apk_val = apk([sq], hypo[ii][jj], k_val)
apk_list.append(apk_val)
return np.mean(apk_list)
def main(k1, b, k3, feature, query_feature, r):
file_list = open(model_dir + "/file-list")
file = file_list.read().split("\n")
file.remove("")
num_doc = len(file)
file_dic = dict()
for i in range(num_doc):
term = file[i].split("/")[-1].lower()
file_dic[term] = i
inf_file = open(model_dir + "/inverted-file")
inf = inf_file.read()
del inf_file
inf = inf.split("\n")
inf.remove("")
dictionaries, que_dic, select_voc, num_term, que = build_term_dic(
model_dir, query_dir, num, k3=k3, F=feature, QF=query_feature)
D = build_doc_vector(num_doc, dictionaries, inf, select_voc, k1=k1, b=b)
#pca = PCA(n_components=int(num_term*0.9), svd_solver='full')
#D_=pca.fit_transform(D)
D_ = D
if if_train:
df = pd.read_csv("queries/ans_train.csv")
truth = []
for i in range(num):
app = []
ret = df["retrieved_docs"][i].split(" ")
for ele in ret:
app.append(file_dic[ele])
truth.append(app)
w_file = open("score.txt", "a")
for i in r:
train_rank = output(
que, num, num_doc, D_,
r=i) #,feature=["title","concepts","question"])
for j in range(num):
print(
ml_metrics.apk(truth[j],
train_rank[j, :100].tolist(),
k=100))
score = ml_metrics.mapk(truth, train_rank[:, :100].tolist(), k=100)
#w_file.write("s=%.3f,k1=%.1f, k3=%d, b=%.2f ,r=%d, f="%(score,k1,k3,b,i)+str(feature)+" q="+str(query_feature)+"\n")
print(i, score)
w_file.close()
else:
train_rank = output(que, num, num_doc, D_, r=r)[:, :100].tolist()
return train_rank, file
def mapk_train_vectors(models, labels, k=10):
"""
Log model MAP@k scores of models.
:params:
models (dict) : dict (name,model)
labels (dict) : dict mapping item to its recommendations
k (int) : K in MAP@K
"""
logger.info("Starting evaluation process for MAP@{}...".format(k))
best_score = 0
best_model = None
for model in models:
predictions = {}
for doc_id in labels.keys():
try:
predictions[doc_id] = [
l[0] for l in model.docvecs.most_similar(doc_id, topn=k)
]
except TypeError:
pass
mapk = np.mean([
apk(labels[doc_id], predictions[doc_id], k)
for doc_id in labels.keys() if doc_id in predictions.keys()
])
logger.info("{0} - MAP@{1} : {2}\n".format(str(model), k, mapk))
if mapk > best_score:
best_score = mapk
best_model = str(model)
logger.info("Best model with MAP@{0} = {1} : {2} \n ".format(
k, best_score, best_model))
np.nan_to_num(testStructure) print "Ranking......." testStructure = np.transpose(testStructure) scores = np.matmul(testStructure, invFileTable) ranks = np.argsort(-scores, axis=1) numImgs = ranks.shape[0] mapest = 0; trainImgsperFolder = invFileTable.shape[1]/84 testImgsperFolder = numImgs/84 for i in range(numImgs): x = (i/testImgsperFolder)*trainImgsperFolder mapest = mapest + metrics.apk(range(x, x+trainImgsperFolder), ranks[i,:], 10) mapest = mapest/numImgs print mapest #for i in range(numImgs): # print i/testImgsperFolder+1,">>>>>", ranks[i,:10]/trainImgsperFolder+1
def test_apk(self):
self.assertAlmostEqual(metrics.apk(range(1,6),[6,4,7,1,2], 2), 0.25)
self.assertAlmostEqual(metrics.apk(range(1,6),[1,1,1,1,1], 5), 0.2)
predicted = range(1,21)
predicted.extend(range(200,600))
self.assertAlmostEqual(metrics.apk(range(1,100),predicted, 20), 1.0)
for epoch in range(0,30):
print(80*'=')
print ('Epoch [{}/{}]'.format(epoch,30-1))
#def get_dataloader(train_root_dir,valid_root_dir,train_csv_name,valid_csv_name,num_train_triplets,num_valid_triplets,batch_size,num_workers)
data_loader,data_size = get_dataloader(train_path,test_path,train_csv_path,test_csv_path,num_train_triplets,num_test_triplets,5,6)
train_valid(model,optimizer,scheduler,epoch,data_loader,data_size)
print(80*"=")
#call run_experiment
run_experiment(train_path,test_path,train_csv_path,test_csv_path)
sumAP =0.0
for i in range(len(actual_list)):
sumAP+=ml_metrics.apk(actual_list[i].tolist(),predicted_list[i].tolist(),2)
print(ml_metrics.apk(actual_list[i].tolist(),predicted_list[i].tolist(),2))
print("mean Average precision ",str(sumAP/len(actual_list)))
sumAP =0.0
for i in range(len(actual_list)):
sumAP+=ml_metrics.apk(actual_list[i].tolist(),predicted_list[i].tolist(),5)
print(ml_metrics.apk(actual_list[i].tolist(),predicted_list[i].tolist(),5))
print("mean Average precision ",str(sumAP/len(actual_list)))
num_epochs=20
fig =plt.figure()
for phase in ['train', 'valid']:
list_epoch = []
list_loss = []
ite += 1
app_score /= len(truth[i])
score += app_score
score /= len(predict)
return score
import pandas as pd
def readans(path):
df = pd.read_csv(path)
truth = []
for i in range(len(df)):
app = []
ret = df["retrieved_docs"][i].split(" ")
for ele in ret:
app.append(ele)
truth.append(app)
return truth
import ml_metrics
import sys
truth = readans("ans_train.csv")
predict = readans("%s.csv" % (sys.argv[1]))
print("overall score", ml_metrics.mapk(truth, predict, k=100))
for i in range(len(truth)):
print(i, len(truth[i]), ml_metrics.apk(truth[i], predict[i], k=100))
def mean_avg_precision_k(tupla, k=5):
return ml_metrics.apk(tupla[0], tupla[1], k)
def main(self):
logger.info("Start")
start = time.clock()
# random users_ids
user_ids = self.loadUserIds()
# users --> friends dict
users_friends, users_friends_set = self.loadUsersFriends(user_ids)
logger.info("len of users_friends_set:%d"%len(users_friends_set))
# user --> business
users_business = self.loadUsersBusiness(users_friends_set)
logger.info("len of users_business:%d"%len(users_business))
# rating
rankRatingDict, predicted_rating = self.readCvs(INFILE_RANK_RATING)
# rank business pagerank
rank_pagerank = self.loadObjJson(self.infile_rank_pagerank)
predicted_pagerank = [x['business_id'] for x in rank_pagerank]
rankPagerankDict = self.createDictForWinners(rank_pagerank)
# rank business indegree
rank_indegree = self.loadObjJson(self.infile_rank_indegree)
predicted_indegree = [x['business_id'] for x in rank_indegree]
rankIndegreeDict = self.createDictForWinners(rank_indegree)
all_users = users_friends.keys()
#pp(all_users)
samples = np.split(np.array(range(TOTAL_USERS)), NUM_SAMPES)
maps = {'pagerank':[],'indegree':[],'rating':[]}
winnersAllCountList = list()
for i,chunk in enumerate(samples):
logger.info("sample: %d"%i)
sample = [all_users[x] for x in chunk]
actual = self.getBusinessFromUsers(sample, users_friends, users_business)
logger.info("len of actual:%d"%len(actual))
map_pagerank = ml.apk(actual, predicted_pagerank, k=K_MAP)
logger.info("map_pagerank[%d]:%6.6f"%(i,map_pagerank))
maps['pagerank'].append(map_pagerank)
map_indegree = ml.apk(actual, predicted_indegree, k=K_MAP)
logger.info("map_indegree[%d]:%6.6f"%(i,map_indegree))
maps['indegree'].append(map_indegree)
map_rating = ml.apk(actual, predicted_rating, k=K_MAP)
logger.info("map_rating[%d]:%6.6f"%(i,map_rating))
maps['rating'].append(map_rating)
winnersCountDict = self.winnersCountWinners(actual, rankRatingDict,rankPagerankDict,rankIndegreeDict)
winnersAllCountList.append(winnersCountDict)
#pp(maps)
maps_stats = {'pagerank':{},'indegree':{},'rating':{}}
for x in maps.keys():
maps_stats[x]['mean'] = np.mean(maps[x])
maps_stats[x]['std'] = np.std(maps[x])
pp(maps_stats)
win_stats = {'pagerank':{},'indegree':{},'rating':{}}
for x in winnersCountDict.keys():
win_stats[x]['mean'] = np.mean(winnersCountDict[x])
win_stats[x]['std'] = np.std(winnersCountDict[x])
pp(win_stats)
# out final rank
self.saveCsv(OUTFILE_RANK,maps)
# out final winners
self.saveWinnerCsv(OUTFILE_WINNER, winnersAllCountList)
elapsed = (time.clock() - start)
logger.info("done in %d secs"%int(elapsed))
def main():
args = parse_args()
feature_folders = args.feature_folders
prob_extensions = args.prob_extensions
fusion_weights = args.fusion_weights
split = args.split
num_features = len(feature_folders)
assert (num_features >= 1)
assert (len(prob_extensions) == 1 or len(prob_extensions) == num_features)
assert (len(fusion_weights) == 1 or len(fusion_weights) == num_features)
if (len(prob_extensions) == 1):
prob_extensions = np.repeat(prob_extensions, num_features)
if (len(fusion_weights) == 1):
fusion_weights = np.repeat(fusion_weights, num_features)
print 'computing multi-features video accuracy of Hollywood2 split %d datasets ...' % split
start_time = datetime.now()
# getting train/test files information
info_folder = '/home/tranlaman/Public/data/video/Hollywood2/ClipSets/'
actions = [
'AnswerPhone', 'DriveCar', 'Eat', 'FightPerson', 'GetOutCar',
'HandShake', 'HugPerson', 'Kiss', 'Run', 'SitDown', 'SitUp', 'StandUp'
]
num_actions = len(actions)
test_info_file = '/home/tranlaman/Public/data/video/Hollywood2/ClipSets/actions_test.txt'
lines = np.loadtxt(test_info_file,
dtype=str,
delimiter=' ',
comments=None)
testFiles = lines[:, 0]
numTestFiles = len(testFiles)
assert (numTestFiles == 884)
list_video_gt_per_action = []
for ind in xrange(len(actions)):
action = actions[ind]
action_split_file = '%s/%s_test.txt' % (info_folder, action)
lines = np.loadtxt(action_split_file,
dtype=str,
delimiter=' ',
comments=None)
video_gt_per_action = []
line_index = 0
for line in lines:
if int(line[1]) == 1:
video_gt_per_action.append(line_index)
line_index += 1
list_video_gt_per_action.append(video_gt_per_action)
# get prediction of videos
list_video_prediction = list()
list_video_score = list()
for i in xrange(numTestFiles):
video = testFiles[i]
mean_prob = np.zeros((num_actions))
for k in xrange(0, num_features):
# load spatial features
feature_path = os.path.join(feature_folders[k], video,
'*.%s' % prob_extensions[k])
feature_files = glob.glob(feature_path)
feature_files.sort()
if len(feature_files) == 0:
print 'There is no clips in video %s' % video
sys.exit(1)
prob_mat = np.zeros((num_actions, len(feature_files)))
for ind in xrange(0, len(feature_files)):
file_path = os.path.join(feature_folders[k], video,
feature_files[ind])
blob = load_blob_from_binary(file_path)
prob_mat[:, ind] = blob
feature_mean_prob = np.mean(prob_mat, axis=1)
mean_prob = mean_prob + fusion_weights[k] * feature_mean_prob
mean_prob = softmax(mean_prob)
prediction = np.argmax(mean_prob)
conf = mean_prob[prediction]
list_video_prediction.append(prediction)
list_video_score.append(conf)
list_video_prediction = np.array(list_video_prediction)
list_video_score = np.array(list_video_score)
# compute average precision
ap = np.zeros(len(actions))
list_video_id = np.array(range(numTestFiles))
for ind in xrange(len(actions)):
pred_index = (list_video_prediction == ind)
video_id_pred = list_video_id[pred_index]
score_pred = list_video_score[pred_index]
#sorted_idx = score_pred.argsort()[::-1]
sorted_idx = sorted(range(len(score_pred)),
key=lambda k: score_pred[k],
reverse=True)
sorted_video_id_pred = video_id_pred[sorted_idx]
video_id_gt = list_video_gt_per_action[ind]
ap[ind] = metrics.apk(video_id_gt, sorted_video_id_pred)
meanAP = ap.mean()
current_time = datetime.now()
run_time = current_time - start_time
print 'Run-time: ', run_time
print "Video meanAP of two-stream on test set: {}".format(meanAP)
# write the results into txt file
expString = 'Video classification (meanAP) with multiple cnn features.\n'
fid = open('results.txt', 'a')
fid.write('\n--------------------------------------------------\n')
fid.write('%s' % (expString))
fid.write('Video meanAP on testset split %d of Hollywood2 is %f\n' %
(split, meanAP))
fid.write('Test on following feature folders:\n')
for k in xrange(num_features):
fid.write('Feature folder: %s\n' % feature_folders[k])
fid.write('Prob extension: %s\n' % prob_extensions[k])
fid.write('Fusion weight: %s\n' % fusion_weights[k])
fid.write('Expriment finished at %s \n' % current_time)
fid.close()
# write per class accuracy
# write the results into txt file
fid = open('perClassAccuracy.txt', 'a')
fid.write('\n--------------------------------------------------\n')
fid.write('%s' % (expString))
fid.write('Video meanAP on testset split %d of Hollywood2 is %f\n' %
(split, meanAP))
fid.write('Test on following feature folders:\n')
for k in xrange(num_features):
fid.write('Feature folder: %s\n' % feature_folders[k])
fid.write('Prob extension: %s\n' % prob_extensions[k])
fid.write('Fusion weight: %s\n' % fusion_weights[k])
fid.write('Expriment finished at %s \n' % current_time)
fid.write('Per class AP\n')
for ind in xrange(0, num_actions):
fid.write('Class %d: ... %0.2f\n' % (ind, ap[ind]))
fid.close()
def main():
args = parse_args()
feature_folders = args.feature_folders
prob_extensions = args.prob_extensions
fusion_weights = args.fusion_weights
split = args.split
num_features = len(feature_folders)
assert (num_features >= 1)
assert (len(prob_extensions) == 1 or len(prob_extensions) == num_features)
assert (len(fusion_weights) == 1 or len(fusion_weights) == num_features)
if (len(prob_extensions) == 1):
prob_extensions = np.repeat(prob_extensions, num_features)
if (len(fusion_weights) == 1):
fusion_weights = np.repeat(fusion_weights, num_features)
print 'computing multi-features video accuracy of Olympic split %d datasets ...' % split
start_time = datetime.now()
# getting train/test files information
info_folder = '/home/tranlaman/Public/data/video/Olympic_Sports_split/test/'
actions = [
'basketball_layup', 'bowling', 'clean_and_jerk', 'discus_throw',
'diving_platform_10m', 'diving_springboard_3m', 'hammer_throw',
'high_jump', 'javelin_throw', 'long_jump', 'pole_vault', 'shot_put',
'snatch', 'tennis_serve', 'triple_jump', 'vault'
]
num_actions = len(actions)
testFiles = []
list_video_gt_per_action = []
video_index = 0
for ind in xrange(len(actions)):
action = actions[ind]
action_split_file = '%s/%s.txt' % (info_folder, action)
lines = np.loadtxt(action_split_file,
dtype=str,
delimiter=' ',
comments=None)
video_gt_per_action = []
for line in lines:
video_gt_per_action.append(video_index)
video_index += 1
testFiles.append(os.path.join(action, line))
list_video_gt_per_action.append(video_gt_per_action)
numTestFiles = len(testFiles)
assert (numTestFiles == 134)
# get prediction of videos
list_video_prediction = list()
list_video_score = list()
for i in xrange(numTestFiles):
video = testFiles[i]
mean_prob = np.zeros((num_actions))
for k in xrange(0, num_features):
# load spatial features
feature_path = os.path.join(feature_folders[k], video,
'*.%s' % prob_extensions[k])
feature_files = glob.glob(feature_path)
feature_files.sort()
if len(feature_files) == 0:
print 'There is no clips in video %s' % video
sys.exit(1)
prob_mat = np.zeros((num_actions, len(feature_files)))
for ind in xrange(0, len(feature_files)):
file_path = os.path.join(feature_folders[k], video,
feature_files[ind])
blob = load_blob_from_binary(file_path)
prob_mat[:, ind] = blob
feature_mean_prob = np.mean(prob_mat, axis=1)
mean_prob = mean_prob + fusion_weights[k] * feature_mean_prob
mean_prob = softmax(mean_prob)
prediction = np.argmax(mean_prob)
conf = mean_prob[prediction]
list_video_prediction.append(prediction)
list_video_score.append(conf)
list_video_prediction = np.array(list_video_prediction)
list_video_score = np.array(list_video_score)
# compute average precision
ap = np.zeros(len(actions))
list_video_id = np.array(range(numTestFiles))
for ind in xrange(len(actions)):
pred_index = (list_video_prediction == ind)
video_id_pred = list_video_id[pred_index]
score_pred = list_video_score[pred_index]
sorted_idx = score_pred.argsort()[::-1]
#sorted_idx = sorted(range(len(score_pred)), key=lambda k: score_pred[k], reverse=True)
sorted_video_id_pred = video_id_pred[sorted_idx]
video_id_gt = list_video_gt_per_action[ind]
ap[ind] = metrics.apk(video_id_gt, sorted_video_id_pred)
meanAP = ap.mean()
current_time = datetime.now()
run_time = current_time - start_time
print 'Run-time: ', run_time
print "Video meanAP of two-stream on test set: {}".format(meanAP)
# write the results into txt file
expString = 'Video classification (meanAP) with multiple cnn features.\n'
fid = open('results.txt', 'a')
fid.write('\n--------------------------------------------------\n')
fid.write('%s' % (expString))
fid.write('Video meanAP on testset split %d of Olympic is %f\n' %
(split, meanAP))
fid.write('Test on following feature folders:\n')
for k in xrange(num_features):
fid.write('Feature folder: %s\n' % feature_folders[k])
fid.write('Prob extension: %s\n' % prob_extensions[k])
fid.write('Fusion weight: %s\n' % fusion_weights[k])
fid.write('Expriment finished at %s \n' % current_time)
fid.close()
# write per class accuracy
# write the results into txt file
fid = open('perClassAccuracy.txt', 'a')
fid.write('\n--------------------------------------------------\n')
fid.write('%s' % (expString))
fid.write('Video meanAP on testset split %d of Olympic is %f\n' %
(split, meanAP))
fid.write('Test on following feature folders:\n')
for k in xrange(num_features):
fid.write('Feature folder: %s\n' % feature_folders[k])
fid.write('Prob extension: %s\n' % prob_extensions[k])
fid.write('Fusion weight: %s\n' % fusion_weights[k])
fid.write('Expriment finished at %s \n' % current_time)
fid.write('Per class AP\n')
for ind in xrange(0, num_actions):
fid.write('Class %d: ... %0.2f\n' % (ind, ap[ind]))
fid.close()
def evaluateAP(actual, predicted):
return metrics.apk(actual, predicted, K)
def main(self):
logger.info("Start")
start = time.clock()
# random users_ids
user_ids = self.loadUserIds()
# users --> friends dict
users_friends, users_friends_set = self.loadUsersFriends(user_ids)
logger.info("len of users_friends_set:%d" % len(users_friends_set))
# user --> business
users_business = self.loadUsersBusiness(users_friends_set)
logger.info("len of users_business:%d" % len(users_business))
# rating
rankRatingDict, predicted_rating = self.readCvs(INFILE_RANK_RATING)
# rank business pagerank
rank_pagerank = self.loadObjJson(self.infile_rank_pagerank)
predicted_pagerank = [x['business_id'] for x in rank_pagerank]
rankPagerankDict = self.createDictForWinners(rank_pagerank)
# rank business indegree
rank_indegree = self.loadObjJson(self.infile_rank_indegree)
predicted_indegree = [x['business_id'] for x in rank_indegree]
rankIndegreeDict = self.createDictForWinners(rank_indegree)
all_users = users_friends.keys()
#pp(all_users)
samples = np.split(np.array(range(TOTAL_USERS)), NUM_SAMPES)
maps = {'pagerank': [], 'indegree': [], 'rating': []}
winnersAllCountList = list()
for i, chunk in enumerate(samples):
logger.info("sample: %d" % i)
sample = [all_users[x] for x in chunk]
actual = self.getBusinessFromUsers(sample, users_friends,
users_business)
logger.info("len of actual:%d" % len(actual))
map_pagerank = ml.apk(actual, predicted_pagerank, k=K_MAP)
logger.info("map_pagerank[%d]:%6.6f" % (i, map_pagerank))
maps['pagerank'].append(map_pagerank)
map_indegree = ml.apk(actual, predicted_indegree, k=K_MAP)
logger.info("map_indegree[%d]:%6.6f" % (i, map_indegree))
maps['indegree'].append(map_indegree)
map_rating = ml.apk(actual, predicted_rating, k=K_MAP)
logger.info("map_rating[%d]:%6.6f" % (i, map_rating))
maps['rating'].append(map_rating)
winnersCountDict = self.winnersCountWinners(
actual, rankRatingDict, rankPagerankDict, rankIndegreeDict)
winnersAllCountList.append(winnersCountDict)
#pp(maps)
maps_stats = {'pagerank': {}, 'indegree': {}, 'rating': {}}
for x in maps.keys():
maps_stats[x]['mean'] = np.mean(maps[x])
maps_stats[x]['std'] = np.std(maps[x])
pp(maps_stats)
win_stats = {'pagerank': {}, 'indegree': {}, 'rating': {}}
for x in winnersCountDict.keys():
win_stats[x]['mean'] = np.mean(winnersCountDict[x])
win_stats[x]['std'] = np.std(winnersCountDict[x])
pp(win_stats)
# out final rank
self.saveCsv(OUTFILE_RANK, maps)
# out final winners
self.saveWinnerCsv(OUTFILE_WINNER, winnersAllCountList)
elapsed = (time.clock() - start)
logger.info("done in %d secs" % int(elapsed))
_Q = 3
### calibrated
### pr=MSD_rec.PredSIc(s2u_tr, _A, _Q, "songs_scores.txt")
### uncalibrated
pr = MSD_rec_shefali.PredSI(s2u_tr, _A, _Q)
pr2 = MSD_rec_shefali.PredSU(
s2u_tr, 0.3,
5) #with only this, 0.0 for 10 users in contrast to 0.33 in SI
print 'Creating recommender..'
cp = MSD_rec_shefali.SReco(songs_ordered)
cp.Add(pr)
cp.Add(pr2)
cp.Gamma = [1.0]
r = cp.RecommendToUsers(users_v[user_min:user_max], u2s_v)
#cp.Valid(uu, u2s_v, u2s_h, n_batch=10)
c = 0
for i, user in enumerate(users_v[user_min:user_max]):
c = c + metrics.apk(u2s_h[user], r[user])
pass
print i
t = c / (i + 1)
print t
MSD_util.save_recommendations(r, "kaggle_songs.txt", osfile)
#print j
if user in utsh:
flag = 0
# print user
rec = []
l1 = uts[user]
l2 = map(int, l1)
#print l2[1]
for i in l2:
if i < 10000:
flag = 1
#print indices[i]
rec.extend(indices[i])
#rec.extend(indices[1])
# print rec
# print utsh[user]
hid = map(int, utsh[user])
if flag > 0:
count = count + 1
#print hid
#print j
c = c + metrics.apk(utsh[user], rec, 10000)
#print j
#print count
#print c
relevant = c / (count)
print relevant
allu = c / j
#print allu
def _get_groupped_action(self, **kwargs):
"""
Groups data by `group_by_field` field and calculates mean average
precision.
Note: `group_by_field` should be specified in request parameters.
"""
from ml_metrics import apk
import numpy as np
from operator import itemgetter
logging.info('Start request for calculating MAP')
group_by_field, count = self._parse_map_params()
if not group_by_field:
return odesk_error_response(400, ERR_INVALID_DATA,
'field parameter is required')
res = []
avps = []
groups = TestExample.get_grouped(
field=group_by_field,
model_id=kwargs.get('model_id'),
test_result_id=kwargs.get('test_result_id')
)
import sklearn.metrics as sk_metrics
import numpy
if len(groups) < 1:
logging.error('Can not group')
return odesk_error_response(400, ERR_INVALID_DATA,
'Can not group')
if 'prob' not in groups[0]['list'][0]:
logging.error('Examples do not contain probabilities')
return odesk_error_response(400, ERR_INVALID_DATA, 'Examples do \
not contain probabilities')
if not isinstance(groups[0]['list'][0]['prob'], list):
logging.error('Examples do not contain probabilities')
return odesk_error_response(400, ERR_INVALID_DATA, 'Examples do \
not contain probabilities')
if groups[0]['list'][0]['label'] in ("True", "False"):
def transform(x):
return int(bool(x))
elif groups[0]['list'][0]['label'] in ("0", "1"):
def transform(x):
return int(x)
else:
logging.error('Type of labels do not support')
return odesk_error_response(400, ERR_INVALID_DATA,
'Type of labels do not support')
logging.info('Calculating avps for groups')
calc_average = True
for group in groups:
group_list = group['list']
labels = [transform(item['label']) for item in group_list]
pred_labels = [transform(item['pred']) for item in group_list]
probs = [item['prob'][1] for item in group_list]
if len(labels) > 1:
labels = numpy.array(labels)
probs = numpy.array(probs)
try:
precision, recall, thresholds = \
sk_metrics.precision_recall_curve(labels, probs)
avp = sk_metrics.auc(recall[:count], precision[:count])
except:
avp = apk(labels, pred_labels, count)
else:
avp = apk(labels, pred_labels, count)
if math.isnan(avp):
calc_average = False
avp = "Can't be calculated"
avps.append(avp)
res.append({'group_by_field': group[group_by_field],
'count': len(group_list),
'avp': avp})
res = sorted(res, key=itemgetter("count"), reverse=True)[:100]
logging.info('Calculating map')
mavp = np.mean(avps) if calc_average else "N/A"
context = {self.list_key: {'items': res},
'field_name': group_by_field,
'mavp': mavp}
logging.info('End request for calculating MAP')
return self._render(context)
listOfReleventList = []
listOfRetrievedList = []
bar = progressbar.ProgressBar(redirect_stdout=True)
for i in bar(range(len(queryList))):
queryImage = np.loadtxt('./' + queryDir + '/' + str(queryList[i]) +
'.vc',
delimiter=',')
relevantList = buildRelevantList(queryDir, queryList[i])
nearestNeighbours = imageUtilities.initList(len(relevantList))
distance = Parallel(n_jobs=-1)(
delayed(worker)(queryImage, referenceList[j], databaseDir)
for j in range(len(referenceList)))
nearestNeighbours = sorted(
distance, key=itemgetter('distance'))[:len(relevantList)]
retrievedList = []
for k in nearestNeighbours:
retrievedList.append(int(k['filename']))
apk = metrics.apk(actual=relevantList, predicted=retrievedList)
listOfReleventList.append(relevantList)
listOfRetrievedList.append(retrievedList)
mapk = metrics.mapk(actual=listOfReleventList,
predicted=listOfRetrievedList)
print(mapk)
text = '\nMAPK for ' + queryDir + ' against ' + \
databaseDir + ' is ' + str(mapk)
file = open("mapk.txt", "a")
file.write(text)
file.close()
def average_precision(self):
from ml_metrics import apk
if not hasattr(self, '_apk'):
self._apk = apk(self._labels, self._preds)
return self._apk
def mAP_(actual, predicted):
AP = [ml_metrics.apk(actual=actual[i], predicted=predicted[i], k=5) if len(actual[i]) else 0 for i in range(len(predicted)) ]
mAP = np.mean(AP)
std = np.std(AP)/np.sqrt(len(AP))
#print(f'Error: {mAP:.4f} +/- {3*std:.4f}')
return mAP
for song in songs_ordered:
if len(songs_to_recommend) >= 500:
break
if not song in user_to_songs[user]:
songs_to_recommend.append(song_to_index[song])
pass
myDic[user] = songs_to_recommend
print user
print myDic[1]
pass
pass
c = 0
for i, user in enumerate(canonical_users):
c = c+metrics.apk(uts[user],myDic[user])
pass
t = c/i
print t
//2nd file starts from here
with open ("song_unsort.txt", "w") as f2:
import ml_metrics as metrics
# ##I am going to show how order matters in MAP@K when there is only 1 answer.
#
# ##This experiment is done by calculating AP@K, which gives 1 value. MAP@K is the average of AP@K.
#
# In[ ]:
actual = [1]
predicted = [1, 2, 3, 4, 5]
print('Answer=', actual, 'predicted=', predicted)
print('AP@5 =', metrics.apk(actual, predicted, 5))
predicted = [2, 1, 3, 4, 5]
print('Answer=', actual, 'predicted=', predicted)
print('AP@5 =', metrics.apk(actual, predicted, 5))
predicted = [3, 2, 1, 4, 5]
print('Answer=', actual, 'predicted=', predicted)
print('AP@5 =', metrics.apk(actual, predicted, 5))
predicted = [4, 2, 3, 1, 5]
print('Answer=', actual, 'predicted=', predicted)
print('AP@5 =', metrics.apk(actual, predicted, 5))
predicted = [4, 2, 3, 5, 1]
print('Answer=', actual, 'predicted=', predicted)
def stop(x, y, z, a):
if z > len(y):
sys.exit()
else:
return metrics.apk(x, y, a)
def get_cluster_recommendations_and_calculate_metrics(
estimator,
user_artists_df: pd.DataFrame(columns=["user_id", "artists"]),
train_matrix: csr_matrix,
test_matrix: csr_matrix,
train: NDArray[(Any, ), int],
test: NDArray[(Any, ), int],
) -> Dict[int, float]:
"""
Обучение модели кластеризации, поиск рекомендаций, оценка качества рекомендаций.
"""
estimator = estimator.fit(train_matrix.toarray())
train_df = pd.DataFrame([], columns=["artists", "clusters"])
train_df["artists"] = train
train_df["clusters"] = estimator.labels_
train_df = train_df.groupby("clusters").agg(list)
print("Основные статистики кластеризации:")
print("Количество исполнителей: ", len(estimator.labels_))
print("Количество кластеров:", len(np.unique(estimator.labels_)))
counts_clusters = np.unique(estimator.labels_, return_counts=True)[1]
print("Наибольший размер кластера:", np.max(counts_clusters))
print("Гистограмма распределения количества кластеров по их величине")
hist = np.histogram(counts_clusters,
bins=[1, 2, 3, 4, 5, 10, 50, 100, 500, 1000])
print("Деления: \n", hist[1])
print("Частоты: \n", hist[0])
test_df = pd.DataFrame([], columns=["artists", "clusters"])
test_df["artists"] = test
test_df["clusters"] = estimator.predict(test_matrix.toarray())
test_df = test_df.groupby("clusters").agg(list).reset_index()
metrics = {}
for k in [1, 5, 10, 20]:
metrics[k] = 0
for cluster, test_artists in zip(test_df["clusters"], test_df["artists"]):
# Формирование рекомендации как топ-21 исполнителей в соответствующем кластере
if cluster in train_df.index:
recommend = get_topk_artists_from_idxs(21, user_artists_df,
train_df.loc[cluster])
for artist_id in test_artists:
# Поиск списков исполнителей, которых слушают те же пользователи,
# что и artist_id:
users = np.unique(user_artists_df.loc[
user_artists_df["artists"] == artist_id, "user_id"].values)
similar_artists = (
user_artists_df.loc[user_artists_df["user_id"].isin(users),
"artists"].drop_duplicates().values)
similar_artists = similar_artists[similar_artists != artist_id]
# Подсчет метрик качества
for key in metrics.keys():
metrics[key] += ml_metrics.apk(
actual=list(similar_artists),
predicted=list(recommend[:key]),
k=key,
)
for key in metrics.keys():
metrics[key] /= len(test)
print("MAP@{}: ".format(key), metrics[key])
return metrics
