/
tfidf.py
57 lines (40 loc) · 1.61 KB
/
tfidf.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
from __future__ import division
import re, string, math, random
import dBDelegate
from bson.objectid import ObjectId
from collections import Counter
from itertools import islice
import itertools
import collections
import crawler
appropriate_punctuation = '!"#$%&()*+,./:;<=>?@[\\]^_`{|}~'
punct = re.compile(r'[\s{}]+'.format(re.escape(appropriate_punctuation)))
k = 2
db = dBDelegate.getDBConnection()
def remove_stopwords():
stopwords_file = open('stopwords.txt', 'r')
stopwords = stopwords_file.read()
stopwords = stopwords.split('\n')
stopwords_file.close()
return stopwords
def calculateTfidf(query, positional_index, song, avg_songlength, collection_length, weight, tfidf_values):
f = open('tfidf_samples.txt', 'a')
# for song in list_of_matching_documents:
similarity = 0
for word in set(query):
querytf = query.count(word)
raw_tf = len(positional_index[word]['document_dict'][song])
songtf = len(positional_index[word]['document_dict'])
similarity += (querytf * (raw_tf)/(raw_tf + (k * len(song)/avg_songlength))) * math.log10(collection_length/songtf)
# print (str(song) + ' 0 ' + str(similarity + weight) + ' 0\n')
tfidf_values[song] = similarity + weight
f.write(str(song) + ' ' + str(similarity + weight) + '\n')
f.close()
return tfidf_values
def sortTfidfValues(tfidf_values):
print "****************************************************************"
print "Top 20 TFIDF scores"
sorted_tfidf = sorted(tfidf_values.items(), key=lambda x: (-x[1], x[0]))
top_10_values = itertools.islice(sorted_tfidf, 0, 20)
for song, tfidf in top_10_values:
print dBDelegate.getSongTitle(db, song), tfidf