-
Notifications
You must be signed in to change notification settings - Fork 0
/
movie_classifier.py
73 lines (46 loc) · 1.68 KB
/
movie_classifier.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
import nltk
import random
import pickle
from nltk.corpus import movie_reviews
def movie_classifier():
featureSets=createFeatureSet(3000)
## # set that we'll train our classifier with
training_set = featureSets[:2500]
##
## # set that we'll test against.
testing_set = featureSets[2500:]
##
classifier = nltk.NaiveBayesClassifier.train(training_set)
print("Classifier accuracy percent:",(nltk.classify.accuracy(classifier, testing_set))*100)
def find_features(document, word_features):
words = set(document)
features = {}
for w in word_features:
features[w] = (w in words)
return features
def createFeatureSet(numOfExamples):
documents = [(list(movie_reviews.words(fileid)), category)
for category in movie_reviews.categories()
for fileid in movie_reviews.fileids(category)[:numOfExamples]]
with open('documents.txt', 'wb') as f:
pickle.dump(documents, f)
## #read from file
## with open('documents.txt', 'rb') as f:
## documents = pickle.load(f)
random.shuffle(documents)
all_words = []
## for w in movie_reviews.words():
## all_words.append(w.lower())
#write to file
## with open('allwords.txt', 'wb') as f:
## pickle.dump(all_words, f)
#read from file
with open('allwords.txt', 'rb') as f:
all_words = pickle.load(f)
freqDist = nltk.FreqDist(all_words)
#print('freq dist')
#print(freqDist.most_common(50))
word_features = freqDist.most_common(3000)
featuresets = [(find_features(rev, word_features), category) for (rev, category) in documents]
return featuresets
movie_classifier()