def __init__(self, csv_file):

self.csv_file = open(csv_file, 'rb')

self.spamreader = csv.reader(self.csv_file, delimiter = ',')

def __iter__(self):

for row in spamreader:

doc = row[3].lower().translate(None, string.punctuation)

yield LabeledSentence(doc.split(), [row[4]])

def to_array(self):

self.sentences = []

new_reader = csv.reader(self.csv_file, delimiter = ',')

for row in new_reader:

doc = row[3].lower().translate(None, string.punctuation)

self.sentences.append(LabeledSentence(doc.split(), [row[4]]))

return self.sentences

def sentences_perm(self):

random.shuffle(self.sentences)

sentences = LabeledLineSentence(filename)

model = Doc2Vec(min_count=1, window=10, size=100, sample=1e-4, negative=5, workers=8)

print "starting vocab building"

model.build_vocab(sentences.to_array())

print "starting training"

for epoch in range(10):

print epoch

model.train(sentences.sentences_perm())

model.save('./imdb.d2v')

tokens = nltk.word_tokenize(text)

stems = []

stemmer = SnowballStemmer("english", ignore_stopwords=True)

for item in tokens:

stems.append(stemmer.stem(item))

model = Doc2Vec.load('../project_snapshot/imdb.d2v')

dataset = pickle.load(open('gensim_data.frmt'))

vecs = []

i = 0

with open(filename, 'rb') as csvfile:

spamreader = csv.reader(csvfile, delimiter = '\t')

for row in spamreader:

i += 1

val = tokenize(row[3].lower().translate(None, string.punctuation))

x = model.infer_vector(val)

print i

vecs.append(x)