words=doc_class[0].strip().split(' ')
#create a binary vector for the document with all the words that appear (0:does not appear,1:appears)
#we can set in a dictionary only the indexes of the words that appear
#and we can use that to build a SparseVector
vector_dict={}
for w in words:
vector_dict[dictionary[w]]=1
return LabeledPoint(doc_class[1], SparseVector(len(dictionary),vector_dict))
def Predict(name_text,dictionary,model):
words=name_text[1].strip().split(' ')
vector_dict={}
for w in words:
if(w in dictionary):
vector_dict[dictionary[w]]=1
return (name_text[0], model.predict(SparseVector(len(dictionary),vector_dict)))
data,Y=lf.loadLabeled("./data/train")
print len(data)
dataRDD=sc.parallelize(data,numSlices=16)
#map data to a binary matrix
#1. get the dictionary of the data
#The dictionary of each document is a list of UNIQUE(set) words
lists=dataRDD.map(lambda x:list(set(x.strip().split(' ')))).collect()
all=[]
#combine all dictionaries together (fastest solution for Python)
for l in lists:
all.extend(l)
dict=set(all)
print len(dict)
#it is faster to know the position of the word if we put it as values in a dictionary
dictionary={}
for i,word in enumerate(dict):
from sklearn.metrics import accuracy_score
from sklearn.cross_validation import train_test_split
#Other
import time
import scipy.sparse as sp
import matplotlib.pyplot as plt
import sys
import loadFiles as lf
sliding_window = 2
#%% Load the data and compute alternative features
data, labels = lf.loadLabeled("./train")
#%%Pre process
#Remove html tags
train = ct.removehtml(data)
#Create the dictionnary
data=ct.stemTokenize(train)
idfs = {}
num_documents = len(data)
print "number of documents %s"%num_documents
sc = SparkContext(appName="Simple App") #initialize the spark context
#since we are not in the command line interface we need to add to the spark context
#some of our classes so that they are available to the workers
sc.addFile("helpers.py")
sc.addFile("exctract_terms.py")
#now if we import these files they will also be available to the workers
from helpers import *
import extract_terms as et
# load data : data is a list with the text per doc in each cell. Y is the respective class value
#1 :positive , 0 negative
print "loading local data"
data,Y=lf.loadLabeled(trainF)
print "preprocessing"
pp.proc(data) #clean up the data from number, html tags, punctuations (except for "?!." ...."?!" are replaced by "."
m = TfidfVectorizer(analyzer=et.terms) # m is a compressed matrix with the tfidf matrix the terms are extracted with our own custom function
'''
we need an array to distribute to the workers ...
the array should be the same size as the number of workers
we need one element per worker only
'''
ex=np.zeros(8)
rp=randint(0,7)
ex[rp]=1 #one random worker will be selected so we set one random element to non-zero
md=sc.broadcast(m) #broadcast the vectorizer so that he will be available to all workers
vector_dict[dictionary[w]] = 1
return LabeledPoint(doc_class[1], SparseVector(len(dictionary),
vector_dict))
def Predict(name_text, dictionary, model):
words = name_text[1].strip().split(' ')
vector_dict = {}
for w in words:
if (w in dictionary):
vector_dict[dictionary[w]] = 1
return (name_text[0],
model.predict(SparseVector(len(dictionary), vector_dict)))
data, Y = lf.loadLabeled("./data/train")
#data,Y=lfp.loadLabeled("./data/train",1000)
print len(data)
dataRDD = sc.parallelize(data, numSlices=16)
#map data to a binary matrix
#1. get the dictionary of the data
#The dictionary of each document is a list of UNIQUE(set) words
lists = dataRDD.map(lambda x: list(set(x.strip().split(' ')))).collect()
all = []
#combine all dictionaries together (fastest solution for Python)
for l in lists:
all.extend(l)
dict = set(all)
print len(dict)
#it is faster to know the position of the word if we put it as values in a dictionary
words = [w for w in words if not any(i in w for i in ['/', '<', '>'])]
return words
def FinalPredict(name_text, dictionary, model):
#words=name_text[1].strip().split(' ')
words = doc2words(name_text[1])
vector_dict = {}
for w in words:
if (w in dictionary):
vector_dict[dictionary[w]] = 1
return (name_text[0],
model.predict(SparseVector(len(dictionary), vector_dict)))
data, Y = lf.loadLabeled("/Users/sofia/Desktop/big-data-project/data/train")
print "data loaded, length {}".format(len(data))
dataRDD = sc.parallelize(data, numSlices=16)
lists = dataRDD.map(doc2words)
#lists=dataRDD.map(doc2words).collect()
# create dict
all = []
for l in lists.collect():
all.extend(l)
dict = set(all)
# TF-IDF
hashingTF = HashingTF(numFeatures=len(dict))
tf = hashingTF.transform(lists)
tf.cache()
import loadFiles as lf
### warning here
import extract_terms as et
import preProcess as pp
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as nps
#### TODO change to the cluster folders small
data,Y=lf.loadLabeled("D://Data//DSSP//Data Camp 2//xcode//trainsmall")
#/home/xavier.callens/DataCamp
#data,Y=lf.loadLabeled("/home/xavier.callens/DataCamp/train")
print data
#analyzer and preprocessor dont work together...one or the other
pp.proc(data) # we apply the preprocessing by ourselves
m = TfidfVectorizer(analyzer=et.terms)
tt = m.fit_transform(data)
print tt
rows,cols=tt.shape
print "number of features :" +str(len(m.get_feature_names())) #this is the same as the number of columns
#the function get_feature_names() returns a list of all the terms found
print "non compressed matrix expected size:" + str(rows*cols*8/(1024*1024*1024))+"GB"