def get_feature_vectors(sc, input_file, feature_dimensions):
"""Get feature vector from the lines in input_file_obj using
TF/IDF.
Returns:
vectors RDD
"""
# Load documents (one per line).
tweet_file = sc.textFile(input_file)
input_text_rdd = tweet_file.map(lambda line: _tokenize(line))
input_text_rdd.cache()
# The default feature dimension is 2^20; for a corpus with million
# tweets recommended dimensions are 50000 or 100000. Use higher
# dimensions for larger corpus of tweets.
hashing_tf = HashingTF(feature_dimensions)
tf = hashing_tf.transform(input_text_rdd)
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
tfidf.cache()
return input_text_rdd, tfidf
def tfidf(self):
self._create_rdd()
hashingTF = HashingTF()
tf = hashingTF.transform(self.token_rdd)
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
return tfidf
def TFIDF(source, destination):
if destination[-1] != '/':
destination=destination+'/'
## typically define the source message
rdd=sc.wholeTextFiles(source).map(lambda (name,text): text.split())
tf=HashingTF()
tfVectors=tf.transform(rdd).cache()
a = tfVectors.collect()
# Storing the TF values above in individual files, one per link
ind = 0
for vector in a:
dest_path = destination + "TF_%d"%ind + ".txt"
ind = ind + 1
file = open(dest_path,'w')
file.write(str(vector))
file.close()
# Calculating IDF Values for each case.
idf=IDF()
idfModel=idf.fit(tfVectors)
tfIdfVectors=idfModel.transform(tfVectors)
# Writing TF-IDF values to a single file.
file = open(destination+"TF-IDF.txt", 'w')
file.write(str(tfIdfVectors.collect()))
try:
for i in range(0,100):
print ""#Testing Printing"
except KeyboardInterrupt:
pass
def TFIDF(source, destination):
if destination[-1] != '/':
destination = destination + '/'
## typically define the source message
rdd = sc.wholeTextFiles(source).map(lambda (name, text): text.split())
tf = HashingTF()
tfVectors = tf.transform(rdd).cache()
a = tfVectors.collect()
# Storing the TF values above in individual files, one per link
ind = 0
for vector in a:
dest_path = destination + "TF_%d" % ind + ".txt"
ind = ind + 1
file = open(dest_path, 'w')
file.write(str(vector))
file.close()
# Calculating IDF Values for each case.
idf = IDF()
idfModel = idf.fit(tfVectors)
tfIdfVectors = idfModel.transform(tfVectors)
# Writing TF-IDF values to a single file.
file = open(destination + "TF-IDF.txt", 'w')
file.write(str(tfIdfVectors.collect()))
try:
for i in range(0, 100):
print "" #Testing Printing"
except KeyboardInterrupt:
pass
def main(sc):
stopset = set(stopwords.words('english'))
tweets = sc.textFile('hdfs:/adi/sample.txt')
words = tweets.map(lambda word: word.split(" "))
wordArr = []
for wArr in words.collect():
tempArr = []
for w in wArr:
if not w in stopset:
tempArr.append(w)
wordArr.append(tempArr)
# Open a file
# print wordArr
#tokens = sc.textFile("hdfs:/adi/tokens1.txt")
# Load documents (one per line).
documents = sc.textFile("hdfs:/adi/tokens1.txt").map(lambda line: line.split(" "))
numDims = 100000
hashingTF = HashingTF(numDims)
tf = hashingTF.transform(documents)
tf.cache()
idf = IDF().fit(tf)
tfidf = idf.transform(tf)
tfidf.count()
model = KMeans.train(tfidf, 5)
model.save(sc,"tweetModel1")
print("Final centers: " + str(model.clusterCenters))
# print("Total Cost: " + str(model.computeCost(data)))
sc.stop()
def process(self, newsRDD):
'''
process for 4 classes
'''
hashingTF = HashingTF(self.n)
self.newsRDD = newsRDD
self.featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
#toto = self.featuresRDD.take(1)[0]
#print(toto.words + toto.bg2 + toto.bg3)
self.labeledPointsRdd = self.featuresRDD.map(lambda x: LabeledPoint(x.giveClasseN(1), hashingTF.transform(x.words + x.bg2 + x.bg3)))
try:
nbVeryPos = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 3).count()
nbPos = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 2).count()
nbNeg = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 1).count()
nbVeryNeg = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 0).count()
nbTot = self.featuresRDD.count()
print("nbTot %d" % nbTot)
print("\tnbVeryNeg %d" % nbVeryNeg)
print("\tnbNeg %d" % nbNeg)
print("\tnbPos %d" % nbPos)
print("\tnbVeryPos %d" % nbVeryPos)
except:
pass # empty rdd
return self.labeledPointsRdd
def analyse_data(self, data):
"""
针对入口数据进行合适的分析
param data: file, unicode, str
"""
words = self.sc.textFile(self.training_words_dir)
# 训练词频矩阵
hashingTF = HashingTF()
tf = hashingTF.transform(words)
# 计算TF-IDF矩阵
idfModel = IDF().fit(tf)
tfidf = idfModel.transform(tf)
tf.cache()
with open(self.NBmodel, 'r') as f:
NBmodel = pickle.load(f)
# 先分词后分析
yourwords = set("/".join(jieba.cut_for_search(data)).split("/"))
print '分词结果：{}'.format(yourwords)
yourtf = hashingTF.transform(yourwords)
yourtfidf = idfModel.transform(yourtf)
return NBmodel.predict(yourtfidf), data
class Searcher():
def __init__(self):
self.conf = SparkConf().setMaster("local").setAppName("Searcher")
self.sc = SparkContext(conf=self.conf)
def load_data(self, data_file):
raw_data = self.sc.textFile(data_file)
fields = raw_data.map(lambda x: x.split("\t"))
self.documents = fields.map(lambda x: x[3].split(" "))
self.document_names = fields.map(lambda x: x[1])
def hashing(self, size):
self.hashing_TF = HashingTF(
size) #100K hash buckets just to save some memory
tf = self.hashing_TF.transform(self.documents)
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
self.tfidf = idf.transform(tf)
def search(self, search_text):
search_text_TF = self.hashing_TF.transform([search_text])
search_text_hash_value = int(search_text_TF.indices[0])
search_text_relevance = self.tfidf.map(
lambda x: x[search_text_hash_value])
return search_text_relevance.zip(self.document_names)
def _compute_tfid(texts: RDD) -> IDFModel:
tf = HashingTF().transform(texts.map(lambda t: t.words))
tf.cache()
idf = IDF().fit(tf)
tfidfs = idf.transform(tf)
text_tfs = texts.zip(tfidfs)
return text_tfs.map(lambda t: t[0].set_tfidf(t[1]))
def main():
# 初始化 SparkContext
sc = spark_context(spark_master)
# 读取文件
data = sc.textFile(hdfs_path)
# 分词
documents = data.map(tokenize)
documents.cache()
# TF
hashingTF = HashingTF()
tf = hashingTF.transform(documents)
# IDF
idf = IDF(minDocFreq=2).fit(tf)
# TFIDF
tfidf = idf.transform(tf)
# 链接到 MongoDB
from pymongo import MongoClient
mongo_client = MongoClient(mongo_host)
mongo_client.admin.authenticate(mongo_user, mongo_pass, mechanism='SCRAM-SHA-1')
clear_mongodb(mongo_client)
# zip
term_tfidf = documents.zip(tfidf).map(doc_tfidf)
articles = term_tfidf.flatMap(lambda i: i).reduceByKey(lambda x, y: x + y)
for article in articles.collect():
item = {}
item['text'] = article[0].encode('utf-8')
item['size'] = int(article[1] * 10)
send_mongodb(mongo_client, item)
def entrenar_spam(sc,
sql_context,
dir_spam,
dir_no_spam,
num_trees=20,
max_depth=8):
input_spam = sc.textFile(dir_spam)
input_no_spam = sc.textFile(dir_no_spam)
spam = sql_context.read.json(input_spam).select("text").withColumn(
"label", F.lit(1.0))
no_spam = sql_context.read.json(input_no_spam).select("text").withColumn(
"label", F.lit(0.0))
training_data = spam.unionAll(no_spam)
tokenizer = Tokenizer(inputCol="text", outputCol="words")
wordsData = tokenizer.transform(training_data)
hashingTF = HashingTF(inputCol="words",
outputCol="rawFeatures",
numFeatures=140)
featurizedData = hashingTF.transform(wordsData)
"""idf = IDF(inputCol="rawFeatures", outputCol="features")
idfModel = idf.fit(featurizedData)
rescaledData = idfModel.transform(featurizedData)"""
seed = 1800009193L
(split_20_df, split_80_df) = featurizedData.randomSplit([20.0, 80.0], seed)
test_set_df = split_20_df.cache()
training_set_df = split_80_df.cache()
rf = RandomForestClassifier().setLabelCol("label") \
.setPredictionCol("predicted_label") \
.setFeaturesCol("rawFeatures") \
.setSeed(100088121L) \
.setMaxDepth(max_depth) \
.setNumTrees(num_trees)
rf_pipeline = Pipeline()
rf_pipeline.setStages([rf])
reg_eval = MulticlassClassificationEvaluator(
predictionCol="predicted_label",
labelCol="label",
metricName="accuracy")
crossval = CrossValidator(estimator=rf_pipeline,
evaluator=reg_eval,
numFolds=5)
param_grid = ParamGridBuilder().addGrid(rf.maxBins, [50, 100]).build()
crossval.setEstimatorParamMaps(param_grid)
modelo = crossval.fit(training_set_df).bestModel
predictions_and_labels_df = modelo.transform(test_set_df)
accuracy = reg_eval.evaluate(predictions_and_labels_df)
return modelo, accuracy
def main(sc):
stopset = set(stopwords.words('english'))
tweets = sc.textFile('hdfs:/adi/sample.txt')
words = tweets.map(lambda word: word.split(" "))
wordArr = []
for wArr in words.collect():
tempArr = []
for w in wArr:
if not w in stopset:
tempArr.append(w)
wordArr.append(tempArr)
# Open a file
# print wordArr
#tokens = sc.textFile("hdfs:/adi/tokens1.txt")
# Load documents (one per line).
documents = sc.textFile("hdfs:/adi/tokens1.txt").map(
lambda line: line.split(" "))
numDims = 100000
hashingTF = HashingTF(numDims)
tf = hashingTF.transform(documents)
tf.cache()
idf = IDF().fit(tf)
tfidf = idf.transform(tf)
tfidf.count()
model = KMeans.train(tfidf, 5)
model.save(sc, "tweetModel1")
print("Final centers: " + str(model.clusterCenters))
# print("Total Cost: " + str(model.computeCost(data)))
sc.stop()
def main():
#Reading the json file
reviews_data = sqlContext.read.json(input)
reviews=reviews_data.select('reviewText')
reviews_rdd=reviews.rdd.cache()
rdd_data=reviews_rdd.map(lambda line:str(line.reviewText))
transformed_data=rdd_data.map(transform_data)
#Finding Tf-IDF representation
hashingTF = HashingTF()
tf = hashingTF.transform(transformed_data)
tf.cache()
idf = IDF().fit(tf)
tfidf = idf.transform(tf).collect()
# Normalization
# tfidf = idf.transform(tf)
# normalizer1 = Normalizer()
# normalized_vector=normalizer1.transform(tfidf).collect()
score_rdd=reviews_data.rdd.map(lambda line:str(line.overall)).cache().collect()
dates_rdd=reviews_data.rdd.map(lambda line:str(line.reviewTime)).map(lambda line:line.split(", ")).map(lambda (a,b):b).cache().collect()
combinedList=zip(tfidf,score_rdd,dates_rdd)
combinedRDD=sc.parallelize(combinedList).cache()
TrainRDD=combinedRDD.filter(lambda (x,y,z):z!='2014').map(lambda (x,y,z):(x,y))
TestRDD=combinedRDD.filter(lambda (x,y,z):z=='2014').map(lambda (x,y,z):(x,y))
#Saving test and training data
TrainRDD.saveAsPickleFile(output+'/Train_data_unnormalized.pickle')
TestRDD.saveAsPickleFile(output+'/Test_data_unnormalized.pickle')
def createHashData(rdd):
original = rdd.map(lambda line: line.split(", "))
# load up the json string
data = rdd.map(lambda line: line.split(", ")).collect()
def fn(line):
label = 0.0
if line[9] == 'title':
label = 1.0
return (label, data[0:9])
# create paired data
data_pared = original.map(fn)
print data_pared
htf = HashingTF(100)
# hash data
data_hashed = data_pared.map(
lambda (label, f): LabeledPoint(label, htf.transform(f)))
return data_hashed
def generatedHashedFeatures(tweet):
#get label from tweet
#get text from tweet
htf = HashingTF(50000)
lp = LabeledPoint("0", htf.transform(text))
return lp
def column_search(words,row_filter):
if row_filter == 'n' or row_filter == 'N':
min_row = 0
else:
min_row = row_filter
rawData = table_cols.join(master_index, master_index["Table_Name"]==table_cols["Name"]).rdd
data = rawData.map(lambda x: (x['Doc_ID'], x['Columns'])).map(parse)
titles = data.map(lambda x: x[0])
documents = data.map(lambda x: x[1])
hashingTF = HashingTF()
tf = hashingTF.transform(documents)
tf.cache()
idf = IDF().fit(tf)
normalizer = Normalizer()
tfidf = normalizer.transform(idf.transform(tf))
tfidfData = titles.zip(tfidf).toDF(["label", "features"])
query = parse((0, words))[1]
queryTF = hashingTF.transform(query)
queryTFIDF = normalizer.transform(idf.transform(queryTF))
queryRelevance = tfidfData.rdd.map(lambda x: (x[0], float(x[1].dot(queryTFIDF)))).sortBy(lambda x: -x[1]).filter(lambda x: x[1] > 0)
queryRelevance = queryRelevance.toDF(["Doc_ID", "scores"])
queryRelevance = queryRelevance.join(table_desc,queryRelevance.Doc_ID == table_desc.Doc_ID).select(table_desc.Doc_ID, queryRelevance.scores, table_desc.Columns)
queryRelevance = queryRelevance.join(master_index, master_index.Doc_ID==queryRelevance.Doc_ID).select(master_index.Table_Name,master_index.Table_Length, queryRelevance.Columns, queryRelevance.scores)
queryRelevance = queryRelevance.rdd.filter(lambda x: int(x['Table_Length']) >= int(min_row))
if (queryRelevance.isEmpty()):
print("Sorry, nothing matched in column search, please try a different keyword")
else:
print("Here is your column search result")
queryRelevance.toDF().show()
'''
def main():
"""
Driver program for a spam filter using Spark and MLLib
"""
# Create the Spark Context for parallel processing
sc = SparkContext(appName="Spam Filter")
# Load the spam and ham data files into RDDs
spam = sc.textFile(
"E:\\Personal\\Imp Docs\\Spark Projects\\Spam-Ham\\20050311_spam_2.tar\\20050311_spam_2\\spam.txt"
)
ham = sc.textFile(
"E:\\Personal\\Imp Docs\\Spark Projects\\Spam-Ham\\20030228_easy_ham.tar\\20030228_easy_ham\\ham.txt"
)
# Create a HashingTF instance to map email text to vectors of 10,000 features.
tf = HashingTF(numFeatures=10000)
# Each email is split into words, and each word is mapped to one feature.
spamFeatures = spam.map(lambda email: tf.transform(email.split(" ")))
hamFeatures = ham.map(lambda email: tf.transform(email.split(" ")))
# Create LabeledPoint datasets for positive (spam) and negative (ham) data points.
positiveExamples = spamFeatures.map(
lambda features: LabeledPoint(1, features))
negativeExamples = hamFeatures.map(
lambda features: LabeledPoint(0, features))
# Combine positive and negative datasets into one
data = positiveExamples.union(negativeExamples)
# Split the data into 70% for training and 30% test data sets
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Cache the training data to optmize the Logistic Regression
trainingData.cache()
# Train the model with Logistic Regression using the SGD algorithm.
model = LogisticRegressionWithSGD.train(trainingData)
# Create tuples of actual and predicted values
labels_and_predictions = testData.map(
lambda email: (email.label, model.predict(email.features)))
# Calculate the error rate as number wrong / total number
error_rate = labels_and_predictions.filter(
lambda (val, pred): val != pred).count() / float(testData.count())
# End the Spark Context
sc.stop()
# Print out the error rate
print("*********** SPAM FILTER RESULTS **********")
print("\n")
print("Error Rate: " + str(error_rate))
print("\n")
# Serialize the model for presistance
pickle.dump(model, open("spamFilter.pkl", "wb"))
def generatedHashedFeatures(tweet):
#get label from tweet
#get text from tweet
htf = HashingTF(50000)
lp = LabeledPoint("0", htf.transform(text))
return lp
def processBinary(self, newsRDD):
'''
Process for 2 classes
'''
hashingTF = HashingTF(self.n)
self.newsRDD = newsRDD
self.featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
#toto = self.featuresRDD.take(1)[0]
#print(toto.words + toto.bg2 + toto.bg3)
self.labeledPointsRdd = self.featuresRDD.map(lambda x: LabeledPoint(x.isGoodN(1), hashingTF.transform(x.words + x.bg2 + x.bg3)))
try:
nbPos = self.featuresRDD.filter(lambda x: x.isGoodN(1) == True).count()
nbNeg = self.featuresRDD.filter(lambda x: x.isGoodN(1) == False).count()
nbTot = self.featuresRDD.count()
print("nbTot %d" % nbTot)
print("\tnbNeg %d" % nbNeg)
print("\tnbPos %d" % nbPos)
except:
pass # empty rdd
return self.labeledPointsRdd
def tfidf(self):
self._create_rdd()
hashingTF = HashingTF()
tf = hashingTF.transform(self.token_rdd)
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
return tfidf
def process_data(data):
print("Processing data ...")
if (not data.isEmpty()):
nbModel = bc_model.value
hashingTF = HashingTF(100000)
tf = hashingTF.transform(
data.map(lambda x: x[0].encode('utf-8', 'ignore')))
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
tfidf.cache()
prediction = nbModel.predict(tfidf)
temp = []
i = 0
for p, q, r in data.collect():
temp.append([])
temp[i].append(p.encode('utf-8', 'ignore'))
temp[i].append(q)
temp[i].append(r)
i += 1
i = 0
for p in prediction.collect():
temp[i].append(p)
i += 1
print(temp)
for i in temp:
insert_tweet(str(i[0]), str(i[1]), "0", int(i[3]), int(i[2]))
else:
print("Empty RDD !!!")
pass
def main():
conf = SparkConf().setAppName("twitterclassifier")
sc = SparkContext(conf=conf)
ssc = StreamingContext(sc, 10)
tweets = ssc.socketTextStream("localhost", PORT) \
.map(lambda x: json.loads(x)) \
.filter(lambda x: 'text' in x) \
.map(lambda x: x['text'].encode('utf-8'))
hasher = HashingTF(DIM)
features = tweets.map(lambda x:
(x, hasher.transform(featurize(x)))).cache()
# We create a model with random clusters and specify the number of clusters to find
# decay = 1: total memory; decay = 0: no memory
model = StreamingKMeans(k=N, decayFactor=0.1).setRandomCenters(DIM, 1.0, 0)
model.trainOn(features.map(lambda x: x[1]))
results = model.predictOnValues(features).cache()
# Need a closure over i here.
def print_group(i):
results.filter(lambda x: x[1] == i).map(lambda x: '%i: %s' %
(x[1], x[0])).pprint(3)
for i in xrange(N):
print_group(i)
ssc.start()
ssc.awaitTermination()
def get_feature_vectors(sc, input_file, feature_dimensions):
"""Get feature vector from the lines in input_file_obj using
TF/IDF.
Returns:
vectors RDD
"""
# Load documents (one per line).
tweet_file = sc.textFile(input_file)
input_text_rdd = tweet_file.map(lambda line: _tokenize(line))
input_text_rdd.cache()
# The default feature dimension is 2^20; for a corpus with million
# tweets recommended dimensions are 50000 or 100000. Use higher
# dimensions for larger corpus of tweets.
hashing_tf = HashingTF(feature_dimensions)
tf = hashing_tf.transform(input_text_rdd)
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
tfidf.cache()
return input_text_rdd, tfidf
def run_tf_idf_spark_mllib(df, numFeatures=1 << 20):
tokenizer = Tokenizer(inputCol="body", outputCol="words")
wordsData = tokenizer.transform(df)
words = wordsData.select("words").rdd.map(lambda x: x.words)
hashingTF = MllibHashingTF(numFeatures)
tf = hashingTF.transform(words)
tf.cache()
idf = MllibIDF().fit(tf)
tfidf = idf.transform(tf)
# @TODO make this nicer
tmp = sqlContext.createDataFrame(wordsData.rdd.zip(tfidf),
["data", "features"])
tmp.registerTempTable("tmp")
old_columns = ', '.join(map(lambda x: 'data.%s' % x, wordsData.columns))
with_features = sqlContext.sql("SELECT %s, features FROM tmp" %
old_columns)
tmp = sqlContext.createDataFrame(with_features.rdd.zip(tf),
["data", "rawFeatures"])
tmp.registerTempTable("tmp")
old_columns = ', '.join(map(lambda x: 'data.%s' % x,
with_features.columns))
return sqlContext.sql("SELECT %s, rawFeatures FROM tmp" % old_columns)
def init_tranining_set(sc):
"""
合并积极/消极的词性
param: sc spark对象的context
"""
words = sc.textFile('traning_words.csv')
# 训练词频矩阵
hashingTF = HashingTF()
tf = hashingTF.transform(words)
# 计算TF-IDF矩阵
idfModel = IDF().fit(tf)
tfidf = idfModel.transform(tf)
tf.cache()
with open('NBmodel.pkl', 'r') as f:
NBmodel = pickle.load(f)
session = get_session(settings.DB_URL)
for r in session.execute('select * from traning_collection').fetchall():
yourDocument = r[3]
print r[3]
yourwords="/".join(jieba.cut_for_search(yourDocument)).split("/")
yourtf = hashingTF.transform(yourwords)
yourtfidf=idfModel.transform(yourtf)
print('NaiveBayes Model Predict:', NBmodel.predict(yourtfidf))
def transform(idf, article):
"""
transform article to a sparse vector
"""
token = tokenizing(article)
hashingTF = HashingTF()
tf_test = hashingTF.transform(token)
return idf.transform(tf_test)
def get_tfidf_features(txt_rdd):
hashingTF = HashingTF()
tf = hashingTF.transform(txt_rdd)
tf.cache()
idf = IDF().fit(tf)
tfidf = idf.transform(tf)
return tfidf
def generate_gender_tf(twProfilesRdd, numFe):
"""
Generate Term Frequency tuple (gender,sparse vector) from rdd containing following tuples:
(gender,(clean words tuple))
"""
tf = HashingTF(numFeatures=numFe)
return twProfilesRdd.map(lambda genderDescrTuple: (genderDict[
genderDescrTuple[0]], tf.transform(genderDescrTuple[1])))
def tfidf(self):
tf = HashingTF().transform(self._sents)
self._tf = tf
tf.cache()
idf = IDF().fit(tf)
self.idf = idf
tfidf = idf.transform(tf)
self._tfidf = dict(enumerate(tfidf.collect()))
def get_tfidf_features(txt):
hashingTF = HashingTF()
tf = hashingTF.transform(txt)
tf.cache()
idf = IDF().fit(tf)
tfidf = idf.transform(tf)
return tfidf
def hashing(self, size):
self.hashing_TF = HashingTF(
size) #100K hash buckets just to save some memory
tf = self.hashing_TF.transform(self.documents)
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
self.tfidf = idf.transform(tf)
def tfidf(rdd_doc):
hasingTF = HashingTF()
trainTf = hasingTF.transform(rdd_doc)
trainTf.cache()
idf = IDF().fit(trainTf)
trainTfidf = idf.transform(trainTf)
trainTfidf.cache()
return trainTfidf, lambda x: hasingTF.indexOf(x)
def create_bayes(self):
""" 创建贝叶斯训练模型 """
if self._check_traning_exists():
return
# 获取积极文本构造rdd
positive_file = os.path.join(settings.DATA_DIR, '分类词库/positive.txt')
positive_data = self.sc.textFile(positive_file)
# 数据去重
positive_data = positive_data.distinct()
positive_data = positive_data.map(
lambda line: line.split('###')).filter(lambda line: len(line) == 2)
# 获取消极文本构造rdd
negative_file = os.path.join(settings.DATA_DIR, '分类词库/negative.txt')
negative_data = self.sc.textFile(negative_file)
negative_data = negative_data.distinct()
negative_data = negative_data.map(
lambda line: line.split('###')).filter(lambda line: len(line) == 2)
# 合并训练集
all_data = negative_data.union(positive_data)
all_data.repartition(1)
# 评分已经提前进行处理只有-1与1
rate = all_data.map(lambda s: s[0])
document = all_data.map(lambda s: s[1])
words = document.map(lambda w:"/".\
join(jieba.cut_for_search(w))).\
map(lambda line: line.split("/"))
# 训练词频矩阵
hashingTF = HashingTF()
tf = hashingTF.transform(words)
# 计算TF-IDF矩阵
idfModel = IDF().fit(tf)
tfidf = idfModel.transform(tf)
tf.cache()
# 生成训练集和测试集
zipped = rate.zip(tfidf)
data = zipped.map(lambda line: LabeledPoint(line[0], line[1]))
training, test = data.randomSplit([0.6, 0.4], seed=0)
# 训练贝叶斯分类模型
NBmodel = NaiveBayes.train(training, 1.0)
predictionAndLabel = test.map(lambda p:
(NBmodel.predict(p.features), p.label))
accuracy = 1.0 * predictionAndLabel.filter(lambda x: 1.0 \
if x[0] == x[1] else 0.0).count() / test.count()
# 存储rdd
words.repartition(1).saveAsTextFile(self.training_words_dir)
# 贝叶斯分类模型以pickle存储
with open(self.NBmodel, 'w') as f:
pickle.dump(NBmodel, f)
def makeDict(x):
global data
hash = HashingTF(100000)
for i in range(len(x)):
word = x[i]
ind = hash.indexOf(word)
if ind not in data:
data.update({ind:word})
print len(data)
def tfidf(self, tokenizer):
"""
Get TFIDF matrix rdd with spark tfidf functions
"""
self._create_rdd(tokenizer)
hashingTF = HashingTF()
tf = hashingTF.transform(self.token_rdd)
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
return self.rdd, idf, tfidf
def tf_idf(sc,title_token):
hashingTF = HashingTF(100)
title_token = sc.parallelize(title_token)
tf = hashingTF.transform(title_token)
print tf, ' tf'
idf = IDF().fit(tf)
tfidf = idf.transform(tf)
return tfidf
def vectorize(sc, rdd_words, size=0):
'''
使用TF将词语向量化
向量的维度需要设定的，默认为2^20
'''
if not size:
size = rdd_words.flatMap(lambda x:x).distinct().count() + 10000
hashingTF = HashingTF(size)
tf = hashingTF.transform(rdd_words)
return tf
def main():
"""
Driver program for a spam filter using Spark and MLLib
"""
# Consolidate the individual email files into a single spam file
# and a single ham file
makeDataFileFromEmails( "data/spam_2/", "data/spam.txt")
makeDataFileFromEmails( "data/easy_ham_2/", "data/ham.txt" )
# Create the Spark Context for parallel processing
sc = SparkContext( appName="Spam Filter")
# Load the spam and ham data files into RDDs
spam = sc.textFile( "data/spam.txt" )
ham = sc.textFile( "data/ham.txt" )
# Create a HashingTF instance to map email text to vectors of 10,000 features.
tf = HashingTF(numFeatures = 10000)
# Each email is split into words, and each word is mapped to one feature.
spamFeatures = spam.map(lambda email: tf.transform(email.split(" ")))
hamFeatures = ham.map(lambda email: tf.transform(email.split(" ")))
# Create LabeledPoint datasets for positive (spam) and negative (ham) data points.
positiveExamples = spamFeatures.map(lambda features: LabeledPoint(1, features))
negativeExamples = hamFeatures.map(lambda features: LabeledPoint(0, features))
# Combine positive and negative datasets into one
data = positiveExamples.union(negativeExamples)
# Split the data into 70% for training and 30% test data sets
( trainingData, testData ) = data.randomSplit( [0.7, 0.3] )
# Cache the training data to optmize the Logistic Regression
trainingData.cache()
# Train the model with Logistic Regression using the SGD algorithm.
model = LogisticRegressionWithSGD.train(trainingData)
# Create tuples of actual and predicted values
labels_and_predictions = testData.map( lambda email: (email.label, model.predict( email.features) ) )
# Calculate the error rate as number wrong / total number
error_rate = labels_and_predictions.filter( lambda (val, pred): val != pred ).count() / float(testData.count() )
print( "*********** SPAM FILTER RESULTS **********" )
print( "\n" )
print( "Error Rate: " + str( error_rate ) )
print( "\n" )
# Serialize the model for presistance
pickle.dump( model, open( "spamFilter.pkl", "wb" ) )
sc.stop()
def task2():
#Print title with Machine Learning Classification
print("-------------------------------------------")
startTitle = time.time()
regex1 = re.compile(".*(title:).*")
find1 = [m.group(0) for l in data for m in [regex1.search(l)] if m]
title = [i.split('title: ', 1)[1] for i in find1]
Programming = sc.textFile(fileProgramming)
Other = sc.textFile(fileOther)
# Create a HashingTF instance to map title text to vectors of 100,000 features.
tf = HashingTF(numFeatures=100000)
# Each title is split into words, and each word is mapped to one feature.
programmingFeatures = Programming.map(
lambda title: tf.transform(title.split(" ")))
otherFeatures = Other.map(lambda title: tf.transform(title.split(" ")))
# Create LabeledPoint datasets for positive (programming) and negative (other) examples.
positiveExamples = programmingFeatures.map(
lambda features: LabeledPoint(1, features))
negativeExamples = otherFeatures.map(
lambda features: LabeledPoint(0, features))
trainingData = positiveExamples.union(negativeExamples)
trainingData.cache()
# Run Logistic Regression using the SGD algorithm.
model = LogisticRegressionWithSGD.train(trainingData)
listResult = []
for row in title:
test = tf.transform(row.split(" "))
result = "null"
if model.predict(test) == 1:
result = "Programmings"
else:
result = "Non-Programming"
joinResult = row + " = " + result
listResult.append(joinResult)
for i in listResult:
if 'Non-Programming' in i:
print(i)
for i in listResult:
if 'Programmings' in i:
print(i)
endTitle = time.time()
elapsedTitle = endTitle - startTitle
print(elapsedTitle)
print("-------------------------------------------")
def test_binary_term_freqs(self):
hashingTF = HashingTF(100).setBinary(True)
doc = "a a b c c c".split(" ")
n = hashingTF.numFeatures
output = hashingTF.transform(doc).toArray()
expected = Vectors.sparse(n, {hashingTF.indexOf("a"): 1.0,
hashingTF.indexOf("b"): 1.0,
hashingTF.indexOf("c"): 1.0}).toArray()
for i in range(0, n):
self.assertAlmostEqual(output[i], expected[i], 14, "Error at " + str(i) +
": expected " + str(expected[i]) + ", got " + str(output[i]))
def extractKeywords_Train(self):
documents = self.sc.textFile(self.trainingfile).map(lambda line: line.split(" ")[1:])
hashingTF = HashingTF()
tf = hashingTF.transform(documents)
tf.cache()
idfIgnore = IDF(minDocFreq=2).fit(tf)
tfidfIgnore = idfIgnore.transform(tf)
tfidfIgnore.saveAsTextFile("AAA")
def test_binary_term_freqs(self):
hashingTF = HashingTF(100).setBinary(True)
doc = "a a b c c c".split(" ")
n = hashingTF.numFeatures
output = hashingTF.transform(doc).toArray()
expected = Vectors.sparse(n, {hashingTF.indexOf("a"): 1.0,
hashingTF.indexOf("b"): 1.0,
hashingTF.indexOf("c"): 1.0}).toArray()
for i in range(0, n):
self.assertAlmostEqual(output[i], expected[i], 14, "Error at " + str(i) +
": expected " + str(expected[i]) + ", got " + str(output[i]))
def processKeepNews(self, newsRDD):
'''
process de vectorization, but keep the news for each record in the RDD
'''
hashingTF = HashingTF(self.n)
self.newsRDD = newsRDD
self.featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
self.labeledPointsRdd = self.featuresRDD.map(lambda x: (x.news, LabeledPoint(x.giveClasseN(1), hashingTF.transform(x.words + x.bg2 + x.bg3))))
return self.labeledPointsRdd
def tf_idf_cal(words_rdd): hashingTF = HashingTF() tf = hashingTF.transform(words_rdd) idf = IDF().fit(tf) tfidf = idf.transform(tf).cache() tfidf_str = tfidf.map(lambda line: str(line)).cache() return tfidf_str
def predictSentiment(tweetText):
nbModel = bc_model.value
hashingTF = HashingTF()
tf = hashingTF.transform(tweetText)
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
tfidf.cache()
prediction = nbModel.predict(tfidf)
print "Predictions for this window :"
for i in range(0, prediction.count()):
print prediction.collect()[i], tweetText.collect()[i]
def generate_gender_tf(twProfilesRdd,numFe):
"""
Generate Term Frequency tuple (gender,sparse vector) from rdd containing following tuples:
(gender,(clean words tuple))
"""
tf = HashingTF(numFeatures = numFe)
return twProfilesRdd.map(lambda genderDescrTuple: (genderDict[genderDescrTuple[0]],tf.transform(genderDescrTuple[1])))
def process(self, newsRDD):
hashingTF = HashingTF(self.n)
self.newsRDD = newsRDD
self.featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
#toto = self.featuresRDD.take(1)[0]
#print(toto.words + toto.bg2 + toto.bg3)
self.labeledPointsRdd = self.featuresRDD.map(lambda x: LabeledPoint(x.giveClasseN(1), hashingTF.transform(x.words + x.bg2 + x.bg3)))
try:
nbVeryPos = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 3).count()
nbPos = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 2).count()
nbNeg = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 1).count()
nbVeryNeg = self.featuresRDD.filter(lambda x: x.giveClasseN(1) == 0).count()
nbTot = self.featuresRDD.count()
print("nbTot %d" % nbTot)
print("\tnbVeryNeg %d" % nbVeryNeg)
print("\tnbNeg %d" % nbNeg)
print("\tnbPos %d" % nbPos)
print("\tnbVeryPos %d" % nbVeryPos)
except:
pass # empty rdd
return self.labeledPointsRdd
def processKeepNews(self, newsRDD):
hashingTF = HashingTF(self.n)
self.newsRDD = newsRDD
self.featuresRDD = newsRDD.map(lambda x: FeaturesV2(x))
self.labeledPointsRdd = self.featuresRDD.map(lambda x: (x.news, LabeledPoint(x.giveClasseN(1), hashingTF.transform(x.words + x.bg2 + x.bg3))))
return self.labeledPointsRdd
def naivebayes_mllib():
AWS_ACCESS_KEY_ID = "XXXXXXXXXXXXXXXXXX"
AWS_SECRET_ACCESS_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
sc._jsc.hadoopConfiguration().set("fs.s3n.awsAccessKeyId", AWS_ACCESS_KEY_ID)
sc._jsc.hadoopConfiguration().set("fs.s3n.awsSecretAccessKey", AWS_SECRET_ACCESS_KEY)
tr_folder = "s3n://usf-ml2/hwspark/train/"
tr_neg_path = tr_folder+ "neg/*.txt"
neg_files = sc.textFile(tr_neg_path)
neg = neg_files.map(lambda x: parsedoc(x))
neg = neg.map(lambda x: x.replace(',',' ').replace('.', ' ').replace('-',' ').lower())
neg1= neg.flatMap(lambda x:x.split())
neg1 = neg1.map(lambda x: removeStopWords(x))
tf = HashingTF().transform(neg1.map(lambda x: x, preservesPartitioning=True))
neg_tr = tf.map(lambda x: LabeledPoint(0.0, x))
tr_pos_path = tr_folder+ "pos/*.txt"
pos_files = sc.textFile(tr_pos_path)
pos = pos_files.map(lambda x: x.replace(',',' ').replace('.', ' ').replace('-',' ').lower())
pos = pos.map(lambda x: parsedoc(x))
pos1= pos.flatMap(lambda x:x.split())
pos1 = pos1.map(lambda x: removeStopWords(x))
tf_pos = HashingTF().transform(pos1.map(lambda x: x, preservesPartitioning=True))
pos_tr = tf_pos.map(lambda x: LabeledPoint(1.0, x))
training = neg_tr.union(pos_tr)
model = NaiveBayes.train(training)
te_folder = "s3n://usf-ml2/hw_spark/test/"
test_Npath = te_folder+"neg/*.txt"
test_Ppath = te_folder+ "pos/*.txt"
test = sc.textFile(test_Npath)
test_p = sc.textFile(test_Ppath)
test = test.map(lambda x: parsedoc(x))
test2= test.flatMap(lambda x:x.split())
test1 = test2.map(lambda x: x.replace(',',' ').replace('.', ' ').replace('-',' ').lower())
test2 = test1.map(lambda x: removeStopWords(x))
tf1 = HashingTF().transform(test2.map(lambda x: x, preservesPartitioning=True))
test5 = tf1.map(lambda x: LabeledPoint(0.0, x))
test_p = test_p.map(lambda x: parsedoc(x))
test_p1 = test_p.map(lambda x: x.replace(',',' ').replace('.', ' ').replace('-',' ').lower())
test_p2= test_p1.flatMap(lambda x:x.split())
test_p2 = test_p2.map(lambda x: removeStopWords(x))
tf_p1 = HashingTF().transform(test_p2.map(lambda x: x, preservesPartitioning=True))
test_p5 = tf_p1.map(lambda x: LabeledPoint(1.0, x))
testpn = test5.union(test_p5)
predictionAndLabel = testpn.map(lambda p: (model.predict(p.features), p.label))
accuracy = predictionAndLabel.filter(lambda (x, v): x == v).count()*1.0 /float(test2.count()+test_p2.count())
print "Accuracy is {}".format(round(accuracy,5))
def mySpark(minFreq, keyWord):
# text cleaning function
def removePunctuation(text):
res=text.lower().strip()
res=re.sub("[^0-9a-zA-Z ]", "", res)
return res.split(" ")
# Function for printing each element in RDD
def println(x):
for i in x:
print i
# Boilerplate Spark stuff:
conf = SparkConf().setMaster("local").setAppName("SparkTFIDF")
sc = SparkContext(conf = conf)
# Load documents content (one per line) + cleaning.
rawData = sc.textFile("list_berita-30.tsv")
fields = rawData.map(lambda x: x.split("\t"))
documents = fields.map(lambda x: removePunctuation(x[3]))
# Get documents content without word mapping
documentNames = fields.map(lambda x: x[3])
# TF processing
hashingTF = HashingTF(100000) #100K hash buckets just to save some memory
tf = hashingTF.transform(documents)
# IDF & TF-IDF processing
tf.cache()
idf = IDF(minDocFreq=int(minFreq)).fit(tf)
tfidf = idf.transform(tf)
# Get keyword relevance with content and zip it
keywordTF = hashingTF.transform(removePunctuation(keyWord))
keywordHashValue = int(keywordTF.indices[0])
keywordRelevance = tfidf.map(lambda x: x[keywordHashValue])
zippedResults = keywordRelevance.zip(documentNames)
# print result
print "Best document for keywords is:"
print zippedResults.max()
def main():
# 初始化 SparkContext
sc = spark_context(spark_master)
# 加载数据
data = sc.textFile(hdfs_path)
# 计算词频
documents = data.map(tokenize)
hashingTF = HashingTF(2 << 10)
tf = hashingTF.transform(documents)
# 对文档词频进行索引
corpus = tf.zipWithIndex().map(lambda x: [x[1], x[0]]).cache()
# 索引和词的映射
mapping = hashing_term_mapping(documents)
mapping.cache()
# 训练 LDA 模型
ldaModel = LDA.train(corpus, k=3)
# 链接到 MongoDB
from pymongo import MongoClient
mongo_client = MongoClient(mongo_host)
mongo_client.admin.authenticate(mongo_user, mongo_pass, mechanism="SCRAM-SHA-1")
clear_mongodb(mongo_client)
# 保存结果到 MongoDB
topics = ldaModel.describeTopics(maxTermsPerTopic=10)
for topic in range(3):
doc = {}
doc["name"] = "topic " + str(topic)
doc["terms"] = []
for i in range(10):
term_index = topics[topic][0][i]
for term in mapping.lookup(term_index):
doc["terms"].append([term.encode("utf8"), topics[topic][1][i]])
send_mongodb(mongo_client, doc)
def run_tf_idf_spark_mllib(df, numFeatures=1 << 20):
tokenizer = Tokenizer(inputCol="body", outputCol="words")
wordsData = tokenizer.transform(df)
words = wordsData.select("words").rdd.map(lambda x: x.words)
hashingTF = MllibHashingTF(numFeatures)
tf = hashingTF.transform(words)
tf.cache()
idf = MllibIDF().fit(tf)
tfidf = idf.transform(tf)
# @TODO make this nicer
tmp = sqlContext.createDataFrame(wordsData.rdd.zip(tfidf), ["data", "features"])
tmp.registerTempTable("tmp")
old_columns = ', '.join(map(lambda x: 'data.%s' % x, wordsData.columns))
with_features = sqlContext.sql("SELECT %s, features FROM tmp" % old_columns)
tmp = sqlContext.createDataFrame(with_features.rdd.zip(tf), ["data", "rawFeatures"])
tmp.registerTempTable("tmp")
old_columns = ', '.join(map(lambda x: 'data.%s' % x, with_features.columns))
return sqlContext.sql("SELECT %s, rawFeatures FROM tmp" % old_columns)
def idf_scores_for_terms(tfidf, group_key):
group_keys = tfidf.select(group_key).collect()
words = tfidf.select("words").collect()
features = tfidf.select("features").collect()
rawFeatures = tfidf.select("rawFeatures").collect()
numFeatures = features[0].features.size
mllibHashingTf = MllibHashingTF(numFeatures=numFeatures)
idfscores = list()
for line_idx, feature_row in enumerate(features):
terms = dict()
for term_array in words[line_idx]:
for term in term_array:
term_index = mllibHashingTf.indexOf(term)
terms[term] = (feature_row.features[term_index], rawFeatures[line_idx].rawFeatures[term_index])
for term, score in terms.iteritems():
idfscores.append((group_keys[line_idx][group_key], term, float(score[0]), float(score[1])))
return idfscores
def featurize(tweet_tuple):
"""
generate features for this tweet text
returns: csv line with a the last field
containing the feature vector for the tweet
"""
ID_FIELD_IDX = 0
CREATED_AT_IDX = 1
TIMESTAMP_MS = 2
LANG_FIELD_IDX = 3
LON_FIELD_IDX = 4
LAT_FIELD_IDX = 5
TEXT_IDX = 6
TWEET_IDX = 1
#split the tweet into components id, lang, text, lon, lat etc
tweet_attrib_list = tweet_tuple[TWEET_IDX].split(",")
#get the text
text = tweet_attrib_list[TEXT_IDX]
#tokenize the text
word_list = tokenize(text)
#remove stop words
word_list = removeStopWords(word_list)
#remove punctuations
word_list = removePunctuation(word_list)
#stemmed the tokens
word_list = stemmed_tokens(word_list)
st = " ".join(word_list)
#hash the words
htf = HashingTF(50000)
hashedfeatures = htf.transform(text)
tweet = tweet_tuple[TWEET_IDX]
results = {'tweet':tweet, 'features':hashedfeatures}
return results
def createHashData(rdd):
original = rdd.map(lambda line : line.split(", "))
# load up the json string
data = rdd.map(lambda line : line.split(", ")).collect();
def fn(line):
label = 0.0
if line[9] == 'title':
label = 1.0
return (label, data[0:9])
# create paired data
data_pared = original.map(fn)
print data_pared
htf = HashingTF(100)
# hash data
data_hashed = data_pared.map(lambda (label, f) : LabeledPoint(label, htf.transform(f)))
return data_hashed
# Initialize a SparkContext
sc = SparkContext()
# Import full dataset of newsgroup posts as text file
#data_raw = sc.textFile('hdfs://ec2-54-213-237-76.us-west-2.compute.amazonaws.com:9000/trainingdata/trainingdata/bbcjsontxt')
data_raw = sc.textFile('bbcdataset.json')
# Parse JSON entries in dataset
data = data_raw.map(lambda line: json.loads(line))
# Extract relevant fields in dataset -- category label and text content
data_pared = data.map(lambda line: (line['label'], line['text']))
# Temporary print statement for testing partial script
print data_pared.first()
# Prepare text for analysis using our tokenize function to clean it up
data_cleaned = data_pared.map(lambda (label, text): (label, tokenize(text)))
# Hashing term frequency vectorizer with 50k features
htf = HashingTF(50000)
# Create an RDD of LabeledPoints using category labels as labels and tokenized, hashed text as feature vectors
data_hashed = data_cleaned.map(lambda (label, text): LabeledPoint(hash(label), htf.transform(text)))
# Ask Spark to persist the RDD so it won't have to be re-created later
data_hashed.persist()
# Train a Naive Bayes model on the training data
model = NaiveBayes.train(data_hashed)
#model.save(sc, "hdfs://ec2-54-213-237-76.us-west-2.compute.amazonaws.com:9000/trainingdata/trainingdata/bbcmodela")
model.save(sc, "bbcmodel")
def println(x):
print x
# Boilerplate Spark stuff:
conf = SparkConf().setMaster("local").setAppName("SparkTFIDF")
sc = SparkContext(conf = conf)
# Load documents (one per line).
rawData = sc.textFile("subset-small.tsv")
fields = rawData.map(lambda x: x.split("\t"))
documents = fields.map(lambda x: x[3].split(" "))
documentNames = fields.map(lambda x: x[1])
hashingTF = HashingTF(100000) #100K hash buckets just to save some memory
tf = hashingTF.transform(documents)
tf.cache()
idf = IDF(minDocFreq=2).fit(tf)
tfidf = idf.transform(tf)
keywordTF = hashingTF.transform(["Apollo"])
keywordHashValue = int(keywordTF.indices[0])
keywordRelevance = tfidf.map(lambda x: x[keywordHashValue])
zippedResults = keywordRelevance.zip(documentNames)
print "Best document for keywords is:"
print zippedResults.max()
def getTFVector(review): htf = HashingTF(1000) doc = review.split() return htf.transform(doc).toArray()
from pyspark import SparkContext
from pyspark.mllib.clustering import KMeans, KMeansModel
from pyspark.mllib.feature import HashingTF, IDF
sc = SparkContext()
rdd = sc.wholeTextFiles("/usr/local/Cellar/BigDataAdvanced/Assignment1/TwitterStuff/TweetData").map(lambda (name,text):text.split())
tf = HashingTF()
tfVectors = tf.transform(rdd).cache()
a = tfVectors.collect()
count = 0
for vec in a:
print vec
count = count + 1
with open("TF_Tweet"+str(count)+".txt","w") as f:
f.write(str(vec))
f.close()
idf = IDF()
idfModel = idf.fit(tfVectors)
tfIdfVectors = idfModel.transform(tfVectors)
file = open("TF-IDF_tweet.txt", 'w')
file.write(str(tfIdfVectors.collect()))
#count = 0
#output=tfIdfVectors.collect()
#for vec in output:
# print vec
# count = count + 1
# with open("TF_Wiki"+str(count)+".txt","w") as f:
# f.write(str(vec))
