def doublekmeans(data,year):
data = data.loc[year,'pm2.5']
#kmeans
data = sc.parallelize(data)
cluster_no = 2
maxIter = 30
clusters = KMeans.train(data,cluster_no,maxIter)
#find 1.0 labels
tdata = data.collect()
cluster_info = np.zeros(len(tdata))
label = []
for i in range(0,len(tdata)):
cluster_info[i]=clusters.predict(np.array(tdata[i]))
if cluster_info[i]==1.0:
label.append(i)
#selecting 1.0 data and preparing the data
data1 = clean_data.drop(['Is','Ir','cbwd','No','month','year','hour','day'],axis = 1)
data2 = data1.iloc[label]
data2 = sc.parallelize(data2.as_matrix())
#bisecting kmeans
data2.collect()
cluster_no = 2
maxiter = 30
model = BisectingKMeans.train(data2,cluster_no,maxiter)
return clusters.centers,model.centers, model.computeCost(data2)
def test_bisecting_kmeans(self):
from pyspark.mllib.clustering import BisectingKMeans
data = array([0.0, 0.0, 1.0, 1.0, 9.0, 8.0, 8.0, 9.0]).reshape(4, 2)
bskm = BisectingKMeans()
model = bskm.train(self.sc.parallelize(data, 2), k=4)
p = array([0.0, 0.0])
rdd_p = self.sc.parallelize([p])
self.assertEqual(model.predict(p), model.predict(rdd_p).first())
self.assertEqual(model.computeCost(p), model.computeCost(rdd_p))
self.assertEqual(model.k, len(model.clusterCenters))
def test_bisecting_kmeans(self):
from pyspark.mllib.clustering import BisectingKMeans
data = array([0.0, 0.0, 1.0, 1.0, 9.0, 8.0, 8.0, 9.0]).reshape(4, 2)
bskm = BisectingKMeans()
model = bskm.train(self.sc.parallelize(data, 2), k=4)
p = array([0.0, 0.0])
rdd_p = self.sc.parallelize([p])
self.assertEqual(model.predict(p), model.predict(rdd_p).first())
self.assertEqual(model.computeCost(p), model.computeCost(rdd_p))
self.assertEqual(model.k, len(model.clusterCenters))
def BisectingKMeans_clustering(parsedData):
parsedData = data.map(
lambda line: array([float(x) for x in line.split(' ')]))
# Build the model (cluster the data)
model = BisectingKMeans.train(parsedData, 2, maxIterations=5)
print "\n-----------------------------------------------------------------------------"
print "\n Cluster Centers (BisectingKMeans)"
print "\n-----------------------------------------------------------------------------"
print model.clusterCenters
def bisecting_k_means(unclustered_data,
number_of_clusters,
max_iterations=5,
seed=None,
min_divisible_cluster_size=1.0):
if number_of_clusters < 1:
raise ValueError("While clustering with BisectingKMeans, \
the given number of clusters is not positive")
model = BisectingKMeans.train(
rdd=unclustered_data,
k=number_of_clusters,
maxIterations=max_iterations,
seed=seed,
minDivisibleClusterSize=min_divisible_cluster_size)
cost = model.computeCost(unclustered_data)
return [model, cost]
def main():
compounds = load_data(sc, dataFile)
compounds.partitionBy(executor_num)
fingerprints = select_fingerprints(compounds).cache()
fp_only = fingerprints.map(lambda (id, smi, fp, name): fp)
for x in [1500, 2000]:
start_time = time.time()
model = BisectingKMeans.train(fp_only, k=x)
#print(model.clusterCenters)
#print("Clusters " ,len(model.clusterCenters))
cost = model.computeCost(fp_only)
#model.save(sc, baseFile + '/btreemodel')
print("Bisecting " + str(cost))
#model.clusterCenters.foreach(lambda ctr : print("Cluster Center"))
all_fps = fingerprints.collect()
cluster_assignment = []
end_time1 = time.time()
print("Clustering Time taken ", x, end_time1 - start_time)
for fp in all_fps:
cluster_assignment.append('{} {} {}'.format(
fp[1], fp[3], model.predict(fp[2])))
#print ( "FP ", fp[0], " SMI: ", fp[1], " ", model.predict(fp[2]))
end_time = time.time()
print("Total Time taken ", x, end_time - start_time)
if EVALUATION:
header = sc.parallelize(["smiles Name Cluster"])
clusters = sc.parallelize(cluster_assignment)
output_file = header.union(clusters)
#output_file.foreach(output)
# output_file.saveAsTextFile("../mols/resultsSpark/result")
current_time_milli = int(round(time.time() * 1000))
outputextension = str(current_time_milli)
output_file.coalesce(1).saveAsTextFile(baseFile + "/output" +
str(x) + "/result" +
outputextension)
sc.stop()
"review_count", "average_stars", "yelping_since").rdd.map(lambda x: (x[
0], x[1], (today - par.parse(x[2])).days)).collect()[:1200])
scaler = MinMaxScaler(inputCol="_1",\
outputCol="scaled_1")
trial_df = spark_df.map(lambda x: pyspark.ml.linalg.Vectors.dense(x)).map(
lambda x: (x, )).toDF()
scalerModel = scaler.fit(trial_df)
vector_df = scalerModel.transform(trial_df).select("scaled_1").rdd.map(
lambda x: Vectors.dense(x))
num_clusters = 4
#Input into the Algorithm
km = BisectingKMeans()
start = timer()
kme = km.train(vector_df, k=num_clusters, maxIterations=20, seed=2018)
end = timer()
print(end - start)
centers = kme.clusterCenters
err = vector_df.map(lambda x: (x[0], findCenter(x[0], centers))).collect()
per_clus = [0] * num_clusters
per_clus_num = [0] * num_clusters
#Silhoutte Value comparison
ag = 0
agi = 1200
for er in err:
avg = [0] * num_clusters
avgi = [0] * num_clusters
from numpy import array
from pyspark import SparkContext
from pyspark.mllib.clustering import BisectingKMeans, BisectingKMeansModel
# spark-submit mllib_k_means.py
# 二分KMeans
# K-Means是聚类算法中的最常用的一种，算法最大的特点是简单，好理解，运算速度快
# K-Means算法是一种无监督分类算法
if __name__ == "__main__":
sc = SparkContext(appName="KMeansExample")
data = sc.textFile("kmeans_data.txt")
parsedData = data.map(lambda line: array([float(x) for x in line.split(' ')]))
model = BisectingKMeans.train(parsedData, 2, maxIterations=5)
cost = model.computeCost(parsedData)
print("Final centers: " + str(model.clusterCenters))
print("Bisecting K-means Cost = " + str(cost))
sc.stop()
#
from __future__ import print_function
# $example on$
from numpy import array
# $example off$
from pyspark import SparkContext
# $example on$
from pyspark.mllib.clustering import BisectingKMeans, BisectingKMeansModel
# $example off$
if __name__ == "__main__":
sc = SparkContext(appName="PythonBisectingKMeansExample") # SparkContext
# $example on$
# Load and parse the data
data = sc.textFile("data/mllib/kmeans_data.txt")
parsedData = data.map(lambda line: array([float(x) for x in line.split(' ')]))
# Build the model (cluster the data)
model = BisectingKMeans.train(parsedData, 2, maxIterations=5)
# Evaluate clustering
cost = model.computeCost(parsedData)
print("Bisecting K-means Cost = " + str(cost))
# $example off$
sc.stop()
tfidf = idf.transform(tf)
# In[84]:
# tfidf.collect()
# In[85]:
if algorithm == "K":
clusters = KMeans.train(tfidf,
8,
maxIterations=20,
initializationMode="random",
seed=42)
else:
clusters = BisectingKMeans.train(tfidf, 8, maxIterations=20, seed=42)
clusterCenters = clusters.clusterCenters
# In[ ]:
# In[86]:
documentModel = documents1.zip(tfidf)
# cluster_broadcast = sc.broadcast(clusters)
# In[87]:
def findErrorWC(document, clusters):
documentWords = document[0]
documentTfidf = document[1]
today = dt.datetime.today()
spark_df = sc.parallelize(
spark.read.json("Data/yelp_academic_dataset_user.json").select(
"review_count", "average_stars", "yelping_since").rdd.map(
lambda x: (x[0], x[1], (today - par.parse(x[2])).days)).take(1700))
scaler = MinMaxScaler(inputCol="_1",\
outputCol="scaled_1")
trial_df = spark_df.map(lambda x: pyspark.ml.linalg.Vectors.dense(x)).map(
lambda x: (x, )).toDF()
scalerModel = scaler.fit(trial_df)
vector_df = scalerModel.transform(trial_df).select("scaled_1").rdd.map(
lambda x: Vectors.dense(x))
#Input into the Algorithm
km = BisectingKMeans()
kme = km.train(vector_df, k=4, maxIterations=20, seed=2018)
centers = kme.clusterCenters
#Creating data structure with fields and cluster center
df_with = spark.createDataFrame(
vector_df.map(lambda x: (float(x[0][0]), float(x[0][1]), float(x[0][2]),
findCenter(x[0], centers)))).toPandas()
#Plotting
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
scatter = ax.scatter(df_with['_1'],
df_with['_2'],
df_with['_3'],
c=df_with['_4'])
ax.set_title('Bisecting K Means Clustering')