def test_trainOn_predictOn(self):
"""Test that prediction happens on the updated model."""
stkm = StreamingKMeans(decayFactor=0.0, k=2)
stkm.setInitialCenters([[0.0], [1.0]], [1.0, 1.0])
# Since decay factor is set to zero, once the first batch
# is passed the clusterCenters are updated to [-0.5, 0.7]
# which causes 0.2 & 0.3 to be classified as 1, even though the
# classification based in the initial model would have been 0
# proving that the model is updated.
batches = [[[-0.5], [0.6], [0.8]], [[0.2], [-0.1], [0.3]]]
batches = [sc.parallelize(batch) for batch in batches]
input_stream = self.ssc.queueStream(batches)
predict_results = []
def collect(rdd):
rdd_collect = rdd.collect()
if rdd_collect:
predict_results.append(rdd_collect)
stkm.trainOn(input_stream)
predict_stream = stkm.predictOn(input_stream)
predict_stream.foreachRDD(collect)
t = time()
self.ssc.start()
self._ssc_wait(t, 6.0, 0.01)
self.assertEqual(predict_results, [[0, 1, 1], [1, 0, 1]])
def test_trainOn_model(self):
"""Test the model on toy data with four clusters."""
stkm = StreamingKMeans()
initCenters = [[1.0, 1.0], [-1.0, 1.0], [-1.0, -1.0], [1.0, -1.0]]
stkm.setInitialCenters(centers=initCenters,
weights=[1.0, 1.0, 1.0, 1.0])
# Create a toy dataset by setting a tiny offest for each point.
offsets = [[0, 0.1], [0, -0.1], [0.1, 0], [-0.1, 0]]
batches = []
for offset in offsets:
batches.append([[offset[0] + center[0], offset[1] + center[1]]
for center in initCenters])
batches = [self.sc.parallelize(batch, 1) for batch in batches]
input_stream = self.ssc.queueStream(batches)
stkm.trainOn(input_stream)
t = time()
self.ssc.start()
# Give enough time to train the model.
self._ssc_wait(t, 6.0, 0.01)
finalModel = stkm.latestModel()
self.assertTrue(all(finalModel.centers == array(initCenters)))
self.assertEquals(finalModel.clusterWeights, [5.0, 5.0, 5.0, 5.0])
def test_trainOn_model(self):
"""Test the model on toy data with four clusters."""
stkm = StreamingKMeans()
initCenters = [[1.0, 1.0], [-1.0, 1.0], [-1.0, -1.0], [1.0, -1.0]]
stkm.setInitialCenters(
centers=initCenters, weights=[1.0, 1.0, 1.0, 1.0])
# Create a toy dataset by setting a tiny offest for each point.
offsets = [[0, 0.1], [0, -0.1], [0.1, 0], [-0.1, 0]]
batches = []
for offset in offsets:
batches.append([[offset[0] + center[0], offset[1] + center[1]]
for center in initCenters])
batches = [self.sc.parallelize(batch, 1) for batch in batches]
input_stream = self.ssc.queueStream(batches)
stkm.trainOn(input_stream)
t = time()
self.ssc.start()
# Give enough time to train the model.
self._ssc_wait(t, 6.0, 0.01)
finalModel = stkm.latestModel()
self.assertTrue(all(finalModel.centers == array(initCenters)))
self.assertEquals(finalModel.clusterWeights, [5.0, 5.0, 5.0, 5.0])
def test_trainOn_predictOn(self):
"""Test that prediction happens on the updated model."""
stkm = StreamingKMeans(decayFactor=0.0, k=2)
stkm.setInitialCenters([[0.0], [1.0]], [1.0, 1.0])
# Since decay factor is set to zero, once the first batch
# is passed the clusterCenters are updated to [-0.5, 0.7]
# which causes 0.2 & 0.3 to be classified as 1, even though the
# classification based in the initial model would have been 0
# proving that the model is updated.
batches = [[[-0.5], [0.6], [0.8]], [[0.2], [-0.1], [0.3]]]
batches = [sc.parallelize(batch) for batch in batches]
input_stream = self.ssc.queueStream(batches)
predict_results = []
def collect(rdd):
rdd_collect = rdd.collect()
if rdd_collect:
predict_results.append(rdd_collect)
stkm.trainOn(input_stream)
predict_stream = stkm.predictOn(input_stream)
predict_stream.foreachRDD(collect)
t = time()
self.ssc.start()
self._ssc_wait(t, 6.0, 0.01)
self.assertEqual(predict_results, [[0, 1, 1], [1, 0, 1]])
def test_accuracy_for_single_center(self):
"""Test that parameters obtained are correct for a single center."""
centers, batches = self.streamingKMeansDataGenerator(batches=5,
numPoints=5,
k=1,
d=5,
r=0.1,
seed=0)
stkm = StreamingKMeans(1)
stkm.setInitialCenters([[0., 0., 0., 0., 0.]], [0.])
input_stream = self.ssc.queueStream(
[self.sc.parallelize(batch, 1) for batch in batches])
stkm.trainOn(input_stream)
self.ssc.start()
def condition():
self.assertEqual(stkm.latestModel().clusterWeights, [25.0])
return True
eventually(condition, catch_assertions=True)
realCenters = array_sum(array(centers), axis=0)
for i in range(5):
modelCenters = stkm.latestModel().centers[0][i]
self.assertAlmostEqual(centers[0][i], modelCenters, 1)
self.assertAlmostEqual(realCenters[i], modelCenters, 1)
class StreamingUpdate(object):
"""
Streaming Update: DStream
"""
def __init__(self, init_clusters, decay_factor, time_unit, sc, ssc):
self.init_clusters=init_clusters
self.decay_factor=decay_factor
self.time_unit=time_unit
self.sc=sc
self.ssc=ssc
# implement
def streaming(self, mnk, clusters, init_clusters):
self.mnk=mnk
self.clusters=clusters
self.init_clusters=init_clusters
self.streaming_kmeans=StreamingKMeans(self.init_clusters, self.decay_factor, self.time_unit)
self.streaming_kmeans.setInitialCenters(self.mnk.cluster_centers_, np.ones([self.init_clusters]))
# update shape for centers in StreamingContext
"""
Từ điển được cập nhật khi có tin tức mới đến thì em cập nhật lại kích thước của các centroid
VD: Từ điển ban đầu có kích thước 10 từ
Em biểu diễn một câu có 5 từ bằng sparse vector kích thước 5x10
Từ điển sau khi cập nhật có 15 từ thì câu trên phải biểu diễn lại bằng sparse vector có kích thước 5x15
Có cách biểu diễn khác mà không phải cập nhật lại biểu diễn của câu không ạ
"""
def update_shape(self, docs, dictionary):
self.streaming_kmeans.setRandomCenters(matrix_tfidf(docs, dictionary).shape[1], 1.0, 0)
# save matrix update
def save_matrix_update(self, docs, dictionary):
np.savetxt('/home/ducvu/input_streaming.txt', matrix_tfidf(docs, dictionary))
# load dstream
def load_dstream(self):
self.dstream = self.sc.textFile("/home/ducvu/input_streaming.txt")\
.map(lambda line: Vectors.dense([float(x) for x in line.strip().split(' ')]))
# make predict
def make_predict(self, docs, dictionary):
self.streaming_kmeans.trainOn(self.load_dstream())
self.pred_stream=[]
matrix=matrix_tfidf(docs)
for x in matrix:
self.pred_stream.append(self.streaming_kmeans.latest_model.predict(x))
self.pred_stream=np.array(self.pred_stream)
df = pd.DataFrame(matrix).groupby(self.pred_stream).mean()
for i, r in df.iterrows():
print('\nCluster {0}:'.format(i))
print(','.join([get_tfidf(dictionary).get_feature_names()[t] for t in np.argsort(r)[-15:]]))
def test_model_params(self):
"""Test that the model params are set correctly"""
stkm = StreamingKMeans()
stkm.setK(5).setDecayFactor(0.0)
self.assertEqual(stkm._k, 5)
self.assertEqual(stkm._decayFactor, 0.0)
# Model not set yet.
self.assertIsNone(stkm.latestModel())
self.assertRaises(ValueError, stkm.trainOn, [0.0, 1.0])
stkm.setInitialCenters(centers=[[0.0, 0.0], [1.0, 1.0]], weights=[1.0, 1.0])
self.assertEqual(stkm.latestModel().centers, [[0.0, 0.0], [1.0, 1.0]])
self.assertEqual(stkm.latestModel().clusterWeights, [1.0, 1.0])
def test_model_params(self):
"""Test that the model params are set correctly"""
stkm = StreamingKMeans()
stkm.setK(5).setDecayFactor(0.0)
self.assertEquals(stkm._k, 5)
self.assertEquals(stkm._decayFactor, 0.0)
# Model not set yet.
self.assertIsNone(stkm.latestModel())
self.assertRaises(ValueError, stkm.trainOn, [0.0, 1.0])
stkm.setInitialCenters(
centers=[[0.0, 0.0], [1.0, 1.0]], weights=[1.0, 1.0])
self.assertEquals(
stkm.latestModel().centers, [[0.0, 0.0], [1.0, 1.0]])
self.assertEquals(stkm.latestModel().clusterWeights, [1.0, 1.0])
def test_accuracy_for_single_center(self):
"""Test that parameters obtained are correct for a single center."""
centers, batches = self.streamingKMeansDataGenerator(
batches=5, numPoints=5, k=1, d=5, r=0.1, seed=0)
stkm = StreamingKMeans(1)
stkm.setInitialCenters([[0., 0., 0., 0., 0.]], [0.])
input_stream = self.ssc.queueStream(
[self.sc.parallelize(batch, 1) for batch in batches])
stkm.trainOn(input_stream)
t = time()
self.ssc.start()
self._ssc_wait(t, 10.0, 0.01)
self.assertEquals(stkm.latestModel().clusterWeights, [25.0])
realCenters = array_sum(array(centers), axis=0)
for i in range(5):
modelCenters = stkm.latestModel().centers[0][i]
self.assertAlmostEqual(centers[0][i], modelCenters, 1)
self.assertAlmostEqual(realCenters[i], modelCenters, 1)
trainingData = sc.textFile("data/datatraining.txt")\
.map(lambda line: line.split(',')[2:-1]).map(lambda arr: Vectors.dense([float(x) for x in arr]))
centers = KMeans.train(trainingData, 2).centers
trainingQueue = [trainingData]
trainingStream = ssc.queueStream(trainingQueue)
# We create a model with random clusters and specify the number of clusters to find
model = StreamingKMeans(k=2, decayFactor=0.3)#.setRandomCenters(5, 1.0, 0)
model.setInitialCenters( centers, [1.0,1.0,1.0,1.0,1.0])
# Now register the streams for training and testing and start the job,
# printing the predicted cluster assignments on new data points as they arrive.
model.trainOn(trainingStream)
def parse(lp):
#label = float(lp[lp.find('(') + 1: lp.find(')')])
#vec = Vectors.dense(lp[lp.find('[') + 1: lp.find(']')].split(','))
arr = lp.split(',')[2:-1]
label = lp.split(',')[0]
label = label[1:-1]
vec = Vectors.dense([float(x) for x in arr])
print(model.latestModel().centers)
return LabeledPoint(label, vec)
