def test_dim(self):
linear_data = LinearDataGenerator.generateLinearInput(
intercept=0.0, weights=[0.0, 0.0, 0.0],
xMean=[0.0, 0.0, 0.0], xVariance=[0.33, 0.33, 0.33],
nPoints=4, seed=0, eps=0.1)
self.assertEqual(len(linear_data), 4)
for point in linear_data:
self.assertEqual(len(point.features), 3)
linear_data = LinearDataGenerator.generateLinearRDD(
sc=sc, nexamples=6, nfeatures=2, eps=0.1,
nParts=2, intercept=0.0).collect()
self.assertEqual(len(linear_data), 6)
for point in linear_data:
self.assertEqual(len(point.features), 2)
def test_dim(self):
linear_data = LinearDataGenerator.generateLinearInput(
intercept=0.0, weights=[0.0, 0.0, 0.0],
xMean=[0.0, 0.0, 0.0], xVariance=[0.33, 0.33, 0.33],
nPoints=4, seed=0, eps=0.1)
self.assertEqual(len(linear_data), 4)
for point in linear_data:
self.assertEqual(len(point.features), 3)
linear_data = LinearDataGenerator.generateLinearRDD(
sc=self.sc, nexamples=6, nfeatures=2, eps=0.1,
nParts=2, intercept=0.0).collect()
self.assertEqual(len(linear_data), 6)
for point in linear_data:
self.assertEqual(len(point.features), 2)
def test_prediction(self):
"""Test prediction on a model with weights already set."""
# Create a model with initial Weights equal to coefs
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([10.0, 10.0])
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0, 10.0], [0.0, 0.0], [1.0 / 3.0, 1.0 / 3.0], 100,
42 + i, 0.1)
batches.append(
self.sc.parallelize(batch).map(lambda lp:
(lp.label, lp.features)))
input_stream = self.ssc.queueStream(batches)
output_stream = slr.predictOnValues(input_stream)
samples = []
output_stream.foreachRDD(lambda x: samples.append(x.collect()))
self.ssc.start()
def condition():
self.assertEqual(len(samples), len(batches))
return True
# We want all batches to finish for this test.
eventually(condition, catch_assertions=True)
# Test that mean absolute error on each batch is less than 0.1
for batch in samples:
true, predicted = zip(*batch)
self.assertTrue(mean(abs(array(true) - array(predicted))) < 0.1)
def test_parameter_convergence(self):
"""Test that the model parameters improve with streaming data."""
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([0.0])
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
batches.append(self.sc.parallelize(batch))
model_weights = []
input_stream = self.ssc.queueStream(batches)
input_stream.foreachRDD(
lambda x: model_weights.append(slr.latestModel().weights[0]))
slr.trainOn(input_stream)
self.ssc.start()
def condition():
self.assertEqual(len(model_weights), len(batches))
return True
# We want all batches to finish for this test.
eventually(condition, 90, catch_assertions=True)
w = array(model_weights)
diff = w[1:] - w[:-1]
self.assertTrue(all(diff >= -0.1))
def test_parameter_accuracy(self):
"""Test that coefs are predicted accurately by fitting on toy data."""
# Test that fitting (10*X1 + 10*X2), (X1, X2) gives coefficients
# (10, 10)
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([0.0, 0.0])
xMean = [0.0, 0.0]
xVariance = [1.0 / 3.0, 1.0 / 3.0]
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0, 10.0], xMean, xVariance, 100, 42 + i, 0.1)
batches.append(self.sc.parallelize(batch))
input_stream = self.ssc.queueStream(batches)
slr.trainOn(input_stream)
self.ssc.start()
def condition():
self.assertArrayAlmostEqual(slr.latestModel().weights.array,
[10., 10.], 1)
self.assertAlmostEqual(slr.latestModel().intercept, 0.0, 1)
return True
eventually(condition, catch_assertions=True)
def test_train_prediction(self):
"""Test that error on test data improves as model is trained."""
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([0.0])
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
batches.append(self.sc.parallelize(batch))
predict_batches = [
b.map(lambda lp: (lp.label, lp.features)) for b in batches]
errors = []
def func(rdd):
true, predicted = zip(*rdd.collect())
errors.append(mean(abs(true) - abs(predicted)))
input_stream = self.ssc.queueStream(batches)
output_stream = self.ssc.queueStream(predict_batches)
slr.trainOn(input_stream)
output_stream = slr.predictOnValues(output_stream)
output_stream.foreachRDD(func)
self.ssc.start()
def condition():
if len(errors) == len(predict_batches):
self.assertGreater(errors[1] - errors[-1], 2)
if len(errors) >= 3 and errors[1] - errors[-1] > 2:
return True
return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
self._eventually(condition)
def test_prediction(self):
"""Test prediction on a model with weights already set."""
# Create a model with initial Weights equal to coefs
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([10.0, 10.0])
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0, 10.0], [0.0, 0.0], [1.0 / 3.0, 1.0 / 3.0],
100, 42 + i, 0.1)
batches.append(
self.sc.parallelize(batch).map(lambda lp: (lp.label, lp.features)))
input_stream = self.ssc.queueStream(batches)
output_stream = slr.predictOnValues(input_stream)
samples = []
output_stream.foreachRDD(lambda x: samples.append(x.collect()))
self.ssc.start()
def condition():
self.assertEqual(len(samples), len(batches))
return True
# We want all batches to finish for this test.
self._eventually(condition, catch_assertions=True)
# Test that mean absolute error on each batch is less than 0.1
for batch in samples:
true, predicted = zip(*batch)
self.assertTrue(mean(abs(array(true) - array(predicted))) < 0.1)
def test_parameter_convergence(self):
"""Test that the model parameters improve with streaming data."""
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([0.0])
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
batches.append(self.sc.parallelize(batch))
model_weights = []
input_stream = self.ssc.queueStream(batches)
input_stream.foreachRDD(
lambda x: model_weights.append(slr.latestModel().weights[0]))
slr.trainOn(input_stream)
self.ssc.start()
def condition():
self.assertEqual(len(model_weights), len(batches))
return True
# We want all batches to finish for this test.
self._eventually(condition, catch_assertions=True)
w = array(model_weights)
diff = w[1:] - w[:-1]
self.assertTrue(all(diff >= -0.1))
def test_parameter_accuracy(self):
"""Test that coefs are predicted accurately by fitting on toy data."""
# Test that fitting (10*X1 + 10*X2), (X1, X2) gives coefficients
# (10, 10)
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([0.0, 0.0])
xMean = [0.0, 0.0]
xVariance = [1.0 / 3.0, 1.0 / 3.0]
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0, 10.0], xMean, xVariance, 100, 42 + i, 0.1)
batches.append(self.sc.parallelize(batch))
input_stream = self.ssc.queueStream(batches)
slr.trainOn(input_stream)
self.ssc.start()
def condition():
self.assertArrayAlmostEqual(
slr.latestModel().weights.array, [10., 10.], 1)
self.assertAlmostEqual(slr.latestModel().intercept, 0.0, 1)
return True
self._eventually(condition, catch_assertions=True)
def test_train_prediction(self):
"""Test that error on test data improves as model is trained."""
slr = StreamingLinearRegressionWithSGD(stepSize=0.2, numIterations=25)
slr.setInitialWeights([0.0])
# Create ten batches with 100 sample points in each.
batches = []
for i in range(10):
batch = LinearDataGenerator.generateLinearInput(
0.0, [10.0], [0.0], [1.0 / 3.0], 100, 42 + i, 0.1)
batches.append(self.sc.parallelize(batch))
predict_batches = [
b.map(lambda lp: (lp.label, lp.features)) for b in batches]
errors = []
def func(rdd):
true, predicted = zip(*rdd.collect())
errors.append(mean(abs(true) - abs(predicted)))
input_stream = self.ssc.queueStream(batches)
output_stream = self.ssc.queueStream(predict_batches)
slr.trainOn(input_stream)
output_stream = slr.predictOnValues(output_stream)
output_stream.foreachRDD(func)
self.ssc.start()
def condition():
if len(errors) == len(predict_batches):
self.assertGreater(errors[1] - errors[-1], 2)
if len(errors) >= 3 and errors[1] - errors[-1] > 2:
return True
return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
self._eventually(condition)
# print(""" ___ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ___
# __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__
# (______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)\n""")
# print("Transpose a numeric matrix")
# print("Before transpose:")
# print(df_numeric.show())
# print("After transpose:")
# print(transposed_df_numeric.show())
# print(""" ___ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ______ ___
# __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__ __)(__
# (______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)(______)\n""")
# MLib example of linear regression
# lp_rdd = df.select('store','Units','UPC','FeatureOnly','DisplayOnly','FeatureDisplay','MultQty','Categoryid','volume','Price','PriU','BasePriu','BasePrice','TrendIndex','CatTrendIndex','RegPriU','RegPrice','WeekOfYear','HLDY_LA_LG','HLDY_LA_HW','HLDY_HA_LG','HLDY_HA_HW','HLDY_TX_LG','HLDY_TX_HW','HLDY_XM_LG','HLDY_XM_HW','HLDY_NY_LG','HLDY_NY_HW','HLDY_SU_LG','HLDY_SU_HW','HLDY_VA_LG','HLDY_VA_HW','HLDY_EA_LG','HLDY_EA_HW','HLDY_ME_LG','HLDY_ME_HW','HLDY_ID_LG','HLDY_ID_HW','LogHolidayIndex','LogWeekofyearIndex','MfrID','BrandID','Lift62','Lift116','Lift119','Lift164','Lift169','Lift301','Lift343','Lift353','Lift363','Lift369','Lift383','Lift401','Lift413','Lift441','Lift443','Lift482','Lift548','Lift570','Lift572','Lift574','Lift578','Lift598','Lift605','Lift725','Lift726','Lift751','Lift838','Lift857','Lift873','Lift1000','AbsPrice62','AbsPrice116','AbsPrice119','AbsPrice164','AbsPrice169','AbsPrice301','AbsPrice343','AbsPrice353','AbsPrice363','AbsPrice369','AbsPrice383','AbsPrice401','AbsPrice413','AbsPrice441','AbsPrice443','AbsPrice482','AbsPrice548','AbsPrice570','AbsPrice572','AbsPrice574','AbsPrice578','AbsPrice598','AbsPrice605','AbsPrice725','AbsPrice726','AbsPrice751','AbsPrice838','AbsPrice857','AbsPrice873','AbsPrice1000','LogVolume','LogUnits','Discount','LogPriceIndex','LogSeason','LogPrice','LogRegPrice','LogBasePrice','LogPriu','LogRegPriu','LogBasePriu','LogSpecialPack','Intercept','ppgid').map(dfToLPRDD)
# model = LinearRegressionWithSGD.train(lp_rdd)
# # model.save(sc, file_path + "model/lin_reg_model")
# pred = lp_rdd.map(lambda p: (p.label, model.predict(p.features)))
# print(pred.collect())
print("generateLinearInput")
data = LinearDataGenerator.generateLinearInput(0, [1,2,3], [23, 45, 12], [.2, .5, .9], 50, 12314, 1)
print(data)
print("generateLinearRDD")
data = LinearDataGenerator.generateLinearRDD(sc, 50, 10, 1)
print(data)
# coefficients model