def test_predictions(self):
"""Test predicted values on a toy model."""
input_batches = []
for i in range(20):
batch = self.sc.parallelize(
self.generateLogisticInput(0, 1.5, 100, 42 + i))
input_batches.append(batch.map(lambda x: (x.label, x.features)))
input_stream = self.ssc.queueStream(input_batches)
slr = StreamingLogisticRegressionWithSGD(stepSize=0.2,
numIterations=25)
slr.setInitialWeights([1.5])
predict_stream = slr.predictOnValues(input_stream)
true_predicted = []
predict_stream.foreachRDD(lambda x: true_predicted.append(x.collect()))
self.ssc.start()
def condition():
self.assertEqual(len(true_predicted), len(input_batches))
return True
eventually(condition, catch_assertions=True)
# Test that the accuracy error is no more than 0.4 on each batch.
for batch in true_predicted:
true, predicted = zip(*batch)
self.assertTrue(
self.calculate_accuracy_error(true, predicted) < 0.4)
def test_training_and_prediction(self):
"""Test that the model improves on toy data with no. of batches"""
input_batches = [
self.sc.parallelize(self.generateLogisticInput(
0, 1.5, 100, 42 + i)) for i in range(20)
]
predict_batches = [
b.map(lambda lp: (lp.label, lp.features)) for b in input_batches
]
slr = StreamingLogisticRegressionWithSGD(stepSize=0.01,
numIterations=25)
slr.setInitialWeights([-0.1])
errors = []
def collect_errors(rdd):
true, predicted = zip(*rdd.collect())
errors.append(self.calculate_accuracy_error(true, predicted))
true_predicted = []
input_stream = self.ssc.queueStream(input_batches)
predict_stream = self.ssc.queueStream(predict_batches)
slr.trainOn(input_stream)
ps = slr.predictOnValues(predict_stream)
ps.foreachRDD(lambda x: collect_errors(x))
t = time()
self.ssc.start()
self._ssc_wait(t, 20.0, 0.01)
# Test that the improvement in error is atleast 0.3
self.assertTrue(errors[1] - errors[-1] > 0.3)
def test_convergence(self):
"""
Test that weights converge to the required value on toy data.
"""
input_batches = [
self.sc.parallelize(self.generateLogisticInput(
0, 1.5, 100, 42 + i)) for i in range(20)
]
input_stream = self.ssc.queueStream(input_batches)
models = []
slr = StreamingLogisticRegressionWithSGD(stepSize=0.2,
numIterations=25)
slr.setInitialWeights([0.0])
slr.trainOn(input_stream)
input_stream.foreachRDD(
lambda x: models.append(slr.latestModel().weights[0]))
self.ssc.start()
def condition():
self.assertEqual(len(models), len(input_batches))
return True
# We want all batches to finish for this test.
eventually(condition, 60.0, catch_assertions=True)
t_models = array(models)
diff = t_models[1:] - t_models[:-1]
# Test that weights improve with a small tolerance
self.assertTrue(all(diff >= -0.1))
self.assertTrue(array_sum(diff > 0) > 1)
def test_training_and_prediction(self):
"""Test that the model improves on toy data with no. of batches"""
input_batches = [
self.sc.parallelize(self.generateLogisticInput(0, 1.5, 100, 42 + i))
for i in range(40)]
predict_batches = [
b.map(lambda lp: (lp.label, lp.features)) for b in input_batches]
slr = StreamingLogisticRegressionWithSGD(
stepSize=0.01, numIterations=25)
slr.setInitialWeights([-0.1])
errors = []
def collect_errors(rdd):
true, predicted = zip(*rdd.collect())
errors.append(self.calculate_accuracy_error(true, predicted))
true_predicted = []
input_stream = self.ssc.queueStream(input_batches)
predict_stream = self.ssc.queueStream(predict_batches)
slr.trainOn(input_stream)
ps = slr.predictOnValues(predict_stream)
ps.foreachRDD(lambda x: collect_errors(x))
self.ssc.start()
def condition():
# Test that the improvement in error is > 0.3
if len(errors) == len(predict_batches):
self.assertGreater(errors[1] - errors[-1], 0.3)
if len(errors) >= 3 and errors[1] - errors[-1] > 0.3:
return True
return "Latest errors: " + ", ".join(map(lambda x: str(x), errors))
self._eventually(condition, timeout=60.0)
def test_convergence(self):
"""
Test that weights converge to the required value on toy data.
"""
input_batches = [
self.sc.parallelize(self.generateLogisticInput(
0, 1.5, 100, 42 + i)) for i in range(20)
]
input_stream = self.ssc.queueStream(input_batches)
models = []
slr = StreamingLogisticRegressionWithSGD(stepSize=0.2,
numIterations=25)
slr.setInitialWeights([0.0])
slr.trainOn(input_stream)
input_stream.foreachRDD(
lambda x: models.append(slr.latestModel().weights[0]))
t = time()
self.ssc.start()
self._ssc_wait(t, 15.0, 0.01)
t_models = array(models)
diff = t_models[1:] - t_models[:-1]
# Test that weights improve with a small tolerance,
self.assertTrue(all(diff >= -0.1))
self.assertTrue(array_sum(diff > 0) > 1)
def get_model(weight, pretrained=True):
""" Initiate a streaming model."""
if pretrained:
trained_model = _load_pre_trained_model()
model = MyStreamingLogisticRegressionWithSGD(
trained_model=trained_model)
else:
model = StreamingLogisticRegressionWithSGD()
model.setInitialWeights(weight)
return model
def get_model(pretrained=True):
''' Initiate a streaming model.
If pretrained=True, init a streaming model with the trained parameters;
if not, set initial weight to be all zeros.
'''
if (pretrained):
trained_model = _load_pre_trained_model()
model = MyStreamingLogisticRegressionWithSGD(
trained_model=trained_model)
else:
model = StreamingLogisticRegressionWithSGD()
model.setInitialWeights([0.0] * NUM_FEATURES)
return model
def test_parameter_accuracy(self):
"""
Test that the final value of weights is close to the desired value.
"""
input_batches = [
self.sc.parallelize(self.generateLogisticInput(
0, 1.5, 100, 42 + i)) for i in range(20)
]
input_stream = self.ssc.queueStream(input_batches)
slr = StreamingLogisticRegressionWithSGD(stepSize=0.2,
numIterations=25)
slr.setInitialWeights([0.0])
slr.trainOn(input_stream)
t = time()
self.ssc.start()
self._ssc_wait(t, 20.0, 0.01)
rel = (1.5 - slr.latestModel().weights.array[0]) / 1.5
self.assertAlmostEqual(rel, 0.1, 1)
def append_key_to_dictionary(dictionary, key, value):
dictionary[key] = value
return dictionary
def insert_into_table(values, table_name, host, port):
pass
if __name__ == '__main__':
sc = SparkContext(appName='PythonSparkStreamingKafka')
sc.setLogLevel("WARN") # avoid printing logs
# setting up a model
lr = StreamingLogisticRegressionWithSGD()
parameters = json.load(open('model.json', 'r'))
# lr.setInitialWeights(parameters['weights'])
lr = create_logistic_regression_skl(parameters['weights'],
parameters['intercept'])
stop_words = load_stopwords()
common_words = load_common_words()
reference_table = create_hash_table(common_words=common_words,
stop_words=stop_words)
ssc = StreamingContext(sparkContext=sc, batchDuration=2)
spark_sql = SQLContext(sparkContext=sc)
kafkaStream = KafkaUtils.createDirectStream(
ssc=ssc,
topics=['trump'],
features_test = test.map(
lambda tweet: (filtering(tweet[0].split(" ")), tweet[1])).map(
lambda tweet: ([model.value.get(word) for word in tweet[0]], tweet[1]))
#SUM among vectors
features_training = features_training.filter(lambda tweet: check_None(tweet[
0])).map(lambda tweet: (media(tweet[0], vectorSize), tweet[1]))
features_test = features_test.filter(lambda tweet: check_None(tweet[0])).map(
lambda tweet: (media(tweet[0], vectorSize), tweet[1]))
features_training = features_training.map(lambda tweet: LabeledPoint(
tweet[1], tweet[0])).filter(lambda labeled: labeled.features)
features_test = features_test.map(lambda tweet: LabeledPoint(
tweet[1], tweet[0])).filter(lambda labeled: labeled.features)
model_2 = StreamingLogisticRegressionWithSGD()
model_2.setInitialWeights([0.0] * vectorSize)
model_2.trainOn(features_training)
# Test
predictions = model_2.predictOnValues(
features_test.map(lambda tweet: (tweet.label, tweet.features)))
# 0 - ITA
# 1 - ENG
true_eng = predictions.window(test_seconds, 1) \
.filter(lambda prediction: prediction[0] == 1.0 and prediction[1] == 1) \
.map(lambda prediction: (prediction, 1)) \
.reduceByKey(lambda a, b: a + b).pprint()
if __name__ == '__main__':
# Get user input first
with open('config/malicious_ips.txt', 'r') as f:
for line in f:
MALICIOUS_IPS.append(str(line.replace('\n', '')))
# First create the streaming context
sc = SparkContext(appName="Realtime Packet Classifier")
sc.setLogLevel("ERROR")
ssc = StreamingContext(sc, UPDATE_TIMER)
# Create the data streams for the training and streaming directory
trainingStream = ssc.textFileStream(TRAINING_DIR).map(processTrainingLine)
secondaryTrainingStream = ssc.textFileStream(SEC_TRAINING_DIR).map(
processGeneratedLine)
testingStream = ssc.textFileStream(STREAMING_DIR).map(processGeneratedLine)
# Create the model and train it on the training data
model = StreamingLogisticRegressionWithSGD(numIterations=500)
model.setInitialWeights([0 for i in range(75)])
model.trainOn(trainingStream)
model.trainOn(secondaryTrainingStream)
# Get the model to predict on values incoming in the streaming directory
model.predictOnValues(testingStream.map(lambda lp: (lp.label, lp.features))\
).pprint(50)
# Start the stream and await manual termination
ssc.start()
ssc.awaitTermination()
