-
Notifications
You must be signed in to change notification settings - Fork 0
/
OneTestScenario-WithServer.py
59 lines (46 loc) · 1.51 KB
/
OneTestScenario-WithServer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
import threading, time
from pyspark import SparkContext, SparkConf
from pyspark.streaming import StreamingContext
from pyspark.mllib.regression import StreamingLinearRegressionWithSGD
sc = SparkContext("local[5]", "Tester")
sc.setLogLevel("OFF")
conf = SparkConf()
model = StreamingLinearRegressionWithSGD(stepSize=0.01)
class SparkThread(threading.Thread):
global sc, model
def __init__(self):
threading.Thread.__init__(self)
pass
def run(self):
"""
from OneTestTrainer import train
ssc = StreamingContext(sc, 5)
train(model=model, Context=sc, streamingContext=ssc)
ssc.stop(stopSparkContext=False)
"""
ssc = StreamingContext(sc, 5)
from OneTestScenario import SparkApp
SparkApp(model=model, Context=sc, streamingContext=ssc)
print "\nSpark thread ended.\n"
class ServerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
pass
def run(self):
import RandomNumberServer
RandomNumberServer.start()
print "\nServer thread stopped.\n"
# NOT USED IN THIS COMMIT.
class TrainingServer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
import RandomNumberTrainer
RandomNumberTrainer.start()
print "\nTrainer server stopped.\n"
threadSpark = SparkThread()
threadServer = ServerThread()
#threadTrainer = TrainingServer()
threadServer.start()
#threadTrainer.start()
threadSpark.start()