def load_parameters(self):
self.amount_prediction_method = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL,
file_name='amount_method')
self.trend_prediction_method = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL,
file_name='trend_method')
self.data_features = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL, file_name='features')
self.stock_symbol = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL, file_name='symbol')
self.data_parser = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL, file_name='data_parser')
amount_model_path = os.path.join(os.path.abspath(self.model_path), 'amount_model')
trend_model_path = os.path.join(os.path.abspath(self.model_path), 'trend_model')
if self.amount_prediction_method == self.RANDOM_FOREST:
amount_model = RandomForestModel.load(sc=self.sc, path=amount_model_path)
elif self.amount_prediction_method == self.LINEAR_REGRESSION:
amount_model = LinearRegressionModel.load(sc=self.sc, path=amount_model_path)
else:
amount_model = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL, file_name='amount_model')
if self.trend_prediction_method == self.RANDOM_FOREST:
trend_model = RandomForestModel.load(sc=self.sc, path=trend_model_path)
elif self.trend_prediction_method == self.LOGISTIC_REGRESSION:
trend_model = LogisticRegressionModel.load(sc=self.sc, path=trend_model_path)
elif self.trend_prediction_method == self.NAIVE_BAYES:
trend_model = NaiveBayesModel.load(sc=self.sc, path=trend_model_path)
elif self.trend_prediction_method == self.SVM:
trend_model = SVMModel.load(sc=self.sc, path=trend_model_path)
else:
trend_model = self.load_data_from_file(data_type=self.SAVE_TYPE_MODEL, file_name='trend_model')
return trend_model, amount_model
def random_forest():
conf = SparkConf().setAppName('RF')
sc = SparkContext(conf=conf)
# print("\npyspark version:" + str(sc.version) + "\n")
data = MLUtils.loadLibSVMFile(sc, './data/sample_libsvm_data.txt')
(trainingData, testData) = data.randomSplit([0.7, 0.3])
model = RandomForest.trainClassifier(trainingData,
numClasses=2,
categoricalFeaturesInfo={},
numTrees=3,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=4,
maxBins=32)
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(lambda v, p: v != p).count() / float(
testData.count())
print('Test Error = ' + str(testErr))
print('Learned classification forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, ".model/myRandomForestClassificationModel")
sameModel = RandomForestModel.load(
sc, "./model/myRandomForestClassificationModel")
def get_rfc_model(filename, bucket, client, sc):
# get download_url
objects_dict = client.list_objects(Bucket=BUCKET_NAME)
filenames = map(lambda x: x["Key"], objects_dict["Contents"])
filenames = filter(
lambda x: True
if re.match(filename + '-v' + "[0-9]+", x) else False, filenames)
if len(filenames) == 0:
print "NO RFC MODELS FOUND"
return False
else:
versions = map(lambda x: int(re.search("[0-9]+", x).group()),
filenames)
download_url = filename + "-v" + str(max(versions)) + '.zip'
# download zip and unzip it
local_url = '/tmp/' + download_url
os.system("rm -r " + local_url) # remove zip just in case
os.system("rm -r " + local_url[:-4]) # remove unzipped folder just in case
if download_file(download_url, bucket):
with zipfile.ZipFile(local_url, "r") as z:
z.extractall(local_url[:-4])
return RandomForestModel.load(sc, local_url[:-4]) #-4 for zip
else:
print "ZIP DOWNLOAD FAILED"
return False
def train():
data = MLUtils.loadLibSVMFile(sc, TEST_DATA_PATH)
print("[INFO] load complete.")
# 划分训练集
data = data.randomSplit([0.2, 0.8])[0]
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
model = RandomForest.trainClassifier(trainingData,
numClasses=NUM_OF_CLASSES,
categoricalFeaturesInfo={},
numTrees=NUM_OF_TREES,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=MAXDEPTH,
maxBins=MAXBINS)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(
lambda lp: lp[0] != lp[1]).count() / float(testData.count())
print('[INFO] Test Error = ' + str(testErr))
print('[INFO] Learned classification forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, TEST_MODEL_PATH)
sameModel = RandomForestModel.load(sc, TEST_MODEL_PATH)
def main():
options = parse_args()
sc = SparkContext(appName="PythonRandomForestClassificationExample")
# $example on$
# Load and parse the data file into an RDD of LabeledPoint.
data = MLUtils.loadLibSVMFile(sc, options.data_file)
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
model = RandomForest.trainClassifier(trainingData,
numClasses=2,
categoricalFeaturesInfo={},
numTrees=3,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=4,
maxBins=32)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(
lambda lp: lp[0] != lp[1]).count() / float(testData.count())
print('Test Error = ' + str(testErr))
print('Learned classification forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, options.output_model)
sameModel = RandomForestModel.load(sc, options.output_model)
def Prediction(self, modelType):
data_point = self.Features
if modelType == 'RF':
model = RandomForestModel.load(
self.sc, self.baseDir + '/fraudModel/Model/' + modelType)
result = np.array(
model.predict(self.sc.parallelize(data_point)).collect())
self.df_PD.insert(len(list(self.df_PD.columns)), 'result', result)
elif modelType == 'GBDT':
model = GradientBoostedTreesModel.load(
self.sc, self.baseDir + '/fraudModel/Model/' + modelType)
result = np.array(
model.predict(self.sc.parallelize(data_point)).collect())
self.df_PD.insert(len(list(self.df_PD.columns)), 'result', result)
elif modelType == 'LRsgd':
model = LogisticRegressionModel.load(
self.sc, self.baseDir + '/fraudModel/Model/' + modelType)
result = np.array(
model.predict(self.sc.parallelize(data_point)).collect())
self.df_PD.insert(len(list(self.df_PD.columns)), 'result', result)
elif modelType == 'LRlbfgs':
model = LogisticRegressionModel.load(
self.sc, self.baseDir + '/fraudModel/Model/' + modelType)
result = np.array(
model.predict(self.sc.parallelize(data_point)).collect())
self.df_PD.insert(len(list(self.df_PD.columns)), 'result', result)
elif modelType == 'SVM':
model = SVMModel.load(
self.sc, self.baseDir + '/fraudModel/Model/' + modelType)
result = np.array(
model.predict(self.sc.parallelize(data_point)).collect())
self.df_PD.insert(len(list(self.df_PD.columns)), 'result', result)
else:
pass
def evaluate_model(type):
if type == 'logistic':
model = LogisticRegressionModel.load(sc, "logit_model.model")
elif type == 'tree':
model = DecisionTreeModel.load(sc, "dt_model.model")
elif type == 'rf':
model = RandomForestModel.load(sc, "rf_model.model")
def predict():
testData = MLUtils.loadLibSVMFile(sc,INPUT_DATA_PATH)
print("[INFO] load complete.")
model = RandomForestModel.load(sc,TEST_MODEL_PATH)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
lst = predictions.collect()
with open(TEST_PREDICT_PATH+"/"+time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime())+".txt",'w') as f:
for k in lst:
f.write(str(k)+"\n")
labelsAndPredictions = testData.map(lambda lp: tobin(lp.label)).zip(predictions.map(lambda lp: tobin(lp)))
#print(labelsAndPredictions.collect())
metrics = BinaryClassificationMetrics(labelsAndPredictions)
# Area under precision-recall curve
print("Area under PR = %s" % metrics.areaUnderPR)
# Area under ROC curve
print("Area under ROC = %s" % metrics.areaUnderROC)
#print(labelsAndPredictions.collect())
testErr = labelsAndPredictions.filter(lambda lp: lp[0] != lp[1]).count() / float(testData.count())
print('[INFO] Test Error = ' + str(testErr))
def prediction(parameter_list):
#function
sameModel = RandomForestModel.load(sc, filename)#"path/myrfModel"
print "read model success"
prediction = sameModel.predict(parameter_list)
#example
#[60.0,37.307741,-121.89593,17108.0,0.0,8400.0,0.0,1938.0,1320.0,6.0,2.0,1.0,95125.0,713.0,753543.2])
return prediction
def main(sc, filename):
'''
The driver for the spark scoring application, it generates predictions for
a given file of features and target variables
'''
rawDataRdd = sc.textFile(filename)
print "Data Size: {}".format(rawDataRdd.count())
labeledPointsRdd = rawDataRdd.map(parse_lines)
#load models
logit_model = LogisticRegressionModel.load(sc, "logit_model.model")
dt_model = DecisionTreeModel.load(sc, "dt_model.model")
rf_model = RandomForestModel.load(sc, "rf_model.model")
#logistic predictions
labels_and_preds = labeledPointsRdd.map(lambda p: (float(logit_model.predict(p.features)), p.label ))
labels_and_preds_collected = labels_and_preds.collect()
print "\n"
print "Predictions: Logistic Regression"
y_true = []
y_pred = []
for row in labels_and_preds_collected:
y_true.append(row[1])
y_pred.append(row[0])
# print "predicted: {0} - actual: {1}\n".format(row[0], row[1])
accuracy = labels_and_preds.filter(lambda (v,p): v == p).count() / float(labeledPointsRdd.count())
print_box()
print "Prediction Accuracy (Logistic): {}".format(round(accuracy, 4))
print_box()
print "\n"
#decision tree predictions
predictions = dt_model.predict(labeledPointsRdd.map(lambda p: p.features))
labels_and_preds_dt = labeledPointsRdd.map(lambda p: p.label).zip(predictions)
labels_and_preds_dt_collected = labels_and_preds.collect()
accuracy_dt = labels_and_preds_dt.filter(lambda (v, p): v == p).count() / float(labeledPointsRdd.count())
print_box()
print "Prediction Accuracy (Decision Tree): {}".format(round(accuracy_dt, 4))
print_box()
print "\n"
#random forest predictions
predictions_rf = rf_model.predict(labeledPointsRdd.map(lambda p: p.features))
labels_and_preds_rf = labeledPointsRdd.map(lambda p: p.label).zip(predictions_rf)
accuracy_rf = labels_and_preds_rf.filter(lambda (v, p): v == p).count() / float(labeledPointsRdd.count())
print_box()
print "Prediction Accuracy (Random Forest): {}".format(round(accuracy_rf, 4))
print_box()
def __init__(self, path):
conf = SparkConf() \
.setAppName("crankshaw-pyspark") \
.set("spark.executor.memory", "2g") \
.set("spark.kryoserializer.buffer.mb", "128") \
.set("master", "local")
sc = SparkContext(conf=conf, batchSize=10)
self.model = RandomForestModel.load(sc, path)
self.path = path
print("started spark")
def get_best_classifier(sc, data_dir):
# model_randfor_class_t04_d04_b024_s7593
# model_randfor_class_t04_d04_b024_s7593_params.pckl
from os import listdir
from os.path import isfile, join
models = [(f, int(f[-4:])) for f in listdir(data_dir)
if f[:6] == 'model_' and len(f) == 39]
best_model = max(models, key=lambda item: item[1])
print('best classifier found is |{0}| score = {1} '.format(*best_model))
params = load_pickle_file(data_dir + best_model[0] + '_params.pckl')
print(params)
#return (RandomForestClassificationModel.load(data_dir + best_model[0]), params)
return (RandomForestModel.load(sc=sc,
path=data_dir + best_model[0]), params)
def main():
parser = OptionParser()
parser.add_option('', '--enriched_data_path', action='store', dest='enriched_data_path', help='path to write enriched data')
parser.add_option('', '--model_path', action='store', dest='model_path', help='path for model data')
parser.add_option('', '--kafka_zookeeper_hosts', action='store', dest='kafka_zookeeper_hosts', help='list of Zookeeper hosts (host:port)')
parser.add_option('', '--kafka_broker_list', action='store', dest='kafka_broker_list', help='list of Kafka brokers (host:port)')
parser.add_option('', '--kafka_message_topic', action='store', dest='kafka_message_topic', help='topic to consume input messages from')
parser.add_option('', '--kafka_alert_topic', action='store', dest='kafka_alert_topic', help='topic to produce alert messages to')
parser.add_option('', '--kafka_enriched_data_topic', action='store', dest='kafka_enriched_data_topic', help='topic to produce enriched data to')
parser.add_option('', '--streaming_batch_duration_sec', type='float', default=15.0,
action='store', dest='streaming_batch_duration_sec', help='Streaming batch duration in seconds')
parser.add_option('', '--max_batches', type='int', default=0,
action='store', dest='max_batches', help='Number of batches to process (0 means forever)')
options, args = parser.parse_args()
sc = SparkContext()
ssc = StreamingContext(sc, options.streaming_batch_duration_sec)
sqlContext = getSqlContextInstance(sc)
# Load saved model.
model = None
if options.model_path:
model = RandomForestModel.load(sc, options.model_path)
else:
print('No model loaded.')
# Create Kafka stream to receive new messages.
kvs = KafkaUtils.createDirectStream(ssc, [options.kafka_message_topic], {
'metadata.broker.list': options.kafka_broker_list,
'group.id': 'spark_streaming_processor.py'})
# Take only the 2nd element of the tuple.
messages = kvs.map(lambda x: x[1])
# Convert RDD of JSON strings to RDD of Rows.
rows = messages.map(json_to_row)
# Process messages.
rows.foreachRDD(lambda time, rdd:
process_messages(time, rdd,
ssc=ssc,
model=model,
enriched_data_path=options.enriched_data_path,
zookeeper_hosts=options.kafka_zookeeper_hosts,
kafka_alert_topic=options.kafka_alert_topic,
kafka_enriched_data_topic=options.kafka_enriched_data_topic,
max_batches=options.max_batches))
ssc.start()
ssc.awaitTermination()
def get_rf_model(sc, train=None):
model_path = 'rf.model'
if train is None:
model = RandomForestModel.load(sc, model_path)
else:
model = RandomForest.trainClassifier(train,
numClasses=2,
numTrees=10,
categoricalFeaturesInfo={},
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=10,
maxBins=100)
model.save(sc, model_path)
return model
def predict():
conf = SparkConf().setMaster("local").setAppName("My App")
sc = SparkContext(conf=conf)
testData = MLUtils.loadLibSVMFile(sc, TEST_DATA_PATH)
model = RandomForestModel.load(sc, TEST_MODEL_PATH)
predictions = model.predict(testData.map(lambda x: x.features))
predictlabel_list = predictions.collect()
rateOFeachSort_dict = analyse_result(predictlabel_list)
save(predictlabel_list)
return rateOFeachSort_dict
def test(sc):
files = ["sounds/flushing/20150227_193109-flushing-04.wav",
"sounds/bike/20150227_193806-bici-14.wav",
"sounds/blender/20150227_193606-licuadora-14.wav"
]
rfmodel = RandomForestModel.load(sc, RF_PATH)
dtmodel = DecisionTreeModel.load(sc, DT_PATH)
print dtmodel.toDebugString()
for f in files:
vec = audio.showFeatures(f)
testfeatures = Vectors.dense([float(x) for x in vec.split(' ')])
print(vec)
pred = dtmodel.predict(testfeatures)
print("DT Prediction is " + str(pred), classes[int(pred)])
pred = rfmodel.predict(testfeatures)
print("RF Prediction is " + str(pred), classes[int(pred)])
def predict(sc, data):
#sc = SparkContext(appName="PythonRandomForestClassificationExample")
# $example on$
# Load and parse the data file into an RDD of LabeledPoint.
#data = MLUtils.loadLibSVMFile(sc, 'deathproject/test1.txt')
#data = [1, 0, 5, 1, 1, 65, 1, 2, 0, 1, 0, 6, 3450]
# Split the data into training and test sets (30% held out for testing)
# (trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
print('Starting...')
#model = RandomForest.trainClassifier(trainingData, numClasses=8, categoricalFeaturesInfo={},
# numTrees=5, featureSubsetStrategy="auto",
# impurity='gini', maxDepth=4, maxBins=32)
sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestClassificationModel2")
print('Predicting...')
# Evaluate model on test instances and compute test error
#predictions = sameModel.predict(data.map(lambda x: x.features))
predictions = sameModel.predict(data)
#labelsAndPredictions = data.map(lambda lp: lp.label).zip(predictions)
#testErr = labelsAndPredictions.filter(lambda (v, p): v != p).count() / float(testData.count())
#print('Test Error = ' + str(testErr))
#print('Learned classification forest model:')
#print(sameModel.toDebugString())
print(predictions)
return int(predictions)
def random_forest():
"""
使用mllib对Spark安装包mllib的测试数据集做随机森林测试
80%数据作为训练数据 20%数据作为测试数据
:return:
"""
data_rdd = MLUtils.loadLibSVMFile(
sc, '{}/mllib/sample_libsvm_data.txt'.format(current_dir))
train_data_rdd, test_data_rdd = data_rdd.randomSplit([0.8, 0.2])
model = RandomForest.trainClassifier(train_data_rdd,
numClasses=2,
categoricalFeaturesInfo={},
numTrees=3)
# 根据测试集的features预测laber值为0还是1
predict_rdd = model.predict(test_data_rdd.map(lambda x: x.features))
# 测试集实际的laber值
labels_rdd = test_data_rdd.map(lambda lp: lp.label).zip(predict_rdd)
# 测试样本中预测值与实际值不符的百分比(错误率)
print(labels_rdd.filter(lambda x: x[0] != x[1]).count())
test_err = labels_rdd.filter(lambda x: x[0] != x[1]).count() / float(
test_data_rdd.count())
print("test error rate:{}".format(test_err))
# 保存 训练好的模型
model_path = "{}/my_random_forest_model".format(current_dir)
if not os.path.exists(model_path):
model.save(sc, model_path)
trained_model = RandomForestModel.load(
sc, "{}/my_random_forest_model".format(current_dir))
print(trained_model.toDebugString())
return trained_model
def get_rfc_model(filename, bucket, client, sc):
# get download_url
objects_dict = client.list_objects(Bucket=BUCKET_NAME)
filenames = map(lambda x: x["Key"], objects_dict["Contents"])
filenames = filter(lambda x: True if re.match(filename + '-v' + "[0-9]+", x) else False, filenames)
if len(filenames) == 0:
print "NO RFC MODELS FOUND"
return False
else:
versions = map(lambda x: int(re.search("[0-9]+", x).group()), filenames)
download_url = filename + "-v" + str(max(versions)) + '.zip'
# download zip and unzip it
local_url = '/tmp/' + download_url
os.system("rm -r " + local_url) # remove zip just in case
os.system("rm -r " + local_url[:-4]) # remove unzipped folder just in case
if download_file(download_url, bucket):
with zipfile.ZipFile(local_url, "r") as z:
z.extractall(local_url[:-4])
return RandomForestModel.load(sc, local_url[:-4]) #-4 for zip
else:
print "ZIP DOWNLOAD FAILED"
return False
from pyspark.mllib.regression import LabeledPoint
from pyspark import SparkContext, SparkConf
from pyspark.sql.session import SparkSession
from pyspark.ml.classification import RandomForestClassifier
from pyspark.mllib.tree import RandomForestModel
from pyspark.mllib.tree import RandomForest
from pyspark.mllib.evaluation import MulticlassMetrics
from prettytable import PrettyTable
sc = SparkContext()
spark = SparkSession(sc)
inputDF = spark.read.csv('s3://assignmentcs643/TrainingDataset.csv',header='true', inferSchema='true', sep=';')
datadf= inputDF.rdd.map(lambda row: LabeledPoint(row[-1], Vectors.dense(row[0:-1])))
model = RandomForestModel.load(sc,"s3://assignmentcs643/randomforestmodel.model")
predictions = model.predict(datadf.map(lambda x: x.features))
labels_and_predictions = datadf.map(lambda x: x.label).zip(predictions)
acc = labels_and_predictions.filter(lambda x: x[0] == x[1]).count() / float(datadf.count())
metrics = MulticlassMetrics(labels_and_predictions)
f1 = metrics.fMeasure()
recall = metrics.recall()
precision = metrics.precision()
#evaluation values
print("Model accuracy: %.3f%%" % (acc * 100))
print("Recall Value = %s" % recall)
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
model = RandomForest.trainClassifier(trainingData,
numClasses=2,
categoricalFeaturesInfo={},
numTrees=3,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=4,
maxBins=32)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(
lambda (v, p): v != p).count() / float(testData.count())
print('Test Error = ' + str(testErr))
print('Learned classification forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, "target/tmp/myRandomForestClassificationModel")
sameModel = RandomForestModel.load(
sc, "target/tmp/myRandomForestClassificationModel")
# $example off$
def writeLumbarTrainingReadings(time, rddTraining):
try:
# Convert RDDs of the words DStream to DataFrame and run SQL query
connectionProperties = MySQLConnection.getDBConnectionProps('/home/erik/mysql_credentials.txt')
sqlContext = SQLContext(rddTraining.context)
if rddTraining.isEmpty() == False:
lumbarTrainingReading = sqlContext.jsonRDD(rddTraining)
lumbarTrainingReadingFinal = lumbarTrainingReading.selectExpr("deviceID","metricTypeID","uomID","positionID","actual.y AS actualYaw","actual.p AS actualPitch","actual.r AS actualRoll","setPoints.y AS setPointYaw","setPoints.p AS setPointPitch","setPoints.r AS setPointRoll")
lumbarTrainingReadingFinal.write.jdbc("jdbc:mysql://localhost/biosensor", "SensorTrainingReadings", properties=connectionProperties)
except:
pass
if __name__ == "__main__":
sc = SparkContext(appName="Process Lumbar Sensor Readings")
ssc = StreamingContext(sc, 2) # 2 second batches
loadedModel = RandomForestModel.load(sc, "../machine_learning/models/IoTBackBraceRandomForest.model")
#Process Readings
streamLumbarSensor = KafkaUtils.createDirectStream(ssc, ["LumbarSensorReadings"], {"metadata.broker.list": "localhost:9092"})
lineSensorReading = streamLumbarSensor.map(lambda x: x[1])
lineSensorReading.foreachRDD(writeLumbarReadings)
#Process Training Readings
streamLumbarSensorTraining = KafkaUtils.createDirectStream(ssc, ["LumbarSensorTrainingReadings"], {"metadata.broker.list": "localhost:9092"})
lineSensorTrainingReading = streamLumbarSensorTraining.map(lambda x: x[1])
lineSensorTrainingReading.foreachRDD(writeLumbarTrainingReadings)
# Run and then wait for termination signal
ssc.start()
ssc.awaitTermination()
def setup_model(sc, model_uri):
model = RandomForestModel.load(sc, model_uri)
return model
res.append({'name': 'Root' + head, 'children': parse(data[1:], 0)})
outstr += json.dumps(res[0])
with open('../vis/data/structure.json', 'w') as outfile:
outfile.write(outstr)
print('Conversion Success !')
if __name__ == "__main__":
# debug
# treeStr = open("../vis/data/modelString",'r').read()
# python3 vis.py ../model 7 3
try:
MAX_DEPTH = int(sys.argv[1])
except:
print("default depth is 7.")
try:
USE_TREE = int(sys.argv[2])
except:
print("default tree to convert is num 3")
sc = SparkContext(appName="RandomForest2Json")
trees = RandomForestModel.load(sc, TEST_MODEL_PATH)
treeStr = trees.toDebugString()
with open("../vis/data/modelString", 'w') as f:
f.write(treeStr)
tree2json(treeStr)
paths = paths\
.replace('.0','')\
.replace('feature ','')\
.replace('Predict: ','#')\
.replace('not in','5')\
.replace(' ','|')\
.replace('in','4')\
.replace('>=','0')\
.replace('<=','1')\
.replace('>','2')\
.replace('<','3')\
.replace('Root:','')
print >> f2, paths.decode('utf8')
else:
walk(dic['children'], path+dic['name']+':')
f2.close()
if __name__ == "__main__":
dtModelFile = "output/RFModel"
dtModelResults = "randomForestModel.txt"
sc = SparkContext("local[20]","RFClassification")
dtModel = RandomForestModel.load(sc, dtModelFile)
dtree = dtModel.toDebugString()
print dtree
tree_json(dtree,dtModelResults)
# $example off$
if __name__ == "__main__":
sc = SparkContext(appName="PythonRandomForestClassificationExample")
# $example on$
# Load and parse the data file into an RDD of LabeledPoint.
data = MLUtils.loadLibSVMFile(sc, 'data/mllib/sample_libsvm_data.txt')
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
model = RandomForest.trainClassifier(trainingData, numClasses=2, categoricalFeaturesInfo={},
numTrees=3, featureSubsetStrategy="auto",
impurity='gini', maxDepth=4, maxBins=32)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(lambda (v, p): v != p).count() / float(testData.count())
print('Test Error = ' + str(testErr))
print('Learned classification forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, "target/tmp/myRandomForestClassificationModel")
sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestClassificationModel")
# $example off$
def test_classification(self):
from pyspark.mllib.classification import LogisticRegressionWithSGD, SVMWithSGD, NaiveBayes
from pyspark.mllib.tree import DecisionTree, DecisionTreeModel, RandomForest,\
RandomForestModel, GradientBoostedTrees, GradientBoostedTreesModel
data = [
LabeledPoint(0.0, [1, 0, 0]),
LabeledPoint(1.0, [0, 1, 1]),
LabeledPoint(0.0, [2, 0, 0]),
LabeledPoint(1.0, [0, 2, 1])
]
rdd = self.sc.parallelize(data)
features = [p.features.tolist() for p in data]
temp_dir = tempfile.mkdtemp()
lr_model = LogisticRegressionWithSGD.train(rdd, iterations=10)
self.assertTrue(lr_model.predict(features[0]) <= 0)
self.assertTrue(lr_model.predict(features[1]) > 0)
self.assertTrue(lr_model.predict(features[2]) <= 0)
self.assertTrue(lr_model.predict(features[3]) > 0)
svm_model = SVMWithSGD.train(rdd, iterations=10)
self.assertTrue(svm_model.predict(features[0]) <= 0)
self.assertTrue(svm_model.predict(features[1]) > 0)
self.assertTrue(svm_model.predict(features[2]) <= 0)
self.assertTrue(svm_model.predict(features[3]) > 0)
nb_model = NaiveBayes.train(rdd)
self.assertTrue(nb_model.predict(features[0]) <= 0)
self.assertTrue(nb_model.predict(features[1]) > 0)
self.assertTrue(nb_model.predict(features[2]) <= 0)
self.assertTrue(nb_model.predict(features[3]) > 0)
categoricalFeaturesInfo = {0: 3} # feature 0 has 3 categories
dt_model = DecisionTree.trainClassifier(
rdd, numClasses=2, categoricalFeaturesInfo=categoricalFeaturesInfo, maxBins=4)
self.assertTrue(dt_model.predict(features[0]) <= 0)
self.assertTrue(dt_model.predict(features[1]) > 0)
self.assertTrue(dt_model.predict(features[2]) <= 0)
self.assertTrue(dt_model.predict(features[3]) > 0)
dt_model_dir = os.path.join(temp_dir, "dt")
dt_model.save(self.sc, dt_model_dir)
same_dt_model = DecisionTreeModel.load(self.sc, dt_model_dir)
self.assertEqual(same_dt_model.toDebugString(), dt_model.toDebugString())
rf_model = RandomForest.trainClassifier(
rdd, numClasses=2, categoricalFeaturesInfo=categoricalFeaturesInfo, numTrees=10,
maxBins=4, seed=1)
self.assertTrue(rf_model.predict(features[0]) <= 0)
self.assertTrue(rf_model.predict(features[1]) > 0)
self.assertTrue(rf_model.predict(features[2]) <= 0)
self.assertTrue(rf_model.predict(features[3]) > 0)
rf_model_dir = os.path.join(temp_dir, "rf")
rf_model.save(self.sc, rf_model_dir)
same_rf_model = RandomForestModel.load(self.sc, rf_model_dir)
self.assertEqual(same_rf_model.toDebugString(), rf_model.toDebugString())
gbt_model = GradientBoostedTrees.trainClassifier(
rdd, categoricalFeaturesInfo=categoricalFeaturesInfo, numIterations=4)
self.assertTrue(gbt_model.predict(features[0]) <= 0)
self.assertTrue(gbt_model.predict(features[1]) > 0)
self.assertTrue(gbt_model.predict(features[2]) <= 0)
self.assertTrue(gbt_model.predict(features[3]) > 0)
gbt_model_dir = os.path.join(temp_dir, "gbt")
gbt_model.save(self.sc, gbt_model_dir)
same_gbt_model = GradientBoostedTreesModel.load(self.sc, gbt_model_dir)
self.assertEqual(same_gbt_model.toDebugString(), gbt_model.toDebugString())
try:
rmtree(temp_dir)
except OSError:
pass
if __name__ == "__main__":
sc = SparkContext(appName="PythonRandomForestRegressionExample")
# $example on$
# Load and parse the data file into an RDD of LabeledPoint.
data = MLUtils.loadLibSVMFile(sc, 'data/mllib/sample_libsvm_data.txt')
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
model = RandomForest.trainRegressor(trainingData, categoricalFeaturesInfo={},
numTrees=3, featureSubsetStrategy="auto",
impurity='variance', maxDepth=4, maxBins=32)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testMSE = labelsAndPredictions.map(lambda lp: (lp[0] - lp[1]) * (lp[0] - lp[1])).sum() /\
float(testData.count())
print('Test Mean Squared Error = ' + str(testMSE))
print('Learned regression forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, "target/tmp/myRandomForestRegressionModel")
sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestRegressionModel")
# $example off$
from pyspark.mllib.tree import RandomForest, RandomForestModel
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.util import MLUtils
sc = SparkContext("local", "Amadeus Random Forest Run", pyFiles=['run_model.py'])
data = sc.textFile("data_timbre.csv")
zipped_data = data.zipWithIndex()
keyed_data = zipped_data.map(lambda line: (line[-1], line[:-1]))
target = sc.textFile("target_timbre.csv")
zipped_target = target.zipWithIndex()
keyed_target = zipped_target.map(lambda line: (line[-1], line[:-1]))
target_data = keyed_data.join(keyed_target)
labled_point_data = target_data.map(lambda tup: LabeledPoint(tup[1][1][0], tup[1][0][0].split(',')))
# Split the data into training and test sets (30% held out for testing)
print("Creating Training and Test Data Split")
(trainingData, testData) = labled_point_data.randomSplit([0.7, 0.3])
model = RandomForestModel.load(sc, "myRFModel")
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testMSE = labelsAndPredictions.map(lambda (v, p): (v - p) * (v - p)).sum() / float(testData.count())
print('Test Mean Squared Error = ' + str(testMSE))
testAccuracy = labelsAndPredictions.map(lambda (v, p): 1 if (abs(v - p) < 10) else 0).sum() / float(testData.count())
print('Total Accuracy = ' + str(testAccuracy))
def load (filename) :
model = RandomForestModel.load(sc, filename)
return model
from pyspark.mllib.feature import IDF
from pyspark.mllib.tree import RandomForest, RandomForestModel
from pyspark.mllib.util import MLUtils
from pyspark.mllib.regression import LabeledPoint
from gensim.models.doc2vec import Doc2Vec
from math import exp
from threading import Thread, Event
sc = SparkContext()
sqlContext = SQLContext(sc)
# this is a large object we cache it on each worker node
gmod_broadcast = sc.broadcast( Doc2Vec.load("/data/_hndata/doc2vec_model/hn") )
tfidf_model = RandomForestModel.load(sc, "hdfs:///hndata/hnrrmodel_tfidf")
doc2vec_model = RandomForestModel.load(sc, "hdfs:///hndata/rrscoremodel")
doc2vec_model2 = RandomForestModel.load(sc, "hdfs:///hndata/rrscoremodel2")
tf = sc.pickleFile("hdfs:///hndata/tf_pickle")
idf = IDF().fit(tf)
hashingTF = HashingTF(1000)
def pred_tfidf(docs):
sents = sc.parallelize(docs).map(lambda d: d.strip().split())
new_tf = hashingTF.transform(sents)
tfidf = idf.transform(new_tf)
return tfidf_model.predict(tfidf)
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
#figPath = os.path.join('static', 'skillradar')
######################## LOAD MODEL ###########################
# import pre-trained models
import pyspark
from pyspark import SparkContext, SparkConf
from pyspark.mllib.tree import RandomForest, RandomForestModel
conf = SparkConf().setAppName('GameAnalysis').setMaster('local')
sc = SparkContext()
# player skill analyzing model
model_player = RandomForestModel.load(
sc, 'model/RandomForestModel/myRandomForestRegressionModel')
# match result predicting model
model_match = RandomForestModel.load(sc, "model/Dota2RandomForestModel")
######################## WEB SERVER ###########################
from flask import Flask, render_template, request, redirect, flash, url_for, make_response
from flask_wtf import FlaskForm
from wtforms import StringField, validators, SelectMultipleField, Form, widgets
from wtforms.validators import DataRequired, InputRequired
from flask_bootstrap import Bootstrap
app = Flask(__name__)
Bootstrap(app)
app.secret_key = 'secret-for-dev'
PORT = os.getenv('PORT', '40555')
from pyspark.mllib.tree import RandomForest, RandomForestModel
from pyspark import SparkConf, SparkContext
from pyspark.sql.session import SparkSession
from pyspark.sql import SQLContext, HiveContext
from pyspark.mllib.regression import LabeledPoint
import pickle
import pandas as pd
sc = SparkContext('local')
model = RandomForestModel.load(sc, "myFirstWorkModel")
data = pd.read_csv('/home/hadoop/hadooptest/test.csv')
Nattr = ['PassengerId', 'Pclass', 'Age', 'SibSp', 'Parch', 'Fare']
data['Age'].fillna(int(data.Age.mean()), inplace=True)
data['Cabin'] = data['Cabin'].fillna('missing').map(lambda x: x[0]).map(
lambda x: x if x != 'm' else 'missing')
data['Embarked'].fillna("missing", inplace=True)
for attr in ['Cabin', 'Sex', 'Embarked']:
dummy = pd.get_dummies(data[attr], prefix=attr)
data = pd.concat([data, dummy], axis=1)
data.pop(attr)
'''
[ u'passengerid', u'survived', u'pclass',
u'age', u'sibsp', u'parch',
u'fare', u'cabin_A', u'cabin_B', u'cabin_C',
u'cabin_D', u'cabin_E', u'cabin_F', u'cabin_G',
u'cabin_T', u'cabin_missing', u'sex_female', u'sex_male',
u'embarked_', u'embarked_C', u'embarked_Q', u'embarked_S']
'''
# 7: solr collection/index name
# 8: number of json doc/item to index per batch
if len(sys.argv) != 8:
exit(-1)
## initiate spark context
sc = SparkContext(appName="PythonStreamingDirectKafkaTweetSentiments")
brokers, topic, solrHost, solrPort, confDirHDFS, solrCollection, solrBatchSize = sys.argv[1:]
## load libraries & model ###########
affinListEng=sc.textFile(confDirHDFS+'/AFINN-111_eng.txt')
affinListMalay=sc.textFile(confDirHDFS+'/AFINN-111_malay.txt')
affinEnglish=loadDict(affinListEng.collect(), ',')
affinMalay=loadDict(affinListMalay.collect(), ',')
twitterSentimentRF = RandomForestModel.load(sc, confDirHDFS+"/twitterSentimentRF.model")
## initiate spark streaming context
ssc = StreamingContext(sc, 2)
# parse tweets and collect text
# brokers, topic, solrHost, solrPort = sys.argv[1:]
kvs = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers, "auto.offset.reset": "smallest"}, keyDecoder=utf8_decoder_ignore_error, valueDecoder=utf8_decoder_ignore_error )
lines = kvs.map(lambda x: x[1])
# keep all the texts between widest curly braces
lines = lines.map(lambda x: "{"+re.findall(r'\{(.+)\}',x)[0]+"}")
# convert to ascii
#texts = lines.map(lambda x: json.loads(x)['text'].encode('ascii', "ignore"))
tweets = lines.map(lambda x :(\
json.loads(x)['id'],
json.loads(x)['created_at'].encode('ascii', "ignore"),\
"deviceID", "metricTypeID", "uomID", "positionID",
"actual.y AS actualYaw", "actual.p AS actualPitch",
"actual.r AS actualRoll", "setPoints.y AS setPointYaw",
"setPoints.p AS setPointPitch", "setPoints.r AS setPointRoll")
lumbarTrainingReadingFinal.write.jdbc(
"jdbc:mysql://localhost/biosensor",
"SensorTrainingReadings",
properties=connectionProperties)
except:
pass
if __name__ == "__main__":
sc = SparkContext(appName="Process Lumbar Sensor Readings")
ssc = StreamingContext(sc, 2) # 2 second batches
loadedModel = RandomForestModel.load(
sc, "../machine_learning/models/IoTBackBraceRandomForest.model")
#Process Readings
streamLumbarSensor = KafkaUtils.createDirectStream(
ssc, ["LumbarSensorReadings"],
{"metadata.broker.list": "localhost:9092"})
lineSensorReading = streamLumbarSensor.map(lambda x: x[1])
lineSensorReading.foreachRDD(writeLumbarReadings)
#Process Training Readings
streamLumbarSensorTraining = KafkaUtils.createDirectStream(
ssc, ["LumbarSensorTrainingReadings"],
{"metadata.broker.list": "localhost:9092"})
lineSensorTrainingReading = streamLumbarSensorTraining.map(lambda x: x[1])
lineSensorTrainingReading.foreachRDD(writeLumbarTrainingReadings)
return prediction
@app.route("/spark_rf", methods=["GET"])
def spark_rest_info():
return "hello, spark random forest! please post with json data.", 200
@app.route("/spark_rf", methods=["POST"])
def rest_prediction():
req_post = json.loads(request.data)
resp = dict()
for i in range(max_para_number):
if standard_info_headers[i] in req_post:
parameter_list[i] = req_post[standard_info_headers[i]]
resp[standard_info_headers[i]] = parameter_list[i]
resp['price'] = prediction(parameter_list)
return jsonify(resp), 200
if __name__ == '__main__':
conf = SparkConf().setAppName(APP_NAME)
conf = conf.setMaster("local[*]")
sc = SparkContext(conf=conf)
global sameModel
sameModel = RandomForestModel.load(sc, filename)
if len(sys.argv) > 1:
filename = sys.argv[1] #pass model path
app.run(host='0.0.0.0', port=10003)
from pyspark.mllib.linalg import Vectors
print ("Successfully imported Spark Modules")
except ImportError as e:
print ("Can not import Spark Modules", e)
sys.exit(1)
import functools
import itertools
from kafka import KafkaConsumer
def parseData(line):
splittedLine = line.split(",")
values = [float(s) for s in splittedLine[4:-1]]
label = splittedLine[-1]
featuresVector = Vectors.dense(values)
return LabeledPoint(label, featuresVector)
if __name__ == "__main__":
conf = SparkConf().setAppName("RandomForest_Anomaly_Detection_Kafka_Consumer")
sc = SparkContext(conf=conf)
savedModel = RandomForestModel.load(sc, "../train_model/model")
consumer = KafkaConsumer('test', group_id='my_group', bootstrap_servers=['localhost:9092'])
print("Waiting for messages...")
for message in consumer:
print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset, message.key, message.value))
data = sc.parallelize([message.value])
testData = data.map(parseData)
predictions = savedModel.predict(testData.map(lambda x: x.features))
print("Prediction: ")
print(predictions.first())
'week_start', date_add(col('week_start_origin'),
7 * (num_weeks_ahead + 1)))
## only do optimization for stores in the treatment group
df_test = df_test.filter(col('group_val') == 'treatment').drop('group_val')
df_test = df_test.select([
'store_id', 'product_id', 'department_id', 'brand_id', 'msrp', 'cost',
'avg_hhi', 'avg_traffic', 'week_start'
]).distinct()
# In[21]:
################################################## 3: load the model built last time
dirfilename_load = spark.read.parquet(modelDir + "model_name").filter(
'_1 == "model_name"').rdd.map(lambda p: p[1]).collect()
dirfilename_load = dirfilename_load[0]
rfModel = RandomForestModel.load(sc, dirfilename_load)
# In[34]:
################################################## 4: optimization
##define categorical features, which used in modeling
features_categorical_train_and_test = ["department_id", "brand_id"]
features_numerical_train_and_test = [
"price", "avg_hhi", "avg_traffic", "rl_price", "discount"
]
##feature index the categorical features
for s in features_categorical_train_and_test:
s_StringIndexed = s + "_StringIndexed"
indexer = StringIndexer(inputCol=s, outputCol=s_StringIndexed)
indexer_trans = indexer.fit(df_train)
df_test = indexer_trans.transform(df_test)
dict_add = pickle.load(handle)
with open('dict_city.pickle', 'rb') as handle:
dict_city = pickle.load(handle)
with open('dict_fac.pickle', 'rb') as handle:
dict_fac = pickle.load(handle)
with open('dict_status.pickle', 'rb') as handle:
dict_status = pickle.load(handle)
with open('dict_bor.pickle', 'rb') as handle:
dict_bor = pickle.load(handle)
model = RandomForestModel.load(sc, "modelCritical")
predictList = []
f = open("outFile3.csv", 'rU')
for line in csv.reader(f, delimiter=","):
predictRow = list(line)
'''
agency = predictRow[agency_index]
complaint = predictRow[comp_index]
location = predictRow[loc_index]
incident = predictRow[incident_index]
address = predictRow[add_index]
city = predictRow[city_index]
facility = predictRow[fac_index]
if __name__ == "__main__":
sc = SparkContext(appName="PythonRandomForestRegressionExample")
# $example on$
# Load and parse the data file into an RDD of LabeledPoint.
data = MLUtils.loadLibSVMFile(sc, 'data/mllib/sample_libsvm_data.txt')
# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3])
# Train a RandomForest model.
# Empty categoricalFeaturesInfo indicates all features are continuous.
# Note: Use larger numTrees in practice.
# Setting featureSubsetStrategy="auto" lets the algorithm choose.
model = RandomForest.trainRegressor(trainingData, categoricalFeaturesInfo={},
numTrees=3, featureSubsetStrategy="auto",
impurity='variance', maxDepth=4, maxBins=32)
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testMSE = labelsAndPredictions.map(lambda lp: (lp[0] - lp[1]) * (lp[0] - lp[1])).sum() /\
float(testData.count())
print('Test Mean Squared Error = ' + str(testMSE))
print('Learned regression forest model:')
print(model.toDebugString())
# Save and load model
model.save(sc, "target/tmp/myRandomForestRegressionModel")
sameModel = RandomForestModel.load(sc, "target/tmp/myRandomForestRegressionModel")
# $example off$
def main():
sc = SparkContext(conf=SparkConf().setAppName("Random Forest"))
bytePath = "/Users/priyanka/Desktop/project2files/train"
byteTestPath = "/Users/priyanka/Desktop/project2files/test"
namePath = "/Users/priyanka/Desktop/X_train_small.txt"
nameTestPath = "/Users/priyanka/Desktop/X_test_small.txt"
classPath = "/Users/priyanka/Desktop/y_train_small.txt"
classTestPath = "/Users/priyanka/Desktop/y_test_small.txt"
docData = sc.wholeTextFiles(
bytePath,
25).map(lambda (x, y): (x.encode("utf-8"), y.encode("utf-8")))
print("docData frankie")
docData.take(1)
#clean docData here - remove 1st word from line and remove
cleanDocData = docData.map(lambda (x, y): (x, clean(y.split())))
nameData = sc.textFile(
namePath, 25).map(lambda x: "file:" + bytePath + "/" + x + ".bytes"
).zipWithIndex().map(lambda (x, y): (y, x))
#nameData.take(5)
labelData = sc.textFile(
classPath, 25).zipWithIndex().map(lambda (x, y): (y, str(int(x) - 1)))
joinNameLabel = nameData.join(labelData).map(lambda (x, y): y)
#joinNameLabel.take(5)
joinCleanDocLabel = joinNameLabel.join(cleanDocData).map(lambda (x, y): y)
print("starting hashingTF rosie")
#joinCleanDocLabel.persist()
#tfFormat = joinDocLabel.map(lambda (x,y): (x,y.split()))
hashingTF = HashingTF()
hashData = joinCleanDocLabel.map(
lambda (label, text): LabeledPoint(label, hashingTF.transform(text)))
hashData.persist()
print "hashing TF done"
# model = NaiveBayes.train(hashData)
print("generating model fliss")
model1 = RandomForest.trainClassifier(hashData,
numClasses=10,
categoricalFeaturesInfo={},
numTrees=50,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=4,
maxBins=32)
#==============================================================================
# Testing starts here
#==============================================================================
docTestData = sc.wholeTextFiles(
byteTestPath,
25).map(lambda (x, y): (x.encode("utf-8"), y.encode("utf-8")))
#docTestData.take(1)
cleanDocTestData = docTestData.map(lambda (x, y): (x, clean(y.split())))
nameTestData = sc.textFile(
nameTestPath,
25).map(lambda x: "file:" + byteTestPath + "/" + x + ".bytes"
).zipWithIndex().map(lambda (x, y): (y, x))
labelTestData = sc.textFile(
classTestPath,
25).zipWithIndex().map(lambda (x, y): (y, str(int(x) - 1)))
#joinTestNameLabel=nameTestData.join(labelTestData).map(lambda (x,y):y).cache()
joinTestDocLabel = joinTestNameLabel.join(cleanDocTestData).map(lambda
(x, y): y)
#joinTestDocLabel.persist()
#tfTestFormat = joinTestDocLabel.map(lambda (x,y): (x,y.split()))
print("hashing test kenny")
hashTestData = joinTestDocLabel.map(
lambda (label, text): LabeledPoint(label, hashingTF.transform(text)))
sc.broadcast(hashTestData)
# #NB prediction and labels and accuracy
# prediction_and_labels = hashTestData.map(lambda point: (model.predict(point.features), point.label))
# correct = prediction_and_labels.filter(lambda (predicted, actual): predicted == actual)
# accuracy = correct.count() / float(hashTestData.count())
# print "Classifier correctly predicted category " + str(accuracy * 100) + " percent of the time"
#Random forest prediction and labels and accuracy
print "prediction part lyndz"
prediction1 = model1.predict(hashTestData.map(lambda x: x.features))
labelsAndPredictions1 = hashTestData.map(lambda lp: lp.label).zip(
prediction1)
testErr1 = labelsAndPredictions1.filter(
lambda (v, p): v != p).count() / float(hashTestData.count())
print('Test Error = ' + str(testErr1))
print('Learned classification forest model:')
print(model.toDebugString())
# Save and load random forest model
model1.save(
sc,
"/Users/priyanka/Desktop/project2files/myRandomForestClassificationModel"
)
sameModel1 = RandomForestModel.load(
sc,
"/Users/priyanka/Desktop/project2files/myRandomForestClassificationModel"
)
file=sys.stderr)
exit(-1)
global totalTime, startTime
conf = (
SparkConf().setMaster(
"local").setAppName("SparkStreamingFaultDetection").set(
"spark.executor.memory",
"16g").set("spark.driver.memory",
"16g").set("spark.executor.instances", "4")
#.set("spark.executor.cores", "4")
)
sc = SparkContext(conf=conf)
model = RandomForestModel.load(sc, '/RandomForest1.model')
ssc = StreamingContext(sc, 1)
brokers, topic = sys.argv[1:]
kvs = KafkaUtils.createDirectStream(ssc, [topic],
{"metadata.broker.list": brokers})
testErr = 0
totalData = 0
Order = 10
#testing Data
data = kvs.filter(lambda row: checkCondition(row[1]) != 'sign')
if (data.count() > 0):
def applyModel(fileName, loadModelName, outlierPercentile = 100):
sc = SparkContext( 'local', 'pyspark')
sqlContext = SQLContext(sc)
#########
# load data
#########
data = sc.textFile(fileName)
#extract header and remove it
header = data.first()
data = data.filter(lambda x:x !=header).cache()
header = header.split('\t')
#parse data
data = data.map(lambda x : x.split('\t'))
#########
# prepare features
#########
df = sqlContext.createDataFrame(data, header)
df = (df.withColumn("ADLOADINGTIME",func.regexp_replace('ADLOADINGTIME', 'null', '0').cast('float'))
.withColumn("TIMESTAMP",func.regexp_replace('TIMESTAMP', 'null', '0').cast('int'))
.withColumn("GEOIP_LAT",func.regexp_replace('GEOIP_LAT', 'null', '0').cast('int'))
.withColumn("GEOIP_LNG",func.regexp_replace('GEOIP_LNG', 'null', '0').cast('int'))
.withColumn("HOSTWINDOWHEIGHT",func.regexp_replace('HOSTWINDOWHEIGHT', 'null', '0').cast('int'))
.withColumn("HOSTWINDOWWIDTH",func.regexp_replace('HOSTWINDOWWIDTH', 'null', '0').cast('int'))
.withColumn("TOPMOSTREACHABLEWINDOWHEIGHT",func.regexp_replace('TOPMOSTREACHABLEWINDOWHEIGHT', 'null', '0').cast('int'))
.withColumn("TOPMOSTREACHABLEWINDOWWIDTH",func.regexp_replace('TOPMOSTREACHABLEWINDOWWIDTH', 'null', '0').cast('int'))
)
thr = np.percentile(df.select("ADLOADINGTIME").rdd.collect(), outlierPercentile)
df = df.filter(func.col('ADLOADINGTIME') < thr)
df = df.withColumn("TOPMOSTREACHABLEWINDOWAREA", func.col("TOPMOSTREACHABLEWINDOWHEIGHT")*func.col("TOPMOSTREACHABLEWINDOWWIDTH"))
df = df.withColumn("INTENDENTISACTUALDEVICETYPE", (func.col("ACTUALDEVICETYPE")==func.col("INTENDEDDEVICETYPE")).cast('int'))
df = df.withColumn("COMBINEDID",
func.concat(
func.col('ACCOUNTID'),
func.col('CAMPAIGNID'),
func.col('CREATIVEID'),
func.col('SDK')) )
#df = df.withColumn("COMBINEDID", func.regexp_replace("COMBINEDID", '^$', 'NA'))
df = df.withColumn("COMBINEDEXTERNALID",
func.concat(
func.regexp_replace('EXTERNALADSERVER', 'null', ''),
func.regexp_replace('EXTERNALPLACEMENTID', 'null', ''),
func.regexp_replace('EXTERNALSITEID', 'null', ''),
func.regexp_replace('EXTERNALSUPPLIERID', 'null', '') ))
#df = df.withColumn("COMBINEDEXTERNALID", func.regexp_replace("COMBINEDEXTERNALID", '^$', 'NA'))
df = df.withColumn("PLATFORMCOMBINED",
func.concat(
func.regexp_replace('PLATFORM', 'null', ''),
func.regexp_replace('PLATFORMVERSION', 'null', '') ))
#df = df.withColumn("PLATFORMCOMBINED", func.regexp_replace("PLATFORMCOMBINED", '^$', 'NA'))
df = df.withColumn("UA_OSCOMB",
func.concat(
func.regexp_replace('UA_OS', 'null', ''),
func.regexp_replace('UA_OSVERSION', 'null', '') ))
#df = df.withColumn("UA_OSCOMB", func.regexp_replace("UA_OSCOMB", '^$', 'NA'))
df = df.withColumn("FILESJSON_SIZE",
func.regexp_replace('FILESJSON', '[^,\d]', '') )
df = df.withColumn("FILESJSON_SIZE",
func.regexp_replace('FILESJSON_SIZE', '^,', '') )
df = df.withColumn("FILESJSON_SIZE",
func.regexp_replace('FILESJSON_SIZE', ',,', ',') )
udf = func.udf(lambda x: int(np.fromstring(x,dtype=int, sep=',').sum()), IntegerType())
df = df.withColumn("FILESJSON_SIZE", udf("FILESJSON_SIZE"))
print('Loaded and prapared %d entries' % df.count())
#########
# keep only needed features
#########
features = ['ADLOADINGTIME',
'PLACEMENTID',
'TIMESTAMP',
'CREATIVETYPE',
'UA_HARDWARETYPE',
'UA_VENDOR',
'UA_MODEL',
'UA_BROWSER',
'UA_BROWSERVERSION',
'FILESJSON',
'ERRORSJSON',
'TOPMOSTREACHABLEWINDOWAREA',
'FILESJSON_SIZE',
'COMBINEDID',
'COMBINEDEXTERNALID',
'PLATFORMCOMBINED',
'UA_OSCOMB',
'SDK',
'EXTERNALADSERVER'
]
df = df.select(features)
#########
# Convert categorical features to numerical
#########
featuresCat = [
'PLACEMENTID',
'CREATIVETYPE',
'UA_HARDWARETYPE',
'UA_VENDOR',
'UA_MODEL',
'UA_BROWSER',
'UA_BROWSERVERSION',
'FILESJSON',
'ERRORSJSON',
'COMBINEDID',
'COMBINEDEXTERNALID',
'PLATFORMCOMBINED',
'UA_OSCOMB',
'SDK',
'EXTERNALADSERVER'
]
for i in range(len(featuresCat)):
indexer = StringIndexer(inputCol=featuresCat[i], outputCol='_'+featuresCat[i]).setHandleInvalid("skip").fit(df)
df = indexer.transform(df).drop(featuresCat[i])
writer = indexer._call_java("write")
writer.overwrite().save("indexer_" + featuresCat[i])
featuresCat = [ '_' + featuresCat[i] for i in range(len(featuresCat))]
features = featuresCat[:]
features.append('TIMESTAMP')
features.append('FILESJSON_SIZE')
features.append('TOPMOSTREACHABLEWINDOWAREA')
#########
# Assemble features
#########
assembler = VectorAssembler(
inputCols=features,
outputCol="features")
df = assembler.transform(df)
#########
# Convert to labeled point
#########
lp = (df.select(func.col("ADLOADINGTIME").alias("label"), func.col("features"))
.map(lambda row: LabeledPoint(row.label, row.features)))
lp.cache()
#########
# Load trained model
#########
model = RandomForestModel.load(sc, loadModelName)
print('Model loaded!')
predictions = model.predict(lp.map(lambda x: x.features)).collect()
return predictions
def main():
sc = SparkContext(conf=SparkConf().setAppName("Random Forest"))
sqlContext = SQLContext(sc)
bytePath = "/Users/priyanka/Desktop/project2files/train"
byteTestPath = "/Users/priyanka/Desktop/project2files/test"
namePath = "/Users/priyanka/Desktop/X_train_small.txt"
nameTestPath = "/Users/priyanka/Desktop/X_test_small.txt"
classPath = "/Users/priyanka/Desktop/y_train_small.txt"
classTestPath = "/Users/priyanka/Desktop/y_test_small.txt"
#docData Output: ('file:/Users/priyanka/Desktop/project2files/train/04mcPSei852tgIKUwTJr.bytes', '00401000 20 FF 58 C0 20 FE 5C 01 F8 00 0F 8B 50 FC 06 01\r\n00401010 8C 01 FF")
docData = sc.wholeTextFiles(
bytePath,
25).map(lambda (x, y): (x.encode("utf-8"), y.encode("utf-8")))
print("docData frankie")
docData.take(1)
#clean docData here - remove 1st word from line and remove /r/n
cleanDocData = docData.map(lambda (x, y): (x, clean(y.split())))
# #Extract bigrams
# print "Bigram extract priyanka"
# dfCleanDocData = sqlContext.createDataFrame(cleanDocData, ["label", "words"])
#
#
# ngram = NGram(inputCol="words", outputCol="ngrams")
# ngramDataFrame = ngram.transform(dfCleanDocData)
#
# for ngrams_label in ngramDataFrame.select("ngrams", "label").take(3):
# print(ngrams_label)
#try calculating tf here
x = 16**2 + 1
hashingTF = HashingTF(x)
tfDocData = cleanDocData.map(lambda (x, y): (x, hashingTF.transform(y)))
tfDocData.take(1)
#Output format : (0, u'file:/Users/priyanka/Desktop/project2files/train/c2hn9edSNJKmw0OukrBv.bytes')
nameData = sc.textFile(
namePath, 25).map(lambda x: "file:" + bytePath + "/" + x + ".bytes"
).zipWithIndex().map(lambda (x, y): (y, x))
#nameData.take(5)
#Output format: [(0, '2'), (1, '3'), (2, '2'), (3, '6'), (4, '2')]
labelData = sc.textFile(
classPath, 25).zipWithIndex().map(lambda (x, y): (y, str(int(x) - 1)))
#Output format: (u'file:/Users/priyanka/Desktop/project2files/train/c2hn9edSNJKmw0OukrBv.bytes', '2'),
joinNameLabel = nameData.join(labelData).map(lambda (x, y): y)
#joinNameLabel.take(5)
joinCleanDocLabel = joinNameLabel.join(tfDocData).map(lambda (x, y): y)
hashData = joinCleanDocLabel.map(lambda
(label, text): LabeledPoint(label, text))
print "hashing TF done"
# model = NaiveBayes.train(hashData)
print("generating model fliss")
model1 = RandomForest.trainClassifier(hashData,
numClasses=9,
categoricalFeaturesInfo={},
numTrees=50,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=8,
maxBins=32)
#error: 31.36
#==============================================================================
# Testing starts here
#==============================================================================
docTestData = sc.wholeTextFiles(
byteTestPath,
25).map(lambda (x, y): (x.encode("utf-8"), y.encode("utf-8")))
#docTestData.take(1)
cleanDocTestData = docTestData.map(lambda (x, y): (x, clean(y.split())))
tfDocTestData = cleanDocTestData.map(lambda (x, y):
(x, hashingTF.transform(y)))
nameTestData = sc.textFile(
nameTestPath,
25).map(lambda x: "file:" + byteTestPath + "/" + x + ".bytes"
).zipWithIndex().map(lambda (x, y): (y, x))
labelTestData = sc.textFile(
classTestPath,
25).zipWithIndex().map(lambda (x, y): (y, str(int(x) - 1)))
joinTestNameLabel = nameTestData.join(labelTestData).map(lambda (x, y): y)
joinTestDocLabel = joinTestNameLabel.join(tfDocTestData).map(lambda
(x, y): y)
print("hashing test kenny")
hashTestData = joinTestDocLabel.map(
lambda (label, text): LabeledPoint(label, text))
hashTestData.persist()
#Random forest prediction and labels and accuracy
print "prediction part lyndz"
prediction1 = model1.predict(hashTestData.map(lambda x: x.features))
labelsAndPredictions1 = hashTestData.map(lambda lp: lp.label).zip(
prediction1)
testErr1 = labelsAndPredictions1.filter(
lambda (v, p): v != p).count() / float(hashTestData.count())
print('Test Error = ' + str(testErr1))
print('Learned classification forest model:')
print(model1.toDebugString())
# Save and load random forest model
model1.save(
sc,
"/Users/priyanka/Desktop/project2files/myRandomForestClassificationModel1"
)
sameModel1 = RandomForestModel.load(
sc,
"/Users/priyanka/Desktop/project2files/myRandomForestClassificationModel1"
)
def run(jobNm, sc, sqlContext, inputFile, dictFile,
bByDate=False,
inputPartitions=-1,
sNum=30,
modelPath=None,
bWriteMonitor=False,
writeFileOutput=False):
# import monitoring if needed
if bWriteMonitor==True:
import plotting
#Create monitoring plot and associated vectors
mPX = range(8)
mPY = [0.]*8
mSL = ["Create Feature Map", "Read in Data", "Aggregate for M.L.", "Read in Model", "Apply Model", "Output Results"]
mInd = 0
t0 = time.time()
#Find the word document frequency for the corpus
#this is used for an idf score used in feature vector formation
t1 = time.time()
revLookup = []
fDict = None
if dictFile[:3] == 's3:' or dictFile[:5] == 'hdfs:':
# read dict file from hdfs
fDict = sc.textFile(dictFile).collect()
else:
# read from local file
fDict = open(dictFile,"r")
for line in fDict:
terms = line.split("\t")
revLookup.append(terms[0])
nVecLen = len(revLookup)
t2 = time.time()
diff = t2-t1
print "Time to read dict", diff
if bWriteMonitor:
mPY[mInd] = diff
mInd = mInd+1
plotting.updateMonitorPlot(mPX, mPY, mSL, jobNm)
#Read in data and filter out entries with no valid words
t1 = time.time()
print 'inputFile ',inputFile
print 'inputPartitions ',inputPartitions
records = aggregatedComparison.loadPoint(sc, sqlContext, inputFile, inputPartitions)
nGoodTweets = records.count()
t2 = time.time()
print "Number of good tweets:",nGoodTweets
diff = t2-t1
print "Time to read in data", diff
if bWriteMonitor:
mPY[mInd] = diff
mInd = mInd+1
plotting.updateMonitorPlot(mPX, mPY, mSL, jobNm)
#Format data for ML input
t1 = time.time()
mlApply = None
if bByDate:
mlApply = records.map(lambda x: (x.key, [LabeledPoint(1.0, x.vector), x.lat, x.lon, x.size, x.binSize, x.dt])).cache()
else:
mlApply = records.map(lambda x: (x.key, [LabeledPoint(1.0, x.vector), x.lat, x.lon, x.size, x.binSize])).cache()
nApp = mlApply.count()
t2 = time.time()
print "Number of collapsed points:", nApp
diff = t2-t1
print "Time to map points", diff
if bWriteMonitor:
mPY[mInd] = diff
mInd = mInd+1
plotting.updateMonitorPlot(mPX, mPY, mSL, jobNm)
# Read in Model
t1 = time.time()
model_Tree = RandomForestModel.load(sc, modelPath)
t2 = time.time()
diff = t2-t1
print "Time to read in model", diff
if bWriteMonitor:
mPY[mInd] = diff
mInd = mInd+1
plotting.updateMonitorPlot(mPX, mPY, mSL, jobNm)
# apply model
t1 = time.time()
predictions_Tree = model_Tree.predict(mlApply.map(lambda x: x[1][0].features))
vecAndPredictions = mlApply.zip(predictions_Tree)
vecAndPredictions.cache()
vecAndPredictions.count()
t2 = time.time()
diff = t2-t1
print "Time to apply model: ", diff
if bWriteMonitor:
mPY[mInd] = diff
mInd = mInd+1
plotting.updateMonitorPlot(mPX, mPY, mSL, jobNm)
#Get the results
t1 = time.time()
resultSet = clustering.locationBasedOutputV2(bByDate, jobNm, vecAndPredictions, sNum, revLookup, writeFileOutput, [])
t2 = time.time()
diff = t2-t1
print "Time to create json objects for output: ", diff
if bWriteMonitor:
mPY[mInd] = diff
mInd = mInd+1
plotting.updateMonitorPlot(mPX, mPY, mSL, jobNm)
diff = time.time() - t0
print "<----------BOOM GOES THE DYNOMITE!---------->"
print "< total number of tweets:,", nGoodTweets
print "< total process Time:", diff
print "< total idf vector length:", nVecLen
print "<------------------------------------------->"
return resultSet
def test_classification(self):
from pyspark.mllib.classification import LogisticRegressionWithSGD, SVMWithSGD, NaiveBayes
from pyspark.mllib.tree import DecisionTree, DecisionTreeModel, RandomForest, \
RandomForestModel, GradientBoostedTrees, GradientBoostedTreesModel
data = [
LabeledPoint(0.0, [1, 0, 0]),
LabeledPoint(1.0, [0, 1, 1]),
LabeledPoint(0.0, [2, 0, 0]),
LabeledPoint(1.0, [0, 2, 1])
]
rdd = self.sc.parallelize(data)
features = [p.features.tolist() for p in data]
temp_dir = tempfile.mkdtemp()
lr_model = LogisticRegressionWithSGD.train(rdd, iterations=10)
self.assertTrue(lr_model.predict(features[0]) <= 0)
self.assertTrue(lr_model.predict(features[1]) > 0)
self.assertTrue(lr_model.predict(features[2]) <= 0)
self.assertTrue(lr_model.predict(features[3]) > 0)
svm_model = SVMWithSGD.train(rdd, iterations=10)
self.assertTrue(svm_model.predict(features[0]) <= 0)
self.assertTrue(svm_model.predict(features[1]) > 0)
self.assertTrue(svm_model.predict(features[2]) <= 0)
self.assertTrue(svm_model.predict(features[3]) > 0)
nb_model = NaiveBayes.train(rdd)
self.assertTrue(nb_model.predict(features[0]) <= 0)
self.assertTrue(nb_model.predict(features[1]) > 0)
self.assertTrue(nb_model.predict(features[2]) <= 0)
self.assertTrue(nb_model.predict(features[3]) > 0)
categoricalFeaturesInfo = {0: 3} # feature 0 has 3 categories
dt_model = DecisionTree.trainClassifier(
rdd,
numClasses=2,
categoricalFeaturesInfo=categoricalFeaturesInfo,
maxBins=4)
self.assertTrue(dt_model.predict(features[0]) <= 0)
self.assertTrue(dt_model.predict(features[1]) > 0)
self.assertTrue(dt_model.predict(features[2]) <= 0)
self.assertTrue(dt_model.predict(features[3]) > 0)
dt_model_dir = os.path.join(temp_dir, "dt")
dt_model.save(self.sc, dt_model_dir)
same_dt_model = DecisionTreeModel.load(self.sc, dt_model_dir)
self.assertEqual(same_dt_model.toDebugString(),
dt_model.toDebugString())
rf_model = RandomForest.trainClassifier(
rdd,
numClasses=2,
categoricalFeaturesInfo=categoricalFeaturesInfo,
numTrees=10,
maxBins=4,
seed=1)
self.assertTrue(rf_model.predict(features[0]) <= 0)
self.assertTrue(rf_model.predict(features[1]) > 0)
self.assertTrue(rf_model.predict(features[2]) <= 0)
self.assertTrue(rf_model.predict(features[3]) > 0)
rf_model_dir = os.path.join(temp_dir, "rf")
rf_model.save(self.sc, rf_model_dir)
same_rf_model = RandomForestModel.load(self.sc, rf_model_dir)
self.assertEqual(same_rf_model.toDebugString(),
rf_model.toDebugString())
gbt_model = GradientBoostedTrees.trainClassifier(
rdd,
categoricalFeaturesInfo=categoricalFeaturesInfo,
numIterations=4)
self.assertTrue(gbt_model.predict(features[0]) <= 0)
self.assertTrue(gbt_model.predict(features[1]) > 0)
self.assertTrue(gbt_model.predict(features[2]) <= 0)
self.assertTrue(gbt_model.predict(features[3]) > 0)
gbt_model_dir = os.path.join(temp_dir, "gbt")
gbt_model.save(self.sc, gbt_model_dir)
same_gbt_model = GradientBoostedTreesModel.load(self.sc, gbt_model_dir)
self.assertEqual(same_gbt_model.toDebugString(),
gbt_model.toDebugString())
try:
rmtree(temp_dir)
except OSError:
pass
numTrees=6,
featureSubsetStrategy="auto",
impurity='gini',
maxDepth=4,
maxBins=32
)
elapsedTime = time() - startTime
print "Classifier trained in {} seconds".format(round(elapsedTime,3))
# Save the madel for use in evaluating readings
model.save(sc,"models/IoTBackBraceRandomForest.model")
# Evaluate model on test instances and compute test error
predictions = model.predict(testData.map(lambda x: x.features))
labelsAndPredictions = testData.map(lambda lp: lp.label).zip(predictions)
testErr = labelsAndPredictions.filter(lambda (v, p): v != p).count() / float(testData.count())
print('Test Error = ' + str(testErr))
print('Learned classification forest model:')
print(model.toDebugString())
loadedModel = RandomForestModel.load(sc, "models/IoTBackBraceRandomForest.model")
for i in range(-50,10):
prediction = loadedModel.predict([i])
positions = {0 : "upright",
1 : "back bent",
2 : "stooped"
}
print str(i) + " => " + str(positions[prediction])