/
wineQualityModelling.py
60 lines (44 loc) · 2.13 KB
/
wineQualityModelling.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
from pyspark.mllib.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml import Pipeline
from pyspark.mllib.evaluation import MulticlassMetrics
import pyspark.sql.functions as func
import pyspark
conf = SparkConf().setAppName("Wine Quality Prediction").setMaster("local[4]")
sc = SparkContext(conf=conf)
spark = SparkSession.builder.getOrCreate()
# Read the data and Print the schema
print("\n\nThe Program has started...\n\n")
traindf = spark_session.read.csv('TrainingDataset.csv',header='true', inferSchema='true', sep=';')
print("\n\nPrinting Training Schema\n\n")
defTrain.printSchema()
defTrain.count()
# Make an array of features excluding the column to classify
featureColumns = [col for col in defTrain.columns if (
col != '""""quality"""""')]
assembler = VectorAssembler(inputCols=featureColumns, outputCol='features')
dataDF = assembler.transform(defTrain)
print("\n\nPrinting Training Schema with Features Table\n\n")
dataDF.printSchema()
# Random Splitting of Data
splitValue = 0.7
trainingDF, testDF = defTrain.randomSplit([splitValue, 1 - splitValue])
print("\nSplitted Data into Training and Testing Dataset\n")
# Random Forest Regression on TrainingDataset
rf = RandomForestClassifier(featuresCol='features', labelCol='""""quality"""""',
numTrees=100, maxBins=484, maxDepth=25, minInstancesPerNode=5, seed=34)
rfPipeline = Pipeline(stages=[assembler, rf])
rfPipelineModel = rfPipeline.fit(trainingDF)
evaluator = RegressionEvaluator(
labelCol='""""quality"""""', predictionCol="prediction", metricName="rmse")
rfTrainingPredictions = rfPipelineModel.transform(defTrain)
rfTestPredictions = rfPipelineModel.transform(testDF)
print("\nCompleted Model Training...\n\nRandom Forest RMSE on traning data = %g\n" %
evaluator.evaluate(rfTrainingPredictions))
print("\nRandom Forest RMSE on test data = %g\n" %
evaluator.evaluate(rfTestPredictions))
rf.save("rfPipelineModel")