from pyspark.ml.evaluation import RegressionEvaluator from pyspark.ml.regression import LinearRegression # Load data dataset = spark.read.format("libsvm").load("data/sample_linear_regression_data.txt") # Split data into training and test sets (trainingData, testData) = dataset.randomSplit([0.7, 0.3]) # Fit model lr = LinearRegression(maxIter=10, regParam=0.3, elasticNetParam=0.8) lrModel = lr.fit(trainingData) # Make predictions on test data predictions = lrModel.transform(testData) # Evaluate model regEval = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse") rmse = regEval.evaluate(predictions) r2 = regEval.evaluate(predictions, {regEval.metricName: "r2"}) mae = regEval.evaluate(predictions, {regEval.metricName: "mae"}) # Output results print("RMSE = ", rmse) print("R2 = ", r2) print("MAE = ", mae)
from pyspark.ml.evaluation import RegressionEvaluator from pyspark.ml.regression import DecisionTreeRegressor # Load data dataset = spark.read.format("libsvm").load("data/sample_linear_regression_data.txt") # Split data into training and test sets (trainingData, testData) = dataset.randomSplit([0.7, 0.3]) # Fit model dt = DecisionTreeRegressor(featuresCol="features", labelCol="label") dtModel = dt.fit(trainingData) # Make predictions on test data predictions = dtModel.transform(testData) # Evaluate model regEval = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse") rmse = regEval.evaluate(predictions) r2 = regEval.evaluate(predictions, {regEval.metricName: "r2"}) mae = regEval.evaluate(predictions, {regEval.metricName: "mae"}) # Output results print("RMSE = ", rmse) print("R2 = ", r2) print("MAE = ", mae)This example demonstrates how to use RegressionEvaluator to evaluate a decision tree regression model. After fitting the model, the 'transform' method is used to make predictions on the test data, and then the RegressionEvaluator is used to compute the RMSE, R2, and MAE metrics for the model. As can be seen, the Pyspark package library contains the RegressionEvaluator class, which can be used to evaluate regression models.