# COMMAND ---------- petalPredictions = petalModels[0].transform(irisPetal) display(petalPredictions) # COMMAND ---------- # MAGIC %md # MAGIC Next, we'll evaluate the model using the `RegressionEvaluator`. # COMMAND ---------- from pyspark.ml.evaluation import RegressionEvaluator regEval = RegressionEvaluator().setLabelCol('petalWidth') print regEval.explainParams() # COMMAND ---------- # MAGIC %md # MAGIC The default value for `RegressionEvaluator` is root mean square error (RMSE). Let's view that first. # COMMAND ---------- print regEval.evaluate(petalPredictions) # COMMAND ---------- # MAGIC %md # MAGIC `RegressionEvaluator` also supports mean square error (MSE), \\( r^2 \\), and mean absolute error (MAE). We'll view the \\( r^2 \\) metric next.
# label: predictions.printSchema() predictions.show(5) # ## Evaluate the linear regression model on the test data # Import the `RegressionEvaluator` class from the `pyspark.ml.evaluation` module: from pyspark.ml.evaluation import RegressionEvaluator # Create an instance of the `RegressionEvaluator` class: evaluator = RegressionEvaluator(predictionCol="prediction", labelCol="duration", metricName="r2") # Call the `explainParams` method to see other metrics: print(evaluator.explainParams()) # Use the `evaluate` method to compute the metric on the `predictions` DataFrame: evaluator.evaluate(predictions) # Use the `setMetricName` method to change the metric: evaluator.setMetricName("rmse").evaluate(predictions) # **Note:** You can also use the `evaluate` method of the `LinearRegressionModel` class. # ## Plot the linear regression model def plot_lr_model(): pdf = predictions.sample(withReplacement=False, fraction=0.1, seed=34512).toPandas()
# COMMAND ---------- petalPredictions = petalModels[0].transform(irisPetal) display(petalPredictions) # COMMAND ---------- # MAGIC %md # MAGIC Next, we'll evaluate the model using the `RegressionEvaluator`. # COMMAND ---------- from pyspark.ml.evaluation import RegressionEvaluator regEval = RegressionEvaluator().setLabelCol('petalWidth') print regEval.explainParams() # COMMAND ---------- # MAGIC %md # MAGIC The default value for `RegressionEvaluator` is root mean square error (RMSE). Let's view that first. # COMMAND ---------- print regEval.evaluate(petalPredictions) # COMMAND ---------- # MAGIC %md # MAGIC `RegressionEvaluator` also supports mean square error (MSE), \\( r^2 \\), and mean absolute error (MAE). We'll view the \\( r^2 \\) metric next.