def regression_metrics(self, predictions: DataFrame): return ComputeModelStatistics( evaluationMetric='regression', labelCol=self.estimator.getLabelCol(), scoresCol=self.estimator.getPredictionCol())\ .transform(predictions)\ .toPandas().to_dict(orient='list')
model = LightGBMRegressionModel.loadNativeModelFromFile("mymodel") # COMMAND ---------- # MAGIC %md View the feature importances of the trained model. # COMMAND ---------- print(model.getFeatureImportances()) # COMMAND ---------- # MAGIC %md Score the regressor on the test data. # COMMAND ---------- scoredData = model.transform(test) display(scoredData) # COMMAND ---------- # MAGIC %md Compute metrics using ComputeModelStatistics # COMMAND ---------- from mmlspark.train import ComputeModelStatistics metrics = ComputeModelStatistics(evaluationMetric='regression', labelCol='label', scoresCol='prediction') \ .transform(scoredData) display(metrics)
labelCol="income", numFeatures=256).fit(train) # COMMAND ---------- # MAGIC %md After the model is trained, we score it against the test dataset and view metrics. # COMMAND ---------- from mmlspark.train import ComputeModelStatistics, TrainedClassifierModel prediction = model.transform(test) prediction.printSchema() # COMMAND ---------- metrics = ComputeModelStatistics().transform(prediction) metrics.limit(10).toPandas() # COMMAND ---------- # MAGIC %md First, we will define the webservice input/output. # MAGIC For more information, you can visit the [documentation for Spark Serving](https://github.com/Azure/mmlspark/blob/master/docs/mmlspark-serving.md) # COMMAND ---------- from pyspark.sql.types import * from mmlspark.io import * import uuid serving_inputs = spark.readStream.server() \ .address("localhost", 8898, "my_api") \
# MAGIC trained models by find the model which performs best on the `test` # MAGIC dataset given the specified metric # MAGIC # MAGIC 3. The **`CompueModelStatistics`** Transformer computes the different # MAGIC metrics on a scored dataset (in our case, the `validation` dataset) # MAGIC at the same time # COMMAND ---------- from mmlspark.train import TrainClassifier, ComputeModelStatistics from mmlspark.automl import FindBestModel # Prepare data for learning train, test, validation = data.randomSplit([0.60, 0.20, 0.20], seed=123) # Train the models on the 'train' data lrHyperParams = [0.05, 0.1, 0.2, 0.4] logisticRegressions = [LogisticRegression(regParam = hyperParam) for hyperParam in lrHyperParams] lrmodels = [TrainClassifier(model=lrm, labelCol="label", numFeatures=10000).fit(train) for lrm in logisticRegressions] # Select the best model bestModel = FindBestModel(evaluationMetric="AUC", models=lrmodels).fit(test) # Get AUC on the validation dataset predictions = bestModel.transform(validation) metrics = ComputeModelStatistics().transform(predictions) print("Best model's AUC on validation set = " + "{0:.2f}%".format(metrics.first()["AUC"] * 100))
# COMMAND ---------- # MAGIC %md Score the regressor on the test data. # COMMAND ---------- scoredData = model.transform(testCat) scoredData.limit(10).toPandas() # COMMAND ---------- # MAGIC %md Compute model metrics against the entire scored dataset # COMMAND ---------- from mmlspark.train import ComputeModelStatistics metrics = ComputeModelStatistics().transform(scoredData) metrics.toPandas() # COMMAND ---------- # MAGIC %md Finally, compute and show statistics on individual predictions in the test # MAGIC dataset, demonstrating the usage of `ComputePerInstanceStatistics` # COMMAND ---------- from mmlspark.train import ComputePerInstanceStatistics evalPerInstance = ComputePerInstanceStatistics().transform(scoredData) evalPerInstance.select("ArrDelay", "Scores", "L1_loss", "L2_loss") \ .limit(10).toPandas()
# MAGIC %md After the models have been trained and scored, compute some basic statistics # MAGIC to evaluate the predictions. The following statistics are calculated for # MAGIC regression models to evaluate: # MAGIC * Mean squared error # MAGIC * Root mean squared error # MAGIC * R^2 # MAGIC * Mean absolute error # MAGIC # MAGIC Use the `ComputeModelStatistics` API to compute basic statistics for # MAGIC the Poisson and the Random Forest models. # COMMAND ---------- from mmlspark.train import ComputeModelStatistics poissonMetrics = ComputeModelStatistics().transform(poissonPrediction) print("Poisson Metrics") poissonMetrics.toPandas() # COMMAND ---------- randomForestMetrics = ComputeModelStatistics().transform( randomForestPrediction) print("Random Forest Metrics") randomForestMetrics.toPandas() # COMMAND ---------- # MAGIC %md We can also compute per instance statistics for `poissonPrediction`: # COMMAND ----------
df_test = df_test.filter(col("label") != 2.0) \ .withColumn("label", when(col("label") > 0, 1.0).otherwise(0.0)) \ .select(["label", "text"]) print("Number of test samples after filtering: ", df_test.count()) # COMMAND ---------- # Make predictions predictions = vw_trained.transform(df_test) predictions.limit(10).toPandas() # COMMAND ---------- # Compute model performance metrics metrics = ComputeModelStatistics(evaluationMetric="classification", labelCol="label", scoredLabelsCol="prediction").transform(predictions) metrics.toPandas() # COMMAND ---------- # Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve) class CurveMetrics(BinaryClassificationMetrics): def __init__(self, *args): super(CurveMetrics, self).__init__(*args) def get_curve(self, method): rdd = getattr(self._java_model, method)().toJavaRDD() points = [] for row in rdd.collect(): points += [(float(row._1()), float(row._2()))]
# COMMAND ---------- # MAGIC %md Featurize images # COMMAND ---------- featurizedImages = cntkModel.transform(imagesWithLabels).select(["features","labels"]) # COMMAND ---------- # MAGIC %md Use featurized images to train a classifier # COMMAND ---------- from mmlspark.train import TrainClassifier from pyspark.ml.classification import RandomForestClassifier train,test = featurizedImages.randomSplit([0.75,0.25]) model = TrainClassifier(model=RandomForestClassifier(),labelCol="labels").fit(train) # COMMAND ---------- # MAGIC %md Evaluate the accuracy of the model # COMMAND ---------- from mmlspark.train import ComputeModelStatistics predictions = model.transform(test) metrics = ComputeModelStatistics(evaluationMetric="accuracy").transform(predictions) metrics.show()
import mmlspark from mmlspark.lightgbm import LightGBMClassifier from mmlspark.train import TrainClassifier, ComputeModelStatistics df = spark.read.csv(r"C:\Users\yanrujing\Desktop\breast_cancer.csv", header=True, inferSchema=True) train_data, test_data = df.randomSplit([0.8, 0.2], seed=0) print(df.limit(10).toPandas()) model = TrainClassifier(model=LogisticRegression(), labelCol="class", numFeatures=256).fit(train_data) prediction = model.transform(test_data) metrics = ComputeModelStatistics().transform(prediction) print(metrics.limit(10).toPandas()) f1 = VectorAssembler(inputCols=[ 'clump_thickness', 'unif_cell_size', 'unif_cell_shape', 'marg_adhesion', 'single_epith_cell_size', 'bare_nuclei', 'bland_chrom', 'norm_nucleoli', 'mitoses' ], outputCol='features') f2 = StringIndexer(inputCol='class', outputCol='label') p = Pipeline(stages=[f1, f2]).fit(df) data = p.transform(df) train_data, test_data = data.randomSplit([0.8, 0.2], seed=0) model = LightGBMClassifier(objective='binary').fit(train_data) test_predict = model.transform(test_data)
lr_predictions = lr_model.transform(lr_test_data) display(lr_predictions.limit(10).toPandas()) # COMMAND ---------- # MAGIC %md We evaluate the prediction result by using `mmlspark.train.ComputeModelStatistics` which returns four metrics: # MAGIC * [MSE (Mean Squared Error)](https://en.wikipedia.org/wiki/Mean_squared_error) # MAGIC * [RMSE (Root Mean Squared Error)](https://en.wikipedia.org/wiki/Root-mean-square_deviation) = sqrt(MSE) # MAGIC * [R quared](https://en.wikipedia.org/wiki/Coefficient_of_determination) # MAGIC * [MAE (Mean Absolute Error)](https://en.wikipedia.org/wiki/Mean_absolute_error) # COMMAND ---------- metrics = ComputeModelStatistics( evaluationMetric='regression', labelCol='target', scoresCol='prediction').transform(lr_predictions) results = metrics.toPandas() results.insert(0, 'model', ['Spark MLlib - Linear Regression']) display(results) # COMMAND ---------- # MAGIC %md ## Vowpal Wabbit # COMMAND ---------- # MAGIC %md Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported. # COMMAND ----------