def task_7(data_io, train_data, test_data): # ---------------------- Your implementation begins------------------------ dt = DecisionTreeRegressor(labelCol="overall", featuresCol="features", maxDepth=5) model = dt.fit(train_data) predictions = model.transform(test_data) evaluator = RegressionEvaluator(labelCol="overall", predictionCol="prediction", metricName="rmse") rmse = evaluator.evaluate(predictions) # ------------------------------------------------------------------------- # ---------------------- Put results in res dict -------------------------- res = {'test_rmse': None} # Modify res: res['test_rmse'] = rmse # ------------------------------------------------------------------------- # ----------------------------- Do not change ----------------------------- data_io.save(res, 'task_7') return res
def build_decision_tree_regression(observation_df, feature_columns): # Create new column with all of the features vector_observation_df = create_feature_column(observation_df, feature_columns, ['features', 'duration_sec']) train_df, test_df = vector_observation_df.randomSplit([0.7, 0.3]) lr = DecisionTreeRegressor(featuresCol="features", labelCol="duration_sec") model = lr.fit(train_df) test_predictions = model.transform(test_df) test_predictions.select("prediction", "duration_sec", "features").show(5) evaluator = RegressionEvaluator(predictionCol='prediction', labelCol="duration_sec", metricName="rmse") print("RMSE on test data = %g" % evaluator.evaluate(test_predictions)) evaluator = RegressionEvaluator(predictionCol='prediction', labelCol="duration_sec", metricName="r2") print("R2 on test data = %g" % evaluator.evaluate(test_predictions)) return model
def test_decision_tree_regressor(self): features = [[0, 1], [1, 1], [2, 0]] features = numpy.array(features, dtype=numpy.float32) labels = [100, -10, 50] dd = [(labels[i], Vectors.dense(features[i])) for i in range(len(labels))] data = self.spark.createDataFrame( self.spark.sparkContext.parallelize(dd), schema=["label", "features"]) dt = DecisionTreeRegressor(labelCol="label", featuresCol="features") model = dt.fit(data) feature_count = data.select('features').first()[0].size model_onnx = convert_sparkml( model, 'Sparkml Decision Tree Regressor', [('features', FloatTensorType([None, feature_count]))], spark_session=self.spark) self.assertTrue(model_onnx is not None) # run the model data_np = data.toPandas().features.apply( lambda x: pandas.Series(x.toArray())).values.astype(numpy.float32) predicted = model.transform(data) expected = [ predicted.toPandas().prediction.values.astype(numpy.float32) ] paths = save_data_models(data_np, expected, model, model_onnx, basename="SparkmlDecisionTreeRegressor") onnx_model_path = paths[-1] output, output_shapes = run_onnx_model(['prediction'], data_np, onnx_model_path) compare_results(expected, output, decimal=5)
def predict_price_of_unit_area_by_decision_tree( real_estate_dataset_df: DataFrame): """ Predict the price per unit area based on house age, distance to MRT (public transportation) and number of convenience stores, using decision tree regression. :param real_estate_dataset_df: :return: """ real_estate_dataset_df = transform_dataset_to_label_feature_form( real_estate_dataset_df) train_test_datasets = real_estate_dataset_df.randomSplit([0.5, 0.5]) train_dataset = train_test_datasets[0] test_dataset = train_test_datasets[1] # setLabelCol, setFeatureCol: Change column name for "label" and "features" columns. decision_tree_regressor = DecisionTreeRegressor().setLabelCol( 'actual_price') model = decision_tree_regressor.fit(train_dataset) # Create predictions for testing dataset. predictions = model.transform(test_dataset).\ select('actual_price', func.round(func.col('prediction'), 2).alias('predicted_price')).\ orderBy(func.desc('actual_price')).cache() return predictions
def decision_tree_regression(train_data, test_data): dt = DecisionTreeRegressor(featuresCol='features', labelCol='MEDV') dt_model = dt.fit(train_data) dt_predictions = dt_model.transform(test_data) dt_evaluator = RegressionEvaluator( labelCol='MEDV', predictionCol='prediction', metricName='rmse', ) rmse = dt_evaluator.evaluate(dt_predictions) print('Root Mean Squared Error (RMSE) on test data = %g' % rmse) print(dt_model.featureImportances)
def task_8(data_io, train_data, test_data): # ---------------------- Your implementation begins------------------------ trainingData, testData = train_data.randomSplit([0.75, 0.25]) best = 0 all_rmse = [] lowest_rmse = 100 for i in [5, 7, 9, 12]: dt = DecisionTreeRegressor(labelCol="overall", featuresCol="features", maxDepth=i) model = dt.fit(trainingData) predictions = model.transform(testData) evaluator = RegressionEvaluator(labelCol="overall", predictionCol="prediction", metricName="rmse") rmse = evaluator.evaluate(predictions) all_rmse = all_rmse + [rmse] if rmse <= lowest_rmse: lowest_rmse = rmse best = i best_model = model predictions = best_model.transform(test_data) evaluator = RegressionEvaluator(labelCol="overall", predictionCol="prediction", metricName="rmse") rmse = evaluator.evaluate(predictions) # ------------------------------------------------------------------------- # ---------------------- Put results in res dict -------------------------- res = { 'test_rmse': None, 'valid_rmse_depth_5': None, 'valid_rmse_depth_7': None, 'valid_rmse_depth_9': None, 'valid_rmse_depth_12': None, } # Modify res: res['test_rmse'] = rmse res['valid_rmse_depth_5'] = all_rmse[0] res['valid_rmse_depth_7'] = all_rmse[1] res['valid_rmse_depth_9'] = all_rmse[2] res['valid_rmse_depth_12'] = all_rmse[3] # ------------------------------------------------------------------------- # ----------------------------- Do not change ----------------------------- data_io.save(res, 'task_8') return res
def decisionTreeRegressor(data, ncolumns, schemaNames): from pyspark.ml import Pipeline from pyspark.ml.regression import DecisionTreeRegressor from pyspark.ml.tuning import ParamGridBuilder from pyspark.ml.feature import StringIndexer, VectorIndexer from pyspark.ml.tuning import CrossValidator from pyspark.ml.evaluation import RegressionEvaluator from pyspark.ml.feature import Binarizer from pyspark.ml.evaluation import BinaryClassificationEvaluator import numpy as np import time binarizer = Binarizer( threshold=0.00001, inputCol="features", outputCol="binarized_features", ) binarizedDataFrame = binarizer.transform(data) (trainingData, testData) = binarizedDataFrame.randomSplit([0.9, 0.1], 50) dtr = DecisionTreeRegressor(labelCol="label", featuresCol="binarized_features", maxDepth=10, maxBins=10, impurity='Variance') timer = '' start = time.time() cvModel = dtr.fit(trainingData) end = time.time() timer = ((end - start) / 60) prediction = cvModel.transform(testData) evaluator = RegressionEvaluator\ (labelCol="label", predictionCol="prediction", metricName="rmse") rmse = evaluator.evaluate(prediction) evaluator = BinaryClassificationEvaluator(rawPredictionCol="prediction") areaUC = evaluator.evaluate(prediction) fi = cvModel.featureImportances imp_feat = np.zeros(ncolumns - 1) imp_feat[fi.indices] = fi.values x = np.arange(ncolumns - 1) idx = (-imp_feat).argsort()[:3] feat = [] for i in idx: feat.append(schemaNames[i]) return feat, rmse, areaUC, timer
def TrainDT(trainingData, testData): # Train a DecisionTree model. dt = DecisionTreeRegressor() # Train model. This also runs the indexer. start = time.time() model = dt.fit(trainingData) end = time.time() print('Training DT model took', end - start) # Make predictions. predictions = model.transform(testData) # Select (prediction, true label) and compute test error evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse") rmse = evaluator.evaluate(predictions) print("Root Mean Squared Error (RMSE) on test data = %g" % rmse) evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="r2") r2 = evaluator.evaluate(predictions) print("R2 on test data = %g" % r2) # Make predictions for train predictions = model.transform(trainingData) # Select (prediction, true label) and compute test error evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse") rmse = evaluator.evaluate(predictions) print("Root Mean Squared Error (RMSE) on train data = %g" % rmse) evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="r2") r2 = evaluator.evaluate(predictions) print("R2 on train data = %g" % r2) return model
def decisionTreeRegression(df,arguments): from pyspark.ml.regression import DecisionTreeRegressor maxDepth = 5 minInstancesPerNode = 1 impurity = "variance" if arguments.maxDepth != None: maxDepth = float(arguments.maxDepth) if arguments.minInstancesPerNode != None: minInstancesPerNode = float(arguments.minInstancesPerNode) if arguments.impurity != None: impurity = arguments.impurity dt = DecisionTreeRegressor(maxDepth=maxDepth, minInstancesPerNode=minInstancesPerNode, impurity=impurity) model = dt.fit(df) return model
def decision_tree_regressor(): spark = SparkSession \ .builder \ .appName("Python Spark SQL basic example") \ .config("spark.some.config.option", "some-value") \ .getOrCreate() df = spark.createDataFrame([(1.0, Vectors.dense(1.0)), (0.0, Vectors.sparse(1, [], []))], ["label", "features"]) dt = DecisionTreeRegressor(maxDepth=2, varianceCol="variance") model = dt.fit(df) model.depth # 1 model.numNodes # 3 model.featureImportances # SparseVector(1, {0: 1.0}) model.numFeatures # 1 test0 = spark.createDataFrame([(Vectors.dense(-1.0), )], ["features"]) model.transform(test0).head().prediction # 0.0 test1 = spark.createDataFrame([(Vectors.sparse(1, [0], [1.0]), )], ["features"]) model.transform(test1).head().prediction # 1.0 temp_path = "./" dtr_path = temp_path + "/dtr" dt.save(dtr_path) dt2 = DecisionTreeRegressor.load(dtr_path) dt2.getMaxDepth() # 2 model_path = temp_path + "/dtr_model" model.save(model_path) model2 = DecisionTreeRegressionModel.load(model_path) model.numNodes == model2.numNodes # True model.depth == model2.depth # True model.transform(test1).head().variance
elasticNetParam=0.8) lr_model = lr.fit(train_df) print("Coefficients: " + str(lr_model.coefficients)) print("Intercept: " + str(lr_model.intercept)) trainingSummary = lr_model.summary print("RMSE: %f" % trainingSummary.rootMeanSquaredError) print("r2: %f" % trainingSummary.r2) train_df.describe().show() lr_predictions = lr_model.transform(test_df) lr_predictions.select("prediction", "PE", "features").show(5) lr_evaluator = RegressionEvaluator(predictionCol="prediction", labelCol="PE", metricName="r2") print("R Squared (R2) on test data = %g" % lr_evaluator.evaluate(lr_predictions)) ## DecisionTreeRegressor portion from pyspark.ml.regression import DecisionTreeRegressor dt = DecisionTreeRegressor(featuresCol='features', labelCol='PE') dt_model = dt.fit(train_df) dt_predictions = dt_model.transform(test_df) dt_evaluator = RegressionEvaluator(labelCol="PE", predictionCol="prediction", metricName="rmse") rmse = dt_evaluator.evaluate(dt_predictions) print( "DecisionTreeRegressor Root Mean Squared Error (RMSE) on test data = %g" % rmse)
lr_evaluator = RegressionEvaluator(predictionCol="prediction", labelCol="MPG", metricName="r2") print("R Squared (R2) for Linear Regression on test data = %g" % lr_evaluator.evaluate(lr_predictions)) # RMSE on test data test_result = lr_model.evaluate(test_df) print( "Root Mean Squared Error (RMSE) for Linear Regression on test data = %g\n" % test_result.rootMeanSquaredError) #############################---DECISION TREE REGRESSION---################################## dt = DecisionTreeRegressor(featuresCol='features', labelCol='MPG') decisionTree_model = dt.fit(train_df) decisionTree_model_predictions = decisionTree_model.transform(test_df) decisionTree_model_evaluator = RegressionEvaluator( labelCol="MPG", predictionCol="prediction", metricName="rmse") rmse = decisionTree_model_evaluator.evaluate( decisionTree_model_predictions) print( "Root Mean Squared Error (RMSE) for Decision Tree on test data = %g" % rmse) r2_dt = ecisionTree_model_evaluator = RegressionEvaluator( labelCol="MPG", predictionCol="prediction", metricName="r2") print("R Squared (R2) for Decision Tree on test data = %g" % r2_dt.evaluate(decisionTree_model_predictions)) ############################---RANDOM FOREST REGRESSION---##################################
def main(): errorsRMSE_LR = [] errorsR2_LR = [] errorsR2_DT = [] errorsR2_DT5 = [] errorsRMSE_DT = [] errorsRMSE_DT5 = [] rows_training = [] rows_testing = [[] for i in range(N_OF_CLUSTERS)] for week_nb in range(FIRST_WEEK, LAST_WEEK + 1): print('week nb : ', week_nb) for day_of_week in range(DAY_IN_WEEK): for time_of_day_code in range(TIME_SLOTS_WITHIN_DAY): for cid in range(N_OF_CLUSTERS): curFeature = demandCache.get_demand( week_nb, day_of_week, time_of_day_code, cid) if curFeature != []: time_of_day_code, origin, day_of_week, day, week, hour, minute, is_manhattan, is_airport, amount = extract_feature( curFeature) if (week_nb < WEEK_NB_TEST): rows_training.append( (time_of_day_code, origin, day_of_week, day, week, hour, minute, is_manhattan, is_airport, amount)) else: rows_testing[cid].append( (time_of_day_code, origin, day_of_week, day, week, hour, minute, is_manhattan, is_airport, amount)) df_training = spark.createDataFrame(rows_training, [ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "is_manhattan", "is_airport", "amount" ]) assembler = VectorAssembler(inputCols=[ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "is_manhattan", "is_airport" ], outputCol='features') output_training = assembler.transform(df_training) final_data_training = output_training.select('features', 'amount') decisionTree = DecisionTreeRegressor(labelCol='amount', maxDepth=3) dt_model = decisionTree.fit(final_data_training) #print(dt_model.toDebugString) decisionTree5 = DecisionTreeRegressor(labelCol='amount', maxDepth=5) dt_model5 = decisionTree5.fit(final_data_training) #print(dt_model5.toDebugString) file = open("DT_final_features_one_model_INFO.txt", "w") file.write("DT maxDepth 3 : \n" + dt_model.toDebugString) file.write("DT maxDepth 5 : \n" + dt_model5.toDebugString) file.close() linearRegression = LinearRegression(labelCol='amount') lr_model = linearRegression.fit(final_data_training) for cid in range(N_OF_CLUSTERS): print('cluster: ', cid) df_testing = spark.createDataFrame(rows_testing[cid], [ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "is_manhattan", "is_airport", "amount" ]) #df_testing.show() output_testing = assembler.transform(df_testing) final_data_testing = output_testing.select('features', 'amount') predictionsDT = dt_model.transform(final_data_testing) predictionsDT5 = dt_model5.transform(final_data_testing) predictionsLR = lr_model.evaluate(final_data_testing) """ Evaluation rmse : """ rmse = predictionsLR.rootMeanSquaredError errorsRMSE_LR.append(rmse) #print("Root Mean Squared Error (RMSE) for LR on test data = %g" % rmse) r2 = predictionsLR.r2 errorsR2_LR.append(r2) #print("R Squared Error (R2) for LR on test data = %g" % r2) """ Evaluation rmse : """ evaluatorRMSE = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="rmse") rmse = evaluatorRMSE.evaluate(predictionsDT) rmse5 = evaluatorRMSE.evaluate(predictionsDT5) errorsRMSE_DT.append(rmse) errorsRMSE_DT5.append(rmse5) #print("Root Mean Squared Error (RMSE) for DT on test data = %g" % rmse) evaluatorR2 = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="r2") r2 = evaluatorR2.evaluate(predictionsDT) r25 = evaluatorR2.evaluate(predictionsDT5) errorsR2_DT.append(r2) errorsR2_DT5.append(r25) #print("R Squared Error (R2) for DT on test data = %g" % r2) return errorsRMSE_LR, errorsR2_LR, errorsRMSE_DT, errorsR2_DT, errorsRMSE_DT5, errorsR2_DT5
#testLFDF.take(10) # COMMAND ---------- #Creating an evaluator measuring our label vs our prediction using RMSE evaluation. evaluator = RegressionEvaluator(metricName="rmse")\ .setLabelCol("price_doc")\ .setPredictionCol("prediction") # COMMAND ---------- #Decision tree regression, testing on both train and test dataset. dt = DecisionTreeRegressor(labelCol='price_doc') #This builds the dt model using the train dataset model = dt.fit(trainLFDF) #This predicts dt model outcomes on train and test dataset trainPredictions = model.transform(trainLFDF) testPredictions = model.transform(testLFDF) trainscore = evaluator.evaluate(trainPredictions) testscore = evaluator.evaluate(testPredictions) print(trainscore, testscore) #DT 8 Vars RMSE 3493522, 3901961 # COMMAND ---------- #Gradient boosted tree regression gbt = GBTRegressor(labelCol='price_doc') model = gbt.fit(trainLFDF)
# COMMAND ---------- # MAGIC %md # MAGIC #### Regression with decision trees # COMMAND ---------- from pyspark.ml.regression import DecisionTreeRegressor dtr = DecisionTreeRegressor().setLabelCol('petalWidth') print dtr.explainParams() # COMMAND ---------- dtrModel = dtr.fit(irisPetal) dtrPredictions = dtrModel.transform(irisPetal) print regEval.evaluate(dtrPredictions, {regEval.metricName: 'r2'}) print regEval.evaluate(dtrPredictions, {regEval.metricName: 'rmse'}) # COMMAND ---------- # MAGIC %md # MAGIC Let's also build a gradient boosted tree. # COMMAND ---------- from pyspark.ml.regression import GBTRegressor gbt = GBTRegressor().setLabelCol('petalWidth') print gbt.explainParams()
def spark_process(sqlContext, sc, validate, path_to_file): ###################### # # HDFS to DataFrame # ###################### ## all fields: # ['vendor_id', 'pickup_datetime', 'dropoff_datetime', 'passenger_count', 'trip_distance', # 'pickup_longitude', 'pickup_latitude', 'rate_code', 'store_and_fwd_flag', 'dropoff_longitude', # 'dropoff_latitude', 'payment_type', 'fare_amount', 'surcharge', 'mta_tax', 'tip_amount', # 'tolls_amount', 'total_amount'] # columns to select feature_columns = [1,2,3,5,6,9,10] # read file and convert to DataFrame # dataframe = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load(path_to_file).cache() customSchema = StructType([ StructField("vendor_id", StringType(), True), StructField("pickup_datetime", TimestampType(), True), StructField("dropoff_datetime", TimestampType(), True), StructField("passenger_count", StringType(), True), StructField("trip_distance", StringType(), True), StructField("pickup_longitude", DoubleType(), True), StructField("pickup_latitude", DoubleType(), True), StructField("rate_code", StringType(), True), StructField("store_and_fwd_flag", StringType(), True), StructField("dropoff_longitude", DoubleType(), True), StructField("dropoff_latitude", DoubleType(), True), StructField("payment_type", StringType(), True), StructField("fare_amount", StringType(), True), StructField("surcharge", StringType(), True), StructField("mta_tax", StringType(), True), StructField("tip_amount", StringType(), True), StructField("tolls_amount", StringType(), True), StructField("total_amount", StringType(), True) ]) dataframe = sqlContext.read.format('com.databricks.spark.csv').options(header='true', schema = customSchema).load(path_to_file) # create dataframe with selected columns dataframe = dataframe.select(*(dataframe.columns[n] for n in feature_columns)) # this number does not include the header # number_of_trips = dataframe.count() sqlContext.clearCache() ###################### # # Preprocess data # ###################### # filter rows with null fields # if passenger count is missing assign it a value of 1 # filter invalid location: keep only areas near NYC dataframe = dataframe.na.drop(how='any',subset=['pickup_datetime','dropoff_datetime','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']) \ .fillna(1,subset=["passenger_count"]) \ .filter(dataframe.pickup_latitude>40.0) \ .filter(dataframe.pickup_latitude<41.0) \ .filter(dataframe.pickup_longitude<-73.0) \ .filter(dataframe.pickup_longitude>-74.0) \ .filter(dataframe.dropoff_latitude>40.0) \ .filter(dataframe.dropoff_latitude<41.0) \ .filter(dataframe.dropoff_longitude<-73.0)\ .filter(dataframe.dropoff_longitude>-74.0) ###################### # # features engineering # ###################### # create new column based on time-delta (minutes) # convert pickup-datetime column to hour time_delta_udf = udf(time_delta_minutes,FloatType()) dataframe = dataframe.withColumn('time_delta', time_delta_udf(dataframe.pickup_datetime,dataframe.dropoff_datetime)) \ .withColumn('pick_up_hour', hour(dataframe.pickup_datetime)) dataframe = dataframe.select(dataframe.pick_up_hour, \ dataframe.passenger_count.cast("integer"), \ dataframe.pickup_longitude.cast("double"), \ dataframe.pickup_latitude.cast("double"), \ dataframe.dropoff_longitude.cast("double"),\ dataframe.dropoff_latitude.cast("double"), \ dataframe.time_delta.cast("double")) dataframe = dataframe.filter(dataframe.time_delta > 1.0).cache() # split dataframe into feature and label vector # create feature vectors and labels for model training feature_assembler = VectorAssembler(inputCols = ['pick_up_hour','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude'],outputCol = 'features') transformed = feature_assembler.transform(dataframe) vector_dataframe = transformed.select(col("time_delta").alias("label"),col("features")).cache() ###################### # # train model # ###################### if validate: ################################ # # validate model on 60/40 split # ################################ # split training, test = vector_dataframe.randomSplit([0.6, 0.4], seed=0) decision_tree_reg = DecisionTreeRegressor(maxDepth=12,maxBins=25) model = decision_tree_reg.fit(training) train_pred = model.transform(training) test_pred = model.transform(test) evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="r2") r2_train = evaluator.evaluate(train_pred) evaluator_test = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="r2") r2_test = evaluator_test.evaluate(test_pred) output = test_pred.select("prediction", "label", "features") return output, r2_test, r2_train else: ################### # # train on all data # ################### decision_tree_reg = DecisionTreeRegressor(maxDepth=12,maxBins=25) model = decision_tree_reg.fit(vector_dataframe) predictions = model.transform(vector_dataframe) output = predictions.select("prediction", "label", "features") ########################### # # process to send to Kafka # ########################### schema = StructType([StructField("prediction_mins", FloatType(), True), StructField("pick_up_hour", IntegerType(), True), StructField("pickup_longitude", DoubleType(), True), StructField("pickup_latitude", DoubleType(), True), StructField("dropoff_longitude", DoubleType(), True), StructField("dropoff_latitude", DoubleType(), True)]) features_from_predictions = output.map(lambda row: (float(row.prediction),int(row.features[0]),float(row.features[1]),float(row.features[2]),float(row.features[3]),float(row.features[4]) ) ).collect() sqlContext.clearCache() dataframe_from_prediction_vector = sqlContext.createDataFrame(features_from_predictions,schema).cache() return dataframe_from_prediction_vector
df = df.selectExpr("fare_amount as label", 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude', 'passenger_count') new_df = vecAssembler.setHandleInvalid("skip").transform(df) # Split the data into training and test sets (30% held out for testing) (trainingData, testData) = new_df.randomSplit([0.7, 0.3]) # Train a DecisionTree model. dt = DecisionTreeRegressor() start_time = datetime.now() # Train model. This also runs the indexer. model = dt.fit(trainingData) time_elapsed = datetime.now() - start_time print('TIME OF DECISION TREE REGRESSION TRAINING (hh:mm:ss.ms) {}'.format( time_elapsed)) # Make predictions. predictions = model.transform(testData) # Select example rows to display. predictions.select("prediction", "label", "features").show(5) # Select (prediction, true label) and compute test error evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse")
def spark_process(sqlContext, sc, validate, path_to_file): ###################### # # HDFS to DataFrame # ###################### ## all fields: # ['vendor_id', 'pickup_datetime', 'dropoff_datetime', 'passenger_count', 'trip_distance', # 'pickup_longitude', 'pickup_latitude', 'rate_code', 'store_and_fwd_flag', 'dropoff_longitude', # 'dropoff_latitude', 'payment_type', 'fare_amount', 'surcharge', 'mta_tax', 'tip_amount', # 'tolls_amount', 'total_amount'] # columns to select feature_columns = [1, 2, 3, 5, 6, 9, 10] # read file and convert to DataFrame # dataframe = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load(path_to_file).cache() customSchema = StructType([ StructField("vendor_id", StringType(), True), StructField("pickup_datetime", TimestampType(), True), StructField("dropoff_datetime", TimestampType(), True), StructField("passenger_count", StringType(), True), StructField("trip_distance", StringType(), True), StructField("pickup_longitude", DoubleType(), True), StructField("pickup_latitude", DoubleType(), True), StructField("rate_code", StringType(), True), StructField("store_and_fwd_flag", StringType(), True), StructField("dropoff_longitude", DoubleType(), True), StructField("dropoff_latitude", DoubleType(), True), StructField("payment_type", StringType(), True), StructField("fare_amount", StringType(), True), StructField("surcharge", StringType(), True), StructField("mta_tax", StringType(), True), StructField("tip_amount", StringType(), True), StructField("tolls_amount", StringType(), True), StructField("total_amount", StringType(), True) ]) dataframe = sqlContext.read.format('com.databricks.spark.csv').options( header='true', schema=customSchema).load(path_to_file) # create dataframe with selected columns dataframe = dataframe.select(*(dataframe.columns[n] for n in feature_columns)) # this number does not include the header # number_of_trips = dataframe.count() sqlContext.clearCache() ###################### # # Preprocess data # ###################### # filter rows with null fields # if passenger count is missing assign it a value of 1 # filter invalid location: keep only areas near NYC dataframe = dataframe.na.drop(how='any',subset=['pickup_datetime','dropoff_datetime','pickup_longitude','pickup_latitude','dropoff_longitude','dropoff_latitude']) \ .fillna(1,subset=["passenger_count"]) \ .filter(dataframe.pickup_latitude>40.0) \ .filter(dataframe.pickup_latitude<41.0) \ .filter(dataframe.pickup_longitude<-73.0) \ .filter(dataframe.pickup_longitude>-74.0) \ .filter(dataframe.dropoff_latitude>40.0) \ .filter(dataframe.dropoff_latitude<41.0) \ .filter(dataframe.dropoff_longitude<-73.0)\ .filter(dataframe.dropoff_longitude>-74.0) ###################### # # features engineering # ###################### # create new column based on time-delta (minutes) # convert pickup-datetime column to hour time_delta_udf = udf(time_delta_minutes, FloatType()) dataframe = dataframe.withColumn('time_delta', time_delta_udf(dataframe.pickup_datetime,dataframe.dropoff_datetime)) \ .withColumn('pick_up_hour', hour(dataframe.pickup_datetime)) dataframe = dataframe.select(dataframe.pick_up_hour, \ dataframe.passenger_count.cast("integer"), \ dataframe.pickup_longitude.cast("double"), \ dataframe.pickup_latitude.cast("double"), \ dataframe.dropoff_longitude.cast("double"),\ dataframe.dropoff_latitude.cast("double"), \ dataframe.time_delta.cast("double")) dataframe = dataframe.filter(dataframe.time_delta > 1.0).cache() # split dataframe into feature and label vector # create feature vectors and labels for model training feature_assembler = VectorAssembler(inputCols=[ 'pick_up_hour', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude' ], outputCol='features') transformed = feature_assembler.transform(dataframe) vector_dataframe = transformed.select( col("time_delta").alias("label"), col("features")).cache() ###################### # # train model # ###################### if validate: ################################ # # validate model on 60/40 split # ################################ # split training, test = vector_dataframe.randomSplit([0.6, 0.4], seed=0) decision_tree_reg = DecisionTreeRegressor(maxDepth=12, maxBins=25) model = decision_tree_reg.fit(training) train_pred = model.transform(training) test_pred = model.transform(test) evaluator = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="r2") r2_train = evaluator.evaluate(train_pred) evaluator_test = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="r2") r2_test = evaluator_test.evaluate(test_pred) output = test_pred.select("prediction", "label", "features") return output, r2_test, r2_train else: ################### # # train on all data # ################### decision_tree_reg = DecisionTreeRegressor(maxDepth=12, maxBins=25) model = decision_tree_reg.fit(vector_dataframe) predictions = model.transform(vector_dataframe) output = predictions.select("prediction", "label", "features") ########################### # # process to send to Kafka # ########################### schema = StructType([ StructField("prediction_mins", FloatType(), True), StructField("pick_up_hour", IntegerType(), True), StructField("pickup_longitude", DoubleType(), True), StructField("pickup_latitude", DoubleType(), True), StructField("dropoff_longitude", DoubleType(), True), StructField("dropoff_latitude", DoubleType(), True) ]) features_from_predictions = output.map(lambda row: ( float(row.prediction), int(row.features[0]), float(row.features[ 1]), float(row.features[2]), float(row.features[3]), float(row.features[4]))).collect() sqlContext.clearCache() dataframe_from_prediction_vector = sqlContext.createDataFrame( features_from_predictions, schema).cache() return dataframe_from_prediction_vector
encoder = OneHotEncoder(inputCol=categoricalCol + "Index", outputCol=categoricalCol + "classVec") # Add stages. These are not run here, but will run all at once later on. stages += [stringIndexer, encoder] #encColumns = ['VendorID','RatecodeID','PULocationID','DOLocationID','payment_type','Peak_Time','weekend'] encColumns = [ 'VendorID', 'RatecodeID', 'PULocationID', 'DOLocationID', 'payment_type' ] for eCol in encColumns: encoder = OneHotEncoder(inputCol=eCol, outputCol=eCol + "classVec") stages += [encoder] #label_stringIdx = StringIndexer(inputCol = "verified_purchase", outputCol = "label") #stages += [label_stringIdx] numericCols = ["trip_distance", "passenger_count", "fare_amount", "tip_amount"] assemblerInputs = map(lambda c: c + "classVec", categoricalColumns) + map( lambda c: c + "classVec", encColumns) + numericCols assembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features") stages += [assembler] pipeline = Pipeline(stages=stages) pipelineModel = pipeline.fit(train_X4) dataset = pipelineModel.transform(train_X4) from pyspark.ml.regression import DecisionTreeRegressor dt = DecisionTreeRegressor(labelCol="total_amount", featuresCol="features", maxBins=32) model = dt.fit(dataset) model.write().overwrite().save("./nyc-01020304-6vm-18-DT-model")
def doGrid_one(): grid_data = getGridData(sqlCtx, '_ngrid2500') errorsRMSE_LR = [] errorsR2_LR = [] errorsRMSE_DT = [] errorsR2_DT = [] hor = grid_data['horizontal_slots'] vert = grid_data['vertical_slots'] print(hor, vert) hor = 24 vert = 24 for x in range(hor): print('grid hor:', x) for y in range(vert): train, test = get_features_for_grid(spark, x, y) assembler = VectorAssembler(inputCols=[ "day", "day_of_week", "hour", "is_airport", "is_manhattan", "minute", 'pickup_lat_slot', 'pickup_long_slot', "time_of_day_code", "week" ], outputCol='features') output_training = assembler.transform(train) output_testing = assembler.transform(test) final_data_training = output_training.select('features', 'amount') final_data_testing = output_testing.select('features', 'amount') final_data_training.describe().show() final_data_testing.describe().show() decisionTree = DecisionTreeRegressor(labelCol='amount', maxDepth=3) dt_model = decisionTree.fit(final_data_training) predictionsDT = dt_model.transform(final_data_testing) print(dt_model.toDebugString) linearRegression = LinearRegression(labelCol='amount') lr_model = linearRegression.fit(final_data_training) predictionsLR = lr_model.evaluate(final_data_testing) """ Evaluation LR : """ rmse = predictionsLR.rootMeanSquaredError errorsRMSE_LR.append(rmse) #print("Root Mean Squared Error (RMSE) for LR on test data = ", rmse) r2 = predictionsLR.r2 errorsR2_LR.append(r2) #print("R Squared Error (R2) for LR on test data = ", r2) """ Evaluation DT : """ evaluatorRMSE = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="rmse") rmse = evaluatorRMSE.evaluate(predictionsDT) errorsRMSE_DT.append(rmse) #print("Root Mean Squared Error (RMSE) for DT on test data = ", rmse) evaluatorR2 = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="r2") r2 = evaluatorR2.evaluate(predictionsDT) errorsR2_DT.append(r2) #print("R Squared Error (R2) for DT on test data = ", r2) return hor, vert, errorsR2_DT, errorsRMSE_DT, errorsRMSE_LR, errorsR2_LR
c) print("\n") print("For the whole dataset, the DecisionTreeRegressor is starting...") evaluator_reg = RegressionEvaluator\ (labelCol="label", predictionCol="prediction", metricName="rmse") print("\n") print( "Fetching the best values of parameters from 25% dataset and using them..." ) rtime = time.time() dtr = DecisionTreeRegressor(labelCol="label", featuresCol="features", maxDepth=maxDepth_dtr, maxBins=maxBins_dtr) model_dtr = dtr.fit(trainingData) predictions_dtr = model_dtr.transform(testData) binarizer = Binarizer(threshold=0.5, inputCol="prediction", outputCol="binarized_prediction") binarizedDataFrame = binarizer.transform(predictions_dtr) binarized = binarizedDataFrame.drop('prediction') bdf_dtr = binarized.withColumnRenamed('binarized_prediction', 'prediction') r = time.time() - rtime evaluator_reg = MulticlassClassificationEvaluator\ (labelCol="label", predictionCol="prediction", metricName="accuracy") accuracy_reg = evaluator_reg.evaluate(bdf_dtr) print("\n") print("Accuracy for DecisionTreeRegressor on the whole dataset = %g " % accuracy_reg) evaluate_area_dtr = BinaryClassificationEvaluator(
def Forecast(df, forecast_days, nLags, \ timeSeriesColumn, regressor, sparksession): # this performs model training # this calls the machine-learning algorithms of Spark ML library #labels for machine-learning LeadWindow = window.Window.rowsBetween(0,forecast_days) df = df.withColumn("label",func.last(df[timeSeriesColumn]).over(LeadWindow)) features = [timeSeriesColumn] #Auto-regression feature LagTransformer = LagGather()\ .setLagLength(nLags)\ .setInputCol(timeSeriesColumn) df = LagTransformer.transform(df) featuresGenerated = LagTransformer.getFeatureNames() features.extend(featuresGenerated) #Other feature generators here: #Moving Average Smoothing #TrendGather #****************************************************************************** # VECTOR ASSEMBLER # this assembles the all the features df = df.dropna() vA = VectorAssembler().setInputCols(features)\ .setOutputCol("features") df_m = vA.transform(df) #****************************************************************************** # Splitting data into train, test splitratio = 0.7 df_train, df_test = TimeSeriesSplit(df_m, splitratio, sparksession) #****************************************************************************** # DECISION-TREE REGRESSOR if(regressor == "DecisionTreeRegression"): dr = DecisionTreeRegressor(featuresCol = "features",\ labelCol = "label", maxDepth = 5) model = dr.fit(df_train) predictions_dr_test = model.transform(df_test) predictions_dr_train = model.transform(df_train) # RMSE is used as evaluation metric evaluator = RegressionEvaluator(predictionCol="prediction",\ labelCol="label",\ metricName ="r2") RMSE_dr_test = evaluator.evaluate(predictions_dr_test) RMSE_dr_train = evaluator.evaluate(predictions_dr_train) return (df_test, df_train, \ predictions_dr_test, predictions_dr_train,\ RMSE_dr_test, RMSE_dr_train) #****************************************************************************** # LINEAR REGRESSOR if(regressor == 'LinearRegression'): lr = LinearRegression(featuresCol = "features", labelCol="label", \ maxIter = 100, regParam = 0.4, \ elasticNetParam = 0.1) model = lr.fit(df_train) predictions_lr_test = model.transform(df_test) predictions_lr_train = model.transform(df_train) # RMSE is used as evaluation metric evaluator = RegressionEvaluator(predictionCol="prediction",\ labelCol="label",\ metricName ="r2") RMSE_lr_test= evaluator.evaluate(predictions_lr_test) RMSE_lr_train = evaluator.evaluate(predictions_lr_train) return (df_test, df_train, \ predictions_lr_test, predictions_lr_train,\ RMSE_lr_test, RMSE_lr_train) #***************************************************************************** # RANDOM FOREST REGRESSOR if(regressor == 'RandomForestRegression'): rfr = RandomForestRegressor(featuresCol="features",\ labelCol="label",\ maxDepth = 5,\ subsamplingRate = 0.8,\ ) model = rfr.fit(df_train) predictions_rfr_test = model.transform(df_test) predictions_rfr_train = model.transform(df_train) # RMSE is used as evaluation metric evaluator = RegressionEvaluator(predictionCol="prediction",\ labelCol="label",\ metricName ="rmse") RMSE_rfr_test= evaluator.evaluate(predictions_rfr_test) RMSE_rfr_train = evaluator.evaluate(predictions_rfr_train) return (df_test, df_train, \ predictions_rfr_test, predictions_rfr_train,\ RMSE_rfr_test, RMSE_rfr_train) #***************************************************************************** # GRADIENT BOOSTING TREE REGRESSOR if(regressor == 'GBTRegression'): gbt = GBTRegressor(featuresCol="features",\ labelCol="label",\ maxDepth=5,\ subsamplingRate=0.8) model = gbt.fit(df_train) predictions_gbt_test = model.transform(df_test) predictions_gbt_train = model.transform(df_train) # RMSE is used as evaluation metric evaluator = RegressionEvaluator(predictionCol="prediction",\ labelCol="label",\ metricName ="rmse") RMSE_gbt_test= evaluator.evaluate(predictions_gbt_test) RMSE_gbt_train = evaluator.evaluate(predictions_gbt_train) return (df_test, df_train, \ predictions_gbt_test, predictions_gbt_train,\ RMSE_gbt_test, RMSE_gbt_train)
trainDF.cache() testDF.cache() # - ##### Entrenar un árbol de regresión para predecir la variable minutos. # In[57]: dt = DecisionTreeRegressor(labelCol='minutos') #toma como inputCo "features" de manera predeterminada # In[58]: model=dt.fit(trainDF) # - ##### Evaluar el modelo resultante usando RMSE tanto en la muestra de entrenamiento como en la muestra de test: Comentar el resultado. # In[59]: predictionDF = model.transform(testDF) # In[23]: evaluator = RegressionEvaluator(labelCol="minutos")
def main(): for cid in range(N_OF_CLUSTERS): rows_training = [] rows_testing = [] for week_nb in range(FIRST_WEEK, LAST_WEEK + 1): print('week nb : ', week_nb) for day_of_week in range(DAY_IN_WEEK): for time_of_day_code in range(TIME_SLOTS_WITHIN_DAY): #for tid in range(TOTAL_SLOTS_FOR_LOOP): #TODO do the loop per week, per day and day slot and change fi_features_cache too curFeature = demandCache.get_demand( week_nb, day_of_week, time_of_day_code, cid) if curFeature != []: time_of_day_code, origin, day_of_week, day, week, hour, minute, is_manhattan, is_airport, amount = extract_feature( curFeature) if (week_nb < WEEK_NB_TEST): rows_training.append( (time_of_day_code, origin, day_of_week, day, week, hour, minute, amount)) else: rows_testing.append( (time_of_day_code, origin, day_of_week, day, week, hour, minute, amount)) df_training = spark.createDataFrame(rows_training, [ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "amount" ]) df_testing = spark.createDataFrame(rows_testing, [ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "amount" ]) assembler = VectorAssembler(inputCols=[ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute" ], outputCol='features') output_training = assembler.transform(df_training) output_testing = assembler.transform(df_testing) final_data_training = output_training.select('features', 'amount') final_data_testing = output_testing.select('features', 'amount') decisionTree = DecisionTreeRegressor(labelCol='amount', maxDepth=3) dt_model = decisionTree.fit(final_data_training) predictionsDT = dt_model.transform(final_data_testing) linearRegression = LinearRegression(labelCol='amount') lr_model = linearRegression.fit(final_data_training) predictionsLR = lr_model.evaluate(final_data_testing) # print("Decision tree model max depth = %g" % decisionTree.getMaxDepth()) # print(dt_model.toDebugString) """ Evaluation rmse : """ rmse = predictionsLR.rootMeanSquaredError errorsRMSE_LR.append(rmse) print("Root Mean Squared Error (RMSE) for LR on test data = %g" % rmse) r2 = predictionsLR.r2 errorsR2_LR.append(r2) print("R Squared Error (R2) for LR on test data = %g" % r2) """ Evaluation rmse : """ evaluatorRMSE = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="rmse") rmse = evaluatorRMSE.evaluate(predictionsDT) errorsRMSE_DT.append(rmse) print("Root Mean Squared Error (RMSE) for DT on test data = %g" % rmse) evaluatorR2 = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="r2") r2 = evaluatorR2.evaluate(predictionsDT) errorsR2_DT.append(r2) print("R Squared Error (R2) for DT on test data = %g" % r2)
def main(): for cid in range(N_OF_CLUSTERS): rows_training = [] rows_testing = [] for week_nb in range(FIRST_WEEK, LAST_WEEK + 1): print('week nb : ', week_nb) for day_of_week in range(DAY_IN_WEEK): for time_of_day_code in range(TIME_SLOTS_WITHIN_DAY): #retrieving the features'values: curFeature = demandCache.get_demand( week_nb, day_of_week, time_of_day_code, cid) if curFeature != []: time_of_day_code, origin, day_of_week, day, week, hour, minute, is_manhattan, is_airport, amount = extract_feature( curFeature) # Checking whether the current row should be added to the training or testing set: if (week_nb < WEEK_NB_TEST): rows_training.append( (time_of_day_code, origin, day_of_week, day, week, hour, minute, amount)) else: rows_testing.append( (time_of_day_code, origin, day_of_week, day, week, hour, minute, amount)) # Creating the dataframe for the model containing all the rows : df_training = spark.createDataFrame(rows_training, [ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "amount" ]) df_testing = spark.createDataFrame(rows_testing, [ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute", "amount" ]) assembler = VectorAssembler(inputCols=[ "time_of_day_code", "origin", "day_of_week", "day", "week", "hour", "minute" ], outputCol='features') output_training = assembler.transform(df_training) output_testing = assembler.transform(df_testing) final_data_training = output_training.select('features', 'amount') final_data_testing = output_testing.select('features', 'amount') # Training the Desition Tree: decisionTree = DecisionTreeRegressor(labelCol='amount', maxDepth=3) dt_model = decisionTree.fit(final_data_training) predictionsDT = dt_model.transform(final_data_testing) # print(dt_model.toDebugString) # showing the decision tree # Training the linear regression: linearRegression = LinearRegression(labelCol='amount') lr_model = linearRegression.fit(final_data_training) predictionsLR = lr_model.evaluate(final_data_testing) """ Evaluation rmse : """ rmse = predictionsLR.rootMeanSquaredError errorsRMSE_LR.append(rmse) print("Root Mean Squared Error (RMSE) for LR on test data = %g" % rmse) r2 = predictionsLR.r2 errorsR2_LR.append(r2) print("R Squared Error (R2) for LR on test data = %g" % r2) """ Evaluation rmse : """ evaluatorRMSE = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="rmse") rmse = evaluatorRMSE.evaluate(predictionsDT) errorsRMSE_DT.append(rmse) print("Root Mean Squared Error (RMSE) for DT on test data = %g" % rmse) evaluatorR2 = RegressionEvaluator(labelCol="amount", predictionCol="prediction", metricName="r2") r2 = evaluatorR2.evaluate(predictionsDT) errorsR2_DT.append(r2) print("R Squared Error (R2) for DT on test data = %g" % r2)
glr = GeneralizedLinearRegression()\ .setFamily("gaussian")\ .setLink("identity")\ .setMaxIter(10)\ .setRegParam(0.3)\ .setLinkPredictionCol("linkOut") print glr.explainParams() glrModel = glr.fit(df) # COMMAND ---------- from pyspark.ml.regression import DecisionTreeRegressor dtr = DecisionTreeRegressor() print dtr.explainParams() dtrModel = dtr.fit(df) # COMMAND ---------- from pyspark.ml.regression import RandomForestRegressor from pyspark.ml.regression import GBTRegressor rf = RandomForestRegressor() print rf.explainParams() rfModel = rf.fit(df) gbt = GBTRegressor() print gbt.explainParams() gbtModel = gbt.fit(df) # COMMAND ----------
# Split the data 70-30 train_test_data = model_data.randomSplit([0.8, 0.2], 16430212) train_data = train_test_data[0] test_data = train_test_data[1] print("Train DT") rmseEvaluator = myRmseEvaluator( RegressionEvaluator(predictionCol="prediction", labelCol="trip_duration", metricName="rmse")) maeEvaluator = RegressionEvaluator(predictionCol="prediction", labelCol="trip_duration", metricName="mae") dtr = DecisionTreeRegressor( maxDepth=3).setFeaturesCol("features").setLabelCol("trip_duration") trained_model = dtr.fit(train_data) predictions = trained_model.transform(test_data) # final_result = predictions.select("prediction", "trip_duration").rdd print(trained_model) print("RMSE for Regression Tree:", rmseEvaluator.evaluate(predictions)) print("MAE for Regression Tree:", maeEvaluator.evaluate(predictions)) """ DecisionTreeRegressionModel: uid=DecisionTreeRegressor_826b1c042824, depth=3, numNodes=15, numFeatures=7 If (feature 6 <= 2.7002733639393384) If (feature 6 <= 1.3071311166631614) If (feature 6 <= 0.825910208978972) Predict: 481.0347939172201 Else (feature 6 > 0.825910208978972) Predict: 704.5021037177617 Else (feature 6 > 1.3071311166631614)
categoricalColumns = ['store_and_fwd_flag'] stages = [] # stages in our Pipeline for categoricalCol in categoricalColumns: stringIndexer = StringIndexer(inputCol=categoricalCol, outputCol=categoricalCol+"Index") encoder = OneHotEncoder(inputCol=categoricalCol+"Index", outputCol=categoricalCol+"classVec") # Add stages. These are not run here, but will run all at once later on. stages += [stringIndexer, encoder] #encColumns = ['VendorID','RatecodeID','PULocationID','DOLocationID','payment_type','Peak_Time','weekend'] encColumns = ['VendorID','RatecodeID','PULocationID','DOLocationID','payment_type'] for eCol in encColumns: encoder = OneHotEncoder(inputCol=eCol, outputCol=eCol+"classVec") stages += [encoder] #label_stringIdx = StringIndexer(inputCol = "verified_purchase", outputCol = "label") #stages += [label_stringIdx] numericCols = ["trip_distance", "passenger_count", "fare_amount","tip_amount"] assemblerInputs = map(lambda c: c + "classVec", categoricalColumns) + map(lambda c: c + "classVec", encColumns) + numericCols assembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features") stages += [assembler] pipeline = Pipeline(stages=stages) pipelineModel = pipeline.fit(train_X4) dataset = pipelineModel.transform(train_X4) from pyspark.ml.regression import DecisionTreeRegressor dt = DecisionTreeRegressor(labelCol="total_amount", featuresCol="features", maxBins=32) model = dt.fit(dataset) model.write().overwrite().save("./nyc-01020304-6vm-18-DT-model")
from pyspark.ml.regression import DecisionTreeRegressor dt_models = {} dt_predictions = {} compute_again = False if compute_again == False: dt_models = loadModels("TreeModel_","tree") for park in park_data_with_date_dict: dt_predictions[park] = dt_models[park].transform(test_ds[park]) else: for park in park_data_with_date_dict: #vectorAssembler = VectorAssembler(inputCols=features, outputCol="features") #data = vectorAssembler.transform(all_tables[park]) #train, test = data.randomSplit([0.8,0.2], seed = 12345) dt = DecisionTreeRegressor() dt_models[park] = dt.fit(train_ds[park]) dt_predictions[park] = dt_models[park].transform(test_ds[park]) saveModels(dt_models,"TreeModel_","tree") # COMMAND ---------- #ATTENZIONE: se vuoi visualizzare proprio gli alberi puoi chiamare display(dt_models[park]) # COMMAND ---------- def printEvaluateModel(park,modelsCollection, predictionsCollection): print("EVALUATE MODEL FOR PARKING "+str(park)) print("OVER TEST SET") print("Features importance:" + str(modelsCollection[park].featureImportances)) eval = RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse") # Root Mean Square Error
import numpy as np import matplotlib.pyplot as plt # In[106]: ### model building process #create a sample for model test sample, x = final_data.randomSplit([0.1, 0.8]) # In[107]: # decision trees r2_dtr = np.zeros(10) for i in np.arange(10): dtr = DecisionTreeRegressor(labelCol='mean_temp', maxDepth=(i + 1) * 3.) dtrModel = dtr.fit(sample) prediction_dtr = dtrModel.transform(sample) r2_dtr[i] = evaluator.evaluate(prediction_dtr) plt.plot(np.arange(3, 33, 3), r2_dtr) # so choose 10 as the maxDepth # In[108]: # Random Forest r2_rfr = np.zeros(10) for i in np.arange(10): rfr = RandomForestRegressor(labelCol='mean_temp', maxDepth=(i + 1) * 3.) rfrModel = rfr.fit(sample) prediction_rfr = rfrModel.transform(sample) r2_rfr[i] = evaluator.evaluate(prediction_rfr) plt.plot(np.arange(3, 33, 3), r2_rfr)
def Train(self): st_global = time.time() CommonUtils.create_update_and_save_progress_message(self._dataframe_context,self._scriptWeightDict,self._scriptStages,self._slug,"initialization","info",display=True,emptyBin=False,customMsg=None,weightKey="total") appType = self._dataframe_context.get_app_type() algosToRun = self._dataframe_context.get_algorithms_to_run() algoSetting = filter(lambda x:x.get_algorithm_slug()==self._slug,algosToRun)[0] categorical_columns = self._dataframe_helper.get_string_columns() uid_col = self._dataframe_context.get_uid_column() if self._metaParser.check_column_isin_ignored_suggestion(uid_col): categorical_columns = list(set(categorical_columns) - {uid_col}) allDateCols = self._dataframe_context.get_date_columns() categorical_columns = list(set(categorical_columns)-set(allDateCols)) print categorical_columns result_column = self._dataframe_context.get_result_column() numerical_columns = self._dataframe_helper.get_numeric_columns() numerical_columns = [x for x in numerical_columns if x != result_column] model_path = self._dataframe_context.get_model_path() if model_path.startswith("file"): model_path = model_path[7:] validationDict = self._dataframe_context.get_validation_dict() print "model_path",model_path pipeline_filepath = "file://"+str(model_path)+"/"+str(self._slug)+"/pipeline/" model_filepath = "file://"+str(model_path)+"/"+str(self._slug)+"/model" pmml_filepath = "file://"+str(model_path)+"/"+str(self._slug)+"/modelPmml" df = self._data_frame if self._mlEnv == "spark": pipeline = MLUtils.create_pyspark_ml_pipeline(numerical_columns,categorical_columns,result_column,algoType="regression") pipelineModel = pipeline.fit(df) indexed = pipelineModel.transform(df) featureMapping = sorted((attr["idx"], attr["name"]) for attr in (chain(*indexed.schema["features"].metadata["ml_attr"]["attrs"].values()))) # print indexed.select([result_column,"features"]).show(5) MLUtils.save_pipeline_or_model(pipelineModel,pipeline_filepath) # OriginalTargetconverter = IndexToString(inputCol="label", outputCol="originalTargetColumn") dtreer = DecisionTreeRegressor(labelCol=result_column, featuresCol='features',predictionCol="prediction") if validationDict["name"] == "kFold": defaultSplit = GLOBALSETTINGS.DEFAULT_VALIDATION_OBJECT["value"] numFold = int(validationDict["value"]) if numFold == 0: numFold = 3 trainingData,validationData = indexed.randomSplit([defaultSplit,1-defaultSplit], seed=12345) paramGrid = ParamGridBuilder()\ .addGrid(dtreer.regParam, [0.1, 0.01]) \ .addGrid(dtreer.fitIntercept, [False, True])\ .addGrid(dtreer.elasticNetParam, [0.0, 0.5, 1.0])\ .build() crossval = CrossValidator(estimator=dtreer, estimatorParamMaps=paramGrid, evaluator=RegressionEvaluator(predictionCol="prediction", labelCol=result_column), numFolds=numFold) st = time.time() cvModel = crossval.fit(indexed) trainingTime = time.time()-st print "cvModel training takes",trainingTime bestModel = cvModel.bestModel elif validationDict["name"] == "trainAndtest": trainingData,validationData = indexed.randomSplit([float(validationDict["value"]),1-float(validationDict["value"])], seed=12345) st = time.time() fit = dtreer.fit(trainingData) trainingTime = time.time()-st print "time to train",trainingTime bestModel = fit featureImportance = bestModel.featureImportances print featureImportance,type(featureImportance) # print featureImportance[0],len(featureImportance[1],len(featureImportance[2])) print len(featureMapping) featuresArray = [(name, featureImportance[idx]) for idx, name in featureMapping] print featuresArray MLUtils.save_pipeline_or_model(bestModel,model_filepath) transformed = bestModel.transform(validationData) transformed = transformed.withColumn(result_column,transformed[result_column].cast(DoubleType())) transformed = transformed.select([result_column,"prediction",transformed[result_column]-transformed["prediction"]]) transformed = transformed.withColumnRenamed(transformed.columns[-1],"difference") transformed = transformed.select([result_column,"prediction","difference",FN.abs(transformed["difference"])*100/transformed[result_column]]) transformed = transformed.withColumnRenamed(transformed.columns[-1],"mape") sampleData = None nrows = transformed.count() if nrows > 100: sampleData = transformed.sample(False, float(100)/nrows, seed=420) else: sampleData = transformed print sampleData.show() evaluator = RegressionEvaluator(predictionCol="prediction",labelCol=result_column) metrics = {} metrics["r2"] = evaluator.evaluate(transformed,{evaluator.metricName: "r2"}) metrics["rmse"] = evaluator.evaluate(transformed,{evaluator.metricName: "rmse"}) metrics["mse"] = evaluator.evaluate(transformed,{evaluator.metricName: "mse"}) metrics["mae"] = evaluator.evaluate(transformed,{evaluator.metricName: "mae"}) runtime = round((time.time() - st_global),2) # print transformed.count() mapeDf = transformed.select("mape") # print mapeDf.show() mapeStats = MLUtils.get_mape_stats(mapeDf,"mape") mapeStatsArr = mapeStats.items() mapeStatsArr = sorted(mapeStatsArr,key=lambda x:int(x[0])) # print mapeStatsArr quantileDf = transformed.select("prediction") # print quantileDf.show() quantileSummaryDict = MLUtils.get_quantile_summary(quantileDf,"prediction") quantileSummaryArr = quantileSummaryDict.items() quantileSummaryArr = sorted(quantileSummaryArr,key=lambda x:int(x[0])) # print quantileSummaryArr self._model_summary.set_model_type("regression") self._model_summary.set_algorithm_name("dtree Regression") self._model_summary.set_algorithm_display_name("Decision Tree Regression") self._model_summary.set_slug(self._slug) self._model_summary.set_training_time(runtime) self._model_summary.set_training_time(trainingTime) self._model_summary.set_target_variable(result_column) self._model_summary.set_validation_method(validationDict["displayName"]) self._model_summary.set_model_evaluation_metrics(metrics) self._model_summary.set_model_params(bestEstimator.get_params()) self._model_summary.set_quantile_summary(quantileSummaryArr) self._model_summary.set_mape_stats(mapeStatsArr) self._model_summary.set_sample_data(sampleData.toPandas().to_dict()) self._model_summary.set_feature_importance(featureImportance) # print CommonUtils.convert_python_object_to_json(self._model_summary) elif self._mlEnv == "sklearn": model_filepath = model_path+"/"+self._slug+"/model.pkl" x_train,x_test,y_train,y_test = self._dataframe_helper.get_train_test_data() x_train = MLUtils.create_dummy_columns(x_train,[x for x in categorical_columns if x != result_column]) x_test = MLUtils.create_dummy_columns(x_test,[x for x in categorical_columns if x != result_column]) x_test = MLUtils.fill_missing_columns(x_test,x_train.columns,result_column) st = time.time() est = DecisionTreeRegressor() CommonUtils.create_update_and_save_progress_message(self._dataframe_context,self._scriptWeightDict,self._scriptStages,self._slug,"training","info",display=True,emptyBin=False,customMsg=None,weightKey="total") if algoSetting.is_hyperparameter_tuning_enabled(): hyperParamInitParam = algoSetting.get_hyperparameter_params() evaluationMetricDict = {"name":hyperParamInitParam["evaluationMetric"]} evaluationMetricDict["displayName"] = GLOBALSETTINGS.SKLEARN_EVAL_METRIC_NAME_DISPLAY_MAP[evaluationMetricDict["name"]] hyperParamAlgoName = algoSetting.get_hyperparameter_algo_name() params_grid = algoSetting.get_params_dict_hyperparameter() params_grid = {k:v for k,v in params_grid.items() if k in est.get_params()} print params_grid if hyperParamAlgoName == "gridsearchcv": estGrid = GridSearchCV(est,params_grid) gridParams = estGrid.get_params() hyperParamInitParam = {k:v for k,v in hyperParamInitParam.items() if k in gridParams} estGrid.set_params(**hyperParamInitParam) estGrid.fit(x_train,y_train) bestEstimator = estGrid.best_estimator_ modelFilepath = "/".join(model_filepath.split("/")[:-1]) sklearnHyperParameterResultObj = SklearnGridSearchResult(estGrid.cv_results_,est,x_train,x_test,y_train,y_test,appType,modelFilepath,evaluationMetricDict=evaluationMetricDict) resultArray = sklearnHyperParameterResultObj.train_and_save_models() self._result_setter.set_hyper_parameter_results(self._slug,resultArray) self._result_setter.set_metadata_parallel_coordinates(self._slug,{"ignoreList":sklearnHyperParameterResultObj.get_ignore_list(),"hideColumns":sklearnHyperParameterResultObj.get_hide_columns(),"metricColName":sklearnHyperParameterResultObj.get_comparison_metric_colname(),"columnOrder":sklearnHyperParameterResultObj.get_keep_columns()}) elif hyperParamAlgoName == "randomsearchcv": estRand = RandomizedSearchCV(est,params_grid) estRand.set_params(**hyperParamInitParam) bestEstimator = None else: evaluationMetricDict = {"name":GLOBALSETTINGS.REGRESSION_MODEL_EVALUATION_METRIC} evaluationMetricDict["displayName"] = GLOBALSETTINGS.SKLEARN_EVAL_METRIC_NAME_DISPLAY_MAP[evaluationMetricDict["name"]] algoParams = algoSetting.get_params_dict() algoParams = {k:v for k,v in algoParams.items() if k in est.get_params().keys()} est.set_params(**algoParams) self._result_setter.set_hyper_parameter_results(self._slug,None) if validationDict["name"] == "kFold": defaultSplit = GLOBALSETTINGS.DEFAULT_VALIDATION_OBJECT["value"] numFold = int(validationDict["value"]) if numFold == 0: numFold = 3 kFoldClass = SkleanrKFoldResult(numFold,est,x_train,x_test,y_train,y_test,appType,evaluationMetricDict=evaluationMetricDict) kFoldClass.train_and_save_result() kFoldOutput = kFoldClass.get_kfold_result() bestEstimator = kFoldClass.get_best_estimator() elif validationDict["name"] == "trainAndtest": est.fit(x_train, y_train) bestEstimator = est trainingTime = time.time()-st y_score = bestEstimator.predict(x_test) try: y_prob = bestEstimator.predict_proba(x_test) except: y_prob = [0]*len(y_score) featureImportance={} objs = {"trained_model":bestEstimator,"actual":y_test,"predicted":y_score,"probability":y_prob,"feature_importance":featureImportance,"featureList":list(x_train.columns),"labelMapping":{}} featureImportance = objs["trained_model"].feature_importances_ featuresArray = [(col_name, featureImportance[idx]) for idx, col_name in enumerate(x_train.columns)] if not algoSetting.is_hyperparameter_tuning_enabled(): modelName = "M"+"0"*(GLOBALSETTINGS.MODEL_NAME_MAX_LENGTH-1)+"1" modelFilepathArr = model_filepath.split("/")[:-1] modelFilepathArr.append(modelName+".pkl") joblib.dump(objs["trained_model"],"/".join(modelFilepathArr)) metrics = {} metrics["r2"] = r2_score(y_test, y_score) metrics["mse"] = mean_squared_error(y_test, y_score) metrics["mae"] = mean_absolute_error(y_test, y_score) metrics["rmse"] = sqrt(metrics["mse"]) transformed = pd.DataFrame({"prediction":y_score,result_column:y_test}) transformed["difference"] = transformed[result_column] - transformed["prediction"] transformed["mape"] = np.abs(transformed["difference"])*100/transformed[result_column] sampleData = None nrows = transformed.shape[0] if nrows > 100: sampleData = transformed.sample(n=100,random_state=420) else: sampleData = transformed print sampleData.head() mapeCountArr = pd.cut(transformed["mape"],GLOBALSETTINGS.MAPEBINS).value_counts().to_dict().items() mapeStatsArr = [(str(idx),dictObj) for idx,dictObj in enumerate(sorted([{"count":x[1],"splitRange":(x[0].left,x[0].right)} for x in mapeCountArr],key = lambda x:x["splitRange"][0]))] predictionColSummary = transformed["prediction"].describe().to_dict() quantileBins = [predictionColSummary["min"],predictionColSummary["25%"],predictionColSummary["50%"],predictionColSummary["75%"],predictionColSummary["max"]] print quantileBins quantileBins = sorted(list(set(quantileBins))) transformed["quantileBinId"] = pd.cut(transformed["prediction"],quantileBins) quantileDf = transformed.groupby("quantileBinId").agg({"prediction":[np.sum,np.mean,np.size]}).reset_index() quantileDf.columns = ["prediction","sum","mean","count"] print quantileDf quantileArr = quantileDf.T.to_dict().items() quantileSummaryArr = [(obj[0],{"splitRange":(obj[1]["prediction"].left,obj[1]["prediction"].right),"count":obj[1]["count"],"mean":obj[1]["mean"],"sum":obj[1]["sum"]}) for obj in quantileArr] print quantileSummaryArr runtime = round((time.time() - st_global),2) self._model_summary.set_model_type("regression") self._model_summary.set_algorithm_name("DTREE Regression") self._model_summary.set_algorithm_display_name("Decision Tree Regression") self._model_summary.set_slug(self._slug) self._model_summary.set_training_time(runtime) self._model_summary.set_training_time(trainingTime) self._model_summary.set_target_variable(result_column) self._model_summary.set_validation_method(validationDict["displayName"]) self._model_summary.set_model_evaluation_metrics(metrics) self._model_summary.set_model_params(bestEstimator.get_params()) self._model_summary.set_quantile_summary(quantileSummaryArr) self._model_summary.set_mape_stats(mapeStatsArr) self._model_summary.set_sample_data(sampleData.to_dict()) self._model_summary.set_feature_importance(featuresArray) self._model_summary.set_feature_list(list(x_train.columns)) try: pmml_filepath = str(model_path)+"/"+str(self._slug)+"/traindeModel.pmml" modelPmmlPipeline = PMMLPipeline([ ("pretrained-estimator", objs["trained_model"]) ]) modelPmmlPipeline.target_field = result_column modelPmmlPipeline.active_fields = np.array([col for col in x_train.columns if col != result_column]) sklearn2pmml(modelPmmlPipeline, pmml_filepath, with_repr = True) pmmlfile = open(pmml_filepath,"r") pmmlText = pmmlfile.read() pmmlfile.close() self._result_setter.update_pmml_object({self._slug:pmmlText}) except: pass if not algoSetting.is_hyperparameter_tuning_enabled(): modelDropDownObj = { "name":self._model_summary.get_algorithm_name(), "evaluationMetricValue":self._model_summary.get_model_accuracy(), "evaluationMetricName":"r2", "slug":self._model_summary.get_slug(), "Model Id":modelName } modelSummaryJson = { "dropdown":modelDropDownObj, "levelcount":self._model_summary.get_level_counts(), "modelFeatureList":self._model_summary.get_feature_list(), "levelMapping":self._model_summary.get_level_map_dict(), "slug":self._model_summary.get_slug(), "name":self._model_summary.get_algorithm_name() } else: modelDropDownObj = { "name":self._model_summary.get_algorithm_name(), "evaluationMetricValue":resultArray[0]["R-Squared"], "evaluationMetricName":"r2", "slug":self._model_summary.get_slug(), "Model Id":resultArray[0]["Model Id"] } modelSummaryJson = { "dropdown":modelDropDownObj, "levelcount":self._model_summary.get_level_counts(), "modelFeatureList":self._model_summary.get_feature_list(), "levelMapping":self._model_summary.get_level_map_dict(), "slug":self._model_summary.get_slug(), "name":self._model_summary.get_algorithm_name() } dtreerCards = [json.loads(CommonUtils.convert_python_object_to_json(cardObj)) for cardObj in MLUtils.create_model_summary_cards(self._model_summary)] for card in dtreerCards: self._prediction_narrative.add_a_card(card) self._result_setter.set_model_summary({"dtreeregression":json.loads(CommonUtils.convert_python_object_to_json(self._model_summary))}) self._result_setter.set_dtree_regression_model_summart(modelSummaryJson) self._result_setter.set_dtreer_cards(dtreerCards) CommonUtils.create_update_and_save_progress_message(self._dataframe_context,self._scriptWeightDict,self._scriptStages,self._slug,"completion","info",display=True,emptyBin=False,customMsg=None,weightKey="total")
assembler = VectorAssembler(inputCols=[ "slot_id", "day_of_week", "day_of_month", "week_nb", "hour", "minute" ], outputCol='features') output_training = assembler.transform(df_training) output_testing = assembler.transform(df_testing) final_data_training = output_training.select('features', 'demand') final_data_testing = output_testing.select('features', 'demand') #final_data_training.describe().show() #final_data_testing.describe().show() """ Model and predictions : """ decisionTree = DecisionTreeRegressor(labelCol='demand', maxDepth=3) dt_model = decisionTree.fit(final_data_training) predictions = dt_model.transform(final_data_testing) #print("Decision tree model max depth = %g" % decisionTree.getMaxDepth()) #print(dt_model.toDebugString) """ Evaluation rmse : """ evaluatorRMSE = RegressionEvaluator(labelCol="demand", predictionCol="prediction", metricName="rmse") rmse = evaluatorRMSE.evaluate(predictions) errorsRMSE.append(rmse) print("Root Mean Squared Error (RMSE) on test data = %g" % rmse) evaluatorR2 = RegressionEvaluator(labelCol="demand", predictionCol="prediction", metricName="r2") r2 = evaluatorR2.evaluate(predictions)
assembler = VectorAssembler().setInputCols( ['HouseAge', 'DistanceToMRT', 'NumberConvenienceStores']).setOutputCol('features') df = assembler.transform(data).select('PriceOfUnitArea', 'features') # Let's split our data into training data and testing data trainTest = df.randomSplit([0.5, 0.5]) trainingDF = trainTest[0] testDF = trainTest[1] # Now create our linear regression model dtr = DecisionTreeRegressor().setFeaturesCol('features').setLabelCol( 'PriceOfUnitArea') # Train the model using our training data model = dtr.fit(trainingDF) # Now see if we can predict values in our test data. # Generate predictions using our linear regression model for all features in our # test dataframe: fullPredictions = model.transform(testDF).cache() # Extract the predictions and the "known" correct labels. predictions = fullPredictions.select("prediction").rdd.map(lambda x: x[0]) labels = fullPredictions.select("label").rdd.map(lambda x: x[0]) # Zip them together predictionAndLabel = predictions.zip(labels).collect() # Print out the predicted and actual values for each point for prediction in predictionAndLabel:
# COMMAND ---------- # MAGIC %md # MAGIC #### Regression with decision trees # COMMAND ---------- from pyspark.ml.regression import DecisionTreeRegressor dtr = DecisionTreeRegressor().setLabelCol('petalWidth') print dtr.explainParams() # COMMAND ---------- dtrModel = dtr.fit(irisPetal) dtrPredictions = dtrModel.transform(irisPetal) print regEval.evaluate(dtrPredictions, {regEval.metricName: 'r2'}) print regEval.evaluate(dtrPredictions, {regEval.metricName: 'rmse'}) # COMMAND ---------- # MAGIC %md # MAGIC Let's also build a gradient boosted tree. # COMMAND ---------- from pyspark.ml.regression import GBTRegressor gbt = GBTRegressor().setLabelCol('petalWidth') print gbt.explainParams()