def testXGBoostParameters(prostateDataset): features = ['AGE', 'RACE', 'DPROS', 'DCAPS', 'PSA'] algorithm = H2OXGBoost(seed=1, labelCol="CAPSULE", featuresCols=features, monotoneConstraints={'AGE': 1, 'RACE': -1}, interactionConstraints=[['AGE', 'RACE', 'DPROS'], ['DCAPS', 'PSA']]) model = algorithm.fit(prostateDataset) compareParameterValues(algorithm, model)
def testFitXgboostWithoutError(prostateDataset): xgboost = H2OXGBoost(ntrees=2, seed=42, distribution="bernoulli", labelCol="capsule") model = xgboost.fit(prostateDataset) model.transform(prostateDataset).repartition(1).collect()
def h2o_xgboost(df, label, columns, **kargs): H2OContext.getOrCreate(Spark.instance.spark) df_sti = string_to_index(df, input_cols=label) df_va = vector_assembler(df_sti, input_cols=columns) h2o_xgboost = H2OXGBoost(convertUnknownCategoricalLevelsToNa=True, featuresCols=columns, labelCol=label, **kargs) model = h2o_xgboost.fit(df_va) df_raw = model.transform(df_va) df_pred = df_raw.withColumn("prediction", when(df_raw.prediction_output["p1"] > 0.5, 1.0).otherwise(0.0)) return df_pred, model
def testPipelineSerializationXGBoost(prostateDataset): gridSearchTester(H2OXGBoost().setLabelCol("AGE"), prostateDataset)
def testPipelineSerializationXGBoost(prostateDataset): gridSearchTester(H2OXGBoost(), prostateDataset)
seed=1, l1=0.001, l2=0.0, hidden=[200, 200], featuresCols=[idf.getOutputCol()], labelCol="label") automl = H2OAutoML( convertUnknownCategoricalLevelsToNa=True, maxRuntimeSecs=60 * 100, # 100 minutes maxModels=10, seed=1, labelCol="label") xgboost = H2OXGBoost(convertUnknownCategoricalLevelsToNa=True, featuresCols=[idf.getOutputCol()], labelCol="label") data = load() def trainPipelineModel(idf, hashingTF, stopWordsRemover, tokenizer, algoStage, data): ## Remove all helper columns colPruner = ColumnPruner(columns=[ idf.getOutputCol(), hashingTF.getOutputCol(), stopWordsRemover.getOutputCol(), tokenizer.getOutputCol() ])
# from pysparkling import * from pyspark.sql import SparkSession import h2o # Start Cluster spark = SparkSession.builder.appName("App name").getOrCreate() hc = H2OContext.getOrCreate() assert h2o.cluster().cloud_size == int( spark.sparkContext.getConf().get("spark.executor.instances")) # Prepare Data frame = h2o.import_file( "https://raw.githubusercontent.com/h2oai/sparkling-water/master/examples/smalldata/prostate/prostate.csv" ) sparkDF = hc.asSparkFrame(frame) sparkDF = sparkDF.withColumn("CAPSULE", sparkDF.CAPSULE.cast("string")) [trainingDF, testingDF] = sparkDF.randomSplit([0.8, 0.2]) # Train Model from pysparkling.ml import H2OXGBoost estimator = H2OXGBoost(labelCol="CAPSULE") model = estimator.fit(trainingDF) # Run Predictions model.transform(testingDF).collect() hc.stop() spark.stop()
l2=0.0, hidden=[200, 200], featuresCols=[idf.getOutputCol()], predictionCol="label") elif algo == "automl": ## Create H2OAutoML model algoStage = H2OAutoML( convertUnknownCategoricalLevelsToNa=True, maxRuntimeSecs=60 * 100, # 100 minutes maxModels=3, seed=1, predictionCol="label") elif algo == "xgboost": ## Create H2OXGBoost model algoStage = H2OXGBoost(convertUnknownCategoricalLevelsToNa=True, featuresCols=[idf.getOutputCol()], predictionCol="label") ## Remove all helper columns colPruner = ColumnPruner(columns=[ idf.getOutputCol(), hashingTF.getOutputCol(), stopWordsRemover.getOutputCol(), tokenizer.getOutputCol() ]) ## Create the pipeline by defining all the stages pipeline = Pipeline( stages=[tokenizer, stopWordsRemover, hashingTF, idf, algoStage, colPruner]) ## Test exporting and importing the pipeline. On Systems where HDFS & Hadoop is not available, this call store the pipeline ## to local file in the current directory. In case HDFS & Hadoop is available, this call stores the pipeline to HDFS home
def testParams(): xgboost = H2OXGBoost(modelId=None, splitRatio=1.0, labelCol="label", weightCol=None, featuresCols=[], allStringColumnsToCategorical=True, columnsToCategorical=[], nfolds=0, keepCrossValidationPredictions=False, keepCrossValidationFoldAssignment=False, parallelizeCrossValidation=True, seed=-1, distribution="AUTO", convertUnknownCategoricalLevelsToNa=False, quietMode=True, ntrees=50, nEstimators=0, maxDepth=6, minRows=1.0, minChildWeight=1.0, learnRate=0.3, eta=0.3, learnRateAnnealing=1.0, sampleRate=1.0, subsample=1.0, colSampleRate=1.0, colSampleByLevel=1.0, colSampleRatePerTree=1.0, colSampleByTree=1.0, maxAbsLeafnodePred=0.0, maxDeltaStep=0.0, scoreTreeInterval=0, initialScoreInterval=4000, scoreInterval=4000, minSplitImprovement=0.0, gamma=0.0, nthread=-1, maxBins=256, maxLeaves=0, minSumHessianInLeaf=100.0, minDataInLeaf=0.0, treeMethod="auto", growPolicy="depthwise", booster="gbtree", dmatrixType="auto", regLambda=0.0, regAlpha=0.0, sampleType="uniform", normalizeType="tree", rateDrop=0.0, oneDrop=False, skipDrop=0.0, gpuId=0, backend="auto", foldCol=None, predictionCol="prediction", detailedPredictionCol="detailed_prediction", withDetailedPredictionCol=False, convertInvalidNumbersToNa=False) assert xgboost.getModelId() == None assert xgboost.getSplitRatio() == 1.0 assert xgboost.getLabelCol() == "label" assert xgboost.getWeightCol() == None assert xgboost.getFeaturesCols() == [] assert xgboost.getAllStringColumnsToCategorical() == True assert xgboost.getColumnsToCategorical() == [] assert xgboost.getNfolds() == 0 assert xgboost.getKeepCrossValidationPredictions() == False assert xgboost.getKeepCrossValidationFoldAssignment() == False assert xgboost.getParallelizeCrossValidation() == True assert xgboost.getSeed() == -1 assert xgboost.getDistribution() == "AUTO" assert xgboost.getConvertUnknownCategoricalLevelsToNa() == False assert xgboost.getQuietMode() == True assert xgboost.getNtrees() == 50 assert xgboost.getNEstimators() == 0 assert xgboost.getMaxDepth() == 6 assert xgboost.getMinRows() == 1.0 assert xgboost.getMinChildWeight() == 1.0 assert xgboost.getLearnRate() == 0.3 assert xgboost.getEta() == 0.3 assert xgboost.getLearnRateAnnealing() == 1.0 assert xgboost.getSampleRate() == 1.0 assert xgboost.getSubsample() == 1.0 assert xgboost.getColSampleRate() == 1.0 assert xgboost.getColSampleByLevel() == 1.0 assert xgboost.getColSampleRatePerTree() == 1.0 assert xgboost.getColSampleByTree() == 1.0 assert xgboost.getMaxAbsLeafnodePred() == 0.0 assert xgboost.getMaxDeltaStep() == 0.0 assert xgboost.getScoreTreeInterval() == 0 assert xgboost.getInitialScoreInterval() == 4000 assert xgboost.getScoreInterval() == 4000 assert xgboost.getMinSplitImprovement() == 0.0 assert xgboost.getGamma() == 0.0 assert xgboost.getNthread() == -1 assert xgboost.getMaxBins() == 256 assert xgboost.getMaxLeaves() == 0 assert xgboost.getMinSumHessianInLeaf() == 100.0 assert xgboost.getMinDataInLeaf() == 0.0 assert xgboost.getTreeMethod() == "auto" assert xgboost.getGrowPolicy() == "depthwise" assert xgboost.getBooster() == "gbtree" assert xgboost.getDmatrixType() == "auto" assert xgboost.getRegLambda() == 0.0 assert xgboost.getRegAlpha() == 0.0 assert xgboost.getSampleType() == "uniform" assert xgboost.getNormalizeType() == "tree" assert xgboost.getRateDrop() == 0.0 assert xgboost.getOneDrop() == False assert xgboost.getSkipDrop() == 0.0 assert xgboost.getGpuId() == 0 assert xgboost.getBackend() == "auto" assert xgboost.getFoldCol() == None assert xgboost.getPredictionCol() == "prediction" assert xgboost.getDetailedPredictionCol() == "detailed_prediction" assert xgboost.getWithDetailedPredictionCol() == False assert xgboost.getConvertInvalidNumbersToNa() == False
spark_frame.show(4) # In[8]: spark_frame.describe() # # 3. Train the model # In[10]: from pysparkling.ml import H2OXGBoost estimator = H2OXGBoost(predictionCol="AGE") model = estimator.fit(spark_frame) # # 4. Run Predictions # In[12]: predictions = model.transform(spark_frame) # In[13]: predictions.show(4)
def createInitialXGBoostDefinition(): return H2OXGBoost(featuresCols=featureCols, labelCol="CAPSULE", seed=1, splitRatio=0.8)