def nb_iris(): print("Importing iris_wheader.csv data...\n") iris = h2o.upload_file(pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) iris.describe() iris_nbayes = H2ONaiveBayesEstimator() iris_nbayes.train(x=list(range(4)), y=4, training_frame=iris, validation_frame=iris) iris_nbayes.show() iris_nbayes = H2ONaiveBayesEstimator(nfolds=3) iris_nbayes.train(x=list(range(4)), y=4, training_frame=iris, validation_frame=iris, seed=1234) iris_nbayes.show() iris_nbayes = H2ONaiveBayesEstimator(nfolds=3) iris_nbayes.train(x=list(range(4)), y=4, training_frame=iris, seed=1234) iris_nbayes.show() iris_nbayes = H2ONaiveBayesEstimator(nfolds=3,fold_assignment="Modulo") iris_nbayes.train(x=list(range(4)), y=4, training_frame=iris) iris_nbayes.show() print("And here it is:") print(iris_nbayes.cross_validation_metrics_summary()) print(iris_nbayes.cross_validation_metrics_summary().as_data_frame()) print(iris_nbayes.cross_validation_metrics_summary().as_data_frame()['mean'])
def nb_init_err(): print "Importing iris_wheader.csv data...\n" iris = h2o.upload_file( pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) iris.describe print "Laplace smoothing parameter is negative" try: H2ONaiveBayesEstimator(laplace=-1).train(x=range(4), y=4, training_frame=iris) assert False, "Expected naive bayes algo to fail on negative laplace training parameter" except: pass print "Minimum standard deviation is zero" try: H2ONaiveBayesEstimator().train(x=range(4), y=4, training_frame=iris, min_sdev=0) assert False, "Expected naive bayes algo to fail on min_sdev = 0" except: pass print "Response column is not categorical" try: H2ONaiveBayesEstimator().train(x=range(3), y=3, min_sdev=0) assert False, "Expected naive bayes algo to fail on response not categorical" except: pass
def naive_bayes(xval=None, sample_size=None, nfolds=None, hparams=None, for_stacking=None): """ create a naive bayes algorithm estimator :param xval: if for cross-validation :param sample_size: training set sample amount :param nfolds: k value for k-fold cross-validation :param hparams: hyper parameters for grid search :param for_stacking: if it is used for stacking :return: a constructed naive bayes estimator, a parameters' dict for grid search """ if sample_size <= 50000: if sample_size <= 10000: default_nfolds = 3 else: default_nfolds = 5 laplace_opts = [0.1, 1, 5, 10] min_sdev_opts = [0.001, 0.005, 0.1] eps_sdev_opts = [0, 0.001, 0.01] elif 50000 < sample_size <= 500000: default_nfolds = 3 laplace_opts = [0.1, 1, 5] min_sdev_opts = [0.001, 0.1] eps_sdev_opts = [0, 0.01] else: default_nfolds = 2 laplace_opts = [0.1, 5] min_sdev_opts = [0.001, 0.005] eps_sdev_opts = [0, 0.01] default_hparams = dict({'laplace': laplace_opts, 'min_sdev': min_sdev_opts, 'eps_sdev': eps_sdev_opts}) if nfolds is None: nfolds = default_nfolds if hparams is None: hparams = default_hparams if xval: if for_stacking: nb_estimator = H2ONaiveBayesEstimator(nfolds=nfolds, fold_assignment="Modulo", seed=1, keep_cross_validation_predictions=True) else: nb_estimator = H2ONaiveBayesEstimator(nfolds=nfolds) else: nb_estimator = H2ONaiveBayesEstimator() return nb_estimator, hparams
def nb_baddata(): rawdata = [[random.gauss(0, 1) for r in range(100)] for c in range(10)] print "Training data with all NA's" train = [["NA" for r in range(100)] for c in range(10)] train_h2o = h2o.H2OFrame(python_obj=train) from h2o.estimators.naive_bayes import H2ONaiveBayesEstimator try: H2ONaiveBayesEstimator().train(x=range(1, 10), y=0, training_frame=train_h2o) assert False, "Expected naive bayes algo to fail on training data of all NA's" except: pass # Response column must be categorical print "Training data with a numeric response column" train_h2o = h2o.H2OFrame(python_obj=rawdata) try: H2ONaiveBayesEstimator().train(x=range(1, 10), y=0, training_frame=train_h2o) assert False, "Expected naive bayes algo to fail on training data with a numeric response column" except: pass # Constant response dropped before model building print "Training data with a constant response: drop and throw error" rawdata[0] = 100 * ["A"] train_h2o = h2o.H2OFrame(python_obj=rawdata) try: H2ONaiveBayesEstimator().train(x=range(1, 10), y=0, training_frame=train_h2o) assert False, "Expected naive bayes algo to fail on training data with a constant response: drop and throw error" except: pass # Predictors with constant value automatically dropped print "Training data with 1 col of all 5's: drop automatically" rawdata = [[random.gauss(0, 1) for r in range(100)] for c in range(10)] rawdata[4] = 100 * [5] rawdata[0] = [random.choice(string.letters) for _ in range(100)] train_h2o = h2o.H2OFrame(python_obj=rawdata) model = H2ONaiveBayesEstimator() model.train(x=range(10), y=0, training_frame=train_h2o) assert len(model._model_json['output']['pcond']) == 8, "Expected 8 predictors, but got {0}" \ "".format(len(model._model_json['output']['pcond']))
def naive_bayes_export(): print("###### NAIVE BAYES ######") frame = h2o.upload_file(pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) model = H2ONaiveBayesEstimator(laplace=0.25) model.train(x=list(range(4)), y=4, training_frame=frame) h2o.download_pojo(model, path=RESULT_DIR) expect_error(model.download_mojo, model="Naive Bayes", format='MOJO')
def test_naivebayes_grid_search_over_params(self): """ test_naivebayes_grid_search_over_params performs the following: run gridsearch model and then build each model manually and see if we receive the same error messages. """ print( "*******************************************************************************************" ) print("test_naivebayes_grid_search_over_params for naivebayes ") h2o.cluster_info() print("Hyper-parameters used here is {0}".format( self.final_hyper_params)) # # start grid search # grid_model = H2OGridSearch(H2ONaiveBayesEstimator(nfolds=self.nfolds), # hyper_params=self.final_hyper_params) # grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) # add parameters into params_dict. Use this to manually build model, one at a time params_dict = dict() params_dict["nfolds"] = self.nfolds manual_model = [None] * self.possible_number_models model_index = 0 for fold_v in self.final_hyper_params["fold_assignment"]: for max_t in self.final_hyper_params["max_runtime_secs"]: for laplace_v in self.final_hyper_params["laplace"]: params_list = dict() params_list["fold_assignment"] = fold_v params_list["max_runtime_secs"] = max_t params_list["laplace"] = laplace_v print("Hyper-parameters used here is {0}\n".format( params_list)) params_list.update(params_dict) model_params = dict() # need to taken out max_runtime_secs from model parameters, it is now set in .train() if "max_runtime_secs" in params_list: model_params["max_runtime_secs"] = params_list[ "max_runtime_secs"] max_runtime = params_list["max_runtime_secs"] del params_list["max_runtime_secs"] else: max_runtime = 0 manual_model[model_index] = H2ONaiveBayesEstimator( **params_list) manual_model[model_index].train( x=self.x_indices, y=self.y_index, training_frame=self.training1_data, **model_params) model_index += 1
def _run_builder(key, algorithm, training_dataset_key, y, x, model_type): try: client = _get_memcached_client() contents = get_dataset_contents(training_dataset_key) with tempfile.TemporaryDirectory() as tmpdir: dataset_path = os.path.join(tmpdir, TRAINING_FILE) with open(dataset_path, 'w') as training_file: training_file.write(contents) h2o.init() training_frame = h2o.import_file(dataset_path) if algorithm == MLAlgorithm.NAIVE_BAYES: # naivebayes expects the prediction response to be categorical training_frame[y] = training_frame[y].asfactor() estimator = H2ONaiveBayesEstimator() elif algorithm == MLAlgorithm.GRADIENT_BOOSTING_MACHINE: estimator = H2OGradientBoostingEstimator() kwargs = {'training_frame': training_frame, 'y': y} if x is not None: kwargs['x'] = x estimator.train(**kwargs) temp_folder = os.path.join(os.path.abspath(os.sep), 'tmp') if model_type.upper() == 'POJO': model_file = estimator.download_pojo( path=temp_folder, get_genmodel_jar=True, genmodel_name='h2o-genmodel.jar') else: model_file = estimator.download_mojo( path=temp_folder, get_genmodel_jar=True, genmodel_name='h2o-genmodel.jar') model_performance = estimator.model_performance() details = {'mse': model_performance.mse()} with zipfile.ZipFile(os.path.join(temp_folder, key), 'w') as zip: zip.write(model_file, os.path.basename(model_file)) zip.write(os.path.join(temp_folder, 'h2o-genmodel.jar'), 'h2o-genmodel.jar') client.set( key, json.dumps({ 'status': 'COMPLETE', 'description': 'Model has been built', 'details': details, 'path': model_file })) except Exception as ex: client.set(key, json.dumps({ 'status': 'FAILED', 'description': str(ex) })) logger.exception("Building model failed")
def test_naivebayes_grid_search_over_params(self): """ test_naivebayes_grid_search_over_params performs the following: run gridsearch model and then build each model manually and see if we receive the same error messages. """ print( "*******************************************************************************************" ) print("test_naivebayes_grid_search_over_params for naivebayes ") h2o.cluster_info() print("Hyper-parameters used here is {0}".format( self.final_hyper_params)) # # start grid search # grid_model = H2OGridSearch(H2ONaiveBayesEstimator(nfolds=self.nfolds), # hyper_params=self.final_hyper_params) # grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) # add parameters into params_dict. Use this to manually build model, one at a time params_dict = dict() params_dict["nfolds"] = self.nfolds params_list = dict() params_list["fold_assignment"] = self.final_hyper_params[ "fold_assignment"][0] # params_list["max_runtime_secs"] = self.final_hyper_params["max_runtime_secs"][1] params_list["max_runtime_secs"] = 10 # this will return full NB model # the field manual_model._model_json['output']['cross_validation_metrics_summary'].cell_values will be empty params_list[ "max_runtime_secs"] = 0.001 # this will not return full NB model params_list["laplace"] = self.final_hyper_params["laplace"][0] print("Hyper-parameters used here is {0}\n".format(params_list)) params_list.update(params_dict) model_params = dict() # need to taken out max_runtime_secs from model parameters, it is now set in .train() if "max_runtime_secs" in params_list: model_params["max_runtime_secs"] = params_list["max_runtime_secs"] max_runtime = params_list["max_runtime_secs"] del params_list["max_runtime_secs"] else: max_runtime = 0 manual_model = H2ONaiveBayesEstimator(**params_list) manual_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data, **model_params) print("Done!")
def nb_iris(): print("Importing iris_wheader.csv data...\n") iris = h2o.upload_file( pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) iris.describe() laplace_range = [0, 1, 0.25] for i in laplace_range: print("H2O Naive Bayes with Laplace smoothing = {0}".format(i)) iris_nbayes = H2ONaiveBayesEstimator(laplace=i) iris_nbayes.train(x=list(range(4)), y=4, training_frame=iris) iris_nbayes.show()
def grid_cars_NB(): cars = h2o.import_file( path=pyunit_utils.locate("smalldata/junit/cars_20mpg.csv")) r = cars[0].runif(seed=42) train = cars[r > .2] problem = random.sample(["binomial", "multinomial"], 1) predictors = ["displacement", "power", "weight", "acceleration", "year"] if problem == "binomial": response_col = "economy_20mpg" else: response_col = "cylinders" print("Predictors: {0}".format(predictors)) print("Response: {0}".format(response_col)) print("Converting the response column to a factor...") train[response_col] = train[response_col].asfactor() max_runtime_secs = 10 # this will return full NB model # the field manual_model._model_json['output']['cross_validation_metrics_summary'].cell_values will be empty max_runtime_secs = 0.001 model_params = { 'compute_metrics': True, 'fold_assignment': 'AUTO', 'laplace': 8.3532975, 'nfolds': 2 } cars_nb = H2ONaiveBayesEstimator(**model_params) cars_nb.train(x=predictors, y=response_col, training_frame=train, max_runtime_secs=max_runtime_secs) if len(cars_nb._model_json['output'] ['cross_validation_metrics_summary'].cell_values) > 0: print("Pass test. Complete metrics returned.") else: print("Failed test. Model metrics is missing.")
def nb_prostate(): print "Importing prostate.csv data..." prostate = h2o.upload_file( pyunit_utils.locate("smalldata/logreg/prostate.csv")) print "Converting CAPSULE, RACE, DCAPS, and DPROS to categorical" prostate['CAPSULE'] = prostate['CAPSULE'].asfactor() prostate['RACE'] = prostate['CAPSULE'].asfactor() prostate['DCAPS'] = prostate['DCAPS'].asfactor() prostate['DPROS'] = prostate['DPROS'].asfactor() print "Compare with Naive Bayes when x = 3:9, y = 2" from h2o.estimators.naive_bayes import H2ONaiveBayesEstimator prostate_nb = H2ONaiveBayesEstimator(laplace=0) prostate_nb.train(x=range(2, 9), y=1, training_frame=prostate) prostate_nb.show() print "Predict on training data" prostate_pred = prostate_nb.predict(prostate) prostate_pred.head()
def test_naivebayes_grid_search_over_params(self): """ test_naivebayes_grid_search_over_params performs the following: a. build H2O naivebayes models using grid search. Count and make sure models are only built for hyper-parameters set to legal values. No model is built for bad hyper-parameters values. We should instead get a warning/error message printed out. b. For each model built using grid search, we will extract the parameters used in building that model and manually build a H2O naivebayes model. Logloss are calculated from a test set to compare the performance of grid search model and our manually built model. If their metrics are close, declare test success. Otherwise, declare test failure. c. we will check and make sure the models are built within the max_runtime_secs time limit that was set for it as well. If max_runtime_secs was exceeded, declare test failure as well. """ print( "*******************************************************************************************" ) print("test_naivebayes_grid_search_over_params for naivebayes ") h2o.cluster_info() try: print("Hyper-parameters used here is {0}".format( self.final_hyper_params)) # start grid search grid_model = H2OGridSearch( H2ONaiveBayesEstimator(nfolds=self.nfolds), hyper_params=self.final_hyper_params) grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) self.correct_model_number = len( grid_model) # store number of models built # make sure the correct number of models are built by gridsearch if not (self.correct_model_number == self.possible_number_models): # wrong grid model number self.test_failed += 1 print( "test_naivebayes_grid_search_over_params for naivebayes failed: number of models built by " "gridsearch does not equal to all possible combinations of hyper-parameters" ) else: # add parameters into params_dict. Use this to manually build model params_dict = dict() params_dict["nfolds"] = self.nfolds params_dict["score_tree_interval"] = 0 total_run_time_limits = 0.0 # calculate upper bound of max_runtime_secs true_run_time_limits = 0.0 manual_run_runtime = 0.0 # compare performance metric of model built by gridsearch with manually built model for each_model in grid_model: params_list = grid_model.get_hyperparams_dict( each_model._id) params_list.update(params_dict) model_params = dict() # need to taken out max_runtime_secs from model parameters, it is now set in .train() if "max_runtime_secs" in params_list: model_params["max_runtime_secs"] = params_list[ "max_runtime_secs"] max_runtime = params_list["max_runtime_secs"] del params_list["max_runtime_secs"] else: max_runtime = 0 if "validation_frame" in params_list: model_params["validation_frame"] = params_list[ "validation_frame"] del params_list["validation_frame"] if "score_tree_interval" in params_list: model_params["score_tree_interval"] = params_list[ "score_tree_interval"] del params_list["score_tree_interval"] if "eps_prob" in params_list: model_params["eps_prob"] = params_list["eps_prob"] del params_list["eps_prob"] if "min_prob" in params_list: model_params["min_prob"] = params_list["min_prob"] del params_list["min_prob"] # make sure manual model was provided the same max_runtime_secs as the grid model each_model_runtime = pyunit_utils.find_grid_runtime( [each_model]) manual_model = H2ONaiveBayesEstimator(**params_list) manual_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data, **model_params) # collect the time taken to manually built all models model_runtime = pyunit_utils.find_grid_runtime( [manual_model]) # time taken to build this model manual_run_runtime += model_runtime if max_runtime > 0: # shortest possible time it takes to build this model if (max_runtime < self.model_run_time): total_run_time_limits += model_runtime else: total_run_time_limits += max_runtime true_run_time_limits += max_runtime # compute and compare test metrics between the two models test_grid_model_metrics = \ each_model.model_performance(test_data=self.training1_data)._metric_json[self.training_metric] test_manual_model_metrics = \ manual_model.model_performance(test_data=self.training1_data)._metric_json[self.training_metric] # just compare the mse in this case within tolerance: if (each_model_runtime > 0) and \ (abs(model_runtime - each_model_runtime)/each_model_runtime < self.allowed_runtime_diff) \ and (abs(test_grid_model_metrics - test_manual_model_metrics) > self.allowed_diff): self.test_failed += 1 # count total number of tests that have failed print( "test_naivebayes_grid_search_over_params for naivebayes failed: grid search model and manually " "built H2O model differ too much in test MSE!") break total_run_time_limits = max( total_run_time_limits, true_run_time_limits) * (1 + self.extra_time_fraction) # make sure the max_runtime_secs is working to restrict model built time if not (manual_run_runtime <= total_run_time_limits): self.test_failed += 1 print( "test_naivebayes_grid_search_over_params for naivebayes failed: time taken to manually build models is {0}." " Maximum allowed time is {1}".format( manual_run_runtime, total_run_time_limits)) if self.test_failed == 0: print( "test_naivebayes_grid_search_over_params for naivebayes has passed!" ) except: if self.possible_number_models > 0: print( "test_naivebayes_grid_search_over_params for naivebayes failed: exception was thrown for no reason." ) self.test_failed += 1
def test_naivebayes_grid_search_over_params(self): """ test_naivebayes_grid_search_over_params performs the following: run gridsearch model and then build each model manually and see if we receive the same error messages. """ print( "*******************************************************************************************" ) print("test_naivebayes_grid_search_over_params for naivebayes ") h2o.cluster_info() print("Hyper-parameters used here is {0}".format( self.final_hyper_params)) # start grid search grid_model = H2OGridSearch(H2ONaiveBayesEstimator(nfolds=self.nfolds), hyper_params=self.final_hyper_params) grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) # add parameters into params_dict. Use this to manually build model, one at a time params_dict = dict() params_dict["nfolds"] = self.nfolds manual_model = [None] * len(grid_model) model_index = 0 for each_model in grid_model: params_list = grid_model.get_hyperparams_dict(each_model._id) params_list.update(params_dict) model_params = dict() # need to taken out max_runtime_secs from model parameters, it is now set in .train() if "max_runtime_secs" in params_list: model_params["max_runtime_secs"] = params_list[ "max_runtime_secs"] max_runtime = params_list["max_runtime_secs"] del params_list["max_runtime_secs"] else: max_runtime = 0 if "validation_frame" in params_list: model_params["validation_frame"] = params_list[ "validation_frame"] del params_list["validation_frame"] if "eps_prob" in params_list: model_params["eps_prob"] = params_list["eps_prob"] del params_list["eps_prob"] if "min_prob" in params_list: model_params["min_prob"] = params_list["min_prob"] del params_list["min_prob"] manual_model[model_index] = H2ONaiveBayesEstimator(**params_list) manual_model[model_index].train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data, **model_params) model_index += 1
# In[167]: cred_train.shape # In[168]: cred_test.shape # In[169]: from h2o.estimators.naive_bayes import H2ONaiveBayesEstimator # In[170]: cred_model = H2ONaiveBayesEstimator(model_id='cred_model', laplace=1) # In[171]: cred_model.train(x=features, y=results, training_frame=cred_train) cred_model.train(x=features, y=results, training_frame=cred_test) # In[172]: cred_model.model_performance(cred_train) # In[173]: cred_model.model_performance(cred_test) # In[174]:
firm_period_predictions = pd.read_csv(os.path.join(model_save_dir,"firm_period_predictions20171228.csv")) avg_predictions = firm_period_predictions.groupby('DealerTIN').mean() betas = predictions2betas_curve(avg_predictions,model_score_col,label_var,return_betas=True,plot_betas=False) betas.to_csv(os.path.join(model_save_dir,"betas20171231.csv")) fs_name = feature_sets_names[i] axes = predictions2betas_curve(avg_predictions,model_score_col,label_var,return_betas=False,plot_betas=True,label=fs_name,axes=axes) axes.set_title('betas curves for various feature sets'.title()) plt.legend() CLASSIFIER_DATE_STR = '20180415' CLASSIFIERS = { 'RandomForest': H2ORandomForestEstimator(ntrees=200, keep_cross_validation_predictions=True, stopping_rounds=2, score_each_iteration=True, model_id="rf_cv_all_folds_"+CLASSIFIER_DATE_STR, seed=1000000), 'RandomForest_depth6': H2ORandomForestEstimator(ntrees=200, max_depth=6,keep_cross_validation_predictions=True, stopping_rounds=2, score_each_iteration=True, model_id="rf_cv_all_folds_"+CLASSIFIER_DATE_STR, seed=1000000), 'GLM': H2OGeneralizedLinearEstimator(family= "binomial", lambda_ = 0, compute_p_values = True, remove_collinear_columns=True, keep_cross_validation_predictions=True, model_id="glm_cv_all_folds_"+CLASSIFIER_DATE_STR, seed=1000000), # todo: regularization? http://docs.h2o.ai/h2o/latest-stable/h2o-docs/data-science/glm.html#regularization-parameters-in-glm 'GBM': H2OGradientBoostingEstimator(ntrees=200, learn_rate=0.2, max_depth=20, stopping_tolerance=0.01, stopping_rounds=2, score_each_iteration=True, keep_cross_validation_predictions=True, model_id="gbm_cv_all_folds_"+CLASSIFIER_DATE_STR, seed=1000000), 'NaiveBayes': H2ONaiveBayesEstimator(keep_cross_validation_predictions=True,model_id="naive_bayes_cv_all_folds_"+CLASSIFIER_DATE_STR, seed=1000000) # 'XGBOOST': H2OXGBoostEstimator(ntrees=200, learn_rate=0.2, max_depth=20, stopping_tolerance=0.01, stopping_rounds=2, score_each_iteration=True, keep_cross_validation_predictions=True, model_id="gbm_cv_all_folds_"+CLASSIFIER_DATE_STR, seed=1000000) } # todo: categorical_encoding='one_hot_internal'? No. Works for GLM automatically, impossible for tree methods. def different_classifiers_save_predictions(ffemq12,model_score_col='model_score_bogus_online',label_var='bogus_online'): """ run different classification algorithms on the data and save CV predictions & models """ for algo_name,model in CLASSIFIERS.items(): print algo_name print '-'*30 print time.ctime() try: model_save_dir=r'D:\shekhar_code_github\BogusFirmCatching\Models\diff_classifiers\{}\classifier_{}'.format(CLASSIFIER_DATE_STR,algo_name) if not os.path.exists(model_save_dir):
# In[98]: cols = ['fac_type', 'risk','service_code','month','day','year','code_islarge'] _ = rf_model.partial_plot(data=model_data, cols=cols, nbins=200, figsize=(18, 20)) # In[99]: #NaiveBayes from h2o.estimators.naive_bayes import H2ONaiveBayesEstimator train, test = model_data.split_frame(ratios=[0.7]) naive_bayes = H2ONaiveBayesEstimator(nfolds=10, seed=42) naive_bayes.train(x=feature, y=target, training_frame=train, validation_frame=test) # In[100]: #Make predictions train_true = train.as_data_frame()['grade'].values test_true = test.as_data_frame()['grade'].values train_pred = naive_bayes.predict(train).as_data_frame()['p1'].values test_pred = naive_bayes.predict(test).as_data_frame()['p1'].values train_fpr, train_tpr, _ = roc_curve(train_true, train_pred) test_fpr, test_tpr, _ = roc_curve(test_true, test_pred) train_auc = np.round(auc(train_fpr, train_tpr), 3)
def javapredict(algo, equality, train, test, x, y, compile_only=False, **kwargs): print "Creating model in H2O" if algo == "gbm": model = H2OGradientBoostingEstimator(**kwargs) elif algo == "random_forest": model = H2ORandomForestEstimator(**kwargs) elif algo == "deeplearning": model = H2ODeepLearningEstimator(**kwargs) elif algo == "glm": model = H2OGeneralizedLinearEstimator(**kwargs) elif algo == "naive_bayes": model = H2ONaiveBayesEstimator(**kwargs) elif algo == "kmeans": model = H2OKMeansEstimator(**kwargs) elif algo == "pca": model = H2OPCA(**kwargs) else: raise (ValueError, "algo {0} is not supported".format(algo)) if algo == "kmeans" or algo == "pca": model.train(x=x, training_frame=train) else: model.train(x=x, y=y, training_frame=train) print model # HACK: munge model._id so that it conforms to Java class name. For example, change K-means to K_means. # TODO: clients should extract Java class name from header. regex = re.compile("[+\\-* !@#$%^&()={}\\[\\]|;:'\"<>,.?/]") pojoname = regex.sub("_", model._id) print "Downloading Java prediction model code from H2O" tmpdir = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "results", pojoname)) os.mkdir(tmpdir) h2o.download_pojo(model, path=tmpdir) h2o_genmodel_jar = os.path.join(tmpdir, "h2o-genmodel.jar") assert os.path.exists( h2o_genmodel_jar ), "Expected file {0} to exist, but it does not.".format(h2o_genmodel_jar) print "h2o-genmodel.jar saved in {0}".format(h2o_genmodel_jar) java_file = os.path.join(tmpdir, pojoname + ".java") assert os.path.exists( java_file), "Expected file {0} to exist, but it does not.".format( java_file) print "java code saved in {0}".format(java_file) print "Compiling Java Pojo" javac_cmd = [ "javac", "-cp", h2o_genmodel_jar, "-J-Xmx12g", "-J-XX:MaxPermSize=256m", java_file ] subprocess.check_call(javac_cmd) if not compile_only: print "Predicting in H2O" predictions = model.predict(test) predictions.summary() predictions.head() out_h2o_csv = os.path.join(tmpdir, "out_h2o.csv") h2o.download_csv(predictions, out_h2o_csv) assert os.path.exists( out_h2o_csv ), "Expected file {0} to exist, but it does not.".format(out_h2o_csv) print "H2O Predictions saved in {0}".format(out_h2o_csv) print "Setting up for Java POJO" in_csv = os.path.join(tmpdir, "in.csv") h2o.download_csv(test[x], in_csv) # hack: the PredictCsv driver can't handle quoted strings, so remove them f = open(in_csv, 'r+') csv = f.read() csv = re.sub('\"', '', csv) f.seek(0) f.write(csv) f.truncate() f.close() assert os.path.exists( in_csv), "Expected file {0} to exist, but it does not.".format( in_csv) print "Input CSV to PredictCsv saved in {0}".format(in_csv) print "Running PredictCsv Java Program" out_pojo_csv = os.path.join(tmpdir, "out_pojo.csv") cp_sep = ";" if sys.platform == "win32" else ":" java_cmd = [ "java", "-ea", "-cp", h2o_genmodel_jar + cp_sep + tmpdir, "-Xmx12g", "-XX:MaxPermSize=2g", "-XX:ReservedCodeCacheSize=256m", "hex.genmodel.tools.PredictCsv", "--header", "--model", pojoname, "--input", in_csv, "--output", out_pojo_csv ] p = subprocess.Popen(java_cmd, stdout=PIPE, stderr=STDOUT) o, e = p.communicate() print "Java output: {0}".format(o) assert os.path.exists( out_pojo_csv ), "Expected file {0} to exist, but it does not.".format(out_pojo_csv) predictions2 = h2o.upload_file(path=out_pojo_csv) print "Pojo predictions saved in {0}".format(out_pojo_csv) print "Comparing predictions between H2O and Java POJO" # Dimensions hr, hc = predictions.dim pr, pc = predictions2.dim assert hr == pr, "Expected the same number of rows, but got {0} and {1}".format( hr, pr) assert hc == pc, "Expected the same number of cols, but got {0} and {1}".format( hc, pc) # Value for r in range(hr): hp = predictions[r, 0] if equality == "numeric": pp = float.fromhex(predictions2[r, 0]) assert abs( hp - pp ) < 1e-4, "Expected predictions to be the same (within 1e-4) for row {0}, but got {1} and {2}".format( r, hp, pp) elif equality == "class": pp = predictions2[r, 0] assert hp == pp, "Expected predictions to be the same for row {0}, but got {1} and {2}".format( r, hp, pp) else: raise (ValueError, "equality type {0} is not supported".format(equality))
def test_naivebayes_grid_search_over_params(self): """ test_naivebayes_grid_search_over_params the following: a. grab all truely griddable parameters and randomly or manually set the parameter values. b. Next, build H2O naivebayes models using grid search. Count and make sure models are only built for hyper-parameters set to legal values. No model is built for bad hyper-parameters values. We should instead get a warning/error message printed out. c. For each model built using grid search, we will extract the parameters used in building that model and manually build a H2O naivebayes model. Training metrics are calculated from the gridsearch model and the manually built model. If their metrics differ by too much, print a warning message but don't fail the test. d. we will check and make sure the models are built within the max_runtime_secs time limit that was set for it as well. If max_runtime_secs was exceeded, declare test failure. """ print("*******************************************************************************************") print("test_naivebayes_grid_search_over_params for naivebayes ") h2o.cluster_info() try: print("Hyper-parameters used here is {0}".format(self.final_hyper_params)) # start grid search grid_model = H2OGridSearch(H2ONaiveBayesEstimator(nfolds=self.nfolds), hyper_params=self.final_hyper_params) grid_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) self.correct_model_number = len(grid_model) # store number of models built # make sure the correct number of models are built by gridsearch if not (self.correct_model_number == self.possible_number_models): # wrong grid model number self.test_failed += 1 print("test_naivebayes_grid_search_over_params for naivebayes failed: number of models built by " "gridsearch {0} does not equal to all possible combinations of hyper-parameters " "{1}".format(self.correct_model_number, self.possible_number_models)) else: # add parameters into params_dict. Use this to manually build model params_dict = dict() params_dict["nfolds"] = self.nfolds total_run_time_limits = 0.0 # calculate upper bound of max_runtime_secs true_run_time_limits = 0.0 manual_run_runtime = 0.0 gridsearch_runtime = 0.0 # compare performance metric of model built by gridsearch with manually built model for each_model in grid_model: params_list = grid_model.get_hyperparams_dict(each_model._id) params_list.update(params_dict) model_params = dict() # need to taken out max_runtime_secs from model parameters, it is now set in .train() if "max_runtime_secs" in params_list: model_params["max_runtime_secs"] = params_list["max_runtime_secs"] max_runtime = params_list["max_runtime_secs"] del params_list["max_runtime_secs"] else: max_runtime = 0 if "validation_frame" in params_list: model_params["validation_frame"] = params_list["validation_frame"] del params_list["validation_frame"] if "eps_prob" in params_list: model_params["eps_prob"] = params_list["eps_prob"] del params_list["eps_prob"] if "min_prob" in params_list: model_params["min_prob"] = params_list["min_prob"] del params_list["min_prob"] # make sure manual model was provided the same max_runtime_secs as the grid model each_model_runtime = pyunit_utils.find_grid_runtime([each_model]) gridsearch_runtime += each_model_runtime manual_model = H2ONaiveBayesEstimator(**params_list) manual_model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data, **model_params) # collect the time taken to manually built all models model_runtime = pyunit_utils.find_grid_runtime([manual_model]) # time taken to build this model manual_run_runtime += model_runtime if max_runtime > 0: # shortest possible time it takes to build this model if (max_runtime < self.model_run_time): total_run_time_limits += model_runtime else: total_run_time_limits += max_runtime true_run_time_limits += max_runtime # compute and compare test metrics between the two models grid_model_metrics = \ each_model.model_performance(test_data=self.training1_data)._metric_json[self.training_metric] manual_model_metrics = \ manual_model.model_performance(test_data=self.training1_data)._metric_json[self.training_metric] # just compare the mse in this case within tolerance: if not((type(grid_model_metrics) == str) or (type(manual_model_metrics) == str)): if (abs(grid_model_metrics) > 0) \ and (abs(grid_model_metrics - manual_model_metrics)/grid_model_metrics > self.allowed_diff): print("test_naivebayes_grid_search_over_params for naivebayes WARNING\ngrid search model " "{0}: {1}, time taken to build (secs): {2}\n and manually built H2O model {3}: {4}, " "time taken to build (secs): {5}\ndiffer too much!" "".format(self.training_metric, grid_model_metrics, each_model_runtime, self.training_metric, manual_model_metrics, model_runtime)) print("Time taken for gridsearch to build all models (sec): {0}\n Time taken to manually build all " "models (sec): {1}, total run time limits (sec): " "{2}".format(gridsearch_runtime, manual_run_runtime, total_run_time_limits)) total_run_time_limits = max(total_run_time_limits, true_run_time_limits) * (1+self.extra_time_fraction) # make sure the max_runtime_secs is working to restrict model built time if not(manual_run_runtime <= total_run_time_limits): self.test_failed += 1 print("test_naivebayes_grid_search_over_params for naivebayes failed: time taken to manually build " "models is {0}. Maximum allowed time " "is {1}".format(manual_run_runtime, total_run_time_limits)) if self.test_failed == 0: print("test_naivebayes_grid_search_over_params for naivebayes has passed!") except Exception as e: if self.possible_number_models > 0: print("test_naivebayes_grid_search_over_params for naivebayes failed: exception ({0}) was thrown for " "no reason.".format(e)) self.test_failed += 1
def setup_model(self): """ This function setup the gridsearch hyper-parameters that will be used later on: 1. It will first try to grab all the parameters that are griddable and parameters used by naivebayes. 2. It will find the intersection of parameters that are both griddable and used by naivebayes. 3. There are several extra parameters that are used by naivebayes that are denoted as griddable but actually are not. These parameters have to be discovered manually and they are captured in self.exclude_parameter_lists. 4. We generate the gridsearch hyper-parameter. For numerical parameters, we will generate those randomly. For enums, we will include all of them. :return: None """ # build bare bone model to get all parameters model = H2ONaiveBayesEstimator(nfolds=self.nfolds, compute_metrics=True) model.train(x=self.x_indices, y=self.y_index, training_frame=self.training1_data) self.model_run_time = pyunit_utils.find_grid_runtime([model]) # find model train time print("Time taken to build a base barebone model is {0}".format(self.model_run_time)) # grab all gridable parameters and its type (self.gridable_parameters, self.gridable_types, self.gridable_defaults) = \ pyunit_utils.get_gridables(model._model_json["parameters"]) # randomly generate griddable parameters including values outside legal range, like setting alpha values to # be outside legal range of 0 and 1 and etc (self.hyper_params, self.gridable_parameters, self.gridable_types, self.gridable_defaults) = \ pyunit_utils.gen_grid_search(model.full_parameters.keys(), self.hyper_params, self.exclude_parameter_lists, self.gridable_parameters, self.gridable_types, self.gridable_defaults, random.randint(1, self.max_int_number), self.max_int_val, self.min_int_val, random.randint(1, self.max_real_number), self.max_real_val, self.min_real_val) # scale the max_runtime_secs parameter and others as well to make sure they make sense time_scale = self.time_scale * self.model_run_time if "max_runtime_secs" in list(self.hyper_params): self.hyper_params["max_runtime_secs"] = [time_scale * x for x in self.hyper_params["max_runtime_secs"]] # generate a new final_hyper_params which only takes a subset of all griddable parameters while # hyper_params take all griddable parameters and generate the grid search hyper-parameters [self.possible_number_models, self.final_hyper_params] = \ pyunit_utils.check_and_count_models(self.hyper_params, self.params_zero_one, self.params_more_than_zero, self.params_more_than_one, self.params_zero_positive, self.max_grid_model) final_hyper_params_keys = list(self.final_hyper_params) # must add max_runtime_secs to restrict unit test run time and as a promise to Arno to test for this if ("max_runtime_secs" not in final_hyper_params_keys) and \ ("max_runtime_secs" in list(self.hyper_params)): self.final_hyper_params["max_runtime_secs"] = self.hyper_params["max_runtime_secs"] len_good_time = len([x for x in self.hyper_params["max_runtime_secs"] if (x >= 0)]) self.possible_number_models = self.possible_number_models*len_good_time # need to check that min_prob >= 1e-10 if "min_prob" in final_hyper_params_keys: old_len_prob = len([x for x in self.final_hyper_params["max_runtime_secs"] if (x >= 0)]) good_len_prob = len([x for x in self.final_hyper_params["max_runtime_secs"] if (x >= 1e-10)]) self.possible_number_models = self.possible_number_models*good_len_prob/old_len_prob if "laplace" in final_hyper_params_keys: self.final_hyper_params["laplace"] = [self.laplace_scale * x for x in self.hyper_params["laplace"]] # write out the hyper-parameters used into json files. pyunit_utils.write_hyper_parameters_json(self.current_dir, self.sandbox_dir, self.json_filename, self.final_hyper_params)
def algo_max_runtime_secs(): ''' This pyunit test is written to ensure that column names and column types are returned in the model output for every algorithm supported by H2O. See PUBDEV-5801. ''' seed = 12345 print("Checking GLM.....") training1_data = h2o.import_file( path=pyunit_utils.locate("smalldata/junit/cars_20mpg.csv")) model = H2OGeneralizedLinearEstimator(family="binomial", alpha=1.0, lambda_search=False, max_iterations=2, seed=seed) checkColumnNamesTypesReturned( training1_data, model, ["displacement", "power", "weight", "acceleration", "year"], y_index="economy_20mpg") print("Checking GLRM.....") irisH2O = h2o.upload_file( pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) glrm_h2o = H2OGeneralizedLowRankEstimator(k=3, loss="Quadratic", gamma_x=0.5, gamma_y=0.5, transform="STANDARDIZE") checkColumnNamesTypesReturned(irisH2O, glrm_h2o, irisH2O.names) print("Checking NaiveBayes......") model = H2ONaiveBayesEstimator(laplace=0.25) x_indices = irisH2O.names y_index = x_indices[-1] x_indices.remove(y_index) checkColumnNamesTypesReturned(irisH2O, model, x_indices, y_index=y_index) # deeplearning print("Checking deeplearning.....") training1_data = h2o.import_file(path=pyunit_utils.locate( "smalldata/gridsearch/gaussian_training1_set.csv")) x_indices = training1_data.names y_index = x_indices[-1] x_indices.remove(y_index) model = H2ODeepLearningEstimator(distribution='gaussian', seed=seed, hidden=[10, 10, 10]) checkColumnNamesTypesReturned(training1_data, model, x_indices, y_index=y_index) # stack ensemble, stacking part is not iterative print( "******************** Skip testing stack ensemble. Test done in pyunit_stackedensemble_regression.py." ) # GBM run print("Checking GBM.....") training1_data = h2o.import_file(path=pyunit_utils.locate( "smalldata/gridsearch/multinomial_training1_set.csv")) x_indices = training1_data.names y_index = x_indices[-1] x_indices.remove(y_index) training1_data[y_index] = training1_data[y_index].round().asfactor() model = H2OGradientBoostingEstimator(distribution="multinomial", seed=seed) checkColumnNamesTypesReturned(training1_data, model, x_indices, y_index=y_index) # random foreset print("Checking Random Forest.....") model = H2ORandomForestEstimator(ntrees=100, score_tree_interval=0) checkColumnNamesTypesReturned(training1_data, model, x_indices, y_index=y_index) # PCA print("Checking PCA.....") training1_data = h2o.import_file( path=pyunit_utils.locate("smalldata/gridsearch/pca1000by25.csv")) x_indices = training1_data.names model = H2OPCA(k=10, transform="STANDARDIZE", pca_method="Power", compute_metrics=True) checkColumnNamesTypesReturned(training1_data, model, x_indices) # kmeans print("Checking kmeans....") training1_data = h2o.import_file(path=pyunit_utils.locate( "smalldata/gridsearch/kmeans_8_centers_3_coords.csv")) x_indices = training1_data.names model = H2OKMeansEstimator(k=10) checkColumnNamesTypesReturned(training1_data, model, x_indices) # word2vec print("Checking word2vec....") train = h2o.import_file(pyunit_utils.locate("bigdata/laptop/text8.gz"), header=1, col_types=["string"]) used = train[0:170000, 0] w2v_model = H2OWord2vecEstimator() checkColumnNamesTypesReturned(train, w2v_model, [], 0)
def basic_inference_works_for_DRF_and_NB_test(): train = h2o.import_file( pyunit_utils.locate("smalldata/iris/iris_train.csv")) test = h2o.import_file(pyunit_utils.locate("smalldata/iris/iris_test.csv")) x_class = train.columns y_class = "species" x_class.remove(y_class) nfolds = 2 nb_class = H2ONaiveBayesEstimator( nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, ) nb_class.train(x=x_class, y=y_class, training_frame=train) gbm_class = H2OGradientBoostingEstimator( nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, ) gbm_class.train(x=x_class, y=y_class, training_frame=train) drf_class = H2ORandomForestEstimator( nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True) drf_class.train(x=x_class, y=y_class, training_frame=train) se_class_0 = H2OStackedEnsembleEstimator( training_frame=train, validation_frame=test, base_models=[nb_class, gbm_class, drf_class], metalearner_algorithm="gbm") se_class_0.train(x_class, y_class, train) assert se_class_0.metalearner().actual_params.get("distribution") == "multinomial", \ "Expected distribution {} but got {}".format("multinomial", se_class_0.metalearner().actual_params.get("distribution")) se_class_1 = H2OStackedEnsembleEstimator( training_frame=train, validation_frame=test, base_models=[gbm_class, drf_class, nb_class], metalearner_algorithm="gbm") se_class_1.train(x_class, y_class, train) assert se_class_1.metalearner().actual_params.get("distribution") == "multinomial", \ "Expected distribution {} but got {}".format("multinomial", se_class_1.metalearner().actual_params.get("distribution")) se_class_2 = H2OStackedEnsembleEstimator( training_frame=train, validation_frame=test, base_models=[drf_class, nb_class, gbm_class], metalearner_algorithm="gbm") se_class_2.train(x_class, y_class, train) assert se_class_2.metalearner().actual_params.get("distribution") == "multinomial", \ "Expected distribution {} but got {}".format("multinomial", se_class_2.metalearner().actual_params.get("distribution")) se_class_3 = H2OStackedEnsembleEstimator( training_frame=train, validation_frame=test, base_models=[nb_class, gbm_class, drf_class]) se_class_3.train(x_class, y_class, train) assert se_class_3.metalearner().actual_params.get("family") == "multinomial", \ "Expected family {} but got {}".format("multinomial", se_class_3.metalearner().actual_params.get("family")) se_class_4 = H2OStackedEnsembleEstimator( training_frame=train, validation_frame=test, base_models=[gbm_class, drf_class, nb_class]) se_class_4.train(x_class, y_class, train) assert se_class_4.metalearner().actual_params.get("family") == "multinomial", \ "Expected family {} but got {}".format("multinomial", se_class_4.metalearner().actual_params.get("family")) se_class_5 = H2OStackedEnsembleEstimator( training_frame=train, validation_frame=test, base_models=[drf_class, nb_class, gbm_class]) se_class_5.train(x_class, y_class, train) assert se_class_5.metalearner().actual_params.get("family") == "multinomial", \ "Expected family {} but got {}".format("multinomial", se_class_5.metalearner().actual_params.get("family"))
def stackedensemble_multinomial_test(): """This test check the following (for multinomial regression): 1) That H2OStackedEnsembleEstimator executes w/o errors on a 6-model manually constructed ensemble. 2) That .predict() works on a stack. 3) That .model_performance() works on a stack. 4) That test performance is better on ensemble vs the base learners. 5) That the validation_frame arg on H2OStackedEnsembleEstimator works correctly. """ df = h2o.import_file( path=pyunit_utils.locate("bigdata/laptop/mnist/test.csv.gz")) y = "C785" x = list(range(784)) df[y] = df[y].asfactor() train = df[0:5000, :] test = df[5000:10000, :] # Number of CV folds (to generate level-one data for stacking) nfolds = 2 # train and cross-validate a GBM my_gbm = H2OGradientBoostingEstimator( distribution="multinomial", nfolds=nfolds, ntrees=10, fold_assignment="Modulo", keep_cross_validation_predictions=True, seed=1) my_gbm.train(x=x, y=y, training_frame=train) # evaluate the performance perf_gbm_train = my_gbm.model_performance() perf_gbm_test = my_gbm.model_performance(test_data=test) print("GBM training performance: ") print(perf_gbm_train) print("GBM test performance: ") print(perf_gbm_test) # train and cross-validate a RF my_rf = H2ORandomForestEstimator(ntrees=10, nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, seed=1) my_rf.train(x=x, y=y, training_frame=train) # evaluate performance perf_rf_train = my_rf.model_performance() perf_rf_test = my_rf.model_performance(test_data=test) print("RF training performance: ") print(perf_rf_train) print("RF test performance: ") print(perf_rf_test) # Train and cross-validate an XGBoost GBM my_xgb = H2OXGBoostEstimator(ntrees=10, nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, seed=1) my_xgb.train(x=x, y=y, training_frame=train) # evaluate performance perf_xgb_train = my_xgb.model_performance() perf_xgb_test = my_xgb.model_performance(test_data=test) print("XGB training performance: ") print(perf_xgb_train) print("XGB test performance: ") print(perf_xgb_test) # Train and cross-validate a Naive Bayes model my_nb = H2ONaiveBayesEstimator(nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, seed=1) my_nb.train(x=x, y=y, training_frame=train) # evaluate performance perf_nb_train = my_nb.model_performance() perf_nb_test = my_nb.model_performance(test_data=test) print("NB training performance: ") print(perf_nb_train) print("NB test performance: ") print(perf_nb_test) # Train and cross-validate a Deep Learning model my_dnn = H2ODeepLearningEstimator(hidden=[10, 10], nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, seed=1) my_dnn.train(x=x, y=y, training_frame=train) # evaluate performance perf_dnn_train = my_dnn.model_performance() perf_dnn_test = my_dnn.model_performance(test_data=test) print("DNN training performance: ") print(perf_dnn_train) print("DNN test performance: ") print(perf_dnn_test) # Train and cross-validate a GLM model my_glm = H2OGeneralizedLinearEstimator( family="multinomial", nfolds=nfolds, fold_assignment="Modulo", keep_cross_validation_predictions=True, seed=1) my_glm.train(x=x, y=y, training_frame=train) # evaluate performance perf_glm_train = my_glm.model_performance() perf_glm_test = my_glm.model_performance(test_data=test) print("GLM training performance: ") print(perf_glm_train) print("GLM test performance: ") print(perf_glm_test) # Train a stacked ensemble using the GBM and GLM above stack = H2OStackedEnsembleEstimator(base_models=[ my_gbm.model_id, my_rf.model_id, my_xgb.model_id, my_nb.model_id, my_dnn.model_id, my_glm.model_id ]) stack.train( x=x, y=y, training_frame=train, validation_frame=test) # also test that validation_frame is working assert isinstance( stack, h2o.estimators.stackedensemble.H2OStackedEnsembleEstimator) assert stack.type == "classifier" # Check that prediction works pred = stack.predict(test_data=test) print(pred) assert pred.nrow == test.nrow, "expected " + str( pred.nrow) + " to be equal to " + str(test.nrow) assert pred.ncol == 11, "expected " + str( pred.ncol) + " to be equal to 1 but it was equal to " + str(pred.ncol) # Evaluate ensemble performance perf_stack_train = stack.model_performance() assert isinstance(perf_stack_train, h2o.model.metrics_base.H2OMultinomialModelMetrics) perf_stack_valid = stack.model_performance(valid=True) assert isinstance(perf_stack_valid, h2o.model.metrics_base.H2OMultinomialModelMetrics) perf_stack_test = stack.model_performance(test_data=test) assert isinstance(perf_stack_test, h2o.model.metrics_base.H2OMultinomialModelMetrics) # Check that stack perf is better (smaller) than the best (smaller) base learner perf: # Test Mean Per Class Error for each base learner baselearner_best_mean_per_class_error_test = min(perf_gbm_test.mean_per_class_error(), \ perf_rf_test.mean_per_class_error(), \ perf_xgb_test.mean_per_class_error(), \ perf_nb_test.mean_per_class_error(), \ perf_dnn_test.mean_per_class_error(), perf_glm_test.mean_per_class_error()) stack_mean_per_class_error_test = perf_stack_test.mean_per_class_error() print("Best Base-learner Test Mean Per Class Error: {0}".format( baselearner_best_mean_per_class_error_test)) print("Ensemble Test Mean Per Class Error: {0}".format( stack_mean_per_class_error_test)) assert stack_mean_per_class_error_test <= baselearner_best_mean_per_class_error_test, + \ "expected stack_mean_per_class_error_test would be less than " \ " baselearner_best_mean_per_class_error_test, found it wasn't " \ "baselearner_best_mean_per_class_error_test = "+ \ str(baselearner_best_mean_per_class_error_test) + \ ",stack_mean_per_class_error_test = "+ \ str(stack_mean_per_class_error_test) # Check that passing `test` as a validation_frame produces the same metric as stack.model_performance(test) # since the metrics object is not exactly the same, we can just test that RSME is the same perf_stack_validation_frame = stack.model_performance(valid=True) assert stack_mean_per_class_error_test == perf_stack_validation_frame.mean_per_class_error(), \ "expected stack_mean_per_class_error_test to be the same as " \ "perf_stack_validation_frame.mean_per_class_error() found it wasn't" \ "perf_stack_validation_frame.mean_per_class_error() = " + \ str(perf_stack_validation_frame.mean_per_class_error()) + \ "stack_mean_per_class_error_test was " + \ str(stack_mean_per_class_error_test)