def pca_export(): print("###### PCA ######") frame = h2o.upload_file(pyunit_utils.locate("smalldata/pca_test/USArrests.csv")) model = H2OPrincipalComponentAnalysisEstimator(k=3, impute_missing=True) model.train(x=list(range(4)), training_frame=frame) h2o.download_pojo(model, path=RESULT_DIR) expect_error(model.download_mojo, model="PCA", format='MOJO')
def scale_pca_rf_pipe_new_import(): from h2o.estimators.pca import H2OPrincipalComponentAnalysisEstimator iris = h2o.import_file(path=pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) # build transformation pipeline using sklearn's Pipeline and H2O transforms pipe = Pipeline([ ("standardize", H2OScaler()), ("pca", H2OPrincipalComponentAnalysisEstimator().init_for_pipeline()), ("rf", H2ORandomForestEstimator()) ]) params = {"standardize__center": [True, False], # Parameters to test "standardize__scale": [True, False], "pca__k": randint(2, iris[1:].shape[1]), "rf__ntrees": randint(50,60), "rf__max_depth": randint(4,8), "rf__min_rows": randint(5,10), "pca__transform": ["none", "standardize"], } custom_cv = H2OKFold(iris, n_folds=5, seed=42) random_search = RandomizedSearchCV(pipe, params, n_iter=5, scoring=make_scorer(h2o_r2_score), cv=custom_cv, random_state=42, n_jobs=1) random_search.fit(iris[1:],iris[0]) print(random_search.best_estimator_)
def kmeans_model(df, xValues): hf = h2o.H2OFrame(df) train, valid, test = hf.split_frame(ratios=[.8, .1]) # kmeans model kmeans = H2OKMeansEstimator(k=3,max_iterations=5,seed = 10,categorical_encoding = "AUTO",max_runtime_secs=10) kmeans.train(xValues, training_frame= hf) # pca model, generate Principal Components for further modelling or plotting pca = H2OPrincipalComponentAnalysisEstimator(k=4) # pca.train(xValues, training_frame= hf) pca.train(list(df.columns), training_frame= hf) pca_features = pca.predict(hf).as_data_frame() pca_metric = pca.summary().as_data_frame() # model metrics cluster_column = kmeans.predict(hf).as_data_frame() # The Between Cluster Sum-of-Square Error inter_cluster_error = kmeans.betweenss() # Within Cluster Sum-of-Square Error intra_cluster_error = kmeans.withinss() # Centroids centroids = kmeans.centers() # Size of clusters cluster_size = kmeans.size() # cluster_column.columns = ['cluster'] frames = [df,cluster_column] transformed_data = pd.concat(frames, axis=1) output = [transformed_data, pca_features, pca_metric, centroids, inter_cluster_error, intra_cluster_error, cluster_size] return output
def scale_pca_rf_pipe_new_import(): from h2o.transforms.preprocessing import H2OScaler from h2o.estimators.pca import H2OPrincipalComponentAnalysisEstimator from h2o.estimators.random_forest import H2ORandomForestEstimator from sklearn.pipeline import Pipeline iris = h2o.import_file( path=pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) # build transformation pipeline using sklearn's Pipeline and H2O estimators without H2OPrincipalComponentAnalysisEstimator.init_for_pipeline() # it should fail # Note: if you use PCA algo in a different combination of pipeline tasks, it could not fail, for example # if you comment line with H2ORandomForestEstimator task, the fit method doesn't fail because the pipeline doesn't # use _fit_transform_one method thus does not use H2OPrincipalComponentAnalysisEstimator.transform method try: pipe = Pipeline([("standardize", H2OScaler()), ("pca", H2OPrincipalComponentAnalysisEstimator(k=2)), ("rf", H2ORandomForestEstimator(seed=42, ntrees=5))]) pipe.fit(iris[:4], iris[4]) assert False, "Pipeline should fail without using H2OPrincipalComponentAnalysisEstimator.init_for_pipeline()" except TypeError: pass # build transformation pipeline using sklearn's Pipeline and H2O estimators with H2OPrincipalComponentAnalysisEstimator.init_for_pipeline() pipe = Pipeline([ ("standardize", H2OScaler()), ("pca", H2OPrincipalComponentAnalysisEstimator(k=2).init_for_pipeline()), ("rf", H2ORandomForestEstimator(seed=42, ntrees=5)) ]) pipe.fit(iris[:4], iris[4]) print(pipe) # set H2OPCA transform property pca = H2OPrincipalComponentAnalysisEstimator(k=2) pca.transform = "standardize" pipe = Pipeline([("standardize", H2OScaler()), ("pca", pca.init_for_pipeline()), ("rf", H2ORandomForestEstimator(seed=42, ntrees=5))]) pipe.fit(iris[:4], iris[4]) print(pipe)
def scale_pca_rf_pipe(): from h2o.transforms.preprocessing import H2OScaler from h2o.estimators.pca import H2OPrincipalComponentAnalysisEstimator from h2o.estimators.random_forest import H2ORandomForestEstimator from sklearn.pipeline import Pipeline iris = h2o.import_file( path=pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) # build transformation pipeline using sklearn's Pipeline and H2O transforms pipe = Pipeline([("standardize", H2OScaler()), ("pca", H2OPrincipalComponentAnalysisEstimator(k=2)), ("rf", H2ORandomForestEstimator(seed=42, ntrees=50))]) pipe.fit(iris[:4], iris[4])
def test_pca_screeplot(): import matplotlib import matplotlib.pyplot as plt matplotlib.use("agg") arrestsH2O = h2o.upload_file( pyunit_utils.locate("smalldata/pca_test/USArrests.csv")) fitH2O = H2OPrincipalComponentAnalysisEstimator(k=4, transform="DEMEAN") fitH2O.train(x=list(range(4)), training_frame=arrestsH2O) # The following should not fail fitH2O.screeplot() fitH2O.screeplot(server=True) fitH2O.screeplot(type="lines", server=True) # Free the memory plt.close("all")
def scale_pca_rf_pipe(): from h2o.transforms.preprocessing import H2OScaler from h2o.estimators.pca import H2OPrincipalComponentAnalysisEstimator from h2o.estimators.random_forest import H2ORandomForestEstimator from sklearn.pipeline import Pipeline from sklearn.grid_search import RandomizedSearchCV from h2o.cross_validation import H2OKFold from h2o.model.regression import h2o_r2_score from sklearn.metrics.scorer import make_scorer from scipy.stats import randint iris = h2o.import_file( path=pyunit_utils.locate("smalldata/iris/iris_wheader.csv")) # build transformation pipeline using sklearn's Pipeline and H2O transforms pipe = Pipeline([("standardize", H2OScaler()), ("pca", H2OPrincipalComponentAnalysisEstimator()), ("rf", H2ORandomForestEstimator())]) params = { "standardize__center": [True, False], # Parameters to test "standardize__scale": [True, False], "pca__k": randint(2, iris[1:].shape[1]), "rf__ntrees": randint(50, 60), "rf__max_depth": randint(4, 8), "rf__min_rows": randint(5, 10), } custom_cv = H2OKFold(iris, n_folds=5, seed=42) random_search = RandomizedSearchCV(pipe, params, n_iter=5, scoring=make_scorer(h2o_r2_score), cv=custom_cv, random_state=42, n_jobs=1) random_search.fit(iris[1:], iris[0]) print(random_search.best_estimator_)
def build_mojo_pipeline(): results_dir = pyunit_utils.locate("results") iris_csv = pyunit_utils.locate('smalldata/iris/iris_train.csv') iris = h2o.import_file(iris_csv) pca = H2OPrincipalComponentAnalysisEstimator(k=2) pca.train(training_frame=iris) principal_components = pca.predict(iris) km = H2OKMeansEstimator(k=3) km.train(training_frame=principal_components) pca_mojo_path = pca.download_mojo(path=results_dir) km_mojo_path = km.download_mojo(get_genmodel_jar=True, path=results_dir) java_cmd = [ "java", "-cp", os.path.join(results_dir, "h2o-genmodel.jar"), "hex.genmodel.tools.BuildPipeline", "--mapping" ] pca_mojo_name = os.path.basename(pca_mojo_path).split('.')[0] for i, pc in enumerate(principal_components.columns): mapping = pc + '=' + pca_mojo_name + ':' + str(i) java_cmd += [mapping] java_cmd += [ "--output", os.path.join(results_dir, "pipe.zip"), "--input", km_mojo_path, pca_mojo_path ] subprocess.Popen(java_cmd, stdout=PIPE, stderr=STDOUT).communicate() h2o_preds = km.predict(principal_components) mojo_preds_raw = h2o.mojo_predict_csv(input_csv_path=iris_csv, mojo_zip_path=os.path.join( results_dir, "pipe.zip")) mojo_preds = h2o.H2OFrame([c['cluster'] for c in mojo_preds_raw], column_names=['predict']) assert (mojo_preds == h2o_preds).mean()[0, "predict"] == 1
def pca_arrests(): print("Importing USArrests.csv data...") arrests = h2o.upload_file( pyunit_utils.locate("smalldata/pca_test/USArrests.csv")) arrests.describe() # import from h2o.transforms.decomposition for i in range(4): print("H2O PCA with " + str(i) + " dimensions:\n") print("Using these columns: {0}".format(arrests.names)) pca_h2o = H2OPCA(k=i + 1) pca_h2o.train(x=list(range(4)), training_frame=arrests) # TODO: pca_h2o.show() # import from h2o.estimators.pca for i in range(4): print("H2O PCA with " + str(i) + " dimensions:\n") print("Using these columns: {0}".format(arrests.names)) pca_h2o = H2OPrincipalComponentAnalysisEstimator(k=i + 1) pca_h2o.train(x=list(range(4)), training_frame=arrests)
def pca_mojo(): h2o.remove_all() NTESTROWS = 200 # number of test dataset rows df = pyunit_utils.random_dataset("regression", ncol_upper=8000, ncol_lower=5000, missing_fraction=0.001, seed=1234) train = df[NTESTROWS:, :] test = df[:NTESTROWS, :] x = df.names transform_types = [ "NONE", "STANDARDIZE", "NORMALIZE", "DEMEAN", "DESCALE" ] # pyunit test loop through transform for transformN in transform_types: # compare H2O predict and mojo predict for all dataset transform types pcaModel = H2OPrincipalComponentAnalysisEstimator( k=3, transform=transformN, seed=1234, impute_missing=True, use_all_factor_levels=False) pcaModel.train(x=x, training_frame=train) pyunit_utils.saveModelMojo(pcaModel) # save mojo model MOJONAME = pyunit_utils.getMojoName(pcaModel._id) TMPDIR = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath('__file__')), "..", "results", MOJONAME)) h2o.download_csv(test[x], os.path.join( TMPDIR, 'in.csv')) # save test file, h2o predict/mojo use same file pred_h2o, pred_mojo = pyunit_utils.mojo_predict( pcaModel, TMPDIR, MOJONAME) # save mojo predict for col in range(pred_h2o.ncols): if pred_h2o[col].isfactor(): pred_h2o[col] = pred_h2o[col].asnumeric() print("Comparing mojo predict and h2o predict...") pyunit_utils.compare_frames_local(pred_h2o, pred_mojo, 1, tol=1e-10)
def kMeans_model(data_temp, Vals, parametersObj, obj_t): print('Performing KMeans modelling') hf = h2o.H2OFrame(data_temp) predictors = list(data_temp.columns) # split into train and validation sets train, valid = hf.split_frame(ratios=[.8], seed=1234) num_data = data_temp.select_dtypes( include=['number', 'int', 'float']).copy() # try using the `k` parameter: # build the model with three clusters # initialize the estimator then train the model try: hf_kmeans = H2OKMeansEstimator( max_iterations=int(parametersObj['max_iterations']), score_each_iteration=bool(parametersObj['score_each_iteration']), ignore_const_cols=bool(parametersObj['ignore_const_cols']), k=int(parametersObj['kvalue']), max_runtime_secs=int(parametersObj['max_runtime_secs']), categorical_encoding=str(parametersObj['categoricalencoding']), standardize=bool(parametersObj['standardize']), estimate_k=bool(parametersObj['estimate_k'])) hf_kmeans.train(x=predictors, training_frame=train, validation_frame=valid) except Exception: hf_kmeans = H2OKMeansEstimator( max_iterations=int(parametersObj['max_iterations']), score_each_iteration=bool(parametersObj['score_each_iteration']), ignore_const_cols=bool(parametersObj['ignore_const_cols']), k=int(parametersObj['kvalue']), max_runtime_secs=int(parametersObj['max_runtime_secs']), categorical_encoding=str(parametersObj['categoricalencoding']), standardize=bool(parametersObj['standardize'])) hf_kmeans.train(x=predictors, training_frame=train, validation_frame=valid) clusters = hf_kmeans.predict(hf) data_temp['cluster'] = clusters.as_data_frame() data_temp['cluster'] = data_temp['cluster'] + 1 #k = int(parametersObj['kvalue']) ab_t = hf_kmeans.centroid_stats()['within_cluster_sum_of_squares'] within = pd.DataFrame(columns=['withiness'], index=range(len(ab_t))) #print(len(within)) within['withiness'] = hf_kmeans.centroid_stats( )['within_cluster_sum_of_squares'] #print(len(hf_kmeans.centroid_stats()['within_cluster_sum_of_squares'])) tss = hf_kmeans.totss() betweenss = hf_kmeans.betweenss() #center = hf_kmeans.centers(train = True) size = pd.DataFrame(columns=['cluster_size'], index=range(len(ab_t))) size['cluster_size'] = hf_kmeans.centroid_stats()['size'] ## PCA pca = H2OPrincipalComponentAnalysisEstimator(k=2, transform="STANDARDIZE", pca_method="GLRM", use_all_factor_levels=True, impute_missing=False, max_iterations=300) pca.train(training_frame=hf) cords = pca.rotation().as_data_frame() cords = cords[['pc1', 'pc2']] cords.columns = ['PC1', 'PC2'] print('here') num_data = data_temp.select_dtypes( include=['int', 'float', 'number']).copy() num_data_cols = list(num_data.columns) print('here1') cat_cols = list(filter(lambda x: x not in num_data_cols, data_temp.columns)) data_temp1 = data_temp[cat_cols] try: num_data_cols.remove('cluster') except Exception: num_data_cols = num_data_cols if num_data.shape[1] > 1: num_data = num_data[num_data_cols] num_data_temp = transformation_inv(num_data, obj_t) frames = [num_data_temp, data_temp1, data_temp['cluster']] data_temp = pd.concat(frames, axis=1) for i in num_data.columns: if data_temp[i].dtypes == 'float' or data_temp[i].dtypes == 'int': for j in range(len(data_temp[i])): data_temp[i].iloc[j] = round(data_temp[i].iloc[j], 2) else: frames = [data_temp1, num_data['cluster']] data_temp = pd.concat(frames, axis=1) print(data_temp.columns) print('KMeans done') listed = [data_temp, cords, [], within, tss, betweenss, size] return listed
# region train validate split + select variables train_with_target = train_df.cbind(target) print("train_with_target: ", train_with_target.shape) if para.TRAIN_LABEL == "": train_label = train_df.col_names else: train_label = para.TRAIN_LABEL train, test = train_with_target.split_frame(ratios=[1-para.TEST_SIZE]) # endregion # region PCA if para.RUN_PCA == 1: pca_decomp = H2OPrincipalComponentAnalysisEstimator(k=124, transform="Standardize", pca_method="Power", impute_missing=True) pca_decomp.train(training_frame=train_df) pred = pca_decomp.predict(train_df) # endregion # region Model print('[{}] Model training starts'.format(util.time_now())) # Define model model = H2ORandomForestEstimator(ntrees=50, max_depth=20, nfolds=10) # Train model model.train(x=train_label, y='TARGET', training_frame=train_with_target) # endregion
gs.get_grid("logloss", decreasing=False) #============================================================================== # integrate with scikit-learn http://scikit-learn.org/stable/modules/pipeline.html #============================================================================== from sklearn.pipeline import Pipeline, make_union from h2o.transforms.preprocessing import H2OScaler from h2o.estimators.pca import H2OPrincipalComponentAnalysisEstimator h2o.__PROGRESS_BAR__=True pipeline = Pipeline([("standardize", H2OScaler()), ("pca", H2OPrincipalComponentAnalysisEstimator(k=2,impute_missing=True)), ("gbm", H2OGradientBoostingEstimator(distribution="multinomial"))]) pipeline.fit? H2OPrincipalComponentAnalysisEstimator? pipeline.fit(X=iris_df[:4], y=iris_df[4]) import sklearn.pipeline dir(sklearn.pipeline) sklearn.pipeline.Parallel? sklearn.pipeline.make_union? from sklearn.decomposition import PCA, TruncatedSVD
def test_pca_importance(): arrestsH2O = h2o.upload_file( pyunit_utils.locate("smalldata/pca_test/USArrests.csv")) fitH2O = H2OPrincipalComponentAnalysisEstimator(k=4, transform="DEMEAN") fitH2O.train(x=list(range(4)), training_frame=arrestsH2O) assert fitH2O.varimp()
print('XGB ranking') grid_res = grid.get_grid(sort_by='rmsle', decreasing=True) print(grid_res) # Get best XGB model # best so far learn_rate:0.01, ntrees:5000, colsample_bytree:0.81, subsample:0.88999, # gamma:0.4, max_depth:4, min_child_weight:4 # with RMSLE: 0.12430670916111265 best_mod = grid_res.models[-1] print('Best XGB:') print(best_mod.params) # Tune PCA model pca_params = {'transform': ['None', 'Standardize', 'Normalize', 'Demean', 'Descale'], 'k': [i for i in range(1, 10)], 'max_iterations': [pow(10, i) for i in range(1, 6)]} pca = H2OPrincipalComponentAnalysisEstimator(seed=seed) grid = H2OGridSearch(model=pca, hyper_params=pca_params, search_criteria={'strategy': 'Cartesian'}) grid.train(list(set(housing.columns) - {'C1', 'Ids'}), y='SalePrice', training_frame=housing, nfolds=10) # Get the grid results, sorted print('PCA ranking') grid_res = grid.get_grid(sort_by='rmsle', decreasing=True) print(grid_res) # Get best PCA model # best so far learn_rate:0.01, ntrees:5000, colsample_bytree:0.81, subsample:0.88999, # gamma:0.4, max_depth:4, min_child_weight:4 # with RMSLE: 0.12430670916111265 best_mod = grid_res.models[-1] print('Best PCA:') print(best_mod.params)