def test_already_transformed_X(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) # Take a third of our test data (a tenth of our overall data) for calibration df_titanic_test, df_titanic_calibration = train_test_split(df_titanic_test, test_size=0.33, random_state=42) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) # pass in return_trans_pipeline, and get the trans pipeline trans_pipeline = ml_predictor.train(df_titanic_train, return_transformation_pipeline=True) # get transformed X through transformation_only X_train_transformed = ml_predictor.transform_only(df_titanic_train) # create a new predictor ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) # pass in trained trans pipeline, and make sure it works ml_predictor.train(df_titanic_train, trained_transformation_pipeline=trans_pipeline) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.14 < test_score < -0.12 # pass in both a trans pipeline and a previously transformed X, and make sure that works ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(None, trained_transformation_pipeline=trans_pipeline, transformed_X=X_train_transformed, transformed_y=df_titanic_train.survived) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.14 < test_score < -0.12
def test_compare_all_models_classification(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, compare_all_models=True) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.16 < test_score < -0.135
def test_list_of_single_model_name_classification(): np.random.seed(0) model_name = 'GradientBoostingClassifier' df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, model_names=[model_name]) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.16 < test_score < -0.135
def test_include_bad_y_vals_predict_classification(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) df_titanic_test.iloc[1]['survived'] = float('nan') df_titanic_test.iloc[8]['survived'] = float('inf') df_titanic_test.iloc[26]['survived'] = None ml_predictor.train(df_titanic_train) test_score = ml_predictor.score(df_titanic_test.to_dict('records'), df_titanic_test.survived) print('test_score') print(test_score) assert -0.16 < test_score < -0.135
def test_pass_in_list_of_dictionaries_predict_classification(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) list_titanic_train = df_titanic_train.to_dict('records') ml_predictor.train(df_titanic_train) test_score = ml_predictor.score(df_titanic_test.to_dict('records'), df_titanic_test.survived) print('test_score') print(test_score) assert -0.16 < test_score < -0.135
def test_all_algos_regression(): # a random seed of 42 has ExtraTreesRegressor getting the best CV score, # and that model doesn't generalize as well as GradientBoostingRegressor. np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train( df_boston_train, model_names=[ 'LinearRegression', 'RandomForestRegressor', 'Ridge', 'GradientBoostingRegressor', 'AdaBoostRegressor', 'SGDRegressor', 'PassiveAggressiveRegressor', 'Lasso', 'LassoLars', 'ElasticNet', 'OrthogonalMatchingPursuit', 'BayesianRidge', 'ARDRegression', 'MiniBatchKMeans', 'DeepLearningRegressor', 'LGBMRegressor', 'XGBClassifier', 'LinearSVR', 'CatBoostRegressor' ]) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) assert -3.4 < test_score < -2.8
def test_categorical_ensemble_basic_classifier(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'pclass': 'categorical', 'embarked': 'categorical', 'sex': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train_categorical_ensemble(df_titanic_train, categorical_column='pclass', optimize_final_model=False) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) # Small sample sizes mean there's a fair bit of noise here assert -0.155 < test_score < -0.135
def test_calibrate_final_model_missing_X_test_y_test_classification(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset() # Take a third of our test data (a tenth of our overall data) for calibration df_titanic_test, df_titanic_calibration = train_test_split( df_titanic_test, test_size=0.33, random_state=42) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor( type_of_estimator='classifier', column_descriptions=column_descriptions) # This should still work, just with warning printed ml_predictor.train(df_titanic_train, calibrate_final_model=True) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.14 < test_score < -0.12
def regression_test(): # a random seed of 42 has ExtraTreesRegressor getting the best CV score, # and that model doesn't generalize as well as GradientBoostingRegressor. np.random.seed(0) model_name = 'LGBMRegressor' df_boston_train, df_boston_test = get_boston_regression_dataset() many_dfs = [] for i in range(100): many_dfs.append(df_boston_train) df_boston_train = pd.concat(many_dfs) column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, model_names=[model_name], perform_feature_scaling=False) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) lower_bound = -3.2 if model_name == 'DeepLearningRegressor': lower_bound = -7.8 if model_name == 'LGBMRegressor': lower_bound = -4.95 if model_name == 'XGBRegressor': lower_bound = -3.4 assert lower_bound < test_score < -2.7
def classification_test(): np.random.seed(0) # model_name = 'GradientBoostingClassifier' model_name = 'LGBMClassifier' df_titanic_train, df_titanic_test = get_titanic_binary_classification_dataset( ) df_titanic_train['DELETE_THIS_FIELD'] = 1 column_descriptions = { 'survived': 'output', 'embarked': 'categorical', 'pclass': 'categorical', 'sex': 'categorical', 'this_does_not_exist': 'ignore', 'DELETE_THIS_FIELD': 'ignore' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, model_names=model_name) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) lower_bound = -0.16 if model_name == 'DeepLearningClassifier': lower_bound = -0.245 if model_name == 'LGBMClassifier': lower_bound = -0.225 assert lower_bound < test_score < -0.135
def ensemble_classifier_basic_test(model_name=None): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ensemble_config = [{ 'model_name': 'LGBMClassifier' }, { 'model_name': 'RandomForestClassifier' }] ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, ensemble_config=ensemble_config) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.15 < test_score < -0.131
def test_all_algos_classification(model_name=None): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train( df_titanic_train, model_names=[ 'LogisticRegression', 'RandomForestClassifier', 'RidgeClassifier', 'GradientBoostingClassifier', 'ExtraTreesClassifier', 'AdaBoostClassifier', 'SGDClassifier', 'Perceptron', 'PassiveAggressiveClassifier', 'DeepLearningClassifier', 'XGBClassifier', 'LGBMClassifier', 'LinearSVC' ]) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) # Linear models aren't super great on this dataset... assert -0.215 < test_score < -0.131
def test_perform_feature_selection_true_regression(model_name=None): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, perform_feature_selection=True, model_names=model_name) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) # Bumping this up since without these features our score drops lower_bound = -4.0 if model_name == 'DeepLearningRegressor': lower_bound = -14.5 if model_name == 'LGBMRegressor': lower_bound = -4.95 assert lower_bound < test_score < -2.8
def test_perform_feature_scaling_true_regression(model_name=None): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, perform_feature_scaling=True) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) assert -3.0 < test_score < -2.7
def test_compare_all_models_regression(): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, compare_all_models=True) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) # ExtraTrees again throws this off assert -3.6 < test_score < -2.8
def optimize_final_model_classification(model_name=None): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) # We just want to make sure these run, not necessarily make sure that they're super accurate # (which takes more time, and is dataset dependent) df_titanic_train = df_titanic_train.sample(frac=0.5) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, optimize_final_model=True, model_names=model_name) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) # Small sample sizes mean there's a fair bit of noise here lower_bound = -0.18 if model_name == 'DeepLearningClassifier': lower_bound = -0.255 if model_name == 'LGBMClassifier': lower_bound = -0.221 if model_name == 'GradientBoostingClassifier': lower_bound = -0.225 if model_name == 'CatBoostClassifier': lower_bound = -0.221 assert lower_bound < test_score < -0.135
def optimize_final_model_regression(model_name=None): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() # We just want to make sure these run, not necessarily make sure that they're super accurate # (which takes more time, and is dataset dependent) df_boston_train = df_boston_train.sample(frac=0.5) column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, optimize_final_model=True, model_names=model_name) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) # the random seed gets a score of -3.21 on python 3.5 # There's a ton of noise here, due to small sample sizes lower_bound = -3.4 if model_name == 'DeepLearningRegressor': lower_bound = -24 if model_name == 'LGBMRegressor': lower_bound = -16 if model_name == 'GradientBoostingRegressor': lower_bound = -5.1 if model_name == 'CatBoostRegressor': lower_bound = -4.5 if model_name == 'XGBRegressor': lower_bound = -4.8 assert lower_bound < test_score < -2.75
def test_input_df_unmodified(): np.random.seed(42) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) df_shape = df_boston_train.shape ml_predictor.train(df_boston_train) training_shape = df_boston_train.shape assert training_shape[0] == df_shape[0] assert training_shape[1] == df_shape[1] test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) assert -3.35 < test_score < -2.8
def test_categorical_ensembling_regression(model_name=None): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train_categorical_ensemble(df_boston_train, perform_feature_selection=True, model_names=model_name, categorical_column='CHAS') test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) lower_bound = -4.2 assert lower_bound < test_score < -2.8
def test_binary_classification(): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, scoring=always_return_ten_thousand) test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert test_score == -10000
def test_linear_model_analytics_classification(model_name=None): np.random.seed(0) df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset( ) column_descriptions = { 'survived': 'output', 'sex': 'categorical', 'embarked': 'categorical', 'pclass': 'categorical' } ml_predictor = Predictor(type_of_estimator='classifier', column_descriptions=column_descriptions) ml_predictor.train(df_titanic_train, model_names='RidgeClassifier') test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived) print('test_score') print(test_score) assert -0.21 < test_score < -0.131
def ensemble_regressor_basic_test(): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ensemble_config = [{ 'model_name': 'LGBMRegressor' }, { 'model_name': 'RandomForestRegressor' }] ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_boston_train, ensemble_config=None) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) assert -3.0 < test_score < -2.8
# Load data df_train, df_test = get_boston_dataset() # Tell auto_ml which column is 'output' # Also note columns that aren't purely numerical # Examples include ['nlp', 'date', 'categorical', 'ignore'] column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) ml_predictor.train(df_train) # Score the model on test data test_score = ml_predictor.score(df_test, df_test.MEDV) # auto_ml is specifically tuned for running in production # It can get predictions on an individual row (passed in as a dictionary) # A single prediction like this takes ~1 millisecond # Here we will demonstrate saving the trained model, and loading it again file_name = ml_predictor.save() trained_model = load_ml_model(file_name) # .predict and .predict_proba take in either: # A pandas DataFrame # A list of dictionaries # A single dictionary (optimized for speed in production evironments) predictions = trained_model.predict(df_test) print(predictions)
def getting_single_predictions_regressor_test(): np.random.seed(0) df_boston_train, df_boston_test = utils.get_boston_regression_dataset() column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'} ensemble_config = [{ 'model_name': 'LGBMRegressor' }, { 'model_name': 'RandomForestRegressor' }] ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions) # NOTE: this is bad practice to pass in our same training set as our fl_data set, but we don't have enough data to do it any other way ml_predictor.train(df_boston_train, ensemble_config=ensemble_config) test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV) print('test_score') print(test_score) assert -3.5 < test_score < -2.8 file_name = ml_predictor.save(str(random.random())) saved_ml_pipeline = load_ml_model(file_name) os.remove(file_name) try: keras_file_name = file_name[:-5] + '_keras_deep_learning_model.h5' os.remove(keras_file_name) except: pass df_boston_test_dictionaries = df_boston_test.to_dict('records') # 1. make sure the accuracy is the same predictions = [] for row in df_boston_test_dictionaries: predictions.append(saved_ml_pipeline.predict(row)) first_score = utils.calculate_rmse(df_boston_test.MEDV, predictions) print('first_score') print(first_score) # Make sure our score is good, but not unreasonably good lower_bound = -3.5 assert lower_bound < first_score < -2.8 # 2. make sure the speed is reasonable (do it a few extra times) data_length = len(df_boston_test_dictionaries) start_time = datetime.datetime.now() for idx in range(1000): row_num = idx % data_length saved_ml_pipeline.predict(df_boston_test_dictionaries[row_num]) end_time = datetime.datetime.now() duration = end_time - start_time print('duration.total_seconds()') print(duration.total_seconds()) # It's very difficult to set a benchmark for speed that will work across all machines. # On my 2013 bottom of the line 15" MacBook Pro, this runs in about 0.8 seconds for 1000 predictions # That's about 1 millisecond per prediction # Assuming we might be running on a test box that's pretty weak, multiply by 3 # Also make sure we're not running unreasonably quickly assert 0.2 < duration.total_seconds() / 1.0 < 60 # 3. make sure we're not modifying the dictionaries (the score is the same after running a few experiments as it is the first time) predictions = [] for row in df_boston_test_dictionaries: predictions.append(saved_ml_pipeline.predict(row)) second_score = utils.calculate_rmse(df_boston_test.MEDV, predictions) print('second_score') print(second_score) # Make sure our score is good, but not unreasonably good assert lower_bound < second_score < -2.8