def test_already_transformed_X():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
# Take a third of our test data (a tenth of our overall data) for calibration
df_titanic_test, df_titanic_calibration = train_test_split(df_titanic_test,
test_size=0.33,
random_state=42)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
# pass in return_trans_pipeline, and get the trans pipeline
trans_pipeline = ml_predictor.train(df_titanic_train,
return_transformation_pipeline=True)
# get transformed X through transformation_only
X_train_transformed = ml_predictor.transform_only(df_titanic_train)
# create a new predictor
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
# pass in trained trans pipeline, and make sure it works
ml_predictor.train(df_titanic_train,
trained_transformation_pipeline=trans_pipeline)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.14 < test_score < -0.12
# pass in both a trans pipeline and a previously transformed X, and make sure that works
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(None,
trained_transformation_pipeline=trans_pipeline,
transformed_X=X_train_transformed,
transformed_y=df_titanic_train.survived)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.14 < test_score < -0.12
def test_compare_all_models_classification():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train, compare_all_models=True)
test_score = ml_predictor.score(df_titanic_test,
df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.16 < test_score < -0.135
def test_list_of_single_model_name_classification():
np.random.seed(0)
model_name = 'GradientBoostingClassifier'
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train, model_names=[model_name])
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.16 < test_score < -0.135
def test_include_bad_y_vals_predict_classification():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
df_titanic_test.iloc[1]['survived'] = float('nan')
df_titanic_test.iloc[8]['survived'] = float('inf')
df_titanic_test.iloc[26]['survived'] = None
ml_predictor.train(df_titanic_train)
test_score = ml_predictor.score(df_titanic_test.to_dict('records'),
df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.16 < test_score < -0.135
def test_pass_in_list_of_dictionaries_predict_classification():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
list_titanic_train = df_titanic_train.to_dict('records')
ml_predictor.train(df_titanic_train)
test_score = ml_predictor.score(df_titanic_test.to_dict('records'),
df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.16 < test_score < -0.135
def test_all_algos_regression():
# a random seed of 42 has ExtraTreesRegressor getting the best CV score,
# and that model doesn't generalize as well as GradientBoostingRegressor.
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train(
df_boston_train,
model_names=[
'LinearRegression', 'RandomForestRegressor', 'Ridge',
'GradientBoostingRegressor', 'AdaBoostRegressor', 'SGDRegressor',
'PassiveAggressiveRegressor', 'Lasso', 'LassoLars', 'ElasticNet',
'OrthogonalMatchingPursuit', 'BayesianRidge', 'ARDRegression',
'MiniBatchKMeans', 'DeepLearningRegressor', 'LGBMRegressor',
'XGBClassifier', 'LinearSVR', 'CatBoostRegressor'
])
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
assert -3.4 < test_score < -2.8
def test_categorical_ensemble_basic_classifier():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'pclass': 'categorical',
'embarked': 'categorical',
'sex': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train_categorical_ensemble(df_titanic_train,
categorical_column='pclass',
optimize_final_model=False)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
# Small sample sizes mean there's a fair bit of noise here
assert -0.155 < test_score < -0.135
def test_calibrate_final_model_missing_X_test_y_test_classification():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset()
# Take a third of our test data (a tenth of our overall data) for calibration
df_titanic_test, df_titanic_calibration = train_test_split(
df_titanic_test, test_size=0.33, random_state=42)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(
type_of_estimator='classifier', column_descriptions=column_descriptions)
# This should still work, just with warning printed
ml_predictor.train(df_titanic_train, calibrate_final_model=True)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.14 < test_score < -0.12
def regression_test():
# a random seed of 42 has ExtraTreesRegressor getting the best CV score,
# and that model doesn't generalize as well as GradientBoostingRegressor.
np.random.seed(0)
model_name = 'LGBMRegressor'
df_boston_train, df_boston_test = get_boston_regression_dataset()
many_dfs = []
for i in range(100):
many_dfs.append(df_boston_train)
df_boston_train = pd.concat(many_dfs)
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train(df_boston_train,
model_names=[model_name],
perform_feature_scaling=False)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
lower_bound = -3.2
if model_name == 'DeepLearningRegressor':
lower_bound = -7.8
if model_name == 'LGBMRegressor':
lower_bound = -4.95
if model_name == 'XGBRegressor':
lower_bound = -3.4
assert lower_bound < test_score < -2.7
def classification_test():
np.random.seed(0)
# model_name = 'GradientBoostingClassifier'
model_name = 'LGBMClassifier'
df_titanic_train, df_titanic_test = get_titanic_binary_classification_dataset(
)
df_titanic_train['DELETE_THIS_FIELD'] = 1
column_descriptions = {
'survived': 'output',
'embarked': 'categorical',
'pclass': 'categorical',
'sex': 'categorical',
'this_does_not_exist': 'ignore',
'DELETE_THIS_FIELD': 'ignore'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train, model_names=model_name)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
lower_bound = -0.16
if model_name == 'DeepLearningClassifier':
lower_bound = -0.245
if model_name == 'LGBMClassifier':
lower_bound = -0.225
assert lower_bound < test_score < -0.135
def ensemble_classifier_basic_test(model_name=None):
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ensemble_config = [{
'model_name': 'LGBMClassifier'
}, {
'model_name': 'RandomForestClassifier'
}]
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train, ensemble_config=ensemble_config)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.15 < test_score < -0.131
def test_all_algos_classification(model_name=None):
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(
df_titanic_train,
model_names=[
'LogisticRegression', 'RandomForestClassifier', 'RidgeClassifier',
'GradientBoostingClassifier', 'ExtraTreesClassifier',
'AdaBoostClassifier', 'SGDClassifier', 'Perceptron',
'PassiveAggressiveClassifier', 'DeepLearningClassifier',
'XGBClassifier', 'LGBMClassifier', 'LinearSVC'
])
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
# Linear models aren't super great on this dataset...
assert -0.215 < test_score < -0.131
def test_perform_feature_selection_true_regression(model_name=None):
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train(df_boston_train,
perform_feature_selection=True,
model_names=model_name)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
# Bumping this up since without these features our score drops
lower_bound = -4.0
if model_name == 'DeepLearningRegressor':
lower_bound = -14.5
if model_name == 'LGBMRegressor':
lower_bound = -4.95
assert lower_bound < test_score < -2.8
def test_perform_feature_scaling_true_regression(model_name=None):
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions)
ml_predictor.train(df_boston_train, perform_feature_scaling=True)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
assert -3.0 < test_score < -2.7
def test_compare_all_models_regression():
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor', column_descriptions=column_descriptions)
ml_predictor.train(df_boston_train, compare_all_models=True)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
# ExtraTrees again throws this off
assert -3.6 < test_score < -2.8
def optimize_final_model_classification(model_name=None):
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
# We just want to make sure these run, not necessarily make sure that they're super accurate
# (which takes more time, and is dataset dependent)
df_titanic_train = df_titanic_train.sample(frac=0.5)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train,
optimize_final_model=True,
model_names=model_name)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
# Small sample sizes mean there's a fair bit of noise here
lower_bound = -0.18
if model_name == 'DeepLearningClassifier':
lower_bound = -0.255
if model_name == 'LGBMClassifier':
lower_bound = -0.221
if model_name == 'GradientBoostingClassifier':
lower_bound = -0.225
if model_name == 'CatBoostClassifier':
lower_bound = -0.221
assert lower_bound < test_score < -0.135
def optimize_final_model_regression(model_name=None):
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
# We just want to make sure these run, not necessarily make sure that they're super accurate
# (which takes more time, and is dataset dependent)
df_boston_train = df_boston_train.sample(frac=0.5)
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train(df_boston_train,
optimize_final_model=True,
model_names=model_name)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
# the random seed gets a score of -3.21 on python 3.5
# There's a ton of noise here, due to small sample sizes
lower_bound = -3.4
if model_name == 'DeepLearningRegressor':
lower_bound = -24
if model_name == 'LGBMRegressor':
lower_bound = -16
if model_name == 'GradientBoostingRegressor':
lower_bound = -5.1
if model_name == 'CatBoostRegressor':
lower_bound = -4.5
if model_name == 'XGBRegressor':
lower_bound = -4.8
assert lower_bound < test_score < -2.75
def test_input_df_unmodified():
np.random.seed(42)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
df_shape = df_boston_train.shape
ml_predictor.train(df_boston_train)
training_shape = df_boston_train.shape
assert training_shape[0] == df_shape[0]
assert training_shape[1] == df_shape[1]
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
assert -3.35 < test_score < -2.8
def test_categorical_ensembling_regression(model_name=None):
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train_categorical_ensemble(df_boston_train,
perform_feature_selection=True,
model_names=model_name,
categorical_column='CHAS')
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
lower_bound = -4.2
assert lower_bound < test_score < -2.8
def test_binary_classification():
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train, scoring=always_return_ten_thousand)
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert test_score == -10000
def test_linear_model_analytics_classification(model_name=None):
np.random.seed(0)
df_titanic_train, df_titanic_test = utils.get_titanic_binary_classification_dataset(
)
column_descriptions = {
'survived': 'output',
'sex': 'categorical',
'embarked': 'categorical',
'pclass': 'categorical'
}
ml_predictor = Predictor(type_of_estimator='classifier',
column_descriptions=column_descriptions)
ml_predictor.train(df_titanic_train, model_names='RidgeClassifier')
test_score = ml_predictor.score(df_titanic_test, df_titanic_test.survived)
print('test_score')
print(test_score)
assert -0.21 < test_score < -0.131
def ensemble_regressor_basic_test():
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ensemble_config = [{
'model_name': 'LGBMRegressor'
}, {
'model_name': 'RandomForestRegressor'
}]
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train(df_boston_train, ensemble_config=None)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
assert -3.0 < test_score < -2.8
# Load data
df_train, df_test = get_boston_dataset()
# Tell auto_ml which column is 'output'
# Also note columns that aren't purely numerical
# Examples include ['nlp', 'date', 'categorical', 'ignore']
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
ml_predictor.train(df_train)
# Score the model on test data
test_score = ml_predictor.score(df_test, df_test.MEDV)
# auto_ml is specifically tuned for running in production
# It can get predictions on an individual row (passed in as a dictionary)
# A single prediction like this takes ~1 millisecond
# Here we will demonstrate saving the trained model, and loading it again
file_name = ml_predictor.save()
trained_model = load_ml_model(file_name)
# .predict and .predict_proba take in either:
# A pandas DataFrame
# A list of dictionaries
# A single dictionary (optimized for speed in production evironments)
predictions = trained_model.predict(df_test)
print(predictions)
def getting_single_predictions_regressor_test():
np.random.seed(0)
df_boston_train, df_boston_test = utils.get_boston_regression_dataset()
column_descriptions = {'MEDV': 'output', 'CHAS': 'categorical'}
ensemble_config = [{
'model_name': 'LGBMRegressor'
}, {
'model_name': 'RandomForestRegressor'
}]
ml_predictor = Predictor(type_of_estimator='regressor',
column_descriptions=column_descriptions)
# NOTE: this is bad practice to pass in our same training set as our fl_data set, but we don't have enough data to do it any other way
ml_predictor.train(df_boston_train, ensemble_config=ensemble_config)
test_score = ml_predictor.score(df_boston_test, df_boston_test.MEDV)
print('test_score')
print(test_score)
assert -3.5 < test_score < -2.8
file_name = ml_predictor.save(str(random.random()))
saved_ml_pipeline = load_ml_model(file_name)
os.remove(file_name)
try:
keras_file_name = file_name[:-5] + '_keras_deep_learning_model.h5'
os.remove(keras_file_name)
except:
pass
df_boston_test_dictionaries = df_boston_test.to_dict('records')
# 1. make sure the accuracy is the same
predictions = []
for row in df_boston_test_dictionaries:
predictions.append(saved_ml_pipeline.predict(row))
first_score = utils.calculate_rmse(df_boston_test.MEDV, predictions)
print('first_score')
print(first_score)
# Make sure our score is good, but not unreasonably good
lower_bound = -3.5
assert lower_bound < first_score < -2.8
# 2. make sure the speed is reasonable (do it a few extra times)
data_length = len(df_boston_test_dictionaries)
start_time = datetime.datetime.now()
for idx in range(1000):
row_num = idx % data_length
saved_ml_pipeline.predict(df_boston_test_dictionaries[row_num])
end_time = datetime.datetime.now()
duration = end_time - start_time
print('duration.total_seconds()')
print(duration.total_seconds())
# It's very difficult to set a benchmark for speed that will work across all machines.
# On my 2013 bottom of the line 15" MacBook Pro, this runs in about 0.8 seconds for 1000 predictions
# That's about 1 millisecond per prediction
# Assuming we might be running on a test box that's pretty weak, multiply by 3
# Also make sure we're not running unreasonably quickly
assert 0.2 < duration.total_seconds() / 1.0 < 60
# 3. make sure we're not modifying the dictionaries (the score is the same after running a few experiments as it is the first time)
predictions = []
for row in df_boston_test_dictionaries:
predictions.append(saved_ml_pipeline.predict(row))
second_score = utils.calculate_rmse(df_boston_test.MEDV, predictions)
print('second_score')
print(second_score)
# Make sure our score is good, but not unreasonably good
assert lower_bound < second_score < -2.8
