def test_foreshadow_integration_data_cleaner_can_drop(filename, problem_type,
X_start, X_end, target,
tmpdir):
from foreshadow.foreshadow import Foreshadow
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
np.random.seed(1337)
data_path = get_file_path("data", filename)
data = pd.read_csv(data_path)
# local_file_folder = "examples"
# data = pd.read_csv("/".join([local_file_folder, filename]))
X_df = data.loc[:, X_start:X_end]
y_df = data.loc[:, target]
X_train, X_test, y_train, y_test = train_test_split(X_df,
y_df,
test_size=0.2)
from foreshadow.estimators import AutoEstimator
estimator = AutoEstimator(
problem_type=problem_type,
auto="tpot",
estimator_kwargs={"max_time_mins": 1},
)
shadow = Foreshadow(estimator=estimator, problem_type=problem_type)
pickled_fitted_pipeline_location = tmpdir.join("fitted_pipeline.p")
shadow.fit(X_train, y_train)
shadow.pickle_fitted_pipeline(pickled_fitted_pipeline_location)
import pickle
with open(pickled_fitted_pipeline_location, "rb") as fopen:
pipeline = pickle.load(fopen)
# If there are new empty columns in the test set, the program should
# not fail.
X_test[X_start] = np.nan
score1 = shadow.score(X_test, y_test)
score2 = pipeline.score(X_test, y_test)
import unittest
assertions = unittest.TestCase("__init__")
# given the randomness of the tpot algorithm and the short run
# time we configured, there is no guarantee the performance can
# converge. The test here aims to evaluate if both cases have
# produced a reasonable score and the difference is small.
# assert score1 > 0.76 and score2 > 0.76
assertions.assertAlmostEqual(score1, score2, places=2)
def test_foreshadow_pickling_and_unpickling_unfitted(tmpdir):
from foreshadow.foreshadow import Foreshadow
from foreshadow.estimators import AutoEstimator
estimator = AutoEstimator(
problem_type=ProblemType.CLASSIFICATION,
auto="tpot",
estimator_kwargs={"max_time_mins": 1},
)
shadow = Foreshadow(estimator=estimator,
problem_type=ProblemType.CLASSIFICATION)
with pytest.raises(ValueError):
shadow.pickle_fitted_pipeline(tmpdir.join("fitted_pipeline.p"))
def test_foreshadow_pickling_and_unpickling_tpot(tmpdir):
from foreshadow.foreshadow import Foreshadow
import pandas as pd
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
np.random.seed(1337)
cancer = load_breast_cancer()
cancerX_df = pd.DataFrame(cancer.data, columns=cancer.feature_names)
cancery_df = pd.DataFrame(cancer.target, columns=["target"])
X_train, X_test, y_train, y_test = train_test_split(cancerX_df,
cancery_df,
test_size=0.2)
from foreshadow.estimators import AutoEstimator
estimator = AutoEstimator(
problem_type=ProblemType.CLASSIFICATION,
auto="tpot",
estimator_kwargs={"max_time_mins": 1},
)
shadow = Foreshadow(estimator=estimator,
problem_type=ProblemType.CLASSIFICATION)
pickled_file_location = tmpdir.join("fitted_pipeline.p")
shadow.fit(X_train, y_train)
shadow.pickle_fitted_pipeline(pickled_file_location)
import pickle
with open(pickled_file_location, "rb") as fopen:
pipeline = pickle.load(fopen)
score1 = shadow.score(X_test, y_test)
score2 = pipeline.score(X_test, y_test)
import unittest
assertions = unittest.TestCase("__init__")
# given the randomness of the tpot algorithm and the short run
# time we configured, there is no guarantee the performance can
# converge. The test here aims to evaluate if both cases have
# produced a reasonable score and the difference is small.
# Changing the decimal point to 1 due to failure on azure pipeline but
# cannot be reproduced locally.
assertions.assertAlmostEqual(score1, score2, places=2)