def test_sample_weight_func():
"""Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline with sample weights"""
tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error')
# Reify pipeline with known scor
pipeline_string = ("ExtraTreesRegressor("
"GradientBoostingRegressor(input_matrix, GradientBoostingRegressor__alpha=0.8,"
"GradientBoostingRegressor__learning_rate=0.1,GradientBoostingRegressor__loss=huber,"
"GradientBoostingRegressor__max_depth=5, GradientBoostingRegressor__max_features=0.5,"
"GradientBoostingRegressor__min_samples_leaf=5, GradientBoostingRegressor__min_samples_split=5,"
"GradientBoostingRegressor__n_estimators=100, GradientBoostingRegressor__subsample=0.25),"
"ExtraTreesRegressor__bootstrap=True, ExtraTreesRegressor__max_features=0.5,"
"ExtraTreesRegressor__min_samples_leaf=5, ExtraTreesRegressor__min_samples_split=5, "
"ExtraTreesRegressor__n_estimators=100)")
tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset)
tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline)
tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r)
tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset)
tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline)
# make up a sample weight
training_classes_r_weight = np.array(range(1, len(training_classes_r)+1))
training_classes_r_weight_dict = set_sample_weight(tpot_obj._fitted_pipeline.steps, training_classes_r_weight)
np.random.seed(42)
cv_score1 = cross_val_score(tpot_obj._fitted_pipeline, training_features_r, training_classes_r, cv=3, scoring='neg_mean_squared_error')
np.random.seed(42)
cv_score2 = cross_val_score(tpot_obj._fitted_pipeline, training_features_r, training_classes_r, cv=3, scoring='neg_mean_squared_error')
np.random.seed(42)
cv_score_weight = cross_val_score(tpot_obj._fitted_pipeline, training_features_r, training_classes_r, cv=3, scoring='neg_mean_squared_error', fit_params=training_classes_r_weight_dict)
np.random.seed(42)
tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r, **training_classes_r_weight_dict)
# Get score from TPOT
known_score = 12.643383517 # Assumes use of mse
score = tpot_obj.score(testing_features_r, testing_classes_r)
# http://stackoverflow.com/questions/5595425/
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
assert np.allclose(cv_score1, cv_score2)
assert not np.allclose(cv_score1, cv_score_weight)
assert isclose(known_score, score)
def test_score_3():
"""Assert that the TPOTRegressor score function outputs a known score for a fix pipeline"""
tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error')
known_score = 12.3727966005 # Assumes use of mse
# Reify pipeline with known score
pipeline_string = (
"ExtraTreesRegressor("
"GradientBoostingRegressor(input_matrix, GradientBoostingRegressor__alpha=0.8,"
"GradientBoostingRegressor__learning_rate=0.1,GradientBoostingRegressor__loss=huber,"
"GradientBoostingRegressor__max_depth=5, GradientBoostingRegressor__max_features=0.5,"
"GradientBoostingRegressor__min_samples_leaf=5, GradientBoostingRegressor__min_samples_split=5,"
"GradientBoostingRegressor__n_estimators=100, GradientBoostingRegressor__subsample=0.25),"
"ExtraTreesRegressor__bootstrap=True, ExtraTreesRegressor__max_features=0.5,"
"ExtraTreesRegressor__min_samples_leaf=5, ExtraTreesRegressor__min_samples_split=5, "
"ExtraTreesRegressor__n_estimators=100)")
tpot_obj._optimized_pipeline = creator.Individual.from_string(
pipeline_string, tpot_obj._pset)
tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(
expr=tpot_obj._optimized_pipeline)
tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r)
# Get score from TPOT
score = tpot_obj.score(testing_features_r, testing_classes_r)
# http://stackoverflow.com/questions/5595425/
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
assert isclose(known_score, score)
def test_score_3():
"""Assert that the TPOTRegressor score function outputs a known score for a fix pipeline"""
tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error')
known_score = 12.3727966005 # Assumes use of mse
# Reify pipeline with known score
pipeline_string = ("ExtraTreesRegressor("
"GradientBoostingRegressor(input_matrix, GradientBoostingRegressor__alpha=0.8,"
"GradientBoostingRegressor__learning_rate=0.1,GradientBoostingRegressor__loss=huber,"
"GradientBoostingRegressor__max_depth=5, GradientBoostingRegressor__max_features=0.5,"
"GradientBoostingRegressor__min_samples_leaf=5, GradientBoostingRegressor__min_samples_split=5,"
"GradientBoostingRegressor__n_estimators=100, GradientBoostingRegressor__subsample=0.25),"
"ExtraTreesRegressor__bootstrap=True, ExtraTreesRegressor__max_features=0.5,"
"ExtraTreesRegressor__min_samples_leaf=5, ExtraTreesRegressor__min_samples_split=5, "
"ExtraTreesRegressor__n_estimators=100)")
tpot_obj._optimized_pipeline = creator.Individual.from_string(pipeline_string, tpot_obj._pset)
tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline)
tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r)
# Get score from TPOT
score = tpot_obj.score(testing_features_r, testing_classes_r)
# http://stackoverflow.com/questions/5595425/
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
assert isclose(known_score, score)
def test_score_3():
"""Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline"""
tpot_obj = TPOTRegressor(scoring='neg_mean_squared_error')
tpot_obj._pbar = tqdm(total=1, disable=True)
known_score = 8.9673743407873712 # Assumes use of mse
# Reify pipeline with known score
tpot_obj._optimized_pipeline = creator.Individual.\
from_string('ExtraTreesRegressor(GradientBoostingRegressor(input_matrix, 100.0, 0.11), 0.17999999999999999)', tpot_obj._pset)
tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(expr=tpot_obj._optimized_pipeline)
tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r)
# Get score from TPOT
score = tpot_obj.score(testing_features_r, testing_classes_r)
# http://stackoverflow.com/questions/5595425/
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
assert isclose(known_score, score)
def test_score_3():
"""Assert that the TPOTRegressor score function outputs a known score for a fixed pipeline"""
tpot_obj = TPOTRegressor(scoring='mean_squared_error')
tpot_obj._pbar = tqdm(total=1, disable=True)
known_score = 8.9673743407873712 # Assumes use of mse
# Reify pipeline with known score
tpot_obj._optimized_pipeline = creator.Individual.\
from_string('ExtraTreesRegressor(GradientBoostingRegressor(input_matrix, 100.0, 0.11), 0.17999999999999999)', tpot_obj._pset)
tpot_obj._fitted_pipeline = tpot_obj._toolbox.compile(
expr=tpot_obj._optimized_pipeline)
tpot_obj._fitted_pipeline.fit(training_features_r, training_classes_r)
# Get score from TPOT
score = tpot_obj.score(testing_features_r, testing_classes_r)
# http://stackoverflow.com/questions/5595425/
def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
assert isclose(known_score, score)
