def test_classification_quality_improvement():
# input data initialization
train_data_path = f'{project_root()}/cases/data/scoring/scoring_train.csv'
test_data_path = f'{project_root()}/cases/data/scoring/scoring_test.csv'
problem = 'classification'
baseline_model = Fedot(problem=problem)
baseline_model.fit(features=train_data_path, target='target', predefined_model='xgboost')
expected_baseline_quality = 0.823
baseline_model.predict_proba(features=test_data_path)
baseline_metrics = baseline_model.get_metrics()
# Define parameters for composing
composer_params = {'max_depth': 3,
'max_arity': 3,
'pop_size': 20,
'num_of_generations': 20,
'learning_time': 10,
'with_tuning': True}
auto_model = Fedot(problem=problem, composer_params=composer_params, seed=42, verbose_level=4)
auto_model.fit(features=train_data_path, target='target')
auto_model.predict_proba(features=test_data_path)
auto_metrics = auto_model.get_metrics()
print(auto_metrics['roc_auc'])
assert auto_metrics['roc_auc'] > baseline_metrics['roc_auc'] >= expected_baseline_quality
def run_one_model_with_specific_evaluation_mod(train_data,
test_data,
mode: str = None):
"""
Runs the example with one model svc.
:param train_data: train data for pipeline training
:param test_data: test data for pipeline training
:param mode: pass gpu flag to make gpu evaluation
"""
problem = 'classification'
if mode == 'gpu':
baseline_model = Fedot(problem=problem, preset='gpu')
else:
baseline_model = Fedot(problem=problem)
svc_node_with_custom_params = PrimaryNode('svc')
# the custom params are needed to make probability evaluation available
# otherwise an error is occurred
svc_node_with_custom_params.custom_params = dict(kernel='rbf',
C=10,
gamma=1,
cache_size=2000,
probability=True)
preset_pipeline = Pipeline(svc_node_with_custom_params)
start = datetime.now()
baseline_model.fit(features=train_data,
target='target',
predefined_model=preset_pipeline)
print(f'Completed with custom params in: {datetime.now() - start}')
baseline_model.predict(features=test_data)
print(baseline_model.get_metrics())
def test_pandas_input_for_api():
train_data, test_data, threshold = get_dataset('classification')
train_features = pd.DataFrame(train_data.features)
train_target = pd.Series(train_data.target)
test_features = pd.DataFrame(test_data.features)
test_target = pd.Series(test_data.target)
# task selection, initialisation of the framework
baseline_model = Fedot(problem='classification')
# fit model without optimisation - single XGBoost node is used
baseline_model.fit(features=train_features,
target=train_target,
predefined_model='xgboost')
# evaluate the prediction with test data
prediction = baseline_model.predict(features=test_features)
assert len(prediction) == len(test_target)
# evaluate quality metric for the test sample
baseline_metrics = baseline_model.get_metrics(metric_names='f1',
target=test_target)
assert baseline_metrics['f1'] > 0
def run_ts_forecasting_example(with_plot=True,
with_pipeline_vis=True,
timeout=None):
train_data_path = f'{fedot_project_root()}/examples/data/salaries.csv'
target = pd.read_csv(train_data_path)['target']
# Define forecast length and define parameters - forecast length
forecast_length = 30
task_parameters = TsForecastingParams(forecast_length=forecast_length)
# init model for the time series forecasting
model = Fedot(problem='ts_forecasting',
task_params=task_parameters,
timeout=timeout)
# run AutoML model design in the same way
pipeline = model.fit(features=train_data_path, target='target')
if with_pipeline_vis:
pipeline.show()
# use model to obtain forecast
forecast = model.predict(features=train_data_path)
print(
model.get_metrics(metric_names=['rmse', 'mae', 'mape'], target=target))
# plot forecasting result
if with_plot:
model.plot_prediction()
return forecast
def run_metocean_forecasting_problem(train_file_path,
test_file_path,
forecast_length=1,
is_visualise=False,
timeout=5):
# Prepare data for train and test
ssh_history, ws_history, ssh_obs = prepare_input_data(
train_file_path, test_file_path)
historical_data = {
'ws': ws_history, # additional variable
'ssh': ssh_history, # target variable
}
fedot = Fedot(
problem='ts_forecasting',
task_params=TsForecastingParams(forecast_length=forecast_length),
timeout=timeout,
verbose_level=4)
pipeline = fedot.fit(features=historical_data, target=ssh_history)
fedot.forecast(historical_data, forecast_length=forecast_length)
metric = fedot.get_metrics(target=ssh_obs)
if is_visualise:
pipeline.show()
fedot.plot_prediction()
return metric
def test_api_predict_correct(task_type: str = 'classification'):
train_data, test_data, _ = get_dataset(task_type)
model = Fedot(problem=task_type, composer_params=composer_params)
fedot_model = model.fit(features=train_data)
prediction = model.predict(features=test_data)
metric = model.get_metrics()
assert isinstance(fedot_model, Pipeline)
assert len(prediction) == len(test_data.target)
assert metric['f1'] > 0
def run_classification_example(timeout=None):
train_data_path = f'{fedot_project_root()}/cases/data/scoring/scoring_train.csv'
test_data_path = f'{fedot_project_root()}/cases/data/scoring/scoring_test.csv'
problem = 'classification'
baseline_model = Fedot(problem=problem, timeout=timeout)
baseline_model.fit(features=train_data_path,
target='target',
predefined_model='xgboost')
baseline_model.predict(features=test_data_path)
print(baseline_model.get_metrics())
auto_model = Fedot(problem=problem, seed=42, timeout=timeout)
auto_model.fit(features=train_data_path, target='target')
prediction = auto_model.predict_proba(features=test_data_path)
print(auto_model.get_metrics())
return prediction
def run_regression_example():
data_path = f'{fedot_project_root()}/cases/data/cholesterol/cholesterol.csv'
data = InputData.from_csv(data_path)
train, test = train_test_data_setup(data)
problem = 'regression'
baseline_model = Fedot(problem=problem)
baseline_model.fit(features=train, predefined_model='xgbreg')
baseline_model.predict(features=test)
print(baseline_model.get_metrics())
auto_model = Fedot(problem=problem, seed=42)
auto_model.fit(features=train, target='target')
prediction = auto_model.predict(features=test)
print(auto_model.get_metrics())
return prediction
def test_multiobj_for_api():
train_data, test_data, _ = get_dataset('classification')
composer_params['composer_metric'] = ['f1', 'node_num']
model = Fedot(problem='classification', composer_params=composer_params)
model.fit(features=train_data)
prediction = model.predict(features=test_data)
metric = model.get_metrics()
assert len(prediction) == len(test_data.target)
assert metric['f1'] > 0
assert model.best_models is not None
def test_api_forecast_correct(task_type: str = 'ts_forecasting'):
# The forecast length must be equal to 12
forecast_length = 12
train_data, test_data, _ = get_dataset(task_type)
model = Fedot(problem='ts_forecasting', composer_params=composer_params,
task_params=TsForecastingParams(forecast_length=forecast_length))
model.fit(features=train_data)
ts_forecast = model.predict(features=train_data)
metric = model.get_metrics(target=test_data.target, metric_names='rmse')
assert len(ts_forecast) == forecast_length
assert metric['rmse'] >= 0
def test_api_forecast_numpy_input_with_static_model_correct(task_type: str = 'ts_forecasting'):
forecast_length = 10
train_data, test_data, _ = get_dataset(task_type)
model = Fedot(problem='ts_forecasting',
task_params=TsForecastingParams(forecast_length=forecast_length))
# Define chain for prediction
node_lagged = PrimaryNode('lagged')
chain = Chain(SecondaryNode('linear', nodes_from=[node_lagged]))
model.fit(features=train_data.features,
target=train_data.target,
predefined_model=chain)
ts_forecast = model.predict(features=train_data)
metric = model.get_metrics(target=test_data.target, metric_names='rmse')
assert len(ts_forecast) == forecast_length
assert metric['rmse'] >= 0
def test_baseline_with_api():
train_data, test_data, threshold = get_dataset('classification')
# task selection, initialisation of the framework
baseline_model = Fedot(problem='classification')
# fit model without optimisation - single XGBoost node is used
baseline_model.fit(features=train_data, target='target', predefined_model='xgboost')
# evaluate the prediction with test data
prediction = baseline_model.predict_proba(features=test_data)
assert len(prediction) == len(test_data.target)
# evaluate quality metric for the test sample
baseline_metrics = baseline_model.get_metrics(metric_names='f1')
assert baseline_metrics['f1'] > 0
def run_credit_scoring_problem(train_file_path, test_file_path,
timeout: float = 5.0,
is_visualise=False,
with_tuning=False,
cache_path=None):
preset = 'light_tun' if with_tuning else 'light'
automl = Fedot(problem='classification', timeout=timeout, verbose_level=4,
preset=preset)
automl.fit(train_file_path)
predict = automl.predict(test_file_path)
metrics = automl.get_metrics()
if is_visualise:
automl.current_pipeline.show()
print(f'Composed ROC AUC is {round(metrics["roc_auc"], 3)}')
return metrics["roc_auc"]
def test_cv_api_correct():
composer_params = {
'max_depth': 1,
'max_arity': 2,
'timeout': 0.0001,
'preset': 'ultra_light',
'cv_folds': 10
}
task = Task(task_type=TaskTypesEnum.classification)
dataset_to_compose, dataset_to_validate = get_data(task)
model = Fedot(problem='classification',
composer_params=composer_params,
verbose_level=2)
fedot_model = model.fit(features=dataset_to_compose)
prediction = model.predict(features=dataset_to_validate)
metric = model.get_metrics()
assert isinstance(fedot_model, Pipeline)
assert len(prediction) == len(dataset_to_validate.target)
assert metric['f1'] > 0
def run_pipeline_with_specific_evaluation_mode(train_data: InputData,
test_data: InputData,
mode: str = None):
"""
Runs the example with 3-node pipeline.
:param train_data: train data for pipeline training
:param test_data: test data for pipeline training
:param mode: pass gpu flag to make gpu evaluation
"""
problem = 'classification'
if mode == 'gpu':
baseline_model = Fedot(problem=problem, preset='gpu')
else:
baseline_model = Fedot(problem=problem)
svc_node_with_custom_params = PrimaryNode('svc')
svc_node_with_custom_params.custom_params = dict(kernel='rbf',
C=10,
gamma=1,
cache_size=2000,
probability=True)
logit_node = PrimaryNode('logit')
rf_node = SecondaryNode(
'rf', nodes_from=[svc_node_with_custom_params, logit_node])
preset_pipeline = Pipeline(rf_node)
start = datetime.now()
baseline_model.fit(features=train_data,
target='target',
predefined_model=preset_pipeline)
print(f'Completed with custom params in: {datetime.now() - start}')
baseline_model.predict(features=test_data)
print(baseline_model.get_metrics())
def run_classification_multiobj_example(with_plot=True):
train_data = pd.read_csv(
f'{project_root()}/examples/data/Hill_Valley_with_noise_Training.data')
test_data = pd.read_csv(
f'{project_root()}/examples/data/Hill_Valley_with_noise_Testing.data')
target = test_data['class']
del test_data['class']
problem = 'classification'
auto_model = Fedot(problem=problem,
learning_time=2,
preset='light',
composer_params={'metric': ['f1', 'node_num']},
seed=42)
auto_model.fit(features=train_data, target='class')
prediction = auto_model.predict_proba(features=test_data)
print(auto_model.get_metrics(target))
if with_plot:
auto_model.best_models.show()
return prediction
def run_additional_learning_example():
train_data_path = f'{fedot_project_root()}/cases/data/scoring/scoring_train.csv'
test_data_path = f'{fedot_project_root()}/cases/data/scoring/scoring_test.csv'
train_data = pd.read_csv(train_data_path)
test_data = pd.read_csv(test_data_path)
test_data_target = test_data['target']
del test_data['target']
problem = 'classification'
auto_model = Fedot(problem=problem,
seed=42,
preset='light',
timeout=5,
composer_params={
'initial_pipeline':
Pipeline(
SecondaryNode(
'logit',
nodes_from=[PrimaryNode('scaling')]))
})
auto_model.fit(features=deepcopy(train_data.head(1000)), target='target')
auto_model.predict_proba(features=deepcopy(test_data))
print('auto_model',
auto_model.get_metrics(target=deepcopy(test_data_target)))
prev_model = auto_model.current_pipeline
prev_model.show()
prev_model.unfit()
atomized_model = Pipeline(
SecondaryNode(operation_type=AtomizedModel(prev_model),
nodes_from=[PrimaryNode('scaling')]))
non_atomized_model = deepcopy(prev_model)
train_data = train_data.head(5000)
timeout = 1
auto_model_from_atomized = Fedot(
problem=problem,
seed=42,
preset='light',
timeout=timeout,
composer_params={'initial_pipeline': atomized_model},
verbose_level=2)
auto_model_from_atomized.fit(features=deepcopy(train_data),
target='target')
auto_model_from_atomized.predict_proba(features=deepcopy(test_data))
auto_model_from_atomized.current_pipeline.show()
print('auto_model_from_atomized',
auto_model_from_atomized.get_metrics(deepcopy(test_data_target)))
auto_model_from_pipeline = Fedot(
problem=problem,
seed=42,
preset='light',
timeout=timeout,
composer_params={'initial_pipeline': non_atomized_model},
verbose_level=2)
auto_model_from_pipeline.fit(features=deepcopy(train_data),
target='target')
auto_model_from_pipeline.predict_proba(features=deepcopy(test_data))
auto_model_from_pipeline.current_pipeline.show()
print('auto_model_from_pipeline',
auto_model_from_pipeline.get_metrics(deepcopy(test_data_target)))