def run_chain_from_automl(train_file_path: str,
test_file_path: str,
max_run_time: timedelta = timedelta(minutes=10)):
train_data = InputData.from_csv(train_file_path)
test_data = InputData.from_csv(test_file_path)
testing_target = test_data.target
chain = Chain()
node_tpot = PrimaryNode('tpot')
node_tpot.model.params = {'max_run_time_sec': max_run_time.seconds}
node_lda = PrimaryNode('lda')
node_rf = SecondaryNode('rf')
node_rf.nodes_from = [node_tpot, node_lda]
chain.add_node(node_rf)
chain.fit(train_data)
results = chain.predict(test_data)
roc_auc_value = roc_auc(y_true=testing_target, y_score=results.predict)
print(roc_auc_value)
return roc_auc_value
def chain_tuning(nodes_to_tune: str,
chain: Chain,
train_data: InputData,
test_data: InputData,
local_iter: int,
tuner_iter_num: int = 50) -> (float, list):
several_iter_scores_test = []
if nodes_to_tune == 'primary':
print('primary_node_tuning')
chain_tune_strategy = chain.fine_tune_primary_nodes
elif nodes_to_tune == 'root':
print('root_node_tuning')
chain_tune_strategy = chain.fine_tune_all_nodes
else:
raise ValueError(
f'Invalid type of nodes. Nodes must be primary or root')
for iteration in range(local_iter):
print(f'current local iteration {iteration}')
# Chain tuning
chain_tune_strategy(train_data, iterations=tuner_iter_num)
# After tuning prediction
chain.fit(train_data)
after_tuning_predicted = chain.predict(test_data)
# Metrics
aft_tun_roc_auc = roc_auc(y_true=test_data.target,
y_score=after_tuning_predicted.predict)
several_iter_scores_test.append(aft_tun_roc_auc)
return float(np.mean(several_iter_scores_test)), several_iter_scores_test
def metric_for_nodes(self, metric_function, train_data: InputData,
test_data: InputData, is_chain_shared: bool,
chain: Chain) -> float:
validate(chain)
if is_chain_shared:
chain = SharedChain(base_chain=chain,
shared_cache=self.shared_cache)
chain.fit(input_data=train_data)
return metric_function(chain, test_data)
def test_secondary_nodes_is_invariant_to_inputs_order(data_setup):
data = data_setup
train, test = train_test_data_setup(data)
first = NodeGenerator.primary_node(model_type=ModelTypesIdsEnum.logit)
second = NodeGenerator.primary_node(model_type=ModelTypesIdsEnum.lda)
third = NodeGenerator.primary_node(model_type=ModelTypesIdsEnum.knn)
final = NodeGenerator.secondary_node(model_type=ModelTypesIdsEnum.xgboost,
nodes_from=[first, second, third])
chain = Chain()
for node in [first, second, third, final]:
chain.add_node(node)
first = deepcopy(first)
second = deepcopy(second)
third = deepcopy(third)
final_shuffled = NodeGenerator.secondary_node(
model_type=ModelTypesIdsEnum.xgboost,
nodes_from=[third, first, second])
chain_shuffled = Chain()
# change order of nodes in list
for node in [final_shuffled, third, first, second]:
chain_shuffled.add_node(node)
train_predicted = chain.fit(input_data=train)
train_predicted_shuffled = chain_shuffled.fit(input_data=train)
# train results should be invariant
assert chain.root_node.descriptive_id == chain_shuffled.root_node.descriptive_id
assert all(
np.equal(train_predicted.predict, train_predicted_shuffled.predict))
test_predicted = chain.predict(input_data=test)
test_predicted_shuffled = chain_shuffled.predict(input_data=test)
# predict results should be invariant
assert all(
np.equal(test_predicted.predict, test_predicted_shuffled.predict))
# change parents order for the nodes fitted chain
nodes_for_change = chain.nodes[3].nodes_from
chain.nodes[3].nodes_from = [
nodes_for_change[2], nodes_for_change[0], nodes_for_change[1]
]
chain.nodes[3].cache.clear()
chain.fit(train)
test_predicted_re_shuffled = chain.predict(input_data=test)
# predict results should be invariant
assert all(
np.equal(test_predicted.predict, test_predicted_re_shuffled.predict))
def run_metocean_forecasting_problem(train_file_path,
test_file_path,
forecast_length=1,
max_window_size=64,
is_visualise=False):
# specify the task to solve
task_to_solve = Task(
TaskTypesEnum.ts_forecasting,
TsForecastingParams(forecast_length=forecast_length,
max_window_size=max_window_size))
full_path_train = os.path.join(str(project_root()), train_file_path)
dataset_to_train = InputData.from_csv(full_path_train,
task=task_to_solve,
data_type=DataTypesEnum.ts)
# a dataset for a final validation of the composed model
full_path_test = os.path.join(str(project_root()), test_file_path)
dataset_to_validate = InputData.from_csv(full_path_test,
task=task_to_solve,
data_type=DataTypesEnum.ts)
chain = get_composite_lstm_chain()
chain_simple = Chain()
node_single = PrimaryNode('ridge')
chain_simple.add_node(node_single)
chain_lstm = Chain()
node_lstm = PrimaryNode('lstm')
chain_lstm.add_node(node_lstm)
chain.fit(input_data=dataset_to_train, verbose=False)
rmse_on_valid = calculate_validation_metric(
chain.predict(dataset_to_validate), dataset_to_validate,
f'full-composite_{forecast_length}', is_visualise)
chain_lstm.fit(input_data=dataset_to_train, verbose=False)
rmse_on_valid_lstm_only = calculate_validation_metric(
chain_lstm.predict(dataset_to_validate), dataset_to_validate,
f'full-lstm-only_{forecast_length}', is_visualise)
chain_simple.fit(input_data=dataset_to_train, verbose=False)
rmse_on_valid_simple = calculate_validation_metric(
chain_simple.predict(dataset_to_validate), dataset_to_validate,
f'full-simple_{forecast_length}', is_visualise)
print(f'RMSE composite: {rmse_on_valid}')
print(f'RMSE simple: {rmse_on_valid_simple}')
print(f'RMSE LSTM only: {rmse_on_valid_lstm_only}')
return rmse_on_valid_simple
def test_regression_chain_fit_correct():
data = get_synthetic_ts_data()
chain = Chain()
node_rfr = PrimaryNode('rfr')
chain.add_node(node_rfr)
train_data, test_data = train_test_data_setup(data)
chain.fit(input_data=train_data)
_, rmse_on_test = get_rmse_value(chain, train_data, test_data)
rmse_threshold = np.std(data.target) * 1.5
assert rmse_on_test < rmse_threshold
def metric_for_nodes(self, metric_function, train_data: InputData,
test_data: InputData, is_chain_shared: bool,
chain: Chain) -> float:
try:
validate(chain)
if is_chain_shared:
chain = SharedChain(base_chain=chain,
shared_cache=self.shared_cache)
chain.fit(input_data=train_data)
return metric_function(chain, test_data)
except Exception as ex:
print(
f'Error in chain assessment during composition: {ex}. Continue.'
)
return max_int_value
def test_regression_chain_with_datamodel_fit_correct():
data = get_synthetic_regression_data()
train_data, test_data = train_test_data_setup(data)
node_data = PrimaryNode('direct_data_model')
node_first = PrimaryNode('ridge')
node_second = SecondaryNode('lasso')
node_second.nodes_from = [node_first, node_data]
chain = Chain(node_second)
chain.fit(train_data)
results = chain.predict(test_data)
assert results.predict.shape == test_data.target.shape
def test_chain_hierarchy_fit_correct(data_setup):
data = data_setup
train, _ = train_test_data_setup(data)
first = NodeGenerator.primary_node(model_type=ModelTypesIdsEnum.logit)
second = NodeGenerator.secondary_node(model_type=ModelTypesIdsEnum.logit,
nodes_from=[first])
third = NodeGenerator.secondary_node(model_type=ModelTypesIdsEnum.logit,
nodes_from=[first])
final = NodeGenerator.secondary_node(model_type=ModelTypesIdsEnum.logit,
nodes_from=[second, third])
chain = Chain()
for node in [first, second, third, final]:
chain.add_node(node)
train_predicted = chain.fit(input_data=train, use_cache=False)
assert chain.root_node.descriptive_id == (
'((/n_ModelTypesIdsEnum.logit_defaultparams;)/'
'n_ModelTypesIdsEnum.logit_defaultparams;;(/'
'n_ModelTypesIdsEnum.logit_defaultparams;)/'
'n_ModelTypesIdsEnum.logit_defaultparams;)/'
'n_ModelTypesIdsEnum.logit_defaultparams')
assert chain.length == 4
assert chain.depth == 3
assert train_predicted.predict.shape == train.target.shape
def test_chain_with_datamodel_fit_correct(data_setup):
data = data_setup
train_data, test_data = train_test_data_setup(data)
chain = Chain()
node_data = PrimaryNode('direct_data_model')
node_first = PrimaryNode('bernb')
node_second = SecondaryNode('rf')
node_second.nodes_from = [node_first, node_data]
chain.add_node(node_data)
chain.add_node(node_first)
chain.add_node(node_second)
chain.fit(train_data)
results = np.asarray(probs_to_labels(chain.predict(test_data).predict))
assert results.shape == test_data.target.shape
def test_chain_with_custom_params_for_model(data_setup):
data = data_setup
custom_params = dict(n_neighbors=1, weights='uniform', p=1)
first = PrimaryNode(model_type='logit')
second = PrimaryNode(model_type='lda')
final = SecondaryNode(model_type='knn', nodes_from=[first, second])
chain = Chain()
chain.add_node(final)
chain_default_params = deepcopy(chain)
chain.root_node.custom_params = custom_params
chain_default_params.fit(data)
chain.fit(data)
custom_params_prediction = chain.predict(data).predict
default_params_prediction = chain_default_params.predict(data).predict
assert not np.array_equal(custom_params_prediction,
default_params_prediction)
def run_tpot_vs_fedot_example(train_file_path: str, test_file_path: str):
train_data = InputData.from_csv(train_file_path)
test_data = InputData.from_csv(test_file_path)
training_features = train_data.features
testing_features = test_data.features
training_target = train_data.target
testing_target = test_data.target
# Average CV score on the training set was: 0.93755
exported_pipeline = make_pipeline(
StackingEstimator(estimator=BernoulliNB()), RandomForestClassifier())
# Fix random state for all the steps in exported pipeline
set_param_recursive(exported_pipeline.steps, 'random_state', 1)
exported_pipeline.fit(training_features, training_target)
results = exported_pipeline.predict_proba(testing_features)[:, 1]
roc_auc_value = roc_auc(y_true=testing_target, y_score=results)
print(roc_auc_value)
chain = Chain()
node_first = PrimaryNode('direct_data_model')
node_second = PrimaryNode('bernb')
node_third = SecondaryNode('rf')
node_third.nodes_from.append(node_first)
node_third.nodes_from.append(node_second)
chain.add_node(node_third)
chain.fit(train_data)
results = chain.predict(test_data)
roc_auc_value = roc_auc(y_true=testing_target, y_score=results.predict)
print(roc_auc_value)
return roc_auc_value
def test_chain_sequential_fit_correct(data_setup):
data = data_setup
train, _ = train_test_data_setup(data)
first = PrimaryNode(model_type='logit')
second = SecondaryNode(model_type='logit', nodes_from=[first])
third = SecondaryNode(model_type='logit', nodes_from=[second])
final = SecondaryNode(model_type='logit', nodes_from=[third])
chain = Chain()
for node in [first, second, third, final]:
chain.add_node(node)
train_predicted = chain.fit(input_data=train, use_cache=False)
assert chain.root_node.descriptive_id == ('(((/n_logit_default_params;)/'
'n_logit_default_params;)/'
'n_logit_default_params;)/'
'n_logit_default_params')
assert chain.length == 4
assert chain.depth == 4
assert train_predicted.predict.shape[0] == train.target.shape[0]
assert final.cache.actual_cached_state is not None
from core.models.model import * from benchmark.benchmark_utils import get_scoring_case_data_paths train_file_path, test_file_path = get_scoring_case_data_paths() train_data = InputData.from_csv(train_file_path) test_data = InputData.from_csv(test_file_path) training_features = train_data.features testing_features = test_data.features training_target = train_data.target testing_target = test_data.target chain = Chain() node0 = NodeGenerator.primary_node(ModelTypesIdsEnum.tpot) node1 = NodeGenerator.primary_node(ModelTypesIdsEnum.lda) node2 = NodeGenerator.secondary_node(ModelTypesIdsEnum.rf) node2.nodes_from.append(node0) node2.nodes_from.append(node1) chain.add_node(node0) chain.add_node(node1) chain.add_node(node2) chain.fit(train_data) results = chain.predict(test_data) roc_auc_value = roc_auc(y_true=testing_target, y_score=results.predict) print(roc_auc_value)