def _eval_strategy_for_task(model_type: str, task_type_for_data: TaskTypesEnum): models_repo = ModelTypesRepository() model_info = models_repo.model_info_by_id(model_type) task_type_for_model = task_type_for_data task_types_acceptable_for_model = model_info.task_type # if the model can't be used directly for the task type from data if task_type_for_model not in task_types_acceptable_for_model: # search the supplementary task types, that can be included in chain which solves original task globally_compatible_task_types = compatible_task_types( task_type_for_model) compatible_task_types_acceptable_for_model = list( set(task_types_acceptable_for_model).intersection( set(globally_compatible_task_types))) if len(compatible_task_types_acceptable_for_model) == 0: raise ValueError( f'Model {model_type} can not be used as a part of {task_type_for_model}.' ) task_type_for_model = compatible_task_types_acceptable_for_model[0] strategy = models_repo.model_info_by_id(model_type).current_strategy( task_type_for_model) return strategy
def get_model(train_file_path: str, cur_lead_time: datetime.timedelta = timedelta(seconds=60)): task = Task(task_type=TaskTypesEnum.classification) dataset_to_compose = InputData.from_csv(train_file_path, task=task) # the search of the models provided by the framework # that can be used as nodes in a chain for the selected task models_repo = ModelTypesRepository() available_model_types, _ = models_repo.suitable_model( task_type=task.task_type, tags=['simple']) metric_function = MetricsRepository(). \ metric_by_id(ClassificationMetricsEnum.ROCAUC_penalty) composer_requirements = GPComposerRequirements( primary=available_model_types, secondary=available_model_types, max_lead_time=cur_lead_time) # Create the genetic programming-based composer, that allow to find # the optimal structure of the composite model builder = GPComposerBuilder(task).with_requirements( composer_requirements).with_metrics(metric_function) composer = builder.build() # run the search of best suitable model chain_evo_composed = composer.compose_chain(data=dataset_to_compose, is_visualise=False) chain_evo_composed.fit(input_data=dataset_to_compose) return chain_evo_composed
def _eval_strategy_for_task(model_type: ModelTypesIdsEnum, task_type_for_data: TaskTypesEnum): strategies_for_tasks = { MachineLearningTasksEnum.classification: [SkLearnClassificationStrategy, AutoMLEvaluationStrategy], MachineLearningTasksEnum.regression: [SkLearnRegressionStrategy], MachineLearningTasksEnum.auto_regression: [StatsModelsAutoRegressionStrategy], MachineLearningTasksEnum.clustering: [SkLearnClusteringStrategy] } models_for_strategies = { SkLearnClassificationStrategy: [ ModelTypesIdsEnum.xgboost, ModelTypesIdsEnum.knn, ModelTypesIdsEnum.logit, ModelTypesIdsEnum.dt, ModelTypesIdsEnum.rf, ModelTypesIdsEnum.mlp, ModelTypesIdsEnum.lda, ModelTypesIdsEnum.qda ], AutoMLEvaluationStrategy: [ModelTypesIdsEnum.tpot, ModelTypesIdsEnum.h2o], SkLearnClusteringStrategy: [ModelTypesIdsEnum.kmeans], SkLearnRegressionStrategy: [ ModelTypesIdsEnum.linear, ModelTypesIdsEnum.ridge, ModelTypesIdsEnum.lasso ], StatsModelsAutoRegressionStrategy: [ModelTypesIdsEnum.ar, ModelTypesIdsEnum.arima] } models_repo = ModelTypesRepository() _, model_info = models_repo.search_models(desired_ids=[model_type]) task_type_for_model = task_type_for_data task_types_acceptable_for_model = model_info[0].task_type # if the model can't be used directly for the task type from data if task_type_for_model not in task_types_acceptable_for_model: # search the supplementary task types, that can be included in chain which solves original task globally_compatible_task_types = compatible_task_types( task_type_for_model) compatible_task_types_acceptable_for_model = list( set(task_types_acceptable_for_model).intersection( set(globally_compatible_task_types))) if len(compatible_task_types_acceptable_for_model) == 0: raise ValueError( f'Model {model_type} can not be used as a part of {task_type_for_model}.' ) task_type_for_model = compatible_task_types_acceptable_for_model[0] eval_strategies = strategies_for_tasks[task_type_for_model] for strategy in eval_strategies: if model_type in models_for_strategies[strategy]: eval_strategy = strategy(model_type) return eval_strategy return None
def test_parameter_free_composer_build_chain_correct(data_fixture, request): random.seed(1) np.random.seed(1) data = request.getfixturevalue(data_fixture) dataset_to_compose = data dataset_to_validate = data available_model_types, _ = ModelTypesRepository().suitable_model( task_type=TaskTypesEnum.classification) metric_function = MetricsRepository().metric_by_id( ClassificationMetricsEnum.ROCAUC) req = GPComposerRequirements(primary=available_model_types, secondary=available_model_types, max_arity=2, max_depth=2, pop_size=2, num_of_generations=1, crossover_prob=0.4, mutation_prob=0.5) opt_params = GPChainOptimiserParameters( genetic_scheme_type=GeneticSchemeTypesEnum.parameter_free) builder = GPComposerBuilder(task=Task( TaskTypesEnum.classification)).with_requirements(req).with_metrics( metric_function).with_optimiser_parameters(opt_params) gp_composer = builder.build() chain_gp_composed = gp_composer.compose_chain(data=dataset_to_compose) chain_gp_composed.fit_from_scratch(input_data=dataset_to_compose) predicted_gp_composed = chain_gp_composed.predict(dataset_to_validate) roc_on_valid_gp_composed = roc_auc(y_true=dataset_to_validate.target, y_score=predicted_gp_composed.predict) assert roc_on_valid_gp_composed > 0.6
def test_random_composer(data_fixture, request): random.seed(1) np.random.seed(1) data = request.getfixturevalue(data_fixture) dataset_to_compose = data dataset_to_validate = data available_model_types, _ = ModelTypesRepository().suitable_model( task_type=TaskTypesEnum.classification) metric_function = MetricsRepository().metric_by_id( ClassificationMetricsEnum.ROCAUC) random_composer = RandomSearchComposer(iter_num=1) req = ComposerRequirements(primary=available_model_types, secondary=available_model_types) chain_random_composed = random_composer.compose_chain( data=dataset_to_compose, initial_chain=None, composer_requirements=req, metrics=metric_function) chain_random_composed.fit_from_scratch(input_data=dataset_to_compose) predicted_random_composed = chain_random_composed.predict( dataset_to_validate) roc_on_valid_random_composed = roc_auc( y_true=dataset_to_validate.target, y_score=predicted_random_composed.predict) assert roc_on_valid_random_composed > 0.6
def test_search_in_repository_by_tag_and_metainfo_correct(): with ModelTypesRepository(mocked_path()) as repo: model_names, _ = repo.suitable_model( task_type=TaskTypesEnum.regression, tags=['ml']) assert 'linear' in model_names assert len(model_names) == 3
def set_default_composer_params(self): if not self._composer.composer_requirements: models, _ = ModelTypesRepository().suitable_model(task_type=self.task.task_type) self._composer.composer_requirements = GPComposerRequirements(primary=models, secondary=models) if not self._composer.metrics: metric_function = MetricsRepository().metric_by_id(ClassificationMetricsEnum.ROCAUC_penalty) if self.task.task_type in (TaskTypesEnum.regression, TaskTypesEnum.ts_forecasting): metric_function = MetricsRepository().metric_by_id(RegressionMetricsEnum.RMSE) self._composer.metrics = metric_function
def test_search_in_repository_by_tag_correct(): with ModelTypesRepository(mocked_path()) as repo: model_names, _ = repo.models_with_tag(tags=['simple', 'linear'], is_full_match=True) assert {'linear', 'logit', 'lasso', 'ridge'}.issubset(model_names) assert len(model_names) == 4 model_names, _ = repo.models_with_tag(tags=['simple', 'linear']) assert {'linear', 'logit', 'knn', 'lda', 'lasso', 'ridge'}.issubset(model_names) assert len(model_names) == 6 model_names, _ = repo.models_with_tag(tags=['non_real_tag']) assert len(model_names) == 0
def test_chain_from_automl_example(): project_root_path = str(project_root()) experimental_repo_path = os.path.join( project_root_path, 'fedot/core/repository/data/model_repository_with_automl.json') with ModelTypesRepository(experimental_repo_path) as _: file_path_train = os.path.join(project_root_path, 'test/data/simple_classification.csv') file_path_test = file_path_train auc = run_chain_from_automl(file_path_train, file_path_test, max_run_time=timedelta(seconds=1)) assert auc > 0.5
def run_credit_scoring_problem(train_file_path, test_file_path, max_lead_time: datetime.timedelta = datetime.timedelta(minutes=5), gp_optimiser_params: Optional[GPChainOptimiserParameters] = None, pop_size=None, generations=None): dataset_to_compose = InputData.from_csv(train_file_path) dataset_to_validate = InputData.from_csv(test_file_path) available_model_types, _ = ModelTypesRepository(). \ suitable_model(task_type=TaskTypesEnum.classification) # the choice of the metric for the chain quality assessment during composition metric_function = MetricsRepository().metric_by_id(ClassificationMetricsEnum.ROCAUC) if gp_optimiser_params: optimiser_parameters = gp_optimiser_params else: selection_types = [SelectionTypesEnum.tournament] crossover_types = [CrossoverTypesEnum.subtree] mutation_types = [MutationTypesEnum.simple, MutationTypesEnum.growth, MutationTypesEnum.reduce] regularization_type = RegularizationTypesEnum.decremental optimiser_parameters = GPChainOptimiserParameters(selection_types=selection_types, crossover_types=crossover_types, mutation_types=mutation_types, regularization_type=regularization_type) composer_requirements = GPComposerRequirements( primary=available_model_types, secondary=available_model_types, max_arity=4, max_depth=3, pop_size=pop_size, num_of_generations=generations, crossover_prob=0.8, mutation_prob=0.8, max_lead_time=max_lead_time) # Create GP-based composer composer = GPComposer() chain_evo_composed = composer.compose_chain(data=dataset_to_compose, initial_chain=None, composer_requirements=composer_requirements, metrics=metric_function, optimiser_parameters=optimiser_parameters, is_visualise=False) chain_evo_composed.fit(input_data=dataset_to_compose, verbose=True) roc_on_valid_evo_composed = calculate_validation_metric(chain_evo_composed, dataset_to_validate) print(f'Composed ROC AUC is {round(roc_on_valid_evo_composed, 3)}') return roc_on_valid_evo_composed, chain_evo_composed, composer
def test_gp_composer_builder(): task = Task(TaskTypesEnum.classification) available_model_types, _ = ModelTypesRepository().suitable_model( task_type=task.task_type) metric_function = MetricsRepository().metric_by_id( ClassificationMetricsEnum.ROCAUC) composer_requirements = GPComposerRequirements( primary=available_model_types, secondary=available_model_types, max_arity=3, max_depth=3, pop_size=5, num_of_generations=4, crossover_prob=0.8, mutation_prob=1, max_lead_time=datetime.timedelta(minutes=5)) scheme_type = GeneticSchemeTypesEnum.steady_state optimiser_parameters = GPChainOptimiserParameters( genetic_scheme_type=scheme_type) builder_with_custom_params = GPComposerBuilder( task=task).with_requirements(composer_requirements).with_metrics( metric_function).with_optimiser_parameters(optimiser_parameters) composer_with_custom_params = builder_with_custom_params.build() assert composer_with_custom_params.optimiser.parameters.genetic_scheme_type == scheme_type assert composer_with_custom_params.metrics == metric_function assert composer_with_custom_params.composer_requirements.pop_size == 5 assert composer_with_custom_params.composer_requirements.mutation_prob == 1 builder_with_default_params = GPComposerBuilder(task=task) composer_with_default_params = builder_with_default_params.build() default_metric = MetricsRepository().metric_by_id( ClassificationMetricsEnum.ROCAUC.ROCAUC_penalty) assert composer_with_default_params.optimiser.parameters.genetic_scheme_type == GeneticSchemeTypesEnum.generational assert composer_with_default_params.metrics == default_metric assert composer_with_default_params.composer_requirements.pop_size == 20 assert composer_with_default_params.composer_requirements.mutation_prob == 0.8
def run_credit_scoring_problem( train_file_path, test_file_path, max_lead_time: datetime.timedelta = datetime.timedelta(minutes=5), is_visualise=False, with_tuning=False): task = Task(TaskTypesEnum.classification) dataset_to_compose = InputData.from_csv(train_file_path, task=task) dataset_to_validate = InputData.from_csv(test_file_path, task=task) # the search of the models provided by the framework that can be used as nodes in a chain for the selected task available_model_types, _ = ModelTypesRepository().suitable_model( task_type=task.task_type) # the choice of the metric for the chain quality assessment during composition metric_function = MetricsRepository().metric_by_id( ClassificationMetricsEnum.ROCAUC_penalty) # the choice and initialisation of the GP search composer_requirements = GPComposerRequirements( primary=available_model_types, secondary=available_model_types, max_arity=3, max_depth=3, pop_size=20, num_of_generations=20, crossover_prob=0.8, mutation_prob=0.8, max_lead_time=max_lead_time) # GP optimiser parameters choice scheme_type = GeneticSchemeTypesEnum.steady_state optimiser_parameters = GPChainOptimiserParameters( genetic_scheme_type=scheme_type) # Create builder for composer and set composer params builder = GPComposerBuilder( task=task).with_requirements(composer_requirements).with_metrics( metric_function).with_optimiser_parameters(optimiser_parameters) # Create GP-based composer composer = builder.build() # the optimal chain generation by composition - the most time-consuming task chain_evo_composed = composer.compose_chain(data=dataset_to_compose, is_visualise=True) if with_tuning: chain_evo_composed.fine_tune_primary_nodes( input_data=dataset_to_compose, iterations=50, verbose=True) chain_evo_composed.fit(input_data=dataset_to_compose, verbose=True) if is_visualise: visualiser = ChainVisualiser() composer.log.info('History visualization started') visualiser.visualise_history(composer.history) composer.log.info('History visualization finished') composer.history.write_composer_history_to_csv() composer.log.info('Best chain visualization started') visualiser.visualise(chain_evo_composed) composer.log.info('Best chain visualization finished') # the quality assessment for the obtained composite models roc_on_valid_evo_composed = calculate_validation_metric( chain_evo_composed, dataset_to_validate) print(f'Composed ROC AUC is {round(roc_on_valid_evo_composed, 3)}') return roc_on_valid_evo_composed
def run_fedot(params: 'ExecutionParams'): train_file_path = params.train_file test_file_path = params.test_file case_label = params.case_label task_type = params.task if task_type == TaskTypesEnum.classification: metric = ClassificationMetricsEnum.ROCAUC elif task_type == TaskTypesEnum.regression: metric = RegressionMetricsEnum.RMSE else: raise NotImplementedError() metric_func = MetricsRepository().metric_by_id(metric) task = Task(task_type) dataset_to_compose = InputData.from_csv(train_file_path, task=task) dataset_to_validate = InputData.from_csv(test_file_path, task=task) models_hyperparameters = get_models_hyperparameters()['FEDOT'] cur_lead_time = models_hyperparameters['MAX_RUNTIME_MINS'] saved_model_name = f'fedot_{case_label}_{task_type.name}_{cur_lead_time}_{metric.name}' loaded_model = load_fedot_model(saved_model_name) if not loaded_model: generations = models_hyperparameters['GENERATIONS'] population_size = models_hyperparameters['POPULATION_SIZE'] # the search of the models provided by the framework that can be used as nodes in a chain' models_repo = ModelTypesRepository() available_model_types, _ = models_repo.suitable_model(task.task_type) heavy_models = ['svc', 'multinb', 'tfidf', 'qda'] available_model_types = [ model for model in available_model_types if model not in heavy_models ] # the choice and initialisation of the GP search composer_requirements = GPComposerRequirements( primary=available_model_types, secondary=available_model_types, max_arity=3, max_depth=2, pop_size=population_size, num_of_generations=generations, crossover_prob=0.8, mutation_prob=0.8, max_lead_time=datetime.timedelta(minutes=cur_lead_time), add_single_model_chains=True) # Create GP-based composer builder = GPComposerBuilder(task).with_requirements( composer_requirements).with_metrics(metric_func) gp_composer = builder.build() chain_gp_composed = gp_composer.compose_chain(data=dataset_to_compose) chain_gp_composed.fit_from_scratch(input_data=dataset_to_compose) save_fedot_model(chain_gp_composed, saved_model_name) else: chain_gp_composed = loaded_model evo_predicted = chain_gp_composed.predict(dataset_to_validate) evo_predicted_labels = chain_gp_composed.predict(dataset_to_validate, output_mode='labels') return dataset_to_validate.target, evo_predicted.predict, evo_predicted_labels.predict
def acceptable_task_types(self): model_info = ModelTypesRepository().model_info_by_id(self.model_type) return model_info.task_type
def test_lazy_load(): with ModelTypesRepository(mocked_path()) as repo: repo_second = ModelTypesRepository() assert repo._repo == repo_second._repo
def metadata(self) -> ModelMetaInfo: model_info = ModelTypesRepository().model_info_by_id(self.model_type) if not model_info: raise ValueError(f'Model {self.model_type} not found') return model_info
return_df: bool = False): df = pd.read_excel(file_path) train, test = split_data(df) file_dir_name = file_path.replace('.', '/').split('/')[-2] file_csv_name = f'{file_dir_name}.csv' directory_names = ['examples', 'data', file_dir_name] ensure_directory_exists(directory_names) if return_df: path = os.path.join(directory_names[0], directory_names[1], directory_names[2], file_csv_name) full_file_path = os.path.join(str(project_root()), path) save_file_to_csv(df, full_file_path) return df, full_file_path else: full_train_file_path, full_test_file_path = get_split_data_paths( directory_names) save_file_to_csv(train, full_train_file_path) save_file_to_csv(train, full_test_file_path) return full_train_file_path, full_test_file_path def print_models_info(repository: ModelTypesRepository, task=TaskTypesEnum.classification): for model in repository.models: print(f'{model.id}, {model.current_strategy(task)}, ' f'{model.current_strategy(task)(model.id).implementation_info}') if __name__ == '__main__': print_models_info(ModelTypesRepository())
def run(dataset, config): log.info("\n**** FEDOT ****\n") is_classification = config.type == 'classification' # Mapping of benchmark metrics to FEDOT metrics metrics_mapping = dict(acc='accuracy', auc='roc_auc', f1='f1', logloss='neg_log_loss', mae='neg_mean_absolute_error', mse='neg_mean_squared_error', msle='neg_mean_squared_log_error', r2='r2', rmse='neg_mean_squared_error') scoring_metric = metrics_mapping[ config.metric] if config.metric in metrics_mapping else None if scoring_metric is None: raise ValueError("Performance metric {} not supported.".format( config.metric)) if is_classification: metric = ClassificationMetricsEnum.ROCAUC task_type = TaskTypesEnum.classification else: metric = RegressionMetricsEnum.RMSE task_type = TaskTypesEnum.regression task = Task(task_type) x_train = dataset.train.X_enc y_train = dataset.train.y_enc x_test = dataset.test.X_enc x_train, y_train = shuffle(dataset.train.X_enc, dataset.train.y_enc, random_state=0) if len(y_train.shape) > 1 and y_train.shape[1] == 1: y_train = np.squeeze(y_train, axis=1) training_params = { k: v for k, v in config.framework_params.items() if not k.startswith('_') } dataset_to_compose = \ InputData(idx=[_ for _ in range(len(y_train))], features=x_train, target=y_train, task=task, data_type=DataTypesEnum.table) dataset_to_test = \ InputData(idx=[_ for _ in range(len(y_train))], features=x_test, target=None, task=task, data_type=DataTypesEnum.table) n_jobs = config.framework_params.get( '_n_jobs', config.cores ) # useful to disable multicore, regardless of the dataset config log.info( 'Running FEDOT with a maximum time of %ss on %s cores, optimizing %s.', config.max_runtime_seconds, n_jobs, scoring_metric) runtime_min = (config.max_runtime_seconds / 60) available_model_types, _ = ModelTypesRepository().suitable_model( task_type=task.task_type) metric_function = MetricsRepository().metric_by_id(metric) # Create GP-based composer composer = GPComposer() if False: # the choice and initialisation of the GP search composer_requirements = GPComposerRequirements( primary=available_model_types, secondary=available_model_types, max_arity=3, max_depth=3, max_lead_time=datetime.timedelta(minutes=runtime_min * 0.8)) # GP optimiser parameters choice scheme_type = GeneticSchemeTypesEnum.parameter_free optimiser_parameters = GPChainOptimiserParameters( genetic_scheme_type=scheme_type) # Create builder for composer and set composer params builder = GPComposerBuilder( task=task).with_requirements(composer_requirements).with_metrics( metric_function).with_optimiser_parameters( optimiser_parameters) composer = builder.build() # the optimal chain generation by composition - the most time-consuming task chain_evo_composed = composer.compose_chain(data=dataset_to_compose, is_visualise=False) else: if is_classification: chain_evo_composed = Chain(PrimaryNode('logit')) else: chain_evo_composed = Chain(PrimaryNode('lasso')) chain_evo_composed.fit(input_data=dataset_to_compose, verbose=False) log.info('Predicting on the test set.') y_test = dataset.test.y_enc predictions = chain_evo_composed.predict(dataset_to_test, output_mode='labels').predict if not is_classification: probabilities = None else: probabilities = chain_evo_composed.predict( dataset_to_test, output_mode='full_probs').predict return result(output_file=config.output_predictions_file, predictions=predictions, truth=y_test, probabilities=probabilities, target_is_encoded=is_classification, models_count=1, training_duration=1)