def _create_eval_feature_cases(params, set_features_to_eval, eval_type): if len(set_features_to_eval) == 0: raise CatboostError("Provide at least one feature to evaluation") # baseline test_cases = list() baseline_case = ExecutionCase( params, ignored_features=list(set_features_to_eval)) # test if eval_type == EvalType.All or eval_type == EvalType.SeqAddAndAll or len( set_features_to_eval) == 1: test_cases.append(ExecutionCase(params, ignored_features=[])) if eval_type == EvalType.SeqRem: for feature_num in set_features_to_eval: test_cases.append( ExecutionCase(params, ignored_features=[feature_num])) elif eval_type == EvalType.SeqAdd or eval_type == EvalType.SeqAddAndAll: for feature_num in set_features_to_eval: cur_features = copy(set_features_to_eval) cur_features.remove(feature_num) test_cases.append( ExecutionCase(params, ignored_features=list(cur_features))) elif eval_type != EvalType.All: raise AttributeError("Don't support {} mode.", eval_type.value) return baseline_case, test_cases
def close(self): if self._file is None: raise CatboostError("Trying to close None {}".format( self._file_path)) self._file.close() self._file = None
def create_metrics_calcer(self, metrics, thread_count, eval_step=1): if not os.path.exists(self._model_path): raise CatboostError("Model was deleted. Can't create calcer now") model = CatBoost() model.load_model(self._model_path) return model.create_metric_calcer(metrics, thread_count=thread_count, eval_period=eval_step)
def __init__(self, metric_results): if len(metric_results) < 1: raise CatboostError("Need at least one result") self._results = dict() self._metrics = dict() self._cases = None for result in metric_results: metric_description = result.get_metric_description() if metric_description in self._results: raise CatboostError( "Duplicate metric {}".format(metric_description)) if self._cases is None: self._cases = result.get_cases() key = metric_description_or_str_to_str(metric_description) self._results[key] = result self._metrics[key] = metric_description
def eval_features(self, learn_config, objective_function, set_features_to_eval, eval_type=EvalType.All, eval_metrics=None, thread_count=None, eval_step=None): """ Evaluate features. Args: learn_config: dict with params or instance of CatBoost. In second case instance params will be used objective_function: objective_function: one of CatBoost loss functions eval_type: Type of feature evaluate (All, SeqAdd, SeqRem) eval_metrics: Additional metrics to calculate thread_count: thread_count to use. If not none will override learn_config values Returns ------- result : Instance of EvaluationResult class """ if eval_metrics is None: eval_metrics = [] if isinstance(learn_config, CatBoost): params = learn_config.get_params() else: params = dict(learn_config) if "loss_function" in params and params[ "loss_function"] != objective_function: raise CatboostError( "Loss function in params {} should be equal to feature evaluation objective " "function {}".format(params["loss_function"], objective_function)) if thread_count is not None: params["thread_count"] = thread_count else: thread_count = 1 if "thread_count" in learn_config: thread_count = params["thread_count"] if eval_step is None: eval_step = 1 params["loss_function"] = objective_function baseline_case, test_cases = self._create_eval_feature_cases( params, set_features_to_eval, eval_type=eval_type) if objective_function not in eval_metrics: eval_metrics.append(objective_function) return self.eval_cases(baseline_case=baseline_case, compare_cases=test_cases, eval_metrics=eval_metrics, thread_count=thread_count, eval_step=eval_step)
def _change_score_config(self, config): if config is not None: if isinstance(config, ScoreType): if config == ScoreType.Abs: config = ScoreConfig.abs_score() elif config == ScoreType.Rel: config = ScoreConfig.rel_score() else: raise CatboostError("Unknown scoreType {}".format(config)) if self._score_config != config: self._score_config = config self.__clear_comparisons()
def __init__(self, case_results): if len(case_results) <= 1: raise CatboostError("Need at least 2 case results, got {} ".format( len(case_results))) self._case_results = dict() self._case_comparisons = dict() self._cases = [case_result.get_case() for case_result in case_results] for case_result in case_results: case = case_result.get_case() self._case_results[case] = case_result
class MetricEvaluationResult: """ Evaluation result for one metric. Stores all ExecutionCases with specified metric scores Computes human-friendly tables with results and some plots """ def __init__(self, case_results): if len(case_results) <= 1: raise CatboostError("Need at least 2 case results, got {} ".format( len(case_results))) self._case_results = dict() self._case_comparisons = dict() self._cases = [case_result.get_case() for case_result in case_results] for case_result in case_results: case = case_result.get_case() self._case_results[case] = case_result self._metric_description = case_results[0].get_metric_description() self._baseline_case = case_results[0].get_case() self._score_config = ScoreConfig(score_type=ScoreType.Rel, score_level=0.01, multiplier=1000) for (case, case_result) in self._case_results.items(): if case_result.get_metric_description( ) != self._metric_description: raise CatboostError( "Metric names should be equal for all case results") if case_result.get_fold_ids() != self.get_fold_ids(): raise CatboostError( "Case results should be computed on the same folds") if case_result.get_eval_step() != self.get_eval_step(): raise CatboostError( "Eval steps should be equal for different cases")
def _calculate_result_metrics(self, cases, metrics, thread_count=-1, evaluation_step=1): """ This method calculate metrics and return them. Args: :param cases: List of the ExecutionCases you want to evaluate :param metrics: List of the metrics to be computed :param thread_count: Count of thread to use. :param: evaluation_step: Step to evaluate metrics :return: instance of EvaluationResult """ cases_set = set(cases) if len(cases_set) != len(cases): raise CatboostError("Found duplicate cases in " + cases) current_wd = self.__go_to_working_dir() try: if self._fold_count <= self._fold_offset: error_msg = 'Count of folds(folds_count - offset) need to be at least one: offset {}, folds_count {}.' raise AttributeError( error_msg.format(self._fold_offset, self._fold_count)) handler = FoldModelsHandler(cases=cases, metrics=metrics, eval_step=evaluation_step, thread_count=thread_count, remove_models=self._remove_models, time_split_mode=self._time_split_mode) reader = _SimpleStreamingFileReader( self._path_to_dataset, sep=self._delimiter, group_feature_num=self._group_feature_num) splitter = _Splitter(reader, self._column_description, seed=self._seed, min_folds_count=self._min_fold_count, time_split_mode=self._time_split_mode) result = handler.proceed(splitter=splitter, fold_size=self._fold_size, folds_count=self._fold_count, fold_offset=self._fold_offset) return self._create_evaluation_results(result) finally: os.chdir(current_wd)
def _add(self, model, learning_curve): if model.get_case() != self._case: raise CatboostError("Model case should be equal to result case") fold_id = model.get_fold_id() self._fold_curves[fold_id] = learning_curve score = max(learning_curve) if self._metric_description.is_max_optimal( ) else min(learning_curve) position = np.argmax( learning_curve) if self._metric_description.is_max_optimal( ) else np.argmin(learning_curve) self._fold_metric.at[fold_id] = score self._fold_metric_iteration.at[fold_id] = position
def _compute_metrics(self, metrics, grouped_by_case_models, learn_folds, skipped_folds, rest_folds): metric_calcers = {} for case, case_models in grouped_by_case_models.items(): metric_calcers[case] = list() for case_model in case_models: metric_calcer = case_model.create_metrics_calcer( metrics, eval_step=self._eval_step, thread_count=self._thread_count) metric_calcers[case].append(metric_calcer) if self._metric_descriptions is None: self._init_case_results( metric_calcer.metric_descriptions()) elif self._metric_descriptions != metric_calcer.metric_descriptions( ): raise CatboostError( "Error: metric names should be consistent") for file_num, fold_file in enumerate(skipped_folds + learn_folds + rest_folds): pool = FoldModelsHandler._create_pool(fold_file, self._thread_count) for case, case_models in grouped_by_case_models.items(): calcers = metric_calcers[case] for model_num, model in enumerate(case_models): if self._time_split_mode: if file_num <= (model_num + len(skipped_folds)): continue elif file_num == (model_num + len(skipped_folds)): continue calcers[model_num].add(pool) for case, case_models in grouped_by_case_models.items(): calcers = metric_calcers[case] case_results = self._case_results[case] for calcer, model in zip(calcers, case_models): scores = calcer.eval_metrics() for metric in self._metric_descriptions: case_results[metric]._add(model, scores.get_result(metric))
def open(self): if self._file is None: self._file = open(self._file_path, mode='a') else: raise CatboostError("File already opened {}".format( self._file_path))
case_results[metric]._add(model, scores.get_result(metric)) @staticmethod def _fit_model(pool, case, fold_id, model_path, time_split_mode=False): from catboost import CatBoost # Learn model make_dirs_if_not_exists(FoldModelsHandler.__MODEL_DIR) feature_count = pool.num_col() if "ignored_features" in case.get_params(): ignored_features = case.get_params()["ignored_features"] if len(ignored_features ) and max(ignored_features) >= feature_count: raise CatboostError( "Error: input parameter contains feature indices wich are not available in pool: " "{}\n " "Check eval_feature set and ignored features options". format(ignored_features)) get_eval_logger().debug('Learn model {} on fold #{}'.format( str(case), fold_id)) cur_time = time.time() params = case.get_params() if time_split_mode: params['has_time'] = True instance = CatBoost(params=params) instance.fit(pool) instance.save_model(fname=model_path) get_eval_logger().debug( 'Operation was done in {} seconds'.format(time.time() - cur_time)) return FoldModel(case, model_path, fold_id)
class FoldModelsHandler(object): """ Class that is responsible for learning models and computing their metrics """ """ All models are placed to the default directory "models". """ __MODEL_DIR = 'models' @staticmethod def _remove_model_dir(): try: if os.path.exists(FoldModelsHandler.__MODEL_DIR): os.rmdir(FoldModelsHandler.__MODEL_DIR) except OSError as err: get_eval_logger().warning(str(err)) def __init__(self, metrics, cases, thread_count, eval_step, remove_models): """ Args: :param remove_models: Set true if you want models to be removed after applying them. """ self._cases = cases self._metrics = metrics self._case_results = dict() for case in self._cases: self._case_results[case] = dict() self._thread_count = thread_count self._eval_step = eval_step self._flag_remove_models = remove_models self._metric_descriptions = None def _init_case_results(self, metric_descriptions): self._metric_descriptions = metric_descriptions for case in self._cases: case_result = self._case_results[case] for metric_description in self._metric_descriptions: case_result[metric_description] = CaseEvaluationResult( case, metric_description, eval_step=self._eval_step) def _compute_metrics(self, metrics, grouped_by_case_models, learn_folds, skipped_folds, rest_folds): metric_calcers = {} for case, case_models in grouped_by_case_models.items(): metric_calcers[case] = list() for case_model in case_models: metric_calcer = case_model.create_metrics_calcer( metrics, eval_step=self._eval_step, thread_count=self._thread_count) metric_calcers[case].append(metric_calcer) if self._metric_descriptions is None: self._init_case_results( metric_calcer.metric_descriptions()) elif self._metric_descriptions != metric_calcer.metric_descriptions( ): raise CatboostError( "Error: metric names should be consistent") for file_num, fold_file in enumerate(learn_folds + skipped_folds + rest_folds): pool = FoldModelsHandler._create_pool(fold_file, self._thread_count) for case, case_models in grouped_by_case_models.items(): calcers = metric_calcers[case] for model_num, model in enumerate(case_models): if file_num != model_num: calcers[model_num].add(pool) for case, case_models in grouped_by_case_models.items(): calcers = metric_calcers[case] case_results = self._case_results[case] for calcer, model in zip(calcers, case_models): scores = calcer.eval_metrics() for metric in self._metric_descriptions: case_results[metric]._add(model, scores.get_result(metric)) @staticmethod def _fit_model(pool, case, fold_id, model_path): from catboost import CatBoost # Learn model make_dirs_if_not_exists(FoldModelsHandler.__MODEL_DIR) feature_count = pool.num_col() if "ignored_features" in case.get_params(): ignored_features = case.get_params()["ignored_features"] if len(ignored_features ) and max(ignored_features) >= feature_count: raise CatboostError( "Error: input parameter contains feature indices wich are not available in pool: " "{}\n " "Check eval_feature set and ignored features options". format(ignored_features))
def delete(self): if self._file is not None: raise CatboostError("Close file before delete") if os.path.exists(self._file_path): os.remove(self._file_path)
def eval_features(self, learn_config, features_to_eval, loss_function=None, eval_type=EvalType.SeqAdd, eval_metrics=None, thread_count=-1, eval_step=None, label_mode=LabelMode.AddFeature): """ Evaluate features. Args: learn_config: dict with params or instance of CatBoost. In second case instance params will be used objective_function: objective_function: one of CatBoost loss functions eval_type: Type of feature evaluate (All, SeqAdd, SeqRem) eval_metrics: Additional metrics to calculate thread_count: thread_count to use. If not none will override learn_config values Returns ------- result : Instance of EvaluationResult class """ features_to_eval = set(features_to_eval) if eval_metrics is None: eval_metrics = [] eval_metrics = eval_metrics if isinstance(eval_metrics, list) else [eval_metrics] if isinstance(learn_config, CatBoost): params = learn_config.get_params() else: params = dict(learn_config) if loss_function is not None: if "loss_function" in params and params["loss_function"] != loss_function: raise CatboostError("Loss function in params {} should be equal to feature evaluation objective " "function {}".format(params["loss_function"], loss_function)) else: if "loss_function" not in params: raise CatboostError("Provide loss function in params or as option to eval_features method") if thread_count is not None and thread_count != -1: params["thread_count"] = thread_count if eval_step is None: eval_step = 1 if loss_function is not None: params["loss_function"] = loss_function else: loss_function = params["loss_function"] if params["loss_function"] == "PairLogit": raise CatboostError("Pair classification is not supported") baseline_case, test_cases = self._create_eval_feature_cases(params, features_to_eval, eval_type=eval_type, label_mode=label_mode) if loss_function not in eval_metrics: eval_metrics.append(loss_function) return self.eval_cases(baseline_case=baseline_case, compare_cases=test_cases, eval_metrics=eval_metrics, thread_count=thread_count, eval_step=eval_step)
def _validate_ignored_features(ignored_features, eval_features): for eval_feature in eval_features: if eval_feature in ignored_features: raise CatboostError( "Feature {} is in ignored set and in tmp-features set at the same time" .format(eval_feature))